|
@@ -55,10 +55,14 @@ def response(resp):
|
55
|
55
|
|
56
|
56
|
# parse results
|
57
|
57
|
for tweet in dom.xpath(results_xpath):
|
58
|
|
- link = tweet.xpath(link_xpath)[0]
|
|
58
|
+ try:
|
|
59
|
+ link = tweet.xpath(link_xpath)[0]
|
|
60
|
+ content = extract_text(tweet.xpath(content_xpath)[0])
|
|
61
|
+ except Exception:
|
|
62
|
+ continue
|
|
63
|
+
|
59
|
64
|
url = urljoin(base_url, link.attrib.get('href'))
|
60
|
65
|
title = extract_text(tweet.xpath(title_xpath))
|
61
|
|
- content = extract_text(tweet.xpath(content_xpath)[0])
|
62
|
66
|
|
63
|
67
|
pubdate = tweet.xpath(timestamp_xpath)
|
64
|
68
|
if len(pubdate) > 0:
|