free
/
searx
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 11 Wiki Activity
Browse Source

[fix] bing unicode encode error - fixes #408

Adam Tauber 9 years ago
parent
b9c8039d74
commit
604f32f672
6 changed files with 18 additions and 18 deletions
Unified View Show Diff Stats
  1. 1
    1
      searx/engines/bing.py
  2. 1
    1
      searx/engines/bing_images.py
  3. 1
    1
      searx/engines/bing_news.py
  4. 4
    4
      searx/tests/engines/test_bing.py
  5. 5
    5
      searx/tests/engines/test_bing_images.py
  6. 6
    6
      searx/tests/engines/test_bing_news.py

+ 1
- 1
searx/engines/bing.py View File

52 
 def response(resp):
 52 
 def response(resp):
53 
     results = []
 53 
     results = []
54
54
55 
-    dom = html.fromstring(resp.content)
55 
+    dom = html.fromstring(resp.text)
56
56
57 
     # parse results
 57 
     # parse results
58 
     for result in dom.xpath('//div[@class="sa_cc"]'):
 58 
     for result in dom.xpath('//div[@class="sa_cc"]'):

+ 1
- 1
searx/engines/bing_images.py View File

63 
 def response(resp):
 63 
 def response(resp):
64 
     results = []
 64 
     results = []
65
65
66 
-    dom = html.fromstring(resp.content)
66 
+    dom = html.fromstring(resp.text)
67
67
68 
     # init regex for yaml-parsing
 68 
     # init regex for yaml-parsing
69 
     p = re.compile('({|,)([a-z]+):(")')
 69 
     p = re.compile('({|,)([a-z]+):(")')

+ 1
- 1
searx/engines/bing_news.py View File

68 
 def response(resp):
 68 
 def response(resp):
69 
     results = []
 69 
     results = []
70
70
71 
-    rss = etree.fromstring(resp.content)
71 
+    rss = etree.fromstring(resp.text)
72
72
73 
     ns = rss.nsmap
 73 
     ns = rss.nsmap
74
74

+ 4
- 4
searx/tests/engines/test_bing.py View File

29 
         self.assertRaises(AttributeError, bing.response, '')
 29 
         self.assertRaises(AttributeError, bing.response, '')
30 
         self.assertRaises(AttributeError, bing.response, '[]')
 30 
         self.assertRaises(AttributeError, bing.response, '[]')
31
31
32 
-        response = mock.Mock(content='<html></html>')
32 
+        response = mock.Mock(text='<html></html>')
33 
         self.assertEqual(bing.response(response), [])
 33 
         self.assertEqual(bing.response(response), [])
34
34
35 
-        response = mock.Mock(content='<html></html>')
35 
+        response = mock.Mock(text='<html></html>')
36 
         self.assertEqual(bing.response(response), [])
 36 
         self.assertEqual(bing.response(response), [])
37
37
38 
         html = """
 38 
         html = """
54 
             </div>
 54 
             </div>
55 
         </div>
 55 
         </div>
56 
         """
 56 
         """
57 
-        response = mock.Mock(content=html)
57 
+        response = mock.Mock(text=html)
58 
         results = bing.response(response)
 58 
         results = bing.response(response)
59 
         self.assertEqual(type(results), list)
 59 
         self.assertEqual(type(results), list)
60 
         self.assertEqual(len(results), 1)
 60 
         self.assertEqual(len(results), 1)
81 
             </div>
 81 
             </div>
82 
         </li>
 82 
         </li>
83 
         """
 83 
         """
84 
-        response = mock.Mock(content=html)
84 
+        response = mock.Mock(text=html)
85 
         results = bing.response(response)
 85 
         results = bing.response(response)
86 
         self.assertEqual(type(results), list)
 86 
         self.assertEqual(type(results), list)
87 
         self.assertEqual(len(results), 1)
 87 
         self.assertEqual(len(results), 1)

+ 5
- 5
searx/tests/engines/test_bing_images.py View File

31 
         self.assertRaises(AttributeError, bing_images.response, '')
 31 
         self.assertRaises(AttributeError, bing_images.response, '')
32 
         self.assertRaises(AttributeError, bing_images.response, '[]')
 32 
         self.assertRaises(AttributeError, bing_images.response, '[]')
33
33
34 
-        response = mock.Mock(content='<html></html>')
34 
+        response = mock.Mock(text='<html></html>')
35 
         self.assertEqual(bing_images.response(response), [])
 35 
         self.assertEqual(bing_images.response(response), [])
36
36
37 
-        response = mock.Mock(content='<html></html>')
37 
+        response = mock.Mock(text='<html></html>')
38 
         self.assertEqual(bing_images.response(response), [])
 38 
         self.assertEqual(bing_images.response(response), [])
39
39
40 
         html = """
 40 
         html = """
52 
         </div>
 52 
         </div>
53 
         """
 53 
         """
54 
         html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
 54 
         html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
55 
-        response = mock.Mock(content=html)
55 
+        response = mock.Mock(text=html)
56 
         results = bing_images.response(response)
 56 
         results = bing_images.response(response)
57 
         self.assertEqual(type(results), list)
 57 
         self.assertEqual(type(results), list)
58 
         self.assertEqual(len(results), 1)
 58 
         self.assertEqual(len(results), 1)
75 
             style="height:144px;" width="178" height="144"/>
 75 
             style="height:144px;" width="178" height="144"/>
76 
         </a>
 76 
         </a>
77 
         """
 77 
         """
78 
-        response = mock.Mock(content=html)
78 
+        response = mock.Mock(text=html)
79 
         results = bing_images.response(response)
 79 
         results = bing_images.response(response)
80 
         self.assertEqual(type(results), list)
 80 
         self.assertEqual(type(results), list)
81 
         self.assertEqual(len(results), 0)
 81 
         self.assertEqual(len(results), 0)
263 
         </div>
 263 
         </div>
264 
         """
 264 
         """
265 
         html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
 265 
         html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')
266 
-        response = mock.Mock(content=html)
266 
+        response = mock.Mock(text=html)
267 
         results = bing_images.response(response)
 267 
         results = bing_images.response(response)
268 
         self.assertEqual(type(results), list)
 268 
         self.assertEqual(type(results), list)
269 
         self.assertEqual(len(results), 10)
 269 
         self.assertEqual(len(results), 10)

+ 6
- 6
searx/tests/engines/test_bing_news.py View File

28 
         self.assertRaises(AttributeError, bing_news.response, '')
 28 
         self.assertRaises(AttributeError, bing_news.response, '')
29 
         self.assertRaises(AttributeError, bing_news.response, '[]')
 29 
         self.assertRaises(AttributeError, bing_news.response, '[]')
30
30
31 
-        response = mock.Mock(content='<html></html>')
31 
+        response = mock.Mock(text='<html></html>')
32 
         self.assertEqual(bing_news.response(response), [])
 32 
         self.assertEqual(bing_news.response(response), [])
33
33
34 
-        response = mock.Mock(content='<html></html>')
34 
+        response = mock.Mock(text='<html></html>')
35 
         self.assertEqual(bing_news.response(response), [])
 35 
         self.assertEqual(bing_news.response(response), [])
36
36
37 
         html = """<?xml version="1.0" encoding="utf-8" ?>
 37 
         html = """<?xml version="1.0" encoding="utf-8" ?>
66 
         </item>
 66 
         </item>
67 
     </channel>
 67 
     </channel>
68 
 </rss>"""  # noqa
 68 
 </rss>"""  # noqa
69 
-        response = mock.Mock(content=html)
69 
+        response = mock.Mock(text=html)
70 
         results = bing_news.response(response)
 70 
         results = bing_news.response(response)
71 
         self.assertEqual(type(results), list)
 71 
         self.assertEqual(type(results), list)
72 
         self.assertEqual(len(results), 2)
 72 
         self.assertEqual(len(results), 2)
105 
         </item>
 105 
         </item>
106 
     </channel>
 106 
     </channel>
107 
 </rss>"""  # noqa
 107 
 </rss>"""  # noqa
108 
-        response = mock.Mock(content=html)
108 
+        response = mock.Mock(text=html)
109 
         results = bing_news.response(response)
 109 
         results = bing_news.response(response)
110 
         self.assertEqual(type(results), list)
 110 
         self.assertEqual(type(results), list)
111 
         self.assertEqual(len(results), 1)
 111 
         self.assertEqual(len(results), 1)
128 
     </channel>
 128 
     </channel>
129 
 </rss>"""  # noqa
 129 
 </rss>"""  # noqa
130
130
131 
-        response = mock.Mock(content=html)
131 
+        response = mock.Mock(text=html)
132 
         results = bing_news.response(response)
 132 
         results = bing_news.response(response)
133 
         self.assertEqual(type(results), list)
 133 
         self.assertEqual(type(results), list)
134 
         self.assertEqual(len(results), 0)
 134 
         self.assertEqual(len(results), 0)
135
135
136 
         html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""
 136 
         html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""
137 
-        response = mock.Mock(content=html)
137 
+        response = mock.Mock(text=html)
138 
         self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)
 138 
         self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)