free
/
searx
Прати 1
Волим 0
Креирај огранак 0
Код Дискусије 0 Захтеви за спајање 0 Издања 11 Вики Activity
2365 Комити
10 Гране
Дрво: dee04b2a21
Гране Ознаке
${ item.name }
Create branch ${ searchTerm }
from 'dee04b2a21'
${ noResults }
searx/tests/unit/engines/test_wikidata.py

test_wikidata.py 20KB
Историја Датотека

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503 
  1. # -*- coding: utf-8 -*-

  2. from lxml.html import fromstring

  3. from collections import defaultdict

  4. import mock

  5. from searx.engines import wikidata

  6. from searx.testing import SearxTestCase

  7. 

  8. 

  9. class TestWikidataEngine(SearxTestCase):

  10. 

  11.     def test_request(self):

  12.         query = 'test_query'

  13.         dicto = defaultdict(dict)

  14.         dicto['language'] = 'en-US'

  15.         params = wikidata.request(query, dicto)

  16.         self.assertIn('url', params)

  17.         self.assertIn(query, params['url'])

  18.         self.assertIn('wikidata.org', params['url'])

  19.         self.assertIn('en', params['url'])

  20. 

  21.         dicto['language'] = 'es-ES'

  22.         params = wikidata.request(query, dicto)

  23.         self.assertIn(query, params['url'])

  24.         self.assertIn('es', params['url'])

  25. 

  26.     # successful cases are not tested here to avoid sending additional requests

  27.     def test_response(self):

  28.         self.assertRaises(AttributeError, wikidata.response, None)

  29.         self.assertRaises(AttributeError, wikidata.response, [])

  30.         self.assertRaises(AttributeError, wikidata.response, '')

  31.         self.assertRaises(AttributeError, wikidata.response, '[]')

  32. 

  33.         response = mock.Mock(text='<html></html>', search_params={"language": "en"})

  34.         self.assertEqual(wikidata.response(response), [])

  35. 

  36.     def test_getDetail(self):

  37.         response = {}

  38.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  39.         self.assertEqual(results, [])

  40. 

  41.         title_html = '<div><div class="wikibase-title-label">Test</div></div>'

  42.         html = """

  43.         <div>

  44.             <div class="wikibase-entitytermsview-heading-description">

  45.             </div>

  46.             <div>

  47.                 <ul class="wikibase-sitelinklistview-listview">

  48.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  49.                 </ul>

  50.             </div>

  51.         </div>

  52.         """

  53.         response = {"parse": {"displaytitle": title_html, "text": html}}

  54. 

  55.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  56.         self.assertEqual(len(results), 1)

  57.         self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')

  58. 

  59.         title_html = """

  60.         <div>

  61.             <div class="wikibase-title-label">

  62.                 <span lang="en">Test</span>

  63.                 <sup class="wb-language-fallback-indicator">English</sup>

  64.             </div>

  65.         </div>

  66.         """

  67.         html = """

  68.         <div>

  69.             <div class="wikibase-entitytermsview-heading-description">

  70.                 <span lang="en">Description</span>

  71.                 <sup class="wb-language-fallback-indicator">English</sup>

  72.             </div>

  73.             <div id="P856">

  74.                 <div class="wikibase-statementgroupview-property-label">

  75.                     <a href="/wiki/Property:P856">

  76.                         <span lang="en">official website</span>

  77.                         <sup class="wb-language-fallback-indicator">English</sup>

  78.                     </a>

  79.                 </div>

  80.                 <div class="wikibase-statementview-mainsnak">

  81.                     <a class="external free" href="https://officialsite.com">

  82.                         https://officialsite.com

  83.                     </a>

  84.                 </div>

  85.             </div>

  86.             <div>

  87.                 <ul class="wikibase-sitelinklistview-listview">

  88.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  89.                 </ul>

  90.             </div>

  91.         </div>

  92.         """

  93.         response = {"parse": {"displaytitle": title_html, "text": html}}

  94. 

  95.         results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")

  96.         self.assertEqual(len(results), 2)

  97.         self.assertEqual(results[0]['title'], 'Official website')

  98.         self.assertEqual(results[0]['url'], 'https://officialsite.com')

  99. 

  100.         self.assertEqual(results[1]['infobox'], 'Test')

  101.         self.assertEqual(results[1]['id'], None)

  102.         self.assertEqual(results[1]['content'], 'Description')

  103.         self.assertEqual(results[1]['attributes'], [])

  104.         self.assertEqual(results[1]['urls'][0]['title'], 'Official website')

  105.         self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')

  106.         self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')

  107.         self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')

  108. 

  109.     def test_add_image(self):

  110.         image_src = wikidata.add_image(fromstring("<div></div>"))

  111.         self.assertEqual(image_src, None)

  112. 

  113.         html = u"""

  114.         <div>

  115.             <div id="P18">

  116.                 <div class="wikibase-statementgroupview-property-label">

  117.                     <a href="/wiki/Property:P18">

  118.                         image

  119.                     </a>

  120.                 </div>

  121.                 <div class="wikibase-statementlistview">

  122.                     <div class="wikibase-statementview listview-item">

  123.                         <div class="wikibase-statementview-rankselector">

  124.                             <span class="wikibase-rankselector-normal"></span>

  125.                         </div>

  126.                         <div class="wikibase-statementview-mainsnak">

  127.                             <div>

  128.                                 <div class="wikibase-snakview-value">

  129.                                     <a href="https://commons.wikimedia.org/wiki/File:image.png">

  130.                                         image.png

  131.                                     </a>

  132.                                 </div>

  133.                             </div>

  134.                         </div>

  135.                     </div>

  136.                 </div>

  137.             </div>

  138.         </div>

  139.         """

  140.         html_etree = fromstring(html)

  141. 

  142.         image_src = wikidata.add_image(html_etree)

  143.         self.assertEqual(image_src,

  144.                          "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")

  145. 

  146.         html = u"""

  147.         <div>

  148.             <div id="P2910">

  149.                 <div class="wikibase-statementgroupview-property-label">

  150.                     <a href="/wiki/Property:P2910">

  151.                         icon

  152.                     </a>

  153.                 </div>

  154.                 <div class="wikibase-statementlistview">

  155.                     <div class="wikibase-statementview listview-item">

  156.                         <div class="wikibase-statementview-rankselector">

  157.                             <span class="wikibase-rankselector-normal"></span>

  158.                         </div>

  159.                         <div class="wikibase-statementview-mainsnak">

  160.                             <div>

  161.                                 <div class="wikibase-snakview-value">

  162.                                     <a href="https://commons.wikimedia.org/wiki/File:icon.png">

  163.                                         icon.png

  164.                                     </a>

  165.                                 </div>

  166.                             </div>

  167.                         </div>

  168.                     </div>

  169.                 </div>

  170.             </div>

  171.             <div id="P154">

  172.                 <div class="wikibase-statementgroupview-property-label">

  173.                     <a href="/wiki/Property:P154">

  174.                         logo

  175.                     </a>

  176.                 </div>

  177.                 <div class="wikibase-statementlistview">

  178.                     <div class="wikibase-statementview listview-item">

  179.                         <div class="wikibase-statementview-rankselector">

  180.                             <span class="wikibase-rankselector-normal"></span>

  181.                         </div>

  182.                         <div class="wikibase-statementview-mainsnak">

  183.                             <div>

  184.                                 <div class="wikibase-snakview-value">

  185.                                     <a href="https://commons.wikimedia.org/wiki/File:logo.png">

  186.                                         logo.png

  187.                                     </a>

  188.                                 </div>

  189.                             </div>

  190.                         </div>

  191.                     </div>

  192.                 </div>

  193.             </div>

  194.         </div>

  195.         """

  196.         html_etree = fromstring(html)

  197. 

  198.         image_src = wikidata.add_image(html_etree)

  199.         self.assertEqual(image_src,

  200.                          "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")

  201. 

  202.     def test_add_attribute(self):

  203.         html = u"""

  204.         <div>

  205.             <div id="P27">

  206.                 <div class="wikibase-statementgroupview-property-label">

  207.                     <a href="/wiki/Property:P27">

  208.                         country of citizenship

  209.                     </a>

  210.                 </div>

  211.                 <div class="wikibase-statementlistview">

  212.                     <div class="wikibase-statementview listview-item">

  213.                         <div class="wikibase-statementview-rankselector">

  214.                             <span class="wikibase-rankselector-normal"></span>

  215.                         </div>

  216.                         <div class="wikibase-statementview-mainsnak">

  217.                             <div>

  218.                                 <div class="wikibase-snakview-value">

  219.                                     <a href="/wiki/Q145">

  220.                                         United Kingdom

  221.                                     </a>

  222.                                 </div>

  223.                             </div>

  224.                         </div>

  225.                     </div>

  226.                 </div>

  227.             </div>

  228.         </div>

  229.         """

  230.         attributes = []

  231.         html_etree = fromstring(html)

  232. 

  233.         wikidata.add_attribute(attributes, html_etree, "Fail")

  234.         self.assertEqual(attributes, [])

  235. 

  236.         wikidata.add_attribute(attributes, html_etree, "P27")

  237.         self.assertEqual(len(attributes), 1)

  238.         self.assertEqual(attributes[0]["label"], "Country of citizenship")

  239.         self.assertEqual(attributes[0]["value"], "United Kingdom")

  240. 

  241.         html = u"""

  242.         <div>

  243.             <div id="P569">

  244.                 <div class="wikibase-statementgroupview-property-label">

  245.                     <a href="/wiki/Property:P569">

  246.                         date of birth

  247.                     </a>

  248.                 </div>

  249.                 <div class="wikibase-statementlistview">

  250.                     <div class="wikibase-statementview listview-item">

  251.                         <div class="wikibase-statementview-rankselector">

  252.                             <span class="wikibase-rankselector-normal"></span>

  253.                         </div>

  254.                         <div class="wikibase-statementview-mainsnak">

  255.                             <div>

  256.                                 <div class="wikibase-snakview-value">

  257.                                     27 January 1832

  258.                                     <sup class="wb-calendar-name">

  259.                                         Gregorian

  260.                                     </sup>

  261.                                 </div>

  262.                             </div>

  263.                         </div>

  264.                     </div>

  265.                 </div>

  266.             </div>

  267.         </div>

  268.         """

  269.         attributes = []

  270.         html_etree = fromstring(html)

  271.         wikidata.add_attribute(attributes, html_etree, "P569", date=True)

  272.         self.assertEqual(len(attributes), 1)

  273.         self.assertEqual(attributes[0]["label"], "Date of birth")

  274.         self.assertEqual(attributes[0]["value"], "27 January 1832")

  275. 

  276.         html = u"""

  277.         <div>

  278.             <div id="P6">

  279.                 <div class="wikibase-statementgroupview-property-label">

  280.                     <a href="/wiki/Property:P27">

  281.                         head of government

  282.                     </a>

  283.                 </div>

  284.                 <div class="wikibase-statementlistview">

  285.                     <div class="wikibase-statementview listview-item">

  286.                         <div class="wikibase-statementview-rankselector">

  287.                             <span class="wikibase-rankselector-normal"></span>

  288.                         </div>

  289.                         <div class="wikibase-statementview-mainsnak">

  290.                             <div>

  291.                                 <div class="wikibase-snakview-value">

  292.                                     <a href="/wiki/Q206">

  293.                                         Old Prime Minister

  294.                                     </a>

  295.                                 </div>

  296.                             </div>

  297.                         </div>

  298.                     </div>

  299.                     <div class="wikibase-statementview listview-item">

  300.                         <div class="wikibase-statementview-rankselector">

  301.                             <span class="wikibase-rankselector-preferred"></span>

  302.                         </div>

  303.                         <div class="wikibase-statementview-mainsnak">

  304.                             <div>

  305.                                 <div class="wikibase-snakview-value">

  306.                                     <a href="/wiki/Q3099714">

  307.                                         Actual Prime Minister

  308.                                     </a>

  309.                                 </div>

  310.                             </div>

  311.                         </div>

  312.                     </div>

  313.                 </div>

  314.             </div>

  315.         </div>

  316.         """

  317.         attributes = []

  318.         html_etree = fromstring(html)

  319.         wikidata.add_attribute(attributes, html_etree, "P6")

  320.         self.assertEqual(len(attributes), 1)

  321.         self.assertEqual(attributes[0]["label"], "Head of government")

  322.         self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")

  323. 

  324.         attributes = []

  325.         html_etree = fromstring(html)

  326.         wikidata.add_attribute(attributes, html_etree, "P6", trim=True)

  327.         self.assertEqual(len(attributes), 1)

  328.         self.assertEqual(attributes[0]["value"], "Actual Prime Minister")

  329. 

  330.     def test_add_url(self):

  331.         html = u"""

  332.         <div>

  333.             <div id="P856">

  334.                 <div class="wikibase-statementgroupview-property-label">

  335.                     <a href="/wiki/Property:P856">

  336.                         official website

  337.                     </a>

  338.                 </div>

  339.                 <div class="wikibase-statementlistview">

  340.                     <div class="wikibase-statementview listview-item">

  341.                         <div class="wikibase-statementview-mainsnak">

  342.                             <div>

  343.                                 <div class="wikibase-snakview-value">

  344.                                     <a class="external free" href="https://searx.me">

  345.                                         https://searx.me/

  346.                                     </a>

  347.                                 </div>

  348.                             </div>

  349.                         </div>

  350.                     </div>

  351.                 </div>

  352.             </div>

  353.         </div>

  354.         """

  355.         urls = []

  356.         html_etree = fromstring(html)

  357.         wikidata.add_url(urls, html_etree, 'P856')

  358.         self.assertEquals(len(urls), 1)

  359.         self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)

  360.         urls = []

  361.         results = []

  362.         wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)

  363.         self.assertEquals(len(urls), 1)

  364.         self.assertEquals(len(results), 1)

  365.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)

  366.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, results)

  367. 

  368.         html = u"""

  369.         <div>

  370.             <div id="P856">

  371.                 <div class="wikibase-statementgroupview-property-label">

  372.                     <a href="/wiki/Property:P856">

  373.                         official website

  374.                     </a>

  375.                 </div>

  376.                 <div class="wikibase-statementlistview">

  377.                     <div class="wikibase-statementview listview-item">

  378.                         <div class="wikibase-statementview-mainsnak">

  379.                             <div>

  380.                                 <div class="wikibase-snakview-value">

  381.                                     <a class="external free" href="http://www.worldofwarcraft.com">

  382.                                         http://www.worldofwarcraft.com

  383.                                     </a>

  384.                                 </div>

  385.                             </div>

  386.                         </div>

  387.                     </div>

  388.                     <div class="wikibase-statementview listview-item">

  389.                         <div class="wikibase-statementview-mainsnak">

  390.                             <div>

  391.                                 <div class="wikibase-snakview-value">

  392.                                     <a class="external free" href="http://eu.battle.net/wow/en/">

  393.                                         http://eu.battle.net/wow/en/

  394.                                     </a>

  395.                                 </div>

  396.                             </div>

  397.                         </div>

  398.                     </div>

  399.                 </div>

  400.             </div>

  401.         </div>

  402.         """

  403.         urls = []

  404.         html_etree = fromstring(html)

  405.         wikidata.add_url(urls, html_etree, 'P856')

  406.         self.assertEquals(len(urls), 2)

  407.         self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)

  408.         self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)

  409. 

  410.     def test_get_imdblink(self):

  411.         html = u"""

  412.         <div>

  413.             <div class="wikibase-statementview-mainsnak">

  414.                 <div>

  415.                     <div class="wikibase-snakview-value">

  416.                         <a class="wb-external-id" href="http://www.imdb.com/tt0433664">

  417.                             tt0433664

  418.                         </a>

  419.                     </div>

  420.                 </div>

  421.             </div>

  422.         </div>

  423.         """

  424.         html_etree = fromstring(html)

  425.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  426. 

  427.         html = u"""

  428.         <div>

  429.             <div class="wikibase-statementview-mainsnak">

  430.                 <div>

  431.                     <div class="wikibase-snakview-value">

  432.                         <a class="wb-external-id"

  433.                            href="href="http://tools.wmflabs.org/...http://www.imdb.com/&id=nm4915994"">

  434.                             nm4915994

  435.                         </a>

  436.                     </div>

  437.                 </div>

  438.             </div>

  439.         </div>

  440.         """

  441.         html_etree = fromstring(html)

  442.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  443.         self.assertIn('https://www.imdb.com/name/nm4915994', imdblink)

  444. 

  445.     def test_get_geolink(self):

  446.         html = u"""

  447.         <div>

  448.             <div class="wikibase-statementview-mainsnak">

  449.                 <div>

  450.                     <div class="wikibase-snakview-value">

  451.                         60°N, 40°E

  452.                     </div>

  453.                 </div>

  454.             </div>

  455.         </div>

  456.         """

  457.         html_etree = fromstring(html)

  458.         geolink = wikidata.get_geolink(html_etree)

  459.         self.assertIn('https://www.openstreetmap.org/', geolink)

  460.         self.assertIn('lat=60&lon=40', geolink)

  461. 

  462.         html = u"""

  463.         <div>

  464.             <div class="wikibase-statementview-mainsnak">

  465.                 <div>

  466.                     <div class="wikibase-snakview-value">

  467.                         34°35'59"S, 58°22'55"W

  468.                     </div>

  469.                 </div>

  470.             </div>

  471.         </div>

  472.         """

  473.         html_etree = fromstring(html)

  474.         geolink = wikidata.get_geolink(html_etree)

  475.         self.assertIn('https://www.openstreetmap.org/', geolink)

  476.         self.assertIn('lat=-34.59', geolink)

  477.         self.assertIn('lon=-58.38', geolink)

  478. 

  479.     def test_get_wikilink(self):

  480.         html = """

  481.         <div>

  482.             <div>

  483.                 <ul class="wikibase-sitelinklistview-listview">

  484.                     <li data-wb-siteid="arwiki"><a href="http://ar.wikipedia.org/wiki/Test">Test</a></li>

  485.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  486.                 </ul>

  487.             </div>

  488.             <div>

  489.                 <ul class="wikibase-sitelinklistview-listview">

  490.                     <li data-wb-siteid="enwikiquote"><a href="https://en.wikiquote.org/wiki/Test">Test</a></li>

  491.                 </ul>

  492.             </div>

  493.         </div>

  494.         """

  495.         html_etree = fromstring(html)

  496.         wikilink = wikidata.get_wikilink(html_etree, 'nowiki')

  497.         self.assertEqual(wikilink, None)

  498.         wikilink = wikidata.get_wikilink(html_etree, 'enwiki')

  499.         self.assertEqual(wikilink, 'https://en.wikipedia.org/wiki/Test')

  500.         wikilink = wikidata.get_wikilink(html_etree, 'arwiki')

  501.         self.assertEqual(wikilink, 'https://ar.wikipedia.org/wiki/Test')

  502.         wikilink = wikidata.get_wikilink(html_etree, 'enwikiquote')

  503.         self.assertEqual(wikilink, 'https://en.wikiquote.org/wiki/Test')