free
/
searx
Volgen 1
Ster 0
Vork 0
Code Kwesties 0 Pull-aanvragen 0 Publicaties 11 Wiki Activiteit
1800 Commits
10 Branches
Tree: 32604be790
Branches Labels
${ item.name }
Maak branch ${ searchTerm }
van '32604be790'
${ noResults }
searx/tests/unit/engines/test_wikidata.py

test_wikidata.py 20KB
Geschiedenis Ruw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504 
  1. # -*- coding: utf-8 -*-

  2. from json import loads

  3. from lxml.html import fromstring

  4. from collections import defaultdict

  5. import mock

  6. from searx.engines import wikidata

  7. from searx.testing import SearxTestCase

  8. 

  9. 

  10. class TestWikidataEngine(SearxTestCase):

  11. 

  12.     def test_request(self):

  13.         query = 'test_query'

  14.         dicto = defaultdict(dict)

  15.         dicto['language'] = 'all'

  16.         params = wikidata.request(query, dicto)

  17.         self.assertIn('url', params)

  18.         self.assertIn(query, params['url'])

  19.         self.assertIn('wikidata.org', params['url'])

  20.         self.assertIn('en', params['url'])

  21. 

  22.         dicto['language'] = 'es_ES'

  23.         params = wikidata.request(query, dicto)

  24.         self.assertIn(query, params['url'])

  25.         self.assertIn('es', params['url'])

  26. 

  27.     # successful cases are not tested here to avoid sending additional requests

  28.     def test_response(self):

  29.         self.assertRaises(AttributeError, wikidata.response, None)

  30.         self.assertRaises(AttributeError, wikidata.response, [])

  31.         self.assertRaises(AttributeError, wikidata.response, '')

  32.         self.assertRaises(AttributeError, wikidata.response, '[]')

  33. 

  34.         response = mock.Mock(content='<html></html>', search_params={"language": "all"})

  35.         self.assertEqual(wikidata.response(response), [])

  36. 

  37.     def test_getDetail(self):

  38.         response = {}

  39.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  40.         self.assertEqual(results, [])

  41. 

  42.         title_html = '<div><div class="wikibase-title-label">Test</div></div>'

  43.         html = """

  44.         <div>

  45.             <div class="wikibase-entitytermsview-heading-description">

  46.             </div>

  47.             <div>

  48.                 <ul class="wikibase-sitelinklistview-listview">

  49.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  50.                 </ul>

  51.             </div>

  52.         </div>

  53.         """

  54.         response = {"parse": {"displaytitle": title_html, "text": html}}

  55. 

  56.         results = wikidata.getDetail(response, "Q123", "en", "en-US")

  57.         self.assertEqual(len(results), 1)

  58.         self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')

  59. 

  60.         title_html = """

  61.         <div>

  62.             <div class="wikibase-title-label">

  63.                 <span lang="en">Test</span>

  64.                 <sup class="wb-language-fallback-indicator">English</sup>

  65.             </div>

  66.         </div>

  67.         """

  68.         html = """

  69.         <div>

  70.             <div class="wikibase-entitytermsview-heading-description">

  71.                 <span lang="en">Description</span>

  72.                 <sup class="wb-language-fallback-indicator">English</sup>

  73.             </div>

  74.             <div id="P856">

  75.                 <div class="wikibase-statementgroupview-property-label">

  76.                     <a href="/wiki/Property:P856">

  77.                         <span lang="en">official website</span>

  78.                         <sup class="wb-language-fallback-indicator">English</sup>

  79.                     </a>

  80.                 </div>

  81.                 <div class="wikibase-statementview-mainsnak">

  82.                     <a class="external free" href="https://officialsite.com">

  83.                         https://officialsite.com

  84.                     </a>

  85.                 </div>

  86.             </div>

  87.             <div>

  88.                 <ul class="wikibase-sitelinklistview-listview">

  89.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  90.                 </ul>

  91.             </div>

  92.         </div>

  93.         """

  94.         response = {"parse": {"displaytitle": title_html, "text": html}}

  95. 

  96.         results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")

  97.         self.assertEqual(len(results), 2)

  98.         self.assertEqual(results[0]['title'], 'Official website')

  99.         self.assertEqual(results[0]['url'], 'https://officialsite.com')

  100. 

  101.         self.assertEqual(results[1]['infobox'], 'Test')

  102.         self.assertEqual(results[1]['id'], None)

  103.         self.assertEqual(results[1]['content'], 'Description')

  104.         self.assertEqual(results[1]['attributes'], [])

  105.         self.assertEqual(results[1]['urls'][0]['title'], 'Official website')

  106.         self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')

  107.         self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')

  108.         self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')

  109. 

  110.     def test_add_image(self):

  111.         image_src = wikidata.add_image(fromstring("<div></div>"))

  112.         self.assertEqual(image_src, None)

  113. 

  114.         html = u"""

  115.         <div>

  116.             <div id="P18">

  117.                 <div class="wikibase-statementgroupview-property-label">

  118.                     <a href="/wiki/Property:P18">

  119.                         image

  120.                     </a>

  121.                 </div>

  122.                 <div class="wikibase-statementlistview">

  123.                     <div class="wikibase-statementview listview-item">

  124.                         <div class="wikibase-statementview-rankselector">

  125.                             <span class="wikibase-rankselector-normal"></span>

  126.                         </div>

  127.                         <div class="wikibase-statementview-mainsnak">

  128.                             <div>

  129.                                 <div class="wikibase-snakview-value">

  130.                                     <a href="https://commons.wikimedia.org/wiki/File:image.png">

  131.                                         image.png

  132.                                     </a>

  133.                                 </div>

  134.                             </div>

  135.                         </div>

  136.                     </div>

  137.                 </div>

  138.             </div>

  139.         </div>

  140.         """

  141.         html_etree = fromstring(html)

  142. 

  143.         image_src = wikidata.add_image(html_etree)

  144.         self.assertEqual(image_src,

  145.                          "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")

  146. 

  147.         html = u"""

  148.         <div>

  149.             <div id="P2910">

  150.                 <div class="wikibase-statementgroupview-property-label">

  151.                     <a href="/wiki/Property:P2910">

  152.                         icon

  153.                     </a>

  154.                 </div>

  155.                 <div class="wikibase-statementlistview">

  156.                     <div class="wikibase-statementview listview-item">

  157.                         <div class="wikibase-statementview-rankselector">

  158.                             <span class="wikibase-rankselector-normal"></span>

  159.                         </div>

  160.                         <div class="wikibase-statementview-mainsnak">

  161.                             <div>

  162.                                 <div class="wikibase-snakview-value">

  163.                                     <a href="https://commons.wikimedia.org/wiki/File:icon.png">

  164.                                         icon.png

  165.                                     </a>

  166.                                 </div>

  167.                             </div>

  168.                         </div>

  169.                     </div>

  170.                 </div>

  171.             </div>

  172.             <div id="P154">

  173.                 <div class="wikibase-statementgroupview-property-label">

  174.                     <a href="/wiki/Property:P154">

  175.                         logo

  176.                     </a>

  177.                 </div>

  178.                 <div class="wikibase-statementlistview">

  179.                     <div class="wikibase-statementview listview-item">

  180.                         <div class="wikibase-statementview-rankselector">

  181.                             <span class="wikibase-rankselector-normal"></span>

  182.                         </div>

  183.                         <div class="wikibase-statementview-mainsnak">

  184.                             <div>

  185.                                 <div class="wikibase-snakview-value">

  186.                                     <a href="https://commons.wikimedia.org/wiki/File:logo.png">

  187.                                         logo.png

  188.                                     </a>

  189.                                 </div>

  190.                             </div>

  191.                         </div>

  192.                     </div>

  193.                 </div>

  194.             </div>

  195.         </div>

  196.         """

  197.         html_etree = fromstring(html)

  198. 

  199.         image_src = wikidata.add_image(html_etree)

  200.         self.assertEqual(image_src,

  201.                          "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")

  202. 

  203.     def test_add_attribute(self):

  204.         html = u"""

  205.         <div>

  206.             <div id="P27">

  207.                 <div class="wikibase-statementgroupview-property-label">

  208.                     <a href="/wiki/Property:P27">

  209.                         country of citizenship

  210.                     </a>

  211.                 </div>

  212.                 <div class="wikibase-statementlistview">

  213.                     <div class="wikibase-statementview listview-item">

  214.                         <div class="wikibase-statementview-rankselector">

  215.                             <span class="wikibase-rankselector-normal"></span>

  216.                         </div>

  217.                         <div class="wikibase-statementview-mainsnak">

  218.                             <div>

  219.                                 <div class="wikibase-snakview-value">

  220.                                     <a href="/wiki/Q145">

  221.                                         United Kingdom

  222.                                     </a>

  223.                                 </div>

  224.                             </div>

  225.                         </div>

  226.                     </div>

  227.                 </div>

  228.             </div>

  229.         </div>

  230.         """

  231.         attributes = []

  232.         html_etree = fromstring(html)

  233. 

  234.         wikidata.add_attribute(attributes, html_etree, "Fail")

  235.         self.assertEqual(attributes, [])

  236. 

  237.         wikidata.add_attribute(attributes, html_etree, "P27")

  238.         self.assertEqual(len(attributes), 1)

  239.         self.assertEqual(attributes[0]["label"], "Country of citizenship")

  240.         self.assertEqual(attributes[0]["value"], "United Kingdom")

  241. 

  242.         html = u"""

  243.         <div>

  244.             <div id="P569">

  245.                 <div class="wikibase-statementgroupview-property-label">

  246.                     <a href="/wiki/Property:P569">

  247.                         date of birth

  248.                     </a>

  249.                 </div>

  250.                 <div class="wikibase-statementlistview">

  251.                     <div class="wikibase-statementview listview-item">

  252.                         <div class="wikibase-statementview-rankselector">

  253.                             <span class="wikibase-rankselector-normal"></span>

  254.                         </div>

  255.                         <div class="wikibase-statementview-mainsnak">

  256.                             <div>

  257.                                 <div class="wikibase-snakview-value">

  258.                                     27 January 1832

  259.                                     <sup class="wb-calendar-name">

  260.                                         Gregorian

  261.                                     </sup>

  262.                                 </div>

  263.                             </div>

  264.                         </div>

  265.                     </div>

  266.                 </div>

  267.             </div>

  268.         </div>

  269.         """

  270.         attributes = []

  271.         html_etree = fromstring(html)

  272.         wikidata.add_attribute(attributes, html_etree, "P569", date=True)

  273.         self.assertEqual(len(attributes), 1)

  274.         self.assertEqual(attributes[0]["label"], "Date of birth")

  275.         self.assertEqual(attributes[0]["value"], "27 January 1832")

  276. 

  277.         html = u"""

  278.         <div>

  279.             <div id="P6">

  280.                 <div class="wikibase-statementgroupview-property-label">

  281.                     <a href="/wiki/Property:P27">

  282.                         head of government

  283.                     </a>

  284.                 </div>

  285.                 <div class="wikibase-statementlistview">

  286.                     <div class="wikibase-statementview listview-item">

  287.                         <div class="wikibase-statementview-rankselector">

  288.                             <span class="wikibase-rankselector-normal"></span>

  289.                         </div>

  290.                         <div class="wikibase-statementview-mainsnak">

  291.                             <div>

  292.                                 <div class="wikibase-snakview-value">

  293.                                     <a href="/wiki/Q206">

  294.                                         Old Prime Minister

  295.                                     </a>

  296.                                 </div>

  297.                             </div>

  298.                         </div>

  299.                     </div>

  300.                     <div class="wikibase-statementview listview-item">

  301.                         <div class="wikibase-statementview-rankselector">

  302.                             <span class="wikibase-rankselector-preferred"></span>

  303.                         </div>

  304.                         <div class="wikibase-statementview-mainsnak">

  305.                             <div>

  306.                                 <div class="wikibase-snakview-value">

  307.                                     <a href="/wiki/Q3099714">

  308.                                         Actual Prime Minister

  309.                                     </a>

  310.                                 </div>

  311.                             </div>

  312.                         </div>

  313.                     </div>

  314.                 </div>

  315.             </div>

  316.         </div>

  317.         """

  318.         attributes = []

  319.         html_etree = fromstring(html)

  320.         wikidata.add_attribute(attributes, html_etree, "P6")

  321.         self.assertEqual(len(attributes), 1)

  322.         self.assertEqual(attributes[0]["label"], "Head of government")

  323.         self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")

  324. 

  325.         attributes = []

  326.         html_etree = fromstring(html)

  327.         wikidata.add_attribute(attributes, html_etree, "P6", trim=True)

  328.         self.assertEqual(len(attributes), 1)

  329.         self.assertEqual(attributes[0]["value"], "Actual Prime Minister")

  330. 

  331.     def test_add_url(self):

  332.         html = u"""

  333.         <div>

  334.             <div id="P856">

  335.                 <div class="wikibase-statementgroupview-property-label">

  336.                     <a href="/wiki/Property:P856">

  337.                         official website

  338.                     </a>

  339.                 </div>

  340.                 <div class="wikibase-statementlistview">

  341.                     <div class="wikibase-statementview listview-item">

  342.                         <div class="wikibase-statementview-mainsnak">

  343.                             <div>

  344.                                 <div class="wikibase-snakview-value">

  345.                                     <a class="external free" href="https://searx.me">

  346.                                         https://searx.me/

  347.                                     </a>

  348.                                 </div>

  349.                             </div>

  350.                         </div>

  351.                     </div>

  352.                 </div>

  353.             </div>

  354.         </div>

  355.         """

  356.         urls = []

  357.         html_etree = fromstring(html)

  358.         wikidata.add_url(urls, html_etree, 'P856')

  359.         self.assertEquals(len(urls), 1)

  360.         self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)

  361.         urls = []

  362.         results = []

  363.         wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)

  364.         self.assertEquals(len(urls), 1)

  365.         self.assertEquals(len(results), 1)

  366.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)

  367.         self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, results)

  368. 

  369.         html = u"""

  370.         <div>

  371.             <div id="P856">

  372.                 <div class="wikibase-statementgroupview-property-label">

  373.                     <a href="/wiki/Property:P856">

  374.                         official website

  375.                     </a>

  376.                 </div>

  377.                 <div class="wikibase-statementlistview">

  378.                     <div class="wikibase-statementview listview-item">

  379.                         <div class="wikibase-statementview-mainsnak">

  380.                             <div>

  381.                                 <div class="wikibase-snakview-value">

  382.                                     <a class="external free" href="http://www.worldofwarcraft.com">

  383.                                         http://www.worldofwarcraft.com

  384.                                     </a>

  385.                                 </div>

  386.                             </div>

  387.                         </div>

  388.                     </div>

  389.                     <div class="wikibase-statementview listview-item">

  390.                         <div class="wikibase-statementview-mainsnak">

  391.                             <div>

  392.                                 <div class="wikibase-snakview-value">

  393.                                     <a class="external free" href="http://eu.battle.net/wow/en/">

  394.                                         http://eu.battle.net/wow/en/

  395.                                     </a>

  396.                                 </div>

  397.                             </div>

  398.                         </div>

  399.                     </div>

  400.                 </div>

  401.             </div>

  402.         </div>

  403.         """

  404.         urls = []

  405.         html_etree = fromstring(html)

  406.         wikidata.add_url(urls, html_etree, 'P856')

  407.         self.assertEquals(len(urls), 2)

  408.         self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)

  409.         self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)

  410. 

  411.     def test_get_imdblink(self):

  412.         html = u"""

  413.         <div>

  414.             <div class="wikibase-statementview-mainsnak">

  415.                 <div>

  416.                     <div class="wikibase-snakview-value">

  417.                         <a class="wb-external-id" href="http://www.imdb.com/tt0433664">

  418.                             tt0433664

  419.                         </a>

  420.                     </div>

  421.                 </div>

  422.             </div>

  423.         </div>

  424.         """

  425.         html_etree = fromstring(html)

  426.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  427. 

  428.         html = u"""

  429.         <div>

  430.             <div class="wikibase-statementview-mainsnak">

  431.                 <div>

  432.                     <div class="wikibase-snakview-value">

  433.                         <a class="wb-external-id"

  434.                            href="href="http://tools.wmflabs.org/...http://www.imdb.com/&id=nm4915994"">

  435.                             nm4915994

  436.                         </a>

  437.                     </div>

  438.                 </div>

  439.             </div>

  440.         </div>

  441.         """

  442.         html_etree = fromstring(html)

  443.         imdblink = wikidata.get_imdblink(html_etree, 'https://www.imdb.com/')

  444.         self.assertIn('https://www.imdb.com/name/nm4915994', imdblink)

  445. 

  446.     def test_get_geolink(self):

  447.         html = u"""

  448.         <div>

  449.             <div class="wikibase-statementview-mainsnak">

  450.                 <div>

  451.                     <div class="wikibase-snakview-value">

  452.                         60°N, 40°E

  453.                     </div>

  454.                 </div>

  455.             </div>

  456.         </div>

  457.         """

  458.         html_etree = fromstring(html)

  459.         geolink = wikidata.get_geolink(html_etree)

  460.         self.assertIn('https://www.openstreetmap.org/', geolink)

  461.         self.assertIn('lat=60&lon=40', geolink)

  462. 

  463.         html = u"""

  464.         <div>

  465.             <div class="wikibase-statementview-mainsnak">

  466.                 <div>

  467.                     <div class="wikibase-snakview-value">

  468.                         34°35'59"S, 58°22'55"W

  469.                     </div>

  470.                 </div>

  471.             </div>

  472.         </div>

  473.         """

  474.         html_etree = fromstring(html)

  475.         geolink = wikidata.get_geolink(html_etree)

  476.         self.assertIn('https://www.openstreetmap.org/', geolink)

  477.         self.assertIn('lat=-34.59', geolink)

  478.         self.assertIn('lon=-58.38', geolink)

  479. 

  480.     def test_get_wikilink(self):

  481.         html = """

  482.         <div>

  483.             <div>

  484.                 <ul class="wikibase-sitelinklistview-listview">

  485.                     <li data-wb-siteid="arwiki"><a href="http://ar.wikipedia.org/wiki/Test">Test</a></li>

  486.                     <li data-wb-siteid="enwiki"><a href="http://en.wikipedia.org/wiki/Test">Test</a></li>

  487.                 </ul>

  488.             </div>

  489.             <div>

  490.                 <ul class="wikibase-sitelinklistview-listview">

  491.                     <li data-wb-siteid="enwikiquote"><a href="https://en.wikiquote.org/wiki/Test">Test</a></li>

  492.                 </ul>

  493.             </div>

  494.         </div>

  495.         """

  496.         html_etree = fromstring(html)

  497.         wikilink = wikidata.get_wikilink(html_etree, 'nowiki')

  498.         self.assertEqual(wikilink, None)

  499.         wikilink = wikidata.get_wikilink(html_etree, 'enwiki')

  500.         self.assertEqual(wikilink, 'https://en.wikipedia.org/wiki/Test')

  501.         wikilink = wikidata.get_wikilink(html_etree, 'arwiki')

  502.         self.assertEqual(wikilink, 'https://ar.wikipedia.org/wiki/Test')

  503.         wikilink = wikidata.get_wikilink(html_etree, 'enwikiquote')

  504.         self.assertEqual(wikilink, 'https://en.wikiquote.org/wiki/Test')