Browse Source

add digbt engine

Unfortunately, it is quite slow so it is disabled.
Furthermore, the display of number of files is wrong
on digbt.org, so it is not displayed on searx.
Noemi Vanyi 8 years ago
parent
commit
3a1c5876b1
4 changed files with 84 additions and 15 deletions
  1. 2
    14
      searx/engines/btdigg.py
  2. 58
    0
      searx/engines/digbt.py
  3. 6
    1
      searx/settings.yml
  4. 18
    0
      searx/utils.py

+ 2
- 14
searx/engines/btdigg.py View File

16
 from lxml import html
16
 from lxml import html
17
 from operator import itemgetter
17
 from operator import itemgetter
18
 from searx.engines.xpath import extract_text
18
 from searx.engines.xpath import extract_text
19
+from searx.utils import get_torrent_size
19
 
20
 
20
 # engine dependent config
21
 # engine dependent config
21
 categories = ['videos', 'music', 'files']
22
 categories = ['videos', 'music', 'files']
68
         leech = 0
69
         leech = 0
69
 
70
 
70
         # convert filesize to byte if possible
71
         # convert filesize to byte if possible
71
-        try:
72
-            filesize = float(filesize)
73
-
74
-            # convert filesize to byte
75
-            if filesize_multiplier == 'TB':
76
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
77
-            elif filesize_multiplier == 'GB':
78
-                filesize = int(filesize * 1024 * 1024 * 1024)
79
-            elif filesize_multiplier == 'MB':
80
-                filesize = int(filesize * 1024 * 1024)
81
-            elif filesize_multiplier == 'KB':
82
-                filesize = int(filesize * 1024)
83
-        except:
84
-            filesize = None
72
+        filesize = get_torrent_size(filesize, filesize_multiplier)
85
 
73
 
86
         # convert files to int if possible
74
         # convert files to int if possible
87
         if files.isdigit():
75
         if files.isdigit():

+ 58
- 0
searx/engines/digbt.py View File

1
+"""
2
+ DigBT (Videos, Music, Files)
3
+
4
+ @website     https://digbt.org
5
+ @provide-api no
6
+
7
+ @using-api   no
8
+ @results     HTML (using search portal)
9
+ @stable      no (HTML can change)
10
+ @parse       url, title, content, magnetlink
11
+"""
12
+
13
+from urlparse import urljoin
14
+from lxml import html
15
+from searx.engines.xpath import extract_text
16
+from searx.utils import get_torrent_size
17
+
18
+categories = ['videos', 'music', 'files']
19
+paging = True
20
+
21
+URL = 'https://digbt.org'
22
+SEARCH_URL = URL + '/search/{query}-time-{pageno}'
23
+FILESIZE = 3
24
+FILESIZE_MULTIPLIER = 4
25
+
26
+
27
+def request(query, params):
28
+    params['url'] = SEARCH_URL.format(query=query, pageno=params['pageno'])
29
+
30
+    return params
31
+
32
+
33
+def response(resp):
34
+    dom = html.fromstring(resp.content)
35
+    search_res = dom.xpath('.//td[@class="x-item"]')
36
+
37
+    if not search_res:
38
+        return list()
39
+
40
+    results = list()
41
+    for result in search_res:
42
+        url = urljoin(URL, result.xpath('.//a[@title]/@href')[0])
43
+        title = result.xpath('.//a[@title]/text()')[0]
44
+        content = extract_text(result.xpath('.//div[@class="files"]'))
45
+        files_data = extract_text(result.xpath('.//div[@class="tail"]')).split()
46
+        filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])
47
+        magnetlink = result.xpath('.//div[@class="tail"]//a[@class="title"]/@href')[0]
48
+
49
+        results.append({'url': url,
50
+                        'title': title,
51
+                        'content': content,
52
+                        'filesize': filesize,
53
+                        'magnetlink': magnetlink,
54
+                        'seed': 'N/A',
55
+                        'leech': 'N/A',
56
+                        'template': 'torrent.html'})
57
+
58
+    return results

+ 6
- 1
searx/settings.yml View File

87
   - name : btdigg
87
   - name : btdigg
88
     engine : btdigg
88
     engine : btdigg
89
     shortcut : bt
89
     shortcut : bt
90
-    
90
+
91
   - name : crossref
91
   - name : crossref
92
     engine : json_engine
92
     engine : json_engine
93
     paging : True
93
     paging : True
118
     weight : 2
118
     weight : 2
119
     disabled : True
119
     disabled : True
120
 
120
 
121
+  - name : digbt
122
+    engine : digbt
123
+    shortcut : dbt
124
+    timeout : 6.0
125
+
121
   - name : digg
126
   - name : digg
122
     engine : digg
127
     engine : digg
123
     shortcut : dg
128
     shortcut : dg

+ 18
- 0
searx/utils.py View File

237
         return a_list[index]
237
         return a_list[index]
238
     else:
238
     else:
239
         return default
239
         return default
240
+
241
+
242
+def get_torrent_size(filesize, filesize_multiplier):
243
+    try:
244
+        filesize = float(filesize)
245
+
246
+        if filesize_multiplier == 'TB':
247
+            filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
248
+        elif filesize_multiplier == 'GB':
249
+            filesize = int(filesize * 1024 * 1024 * 1024)
250
+        elif filesize_multiplier == 'MB':
251
+            filesize = int(filesize * 1024 * 1024)
252
+        elif filesize_multiplier == 'KB':
253
+            filesize = int(filesize * 1024)
254
+    except:
255
+        filesize = None
256
+
257
+    return filesize