[enh] improve translation documentation

_sources/dev/translation.txt

@@ -6,23 +6,29 @@ run these commands in the root directory of searx
 Add new language
 ~~~~~~~~~~~~~~~~
 
-``pybabel init -i messages.pot -d searx/translations -l it``
+.. code:: shell
+
+    pybabel init -i messages.pot -d searx/translations -l it
 
 Update .po files
 ~~~~~~~~~~~~~~~~
 
-``./utils/update-translations.sh``
+.. code:: shell
+
+    ./utils/update-translations.sh
 
 You may have errors here. In that case, edit the
 ``update-translations.sh`` script to change ``pybabel`` to
-``pybabel-python2``
+``pybabel-python2 or pybabel2``
 
 After this step, you can modify the .po files.
 
 Compile translations
 ~~~~~~~~~~~~~~~~~~~~
 
-``pybabel compile -d searx/translations``
+.. code:: shell
+
+    pybabel compile -d searx/translations
 
 Transifex stuff
 ~~~~~~~~~~~~~~~
@@ -32,12 +38,14 @@ Init Project
 
 .. code:: shell
 
+    tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
+
     tx set --auto-local -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
     --source-lang en --type PO --source-file messages.pot --execute
 
-http://docs.transifex.com/developer/client/set
+http://docs.transifex.com/client/init/
 
-*TODO: mapping between transifex and searx*
+http://docs.transifex.com/client/set/
 
 Get translations
 ^^^^^^^^^^^^^^^^
@@ -46,29 +54,27 @@ Get translations
 
     tx pull -a
 
-http://docs.transifex.com/developer/client/pull
+http://docs.transifex.com/client/pull
 
 Upload source File
 ^^^^^^^^^^^^^^^^^^
 
-::
+.. code:: shell
 
     tx push -s
 
 Upload all Translation
 ^^^^^^^^^^^^^^^^^^^^^^
 
-::
+.. code:: shell
 
     tx push -s -t
 
-upload specifc Translation (only for admins)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+upload specifc Translation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-::
+.. code:: shell
 
     tx push -t -l tr
 
-http://docs.transifex.com/developer/client/push
-
-*TODO: upload empty files? (new translations)*
+http://docs.transifex.com/client/push

_static/pygments.css

@@ -4,8 +4,10 @@
 .highlight .err { border: 1px solid #FF0000 } /* Error */
 .highlight .k { color: #007020; font-weight: bold } /* Keyword */
 .highlight .o { color: #666666 } /* Operator */
+.highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */
 .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */
 .highlight .cp { color: #007020 } /* Comment.Preproc */
+.highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */
 .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */
 .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */
 .highlight .gd { color: #A00000 } /* Generic.Deleted */

_static/searchtools.js

@@ -10,6 +10,7 @@
  */
 
 
+/* Non-minified version JS is _stemmer.js if file is provided */ 
 /**
  * Porter Stemmer
  */
@@ -373,8 +374,7 @@ var Search = {
     }
 
     // lookup as search terms in fulltext
-    results = results.concat(this.performTermsSearch(searchterms, excluded, terms, Scorer.term))
-                     .concat(this.performTermsSearch(searchterms, excluded, titleterms, Scorer.title));
+    results = results.concat(this.performTermsSearch(searchterms, excluded, terms, titleterms));
 
     // let the scorer override scores with a custom scoring function
     if (Scorer.score) {
@@ -538,23 +538,47 @@ var Search = {
   /**
    * search for full-text terms in the index
    */
-  performTermsSearch : function(searchterms, excluded, terms, score) {
+  performTermsSearch : function(searchterms, excluded, terms, titleterms) {
     var filenames = this._index.filenames;
     var titles = this._index.titles;
 
-    var i, j, file, files;
+    var i, j, file;
     var fileMap = {};
+    var scoreMap = {};
     var results = [];
 
     // perform the search on the required terms
     for (i = 0; i < searchterms.length; i++) {
       var word = searchterms[i];
+      var files = [];
+      var _o = [
+        {files: terms[word], score: Scorer.term},
+        {files: titleterms[word], score: Scorer.title}
+      ];
+
       // no match but word was a required one
-      if ((files = terms[word]) === undefined)
+      if ($u.every(_o, function(o){return o.files === undefined;})) {
         break;
-      if (files.length === undefined) {
-        files = [files];
       }
+      // found search word in contents
+      $u.each(_o, function(o) {
+        var _files = o.files;
+        if (_files === undefined)
+          return
+
+        if (_files.length === undefined)
+          _files = [_files];
+        files = files.concat(_files);
+
+        // set score for the word in each file to Scorer.term
+        for (j = 0; j < _files.length; j++) {
+          file = _files[j];
+          if (!(file in scoreMap))
+            scoreMap[file] = {}
+          scoreMap[file][word] = o.score;
+        }
+      });
+
       // create the mapping
       for (j = 0; j < files.length; j++) {
         file = files[j];
@@ -576,7 +600,9 @@ var Search = {
       // ensure that none of the excluded terms is in the search result
       for (i = 0; i < excluded.length; i++) {
         if (terms[excluded[i]] == file ||
-          $u.contains(terms[excluded[i]] || [], file)) {
+            titleterms[excluded[i]] == file ||
+            $u.contains(terms[excluded[i]] || [], file) ||
+            $u.contains(titleterms[excluded[i]] || [], file)) {
           valid = false;
           break;
         }
@@ -584,6 +610,9 @@ var Search = {
 
       // if we have still a valid result we can add it to the result list
       if (valid) {
+        // select one (max) score for the file.
+        // for better ranking, we should calculate ranking by using words statistics like basic tf-idf...
+        var score = $u.max($u.map(fileMap[file], function(w){return scoreMap[file][w]}));
         results.push([filenames[file], titles[file], '', null, score]);
       }
     }

dev/engine_overview.html

@@ -190,8 +190,8 @@ overrides are the following:</p>
 </div>
 <div class="section" id="example-code">
 <h3><a class="toc-backref" href="#id7">example code</a><a class="headerlink" href="#example-code" title="Permalink to this headline">¶</a></h3>
-<div class="code python highlight-python"><div class="highlight"><pre><span class="c"># engine dependent config</span>
-<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;general&#39;</span><span class="p">]</span>
+<div class="code python highlight-python"><div class="highlight"><pre><span class="c1"># engine dependent config</span>
+<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;general&#39;</span><span class="p">]</span>
 <span class="n">paging</span> <span class="o">=</span> <span class="bp">True</span>
 <span class="n">language_support</span> <span class="o">=</span> <span class="bp">True</span>
 </pre></div>
@@ -313,17 +313,17 @@ used to specify a search request:</p>
 </div>
 <div class="section" id="id1">
 <h3><a class="toc-backref" href="#id11">example code</a><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
-<div class="code python highlight-python"><div class="highlight"><pre><span class="c"># search-url</span>
-<span class="n">base_url</span> <span class="o">=</span> <span class="s">&#39;https://example.com/&#39;</span>
-<span class="n">search_string</span> <span class="o">=</span> <span class="s">&#39;search?{query}&amp;page={page}&#39;</span>
+<div class="code python highlight-python"><div class="highlight"><pre><span class="c1"># search-url</span>
+<span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;https://example.com/&#39;</span>
+<span class="n">search_string</span> <span class="o">=</span> <span class="s1">&#39;search?{query}&amp;page={page}&#39;</span>
 
-<span class="c"># do search-request</span>
+<span class="c1"># do search-request</span>
 <span class="k">def</span> <span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
     <span class="n">search_path</span> <span class="o">=</span> <span class="n">search_string</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-        <span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span>
-        <span class="n">page</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s">&#39;pageno&#39;</span><span class="p">])</span>
+        <span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">&#39;q&#39;</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span>
+        <span class="n">page</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;pageno&#39;</span><span class="p">])</span>
 
-    <span class="n">params</span><span class="p">[</span><span class="s">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">search_path</span>
+    <span class="n">params</span><span class="p">[</span><span class="s1">&#39;url&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">search_path</span>
 
     <span class="k">return</span> <span class="n">params</span>
 </pre></div>

dev/plugins.html

@@ -45,19 +45,19 @@
 searx.</p>
 <div class="section" id="example-plugin-py">
 <h2>example_plugin.py<a class="headerlink" href="#example-plugin-py" title="Permalink to this headline">¶</a></h2>
-<div class="code python highlight-python"><div class="highlight"><pre><span class="n">name</span> <span class="o">=</span> <span class="s">&#39;Example plugin&#39;</span>
-<span class="n">description</span> <span class="o">=</span> <span class="s">&#39;This plugin extends the suggestions with the word &quot;example&quot;&#39;</span>
-<span class="n">default_on</span> <span class="o">=</span> <span class="bp">False</span>  <span class="c"># disable by default</span>
+<div class="code python highlight-python"><div class="highlight"><pre><span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;Example plugin&#39;</span>
+<span class="n">description</span> <span class="o">=</span> <span class="s1">&#39;This plugin extends the suggestions with the word &quot;example&quot;&#39;</span>
+<span class="n">default_on</span> <span class="o">=</span> <span class="bp">False</span>  <span class="c1"># disable by default</span>
 
-<span class="n">js_dependencies</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>  <span class="c"># optional, list of static js files</span>
-<span class="n">css_dependencies</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>  <span class="c"># optional, list of static css files</span>
+<span class="n">js_dependencies</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>  <span class="c1"># optional, list of static js files</span>
+<span class="n">css_dependencies</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>  <span class="c1"># optional, list of static css files</span>
 
 
-<span class="c"># attach callback to the post search hook</span>
-<span class="c">#  request: flask request object</span>
-<span class="c">#  ctx: the whole local context of the post search hook</span>
+<span class="c1"># attach callback to the post search hook</span>
+<span class="c1">#  request: flask request object</span>
+<span class="c1">#  ctx: the whole local context of the post search hook</span>
 <span class="k">def</span> <span class="nf">post_search</span><span class="p">(</span><span class="n">request</span><span class="p">,</span> <span class="n">ctx</span><span class="p">):</span>
-    <span class="n">ctx</span><span class="p">[</span><span class="s">&#39;search&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">suggestions</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s">&#39;example&#39;</span><span class="p">)</span>
+    <span class="n">ctx</span><span class="p">[</span><span class="s1">&#39;search&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">suggestions</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s1">&#39;example&#39;</span><span class="p">)</span>
     <span class="k">return</span> <span class="bp">True</span>
 </pre></div>
 </div>

dev/search_api.html

@@ -25,7 +25,7 @@
     <script type="text/javascript" src="../_static/doctools.js"></script>
     <link rel="top" title="searx 0.8.0 documentation" href="../index.html" />
     <link rel="next" title="Plugins" href="plugins.html" />
-    <link rel="prev" title="Installation" href="install/installation.html" />
+    <link rel="prev" title="Engine overview" href="engine_overview.html" />
    
   
   <link media="only screen and (max-device-width: 480px)" href="../_static/small_flask.css" type= "text/css" rel="stylesheet" />

dev/translation.html

@@ -43,58 +43,65 @@
 <p>run these commands in the root directory of searx</p>
 <div class="section" id="add-new-language">
 <h2>Add new language<a class="headerlink" href="#add-new-language" title="Permalink to this headline">¶</a></h2>
-<p><code class="docutils literal"><span class="pre">pybabel</span> <span class="pre">init</span> <span class="pre">-i</span> <span class="pre">messages.pot</span> <span class="pre">-d</span> <span class="pre">searx/translations</span> <span class="pre">-l</span> <span class="pre">it</span></code></p>
+<div class="code shell highlight-python"><div class="highlight"><pre>pybabel init -i messages.pot -d searx/translations -l it
+</pre></div>
+</div>
 </div>
 <div class="section" id="update-po-files">
 <h2>Update .po files<a class="headerlink" href="#update-po-files" title="Permalink to this headline">¶</a></h2>
-<p><code class="docutils literal"><span class="pre">./utils/update-translations.sh</span></code></p>
+<div class="code shell highlight-python"><div class="highlight"><pre>./utils/update-translations.sh
+</pre></div>
+</div>
 <p>You may have errors here. In that case, edit the
 <code class="docutils literal"><span class="pre">update-translations.sh</span></code> script to change <code class="docutils literal"><span class="pre">pybabel</span></code> to
-<code class="docutils literal"><span class="pre">pybabel-python2</span></code></p>
+<code class="docutils literal"><span class="pre">pybabel-python2</span> <span class="pre">or</span> <span class="pre">pybabel2</span></code></p>
 <p>After this step, you can modify the .po files.</p>
 </div>
 <div class="section" id="compile-translations">
 <h2>Compile translations<a class="headerlink" href="#compile-translations" title="Permalink to this headline">¶</a></h2>
-<p><code class="docutils literal"><span class="pre">pybabel</span> <span class="pre">compile</span> <span class="pre">-d</span> <span class="pre">searx/translations</span></code></p>
+<div class="code shell highlight-python"><div class="highlight"><pre>pybabel compile -d searx/translations
+</pre></div>
+</div>
 </div>
 <div class="section" id="transifex-stuff">
 <h2>Transifex stuff<a class="headerlink" href="#transifex-stuff" title="Permalink to this headline">¶</a></h2>
 <div class="section" id="init-project">
 <h3>Init Project<a class="headerlink" href="#init-project" title="Permalink to this headline">¶</a></h3>
-<div class="code shell highlight-python"><div class="highlight"><pre>tx set --auto-local -r searx.messagespo &#39;searx/translations/&lt;lang&gt;/LC_MESSAGES/messages.po&#39; \
+<div class="code shell highlight-python"><div class="highlight"><pre>tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
+
+tx set --auto-local -r searx.messagespo &#39;searx/translations/&lt;lang&gt;/LC_MESSAGES/messages.po&#39; \
 --source-lang en --type PO --source-file messages.pot --execute
 </pre></div>
 </div>
-<p><a class="reference external" href="http://docs.transifex.com/developer/client/set">http://docs.transifex.com/developer/client/set</a></p>
-<p><em>TODO: mapping between transifex and searx</em></p>
+<p><a class="reference external" href="http://docs.transifex.com/client/init/">http://docs.transifex.com/client/init/</a></p>
+<p><a class="reference external" href="http://docs.transifex.com/client/set/">http://docs.transifex.com/client/set/</a></p>
 </div>
 <div class="section" id="get-translations">
 <h3>Get translations<a class="headerlink" href="#get-translations" title="Permalink to this headline">¶</a></h3>
 <div class="code shell highlight-python"><div class="highlight"><pre>tx pull -a
 </pre></div>
 </div>
-<p><a class="reference external" href="http://docs.transifex.com/developer/client/pull">http://docs.transifex.com/developer/client/pull</a></p>
+<p><a class="reference external" href="http://docs.transifex.com/client/pull">http://docs.transifex.com/client/pull</a></p>
 </div>
 <div class="section" id="upload-source-file">
 <h3>Upload source File<a class="headerlink" href="#upload-source-file" title="Permalink to this headline">¶</a></h3>
-<div class="highlight-python"><div class="highlight"><pre>tx push -s
+<div class="code shell highlight-python"><div class="highlight"><pre>tx push -s
 </pre></div>
 </div>
 </div>
 <div class="section" id="upload-all-translation">
 <h3>Upload all Translation<a class="headerlink" href="#upload-all-translation" title="Permalink to this headline">¶</a></h3>
-<div class="highlight-python"><div class="highlight"><pre>tx push -s -t
+<div class="code shell highlight-python"><div class="highlight"><pre>tx push -s -t
 </pre></div>
 </div>
 </div>
 </div>
-<div class="section" id="upload-specifc-translation-only-for-admins">
-<h2>upload specifc Translation (only for admins)<a class="headerlink" href="#upload-specifc-translation-only-for-admins" title="Permalink to this headline">¶</a></h2>
-<div class="highlight-python"><div class="highlight"><pre>tx push -t -l tr
+<div class="section" id="upload-specifc-translation">
+<h2>upload specifc Translation<a class="headerlink" href="#upload-specifc-translation" title="Permalink to this headline">¶</a></h2>
+<div class="code shell highlight-python"><div class="highlight"><pre>tx push -t -l tr
 </pre></div>
 </div>
-<p><a class="reference external" href="http://docs.transifex.com/developer/client/push">http://docs.transifex.com/developer/client/push</a></p>
-<p><em>TODO: upload empty files? (new translations)</em></p>
+<p><a class="reference external" href="http://docs.transifex.com/client/push">http://docs.transifex.com/client/push</a></p>
 </div>
 </div>
 

docs/dev/translation.rst

@@ -6,23 +6,29 @@ run these commands in the root directory of searx
 Add new language
 ~~~~~~~~~~~~~~~~
 
-``pybabel init -i messages.pot -d searx/translations -l it``
+.. code:: shell
+
+    pybabel init -i messages.pot -d searx/translations -l it
 
 Update .po files
 ~~~~~~~~~~~~~~~~
 
-``./utils/update-translations.sh``
+.. code:: shell
+
+    ./utils/update-translations.sh
 
 You may have errors here. In that case, edit the
 ``update-translations.sh`` script to change ``pybabel`` to
-``pybabel-python2``
+``pybabel-python2 or pybabel2``
 
 After this step, you can modify the .po files.
 
 Compile translations
 ~~~~~~~~~~~~~~~~~~~~
 
-``pybabel compile -d searx/translations``
+.. code:: shell
+
+    pybabel compile -d searx/translations
 
 Transifex stuff
 ~~~~~~~~~~~~~~~
@@ -32,12 +38,14 @@ Init Project
 
 .. code:: shell
 
+    tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
+
     tx set --auto-local -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
     --source-lang en --type PO --source-file messages.pot --execute
 
-http://docs.transifex.com/developer/client/set
+http://docs.transifex.com/client/init/
 
-*TODO: mapping between transifex and searx*
+http://docs.transifex.com/client/set/
 
 Get translations
 ^^^^^^^^^^^^^^^^
@@ -46,29 +54,27 @@ Get translations
 
     tx pull -a
 
-http://docs.transifex.com/developer/client/pull
+http://docs.transifex.com/client/pull
 
 Upload source File
 ^^^^^^^^^^^^^^^^^^
 
-::
+.. code:: shell
 
     tx push -s
 
 Upload all Translation
 ^^^^^^^^^^^^^^^^^^^^^^
 
-::
+.. code:: shell
 
     tx push -s -t
 
-upload specifc Translation (only for admins)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+upload specifc Translation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-::
+.. code:: shell
 
     tx push -t -l tr
 
-http://docs.transifex.com/developer/client/push
-
-*TODO: upload empty files? (new translations)*
+http://docs.transifex.com/client/push