WikiExtractor · svamberg · Mar 1, 2017 · Mar 1, 2017 · Mar 1, 2017 · Mar 1, 2017
diff --git a/ChangeLog b/ChangeLog
@@ -1,3 +1,22 @@
+2017-01-03  Michal Svamberg  <svamberg@civ.zcu.cz>
+
+	* makehtmlfiles.pl: add new script for building web browsed content
+	This script build simple pages from output of WikiExtractor.
+
+	For more information read example section in README.md
+
+	* WikiExtractor.py (keepPage): check for -xns option (or default = 0)
+	* WikiExtractor.py (replaceInternalLinks): 
+	  - Fix URL links to other pages
+	  - Add support for Image links (make html tag <img src="%s" alt="%s">)
+	* WikiExtractor.py (compact): add lines beginned by space into <tt> tag
+	* WikiExtractor.py (pages_from): insert redirect pages if option
+	--redirect_insert is enabled
+	* WikiExtractor.py (main): add new options used in above changes
+	  --all_links print links to all namespaces (independent of -xns or -ns)
+	  --xml_namespaces add numeric namespaces for search pages
+	  --redirect_insert create pages where used #REDIRECT wiki command
+
 2017-01-15  Giuseppe Attardi  <attardi@di.unipi.it>
 
 	* WikiExtractor.py (process_dump): use text_type to decide whether

diff --git a/README.md b/README.md
@@ -33,79 +33,106 @@ The script is invoked with a Wikipedia dump file as an argument.
 The output is stored in several files of similar size in a given directory.
 Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format).
 
-    usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--html] [-l] [-s]
-                            [--lists] [-ns ns1,ns2] [-xns ns1,ns2]
-                            [--templates TEMPLATES] [--no-templates]
-                            [-r] [--min_text_length MIN_TEXT_LENGTH]
-                            [--filter_disambig_pages] [--processes PROCESSES] [-q]
-                            [--debug] [-a] [-v]
-                            input
-
-    Wikipedia Extractor:
-    Extracts and cleans text from a Wikipedia database dump and stores output in a
-    number of files of similar size in a given directory.
-    Each file will contain several documents in the format:
-
-        <doc id="" revid="" url="" title="">
-            ...
-            </doc>
-
-    Template expansion requires preprocesssng first the whole dump and
-    collecting template definitions.
-
-    positional arguments:
-      input                 XML wiki dump file
-
-    optional arguments:
-      -h, --help            show this help message and exit
-      --processes PROCESSES number of processes to use (default: number of CPU cores)
-
-    Output:
-      -o OUTPUT, --output OUTPUT
-                            directory for extracted files (or '-' for dumping to
-                            stdout)
-      -b n[KMG], --bytes n[KMG]
-                            maximum bytes per output file (default 1M)
-      -c, --compress        compress output files using bzip
-
-    Processing:
-      --html                produce HTML output, subsumes --links
-      -l, --links           preserve links
-      -s, --sections        preserve sections
-      --lists               preserve lists
-      -ns ns1,ns2, --namespaces ns1,ns2
-                            accepted link namespaces
-      -xns ns1,ns2, --xml_namespaces ns1,ns2
-                            accepted page xml namespaces -- 0 for main/articles
-      --templates TEMPLATES
-                            use or create file containing templates
-      --no-templates        Do not expand templates
-      -r, --revision        Include the document revision id (default=False)
-      --min_text_length MIN_TEXT_LENGTH
-                            Minimum expanded text length required to write
-                            document (default=0)
-      --filter_disambig_pages
-                            Remove pages from output that contain disabmiguation
-                            markup (default=False)
-      -it, --ignored_tags
-                            comma separated list of tags that will be dropped, keeping their content
-      -de, --discard_elements
-                            comma separated list of elements that will be removed from the article text
-      --keep_tables
-                            Preserve tables in the output article text (default=False)
-
-    Special:
-      -q, --quiet           suppress reporting progress info
-      --debug               print debug info
-      -a, --article         analyze a file containing a single article (debug
-                            option)
-      -v, --version         print program version
-
-
-Saving templates to a file will speed up performing extraction the next time,
-assuming template definitions have not changed.
-
-Option --no-templates significantly speeds up the extractor, avoiding the cost
-of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates).
+        usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html]
+                                [-l] [--all_links] [-s] [--lists] [-ns ns1,ns2]
+                                [-xns ns1,ns2] [--templates TEMPLATES]
+                                [--no-templates] [-r]
+                                [--min_text_length MIN_TEXT_LENGTH]
+                                [--filter_disambig_pages] [-it abbr,b,big]
+                                [-de gallery,timeline,noinclude] [--keep_tables]
+                                [--processes PROCESSES] [--redirect_insert] [-q]
+                                [--debug] [-a] [-v]
+                                input
+
+        Wikipedia Extractor:
+        Extracts and cleans text from a Wikipedia database dump and stores output in a number of files 
+        of similar size in a given directory.
+        Each file will contain several documents in the format:
+
+            <doc id="" revid="" url="" title="">
+                ...
+                </doc>
+
+        If the program is invoked with the --json flag, then each file will contain several documents 
+        formatted as json ojects, one per line, with the following structure
+
+            {"id": "", "revid": "", "url":"", "title": "", "text": "..."}
+
+        Template expansion requires preprocesssng first the whole dump and collecting template definitions.
+
+        positional arguments:
+          input                 XML wiki dump file
+
+        optional arguments:
+          -h, --help            show this help message and exit
+          --processes PROCESSES
+                                Number of processes to use (default 3)
+
+        Output:
+          -o OUTPUT, --output OUTPUT
+                                directory for extracted files (or '-' for dumping to stdout)
+          -b n[KMG], --bytes n[KMG]
+                                maximum bytes per output file (default 1M)
+          -c, --compress        compress output files using bzip
+          --json                write output in json format instead of the default one
+
+        Processing:
+          --html                produce HTML output, subsumes --links
+          -l, --links           preserve links
+          --all_links           preserve all links (with links to all namespaces)
+          -s, --sections        preserve sections
+          --lists               preserve lists
+          -ns ns1,ns2, --namespaces ns1,ns2
+                                accepted namespaces in links
+          -xns ns1,ns2, --xml_namespaces ns1,ns2
+                                accepted page xml namespaces -- 0 for main/articles
+          --templates TEMPLATES
+                                use or create file containing templates
+          --no-templates        Do not expand templates
+          -r, --revision        Include the document revision id (default=False)
+          --min_text_length MIN_TEXT_LENGTH
+                                Minimum expanded text length required to write document (default=0)
+          --filter_disambig_pages
+                                Remove pages from output that contain disabmiguation markup (default=False)
+          -it abbr,b,big, --ignored_tags abbr,b,big
+                                comma separated list of tags that will be dropped, keeping their content
+          -de gallery,timeline,noinclude, --discard_elements gallery,timeline,noinclude
+                                comma separated list of elements that will be removed from the article text
+          --keep_tables         Preserve tables in the output article text (default=False)
+          --redirect_insert     Insert redirect pages to output
+
+        Special:
+          -q, --quiet           suppress reporting progress info
+          --debug               print debug info
+          -a, --article         analyze a file containing a single article (debug option)
+          -v, --version         print program version
+
+
+Saving templates to a file will speed up performing extraction the next time, assuming template definitions have not changed.
+
+Option --no-templates significantly speeds up the extractor, avoiding the cost of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates).
 
 For further information, visit [the documentation](http://attardi.github.io/wikiextractor).
+
+## Example: Howto make offline content of your MediaWiki as web pages
+1. Make dump of your mediawiki:
+
+        php5 /var/lib/mediawiki/maintenance/dumpBackup.php --current > /tmp/dump.xml
+
+2. Extract data from dump to one big file (param --html is necessary):
+
+        ./WikiExtractor.py -o - -xns 0,2,14,100,102,104,106,108,110,112,114,116 --links --all_links -b 200M --html --redirect_insert --keep_tables -q /tmp/dump.xml > /tmp/wiki.data 
+
+3. Create single html pages and index.html:
+
+        cat /tmp/wiki.data | ./makehtmlfiles.pl /tmp/offline
+
+4. Optional you can copy images:
+
+        cd /var/lib/mediawiki/images
+        find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \;
+
+5. View offline content ONLY (without skins, templates, scripts, ...) of your wikimedia:
+
+        links file:///tmp/offline/index.html
+
diff --git a/WikiExtractor.py b/WikiExtractor.py
@@ -133,6 +133,9 @@ def __eq__ (self, other):
     #
     acceptedNamespaces=['w', 'wiktionary', 'wikt'],
 
+
+    acceptedXMLNamespaces=['0'],
+
     # This is obtained from <siteinfo>
     urlbase='',
 
@@ -212,7 +215,8 @@ def __eq__ (self, other):
 ##
 # page filtering logic -- remove templates, undesired xml namespaces, and disambiguation pages
 def keepPage(ns, page):
-    if ns != '0':               # Aritcle
+    #if ns != '0':               # Aritcle
+    if ns not in acceptedXMLNamespaces:
         return False
     # remove disambig pages if desired
     if options.filter_disambig_pages:
@@ -2055,6 +2059,7 @@ def replaceInternalLinks(text):
                 curp = e1
             label = inner[pipe + 1:].strip()
         res += text[cur:s] + makeInternalLink(title, label) + trail
+#        logging.error('%s',  makeInternalLink(title, label))
         cur = end
     return res + text[cur:]
 
@@ -2326,15 +2331,27 @@ def replaceInternalLinks(text):
 
 def makeInternalLink(title, label):
     colon = title.find(':')
-    if colon > 0 and title[:colon] not in options.acceptedNamespaces:
-        return ''
+    if not options.keepAllLinks:
+        if colon > 0 and title[:colon] not in options.acceptedNamespaces:
+            return '['+ title + ']'
     if colon == 0:
         # drop also :File:
         colon2 = title.find(':', colon + 1)
         if colon2 > 1 and title[colon + 1:colon2] not in options.acceptedNamespaces:
-            return ''
+            if options.keepAllLinks:
+                # remove first colons
+                title = title.lstrip(':')
+                label = label.lstrip(':')
+            else:
+                return '['+ title + ']'
     if options.keepLinks:
-        return '<a href="%s">%s</a>' % (quote(title.encode('utf-8')), label)
+        if title[:colon] == 'Image':
+            #logging.error('%s: %s -> %s', title[:colon], title, label)
+            title_short = title.split(':', 1)
+            label_short = label.split('|')
+            return '<img src="%s" alt="%s">' % (quote(title_short[1].encode('utf-8')), label_short[-1])
+        else:
+            return '<a href="%s">%s</a>' % (quote(title.encode('utf-8')), label)
     else:
         return label
 
@@ -2542,6 +2559,11 @@ def compact(text):
             listLevel = []
             listCount = []
             page.append(line)
+        elif line[0] == ' ':
+            if options.toHTML:
+                page.append("<pre>%s</pre>" % (line))
+            else:
+                page.append(line)
 
         # Drop residuals of lists
         elif line[0] in '{|' or line[-1] == '}':
@@ -2731,7 +2753,7 @@ def pages_from(input):
         elif tag == 'ns':
             ns = m.group(3)
         elif tag == 'redirect':
-            redirect = True
+            redirect = not options.redirect_insert
         elif tag == 'text':
             if m.lastindex == 3 and line[m.start(3)-2] == '/': # self closing
                 # <text xml:space="preserve" />
@@ -3005,6 +3027,7 @@ def reduce_process(opts, output_queue, spool_length,
 minFileSize = 200 * 1024
 
 def main():
+    global acceptedXMLNamespaces
 
     parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),
                                      formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -3028,12 +3051,16 @@ def main():
                         help="produce HTML output, subsumes --links")
     groupP.add_argument("-l", "--links", action="store_true",
                         help="preserve links")
+    groupP.add_argument("--all_links", action="store_true",
+                        help="preserve all links (with links to all namespaces)")
     groupP.add_argument("-s", "--sections", action="store_true",
                         help="preserve sections")
     groupP.add_argument("--lists", action="store_true",
                         help="preserve lists")
     groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2",
                         help="accepted namespaces in links")
+    groupP.add_argument("-xns", "--xml_namespaces", default="", metavar="ns1,ns2",
+                        help="accepted page xml namespaces -- 0 for main/articles")
     groupP.add_argument("--templates",
                         help="use or create file containing templates")
     groupP.add_argument("--no-templates", action="store_false",
@@ -3053,6 +3080,8 @@ def main():
     default_process_count = max(1, cpu_count() - 1)
     parser.add_argument("--processes", type=int, default=default_process_count,
                         help="Number of processes to use (default %(default)s)")
+    groupP.add_argument("--redirect_insert", action="store_true",
+                        help="Insert redirect pages to output")
 
     groupS = parser.add_argument_group('Special')
     groupS.add_argument("-q", "--quiet", action="store_true",
@@ -3068,12 +3097,14 @@ def main():
     args = parser.parse_args()
 
     options.keepLinks = args.links
+    options.keepAllLinks = args.all_links
     options.keepSections = args.sections
     options.keepLists = args.lists
     options.toHTML = args.html
     options.write_json = args.json
     options.print_revision = args.revision
     options.min_text_length = args.min_text_length
+    options.redirect_insert = args.redirect_insert
     if args.html:
         options.keepLinks = True
 
@@ -3111,6 +3142,9 @@ def main():
     if args.discard_elements:
         options.discardElements = set(args.discard_elements.split(','))
 
+    if args.xml_namespaces:
+        acceptedXMLNamespaces = set([unicode(ns) for ns in args.xml_namespaces.split(',')])
+
     FORMAT = '%(levelname)s: %(message)s'
     logging.basicConfig(format=FORMAT)