diff --git a/ChangeLog b/ChangeLog index c8167829..0ac1a222 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +2017-01-03 Michal Svamberg + + * makehtmlfiles.pl: add new script for building web browsed content + This script build simple pages from output of WikiExtractor. + + For more information read example section in README.md + + * WikiExtractor.py (keepPage): check for -xns option (or default = 0) + * WikiExtractor.py (replaceInternalLinks): + - Fix URL links to other pages + - Add support for Image links (make html tag %s) + * WikiExtractor.py (compact): add lines beginned by space into tag + * WikiExtractor.py (pages_from): insert redirect pages if option + --redirect_insert is enabled + * WikiExtractor.py (main): add new options used in above changes + --all_links print links to all namespaces (independent of -xns or -ns) + --xml_namespaces add numeric namespaces for search pages + --redirect_insert create pages where used #REDIRECT wiki command + 2017-01-15 Giuseppe Attardi * WikiExtractor.py (process_dump): use text_type to decide whether diff --git a/README.md b/README.md index a2ec97d1..8ecb0fe6 100644 --- a/README.md +++ b/README.md @@ -33,79 +33,106 @@ The script is invoked with a Wikipedia dump file as an argument. The output is stored in several files of similar size in a given directory. Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format). - usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--html] [-l] [-s] - [--lists] [-ns ns1,ns2] [-xns ns1,ns2] - [--templates TEMPLATES] [--no-templates] - [-r] [--min_text_length MIN_TEXT_LENGTH] - [--filter_disambig_pages] [--processes PROCESSES] [-q] - [--debug] [-a] [-v] - input - - Wikipedia Extractor: - Extracts and cleans text from a Wikipedia database dump and stores output in a - number of files of similar size in a given directory. - Each file will contain several documents in the format: - - - ... - - - Template expansion requires preprocesssng first the whole dump and - collecting template definitions. - - positional arguments: - input XML wiki dump file - - optional arguments: - -h, --help show this help message and exit - --processes PROCESSES number of processes to use (default: number of CPU cores) - - Output: - -o OUTPUT, --output OUTPUT - directory for extracted files (or '-' for dumping to - stdout) - -b n[KMG], --bytes n[KMG] - maximum bytes per output file (default 1M) - -c, --compress compress output files using bzip - - Processing: - --html produce HTML output, subsumes --links - -l, --links preserve links - -s, --sections preserve sections - --lists preserve lists - -ns ns1,ns2, --namespaces ns1,ns2 - accepted link namespaces - -xns ns1,ns2, --xml_namespaces ns1,ns2 - accepted page xml namespaces -- 0 for main/articles - --templates TEMPLATES - use or create file containing templates - --no-templates Do not expand templates - -r, --revision Include the document revision id (default=False) - --min_text_length MIN_TEXT_LENGTH - Minimum expanded text length required to write - document (default=0) - --filter_disambig_pages - Remove pages from output that contain disabmiguation - markup (default=False) - -it, --ignored_tags - comma separated list of tags that will be dropped, keeping their content - -de, --discard_elements - comma separated list of elements that will be removed from the article text - --keep_tables - Preserve tables in the output article text (default=False) - - Special: - -q, --quiet suppress reporting progress info - --debug print debug info - -a, --article analyze a file containing a single article (debug - option) - -v, --version print program version - - -Saving templates to a file will speed up performing extraction the next time, -assuming template definitions have not changed. - -Option --no-templates significantly speeds up the extractor, avoiding the cost -of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). + usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] + [-l] [--all_links] [-s] [--lists] [-ns ns1,ns2] + [-xns ns1,ns2] [--templates TEMPLATES] + [--no-templates] [-r] + [--min_text_length MIN_TEXT_LENGTH] + [--filter_disambig_pages] [-it abbr,b,big] + [-de gallery,timeline,noinclude] [--keep_tables] + [--processes PROCESSES] [--redirect_insert] [-q] + [--debug] [-a] [-v] + input + + Wikipedia Extractor: + Extracts and cleans text from a Wikipedia database dump and stores output in a number of files + of similar size in a given directory. + Each file will contain several documents in the format: + + + ... + + + If the program is invoked with the --json flag, then each file will contain several documents + formatted as json ojects, one per line, with the following structure + + {"id": "", "revid": "", "url":"", "title": "", "text": "..."} + + Template expansion requires preprocesssng first the whole dump and collecting template definitions. + + positional arguments: + input XML wiki dump file + + optional arguments: + -h, --help show this help message and exit + --processes PROCESSES + Number of processes to use (default 3) + + Output: + -o OUTPUT, --output OUTPUT + directory for extracted files (or '-' for dumping to stdout) + -b n[KMG], --bytes n[KMG] + maximum bytes per output file (default 1M) + -c, --compress compress output files using bzip + --json write output in json format instead of the default one + + Processing: + --html produce HTML output, subsumes --links + -l, --links preserve links + --all_links preserve all links (with links to all namespaces) + -s, --sections preserve sections + --lists preserve lists + -ns ns1,ns2, --namespaces ns1,ns2 + accepted namespaces in links + -xns ns1,ns2, --xml_namespaces ns1,ns2 + accepted page xml namespaces -- 0 for main/articles + --templates TEMPLATES + use or create file containing templates + --no-templates Do not expand templates + -r, --revision Include the document revision id (default=False) + --min_text_length MIN_TEXT_LENGTH + Minimum expanded text length required to write document (default=0) + --filter_disambig_pages + Remove pages from output that contain disabmiguation markup (default=False) + -it abbr,b,big, --ignored_tags abbr,b,big + comma separated list of tags that will be dropped, keeping their content + -de gallery,timeline,noinclude, --discard_elements gallery,timeline,noinclude + comma separated list of elements that will be removed from the article text + --keep_tables Preserve tables in the output article text (default=False) + --redirect_insert Insert redirect pages to output + + Special: + -q, --quiet suppress reporting progress info + --debug print debug info + -a, --article analyze a file containing a single article (debug option) + -v, --version print program version + + +Saving templates to a file will speed up performing extraction the next time, assuming template definitions have not changed. + +Option --no-templates significantly speeds up the extractor, avoiding the cost of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). For further information, visit [the documentation](http://attardi.github.io/wikiextractor). + +## Example: Howto make offline content of your MediaWiki as web pages +1. Make dump of your mediawiki: + + php5 /var/lib/mediawiki/maintenance/dumpBackup.php --current > /tmp/dump.xml + +2. Extract data from dump to one big file (param --html is necessary): + + ./WikiExtractor.py -o - -xns 0,2,14,100,102,104,106,108,110,112,114,116 --links --all_links -b 200M --html --redirect_insert --keep_tables -q /tmp/dump.xml > /tmp/wiki.data + +3. Create single html pages and index.html: + + cat /tmp/wiki.data | ./makehtmlfiles.pl /tmp/offline + +4. Optional you can copy images: + + cd /var/lib/mediawiki/images + find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \; + +5. View offline content ONLY (without skins, templates, scripts, ...) of your wikimedia: + + links file:///tmp/offline/index.html + diff --git a/WikiExtractor.py b/WikiExtractor.py index ae448364..fa1ab359 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -133,6 +133,9 @@ def __eq__ (self, other): # acceptedNamespaces=['w', 'wiktionary', 'wikt'], + + acceptedXMLNamespaces=['0'], + # This is obtained from urlbase='', @@ -212,7 +215,8 @@ def __eq__ (self, other): ## # page filtering logic -- remove templates, undesired xml namespaces, and disambiguation pages def keepPage(ns, page): - if ns != '0': # Aritcle + #if ns != '0': # Aritcle + if ns not in acceptedXMLNamespaces: return False # remove disambig pages if desired if options.filter_disambig_pages: @@ -2055,6 +2059,7 @@ def replaceInternalLinks(text): curp = e1 label = inner[pipe + 1:].strip() res += text[cur:s] + makeInternalLink(title, label) + trail +# logging.error('%s', makeInternalLink(title, label)) cur = end return res + text[cur:] @@ -2326,15 +2331,27 @@ def replaceInternalLinks(text): def makeInternalLink(title, label): colon = title.find(':') - if colon > 0 and title[:colon] not in options.acceptedNamespaces: - return '' + if not options.keepAllLinks: + if colon > 0 and title[:colon] not in options.acceptedNamespaces: + return '['+ title + ']' if colon == 0: # drop also :File: colon2 = title.find(':', colon + 1) if colon2 > 1 and title[colon + 1:colon2] not in options.acceptedNamespaces: - return '' + if options.keepAllLinks: + # remove first colons + title = title.lstrip(':') + label = label.lstrip(':') + else: + return '['+ title + ']' if options.keepLinks: - return '%s' % (quote(title.encode('utf-8')), label) + if title[:colon] == 'Image': + #logging.error('%s: %s -> %s', title[:colon], title, label) + title_short = title.split(':', 1) + label_short = label.split('|') + return '%s' % (quote(title_short[1].encode('utf-8')), label_short[-1]) + else: + return '%s' % (quote(title.encode('utf-8')), label) else: return label @@ -2542,6 +2559,11 @@ def compact(text): listLevel = [] listCount = [] page.append(line) + elif line[0] == ' ': + if options.toHTML: + page.append("
%s
" % (line)) + else: + page.append(line) # Drop residuals of lists elif line[0] in '{|' or line[-1] == '}': @@ -2731,7 +2753,7 @@ def pages_from(input): elif tag == 'ns': ns = m.group(3) elif tag == 'redirect': - redirect = True + redirect = not options.redirect_insert elif tag == 'text': if m.lastindex == 3 and line[m.start(3)-2] == '/': # self closing # @@ -3005,6 +3027,7 @@ def reduce_process(opts, output_queue, spool_length, minFileSize = 200 * 1024 def main(): + global acceptedXMLNamespaces parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), formatter_class=argparse.RawDescriptionHelpFormatter, @@ -3028,12 +3051,16 @@ def main(): help="produce HTML output, subsumes --links") groupP.add_argument("-l", "--links", action="store_true", help="preserve links") + groupP.add_argument("--all_links", action="store_true", + help="preserve all links (with links to all namespaces)") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") groupP.add_argument("--lists", action="store_true", help="preserve lists") groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", help="accepted namespaces in links") + groupP.add_argument("-xns", "--xml_namespaces", default="", metavar="ns1,ns2", + help="accepted page xml namespaces -- 0 for main/articles") groupP.add_argument("--templates", help="use or create file containing templates") groupP.add_argument("--no-templates", action="store_false", @@ -3053,6 +3080,8 @@ def main(): default_process_count = max(1, cpu_count() - 1) parser.add_argument("--processes", type=int, default=default_process_count, help="Number of processes to use (default %(default)s)") + groupP.add_argument("--redirect_insert", action="store_true", + help="Insert redirect pages to output") groupS = parser.add_argument_group('Special') groupS.add_argument("-q", "--quiet", action="store_true", @@ -3068,12 +3097,14 @@ def main(): args = parser.parse_args() options.keepLinks = args.links + options.keepAllLinks = args.all_links options.keepSections = args.sections options.keepLists = args.lists options.toHTML = args.html options.write_json = args.json options.print_revision = args.revision options.min_text_length = args.min_text_length + options.redirect_insert = args.redirect_insert if args.html: options.keepLinks = True @@ -3111,6 +3142,9 @@ def main(): if args.discard_elements: options.discardElements = set(args.discard_elements.split(',')) + if args.xml_namespaces: + acceptedXMLNamespaces = set([unicode(ns) for ns in args.xml_namespaces.split(',')]) + FORMAT = '%(levelname)s: %(message)s' logging.basicConfig(format=FORMAT) diff --git a/makehtmlfiles.pl b/makehtmlfiles.pl new file mode 100755 index 00000000..e8ffd80b --- /dev/null +++ b/makehtmlfiles.pl @@ -0,0 +1,109 @@ +#!/usr/bin/perl + +# vystup z wikiextractor ve formatu html rozseka podle tagu 'doc' na jednotlive +# soubory, prida k nim html hlavicku/paticku a upravi odkazy +use strict; +use URI::Escape; +use File::Basename; +use File::Path qw(make_path); +use HTML::Entities; + +my ($out_dir) = @ARGV; +if (not defined $out_dir) { + die "Missing param, usage: makehtmlfiles.pl \n"; +} + +my $html_head_tmpl = ' + + + + OFFLINE MEDIAWIKI + +'; +my $html_head = ''; + +my $html_foot = "\n\n"; + +my $basedir = ''; +my $title = ''; +my $suffix = 'html'; +my $filename = ''; +my $filename_index = "index.$suffix"; +my $line = ''; +my $FILE; +my $INDEX; +my %index = (); +my $indx = ''; # sorted index +my $relative_path = ''; + +make_path("$out_dir/pages", 0700) unless(-d "$out_dir/pages" ); +make_path("$out_dir/images", 0700) unless(-d "$out_dir/images" ); + +foreach $line ( ) { + chomp( $line ); + if ($line =~ /^\s*\s*$/) { + $title = "$1"; + #$filename = uri_escape("$title") . ".$suffix"; + $filename = "$title" . ".$suffix"; + # print "$title \t-> $filename\n"; + + $filename =~ s/\s/_/g; + $basedir = dirname("$filename"); + # print "MKDIR $basedir\n"; + make_path("$out_dir/pages/$basedir", 0700) unless(-d "$out_dir/pages/$basedir" ); + + open($FILE, ">", "$out_dir/pages/$filename") or die "cannot open > $out_dir/$filename: $!"; + + $html_head = $html_head_tmpl; + $html_head =~ s#OFFLINE MEDIAWIKI#$title#; + print $FILE "$html_head"; + print $FILE "\n\n"; + # insert to index + $index{"$title"} = "$filename"; + + # prepare relative path - update URI references + $relative_path = $basedir; + $relative_path =~ s#[^/]+#..#g if $relative_path !~ /^\.$/; + $relative_path .= "/"; + $relative_path = '' if $relative_path =~ m#^./$#; + # print "$basedir -> $relative_path\n"; + + next; + } + + if ($line =~ /^\s*<\/doc>\s*$/) { + print $FILE "$html_foot"; + close $FILE; + next; + } + + # fix URI + decode_entities($line); + $line =~ s#href="(.*?)"#href="$relative_path\1.$suffix"#g; + $line =~ s/(?<=href=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge; + $line =~ s/(?<=href="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge; + $line =~ s#(?<=href="http)([^"]*)#($a=$1)=~s,%2F,/,g;"$a"#ge; + + # fix images + $line =~ s#img src="(.*?)"#img src="$relative_path../images/\u\1"#g; + $line =~ s/(?<=img src=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge; + $line =~ s/(?<=img src="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge; + + print $FILE "$line"; +} + +# make index.html +open($INDEX, ">", "$out_dir/$filename_index") or die "cannot open > $out_dir/$filename_index: $!"; +$html_head = $html_head_tmpl; +$html_head =~ s#OFFLINE MEDIAWIKI#Index of MediaWiki dump#; +print $INDEX "$html_head"; + +foreach $indx (sort keys %index) { + print $INDEX "$indx
"; +} + +print $INDEX "$html_foot"; +close $INDEX; + +exit 0; +