From e8fdf5ca5f66638ea9a6b95efd09e63c59b401c3 Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 16:24:33 +0100 Subject: [PATCH 1/9] Add makehtmlfiles.pl script for offline browsing This script build tree of files of html pages (one page of wiki equal one html page) from WikiExtractor output. This is very simple offline access to content of your MediaWiki. Also make index.html with sorted all stored pages. --- ChangeLog | 7 +++ README.md | 18 ++++++++ makehtmlfiles.pl | 108 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100755 makehtmlfiles.pl diff --git a/ChangeLog b/ChangeLog index c8167829..cdfeb1c2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +2017-01-03 Michal Svamberg + + * makehtmlfiles.pl: add new script for building web browsed content + This script build simple pages from output of WikiExtractor. + + For more information read example section in README.md + 2017-01-15 Giuseppe Attardi * WikiExtractor.py (process_dump): use text_type to decide whether diff --git a/README.md b/README.md index a2ec97d1..c2151d54 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,21 @@ Option --no-templates significantly speeds up the extractor, avoiding the cost of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). For further information, visit [the documentation](http://attardi.github.io/wikiextractor). + +## Example: Howto make offline content of your MediaWiki as web pages +Make dump of your mediawiki: + php5 /var/lib/mediawiki/maintenance/dumpBackup.php --current > /tmp/dump.xml + +Extract data from dump to one big file (param --html is necessary): + ./WikiExtractor.py -o - -xns 0,2,14,100,102,104,106,108,110,112,114,116 --links --all_links -b 200M --html --redirect_insert --keep_tables -q /tmp/dump.xml > /tmp/wiki.data + +Create single html pages and index.html: + cat /tmp/wiki.data | ./makehtmlfiles.pl /tmp/offline + +Optional you can copy images: + cd /var/lib/mediawiki/images + find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \; + +View offline content of yout wikimedia (without skins, images, templates, ...): + links file:///tmp/offline/index.html + diff --git a/makehtmlfiles.pl b/makehtmlfiles.pl new file mode 100755 index 00000000..504a80a9 --- /dev/null +++ b/makehtmlfiles.pl @@ -0,0 +1,108 @@ +#!/usr/bin/perl + +# vystup z wikiextractor ve formatu html rozseka podle tagu 'doc' na jednotlive +# soubory, prida k nim html hlavicku/paticku a upravi odkazy +use strict; +use URI::Escape; +use File::Basename; +use File::Path qw(make_path); +use HTML::Entities; + +my ($out_dir) = @ARGV; +if (not defined $out_dir) { + die "Missing param, usage: makehtmlfiles.pl \n"; +} + +my $html_head_tmpl = ' + + + + OFFLINE MEDIAWIKI + +'; +my $html_head = ''; + +my $html_foot = "\n\n"; + +my $basedir = ''; +my $title = ''; +my $suffix = 'html'; +my $filename = ''; +my $filename_index = "index.$suffix"; +my $line = ''; +my $FILE; +my $INDEX; +my %index = (); +my $indx = ''; # sorted index +my $relative_path = ''; + +make_path("$out_dir/pages", 0700) unless(-d "$out_dir/pages" ); +make_path("$out_dir/images", 0700) unless(-d "$out_dir/images" ); + +foreach $line ( ) { + chomp( $line ); + if ($line =~ /^\s*\s*$/) { + $title = "$1"; + #$filename = uri_escape("$title") . ".$suffix"; + $filename = "$title" . ".$suffix"; + # print "$title \t-> $filename\n"; + + $filename =~ s/\s/_/g; + $basedir = dirname("$filename"); + # print "MKDIR $basedir\n"; + make_path("$out_dir/pages/$basedir", 0700) unless(-d "$out_dir/pages/$basedir" ); + + open($FILE, ">", "$out_dir/pages/$filename") or die "cannot open > $out_dir/$filename: $!"; + + $html_head = $html_head_tmpl; + $html_head =~ s#OFFLINE MEDIAWIKI#$title#; + print $FILE "$html_head"; + print $FILE "\n\n"; + # insert to index + $index{"$title"} = uri_escape("$filename"); + + # prepare relative path - update URI references + $relative_path = $basedir; + $relative_path =~ s#[^/]+#..#g if $relative_path !~ /^\.$/; + $relative_path .= "/"; + $relative_path = '' if $relative_path =~ m#^./$#; + # print "$basedir -> $relative_path\n"; + + next; + } + + if ($line =~ /^\s*<\/doc>\s*$/) { + print $FILE "$html_foot"; + close $FILE; + next; + } + + # fix URI + decode_entities($line); + $line =~ s#href="(.*?)"#href="$relative_path\1.$suffix"#g; + $line =~ s/(?<=href=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge ; + $line =~ s/(?<=href="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge ; + + # fix images + $line =~ s#img src="(.*?)"#img src="$relative_path../images/\u\1"#g; + $line =~ s/(?<=img src=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge ; + $line =~ s/(?<=img src="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge ; + + print $FILE "$line"; +} + +# make index.html +open($INDEX, ">", "$out_dir/$filename_index") or die "cannot open > $out_dir/$filename_index: $!"; +$html_head = $html_head_tmpl; +$html_head =~ s#OFFLINE MEDIAWIKI#Index of MediaWiki dump#; +print $INDEX "$html_head"; + +foreach $indx (sort keys %index) { + print $INDEX "$indx
"; +} + +print $INDEX "$html_foot"; +close $INDEX; + +exit 0; + From bddbcdc04c2e002de9a972b717d3ff2faee3a55a Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 16:49:05 +0100 Subject: [PATCH 2/9] Changes for using makehtmlfiles.pl Add some options and better output for linking pages between themselves. --- ChangeLog | 12 ++++ README.md | 149 ++++++++++++++++++++++++++--------------------- WikiExtractor.py | 46 +++++++++++++-- 3 files changed, 134 insertions(+), 73 deletions(-) diff --git a/ChangeLog b/ChangeLog index cdfeb1c2..0ac1a222 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,18 @@ For more information read example section in README.md + * WikiExtractor.py (keepPage): check for -xns option (or default = 0) + * WikiExtractor.py (replaceInternalLinks): + - Fix URL links to other pages + - Add support for Image links (make html tag %s) + * WikiExtractor.py (compact): add lines beginned by space into tag + * WikiExtractor.py (pages_from): insert redirect pages if option + --redirect_insert is enabled + * WikiExtractor.py (main): add new options used in above changes + --all_links print links to all namespaces (independent of -xns or -ns) + --xml_namespaces add numeric namespaces for search pages + --redirect_insert create pages where used #REDIRECT wiki command + 2017-01-15 Giuseppe Attardi * WikiExtractor.py (process_dump): use text_type to decide whether diff --git a/README.md b/README.md index c2151d54..3820dfae 100644 --- a/README.md +++ b/README.md @@ -33,73 +33,88 @@ The script is invoked with a Wikipedia dump file as an argument. The output is stored in several files of similar size in a given directory. Each file will contains several documents in this [document format](http://medialab.di.unipi.it/wiki/Document_Format). - usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--html] [-l] [-s] - [--lists] [-ns ns1,ns2] [-xns ns1,ns2] - [--templates TEMPLATES] [--no-templates] - [-r] [--min_text_length MIN_TEXT_LENGTH] - [--filter_disambig_pages] [--processes PROCESSES] [-q] - [--debug] [-a] [-v] - input - - Wikipedia Extractor: - Extracts and cleans text from a Wikipedia database dump and stores output in a - number of files of similar size in a given directory. - Each file will contain several documents in the format: - - - ... - - - Template expansion requires preprocesssng first the whole dump and - collecting template definitions. - - positional arguments: - input XML wiki dump file - - optional arguments: - -h, --help show this help message and exit - --processes PROCESSES number of processes to use (default: number of CPU cores) - - Output: - -o OUTPUT, --output OUTPUT - directory for extracted files (or '-' for dumping to - stdout) - -b n[KMG], --bytes n[KMG] - maximum bytes per output file (default 1M) - -c, --compress compress output files using bzip - - Processing: - --html produce HTML output, subsumes --links - -l, --links preserve links - -s, --sections preserve sections - --lists preserve lists - -ns ns1,ns2, --namespaces ns1,ns2 - accepted link namespaces - -xns ns1,ns2, --xml_namespaces ns1,ns2 - accepted page xml namespaces -- 0 for main/articles - --templates TEMPLATES - use or create file containing templates - --no-templates Do not expand templates - -r, --revision Include the document revision id (default=False) - --min_text_length MIN_TEXT_LENGTH - Minimum expanded text length required to write - document (default=0) - --filter_disambig_pages - Remove pages from output that contain disabmiguation - markup (default=False) - -it, --ignored_tags - comma separated list of tags that will be dropped, keeping their content - -de, --discard_elements - comma separated list of elements that will be removed from the article text - --keep_tables - Preserve tables in the output article text (default=False) - - Special: - -q, --quiet suppress reporting progress info - --debug print debug info - -a, --article analyze a file containing a single article (debug - option) - -v, --version print program version + usage: WikiExtractor.py [-h] [-o OUTPUT] [-b n[KMG]] [-c] [--json] [--html] + [-l] [--all_links] [-s] [--lists] [-ns ns1,ns2] + [-xns ns1,ns2] [--templates TEMPLATES] + [--no-templates] [-r] + [--min_text_length MIN_TEXT_LENGTH] + [--filter_disambig_pages] [-it abbr,b,big] + [-de gallery,timeline,noinclude] [--keep_tables] + [--processes PROCESSES] [--redirect_insert] [-q] + [--debug] [-a] [-v] + input + + Wikipedia Extractor: + Extracts and cleans text from a Wikipedia database dump and stores output in a + number of files of similar size in a given directory. + Each file will contain several documents in the format: + + + ... + + + If the program is invoked with the --json flag, then each file will + contain several documents formatted as json ojects, one per line, with + the following structure + + {"id": "", "revid": "", "url":"", "title": "", "text": "..."} + + Template expansion requires preprocesssng first the whole dump and + collecting template definitions. + + positional arguments: + input XML wiki dump file + + optional arguments: + -h, --help show this help message and exit + --processes PROCESSES + Number of processes to use (default 3) + + Output: + -o OUTPUT, --output OUTPUT + directory for extracted files (or '-' for dumping to + stdout) + -b n[KMG], --bytes n[KMG] + maximum bytes per output file (default 1M) + -c, --compress compress output files using bzip + --json write output in json format instead of the default one + + Processing: + --html produce HTML output, subsumes --links + -l, --links preserve links + --all_links preserve all links (with links to all namespaces) + -s, --sections preserve sections + --lists preserve lists + -ns ns1,ns2, --namespaces ns1,ns2 + accepted namespaces in links + -xns ns1,ns2, --xml_namespaces ns1,ns2 + accepted page xml namespaces -- 0 for main/articles + --templates TEMPLATES + use or create file containing templates + --no-templates Do not expand templates + -r, --revision Include the document revision id (default=False) + --min_text_length MIN_TEXT_LENGTH + Minimum expanded text length required to write + document (default=0) + --filter_disambig_pages + Remove pages from output that contain disabmiguation + markup (default=False) + -it abbr,b,big, --ignored_tags abbr,b,big + comma separated list of tags that will be dropped, + keeping their content + -de gallery,timeline,noinclude, --discard_elements gallery,timeline,noinclude + comma separated list of elements that will be removed + from the article text + --keep_tables Preserve tables in the output article text + (default=False) + --redirect_insert Insert redirect pages to output + + Special: + -q, --quiet suppress reporting progress info + --debug print debug info + -a, --article analyze a file containing a single article (debug + option) + -v, --version print program version Saving templates to a file will speed up performing extraction the next time, diff --git a/WikiExtractor.py b/WikiExtractor.py index ae448364..5357473c 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -133,6 +133,9 @@ def __eq__ (self, other): # acceptedNamespaces=['w', 'wiktionary', 'wikt'], + + acceptedXMLNamespaces=['0'], + # This is obtained from urlbase='', @@ -212,7 +215,8 @@ def __eq__ (self, other): ## # page filtering logic -- remove templates, undesired xml namespaces, and disambiguation pages def keepPage(ns, page): - if ns != '0': # Aritcle + #if ns != '0': # Aritcle + if ns not in acceptedXMLNamespaces: return False # remove disambig pages if desired if options.filter_disambig_pages: @@ -2055,6 +2059,7 @@ def replaceInternalLinks(text): curp = e1 label = inner[pipe + 1:].strip() res += text[cur:s] + makeInternalLink(title, label) + trail +# logging.error('%s', makeInternalLink(title, label)) cur = end return res + text[cur:] @@ -2326,15 +2331,27 @@ def replaceInternalLinks(text): def makeInternalLink(title, label): colon = title.find(':') - if colon > 0 and title[:colon] not in options.acceptedNamespaces: - return '' + if not options.keepAllLinks: + if colon > 0 and title[:colon] not in options.acceptedNamespaces: + return '['+ title + ']' if colon == 0: # drop also :File: colon2 = title.find(':', colon + 1) if colon2 > 1 and title[colon + 1:colon2] not in options.acceptedNamespaces: - return '' + if options.keepAllLinks: + # remove first colons + title = title.lstrip(':') + label = label.lstrip(':') + else: + return '['+ title + ']' if options.keepLinks: - return '%s' % (quote(title.encode('utf-8')), label) + if title[:colon] == 'Image': + #logging.error('%s: %s -> %s', title[:colon], title, label) + title_short = title.split(':', 1) + label_short = label.split('|') + return '%s' % (quote(title_short[1].encode('utf-8')), label_short[-1]) + else: + return '%s' % (quote(title.encode('utf-8')), label) else: return label @@ -2542,6 +2559,11 @@ def compact(text): listLevel = [] listCount = [] page.append(line) + elif line[0] == ' ': + if options.toHTML: + page.append("
%s
" % (line)) + else: + page.append(line) # Drop residuals of lists elif line[0] in '{|' or line[-1] == '}': @@ -2731,7 +2753,7 @@ def pages_from(input): elif tag == 'ns': ns = m.group(3) elif tag == 'redirect': - redirect = True + redirect = False if options.redirect_insert else True elif tag == 'text': if m.lastindex == 3 and line[m.start(3)-2] == '/': # self closing # @@ -3005,6 +3027,7 @@ def reduce_process(opts, output_queue, spool_length, minFileSize = 200 * 1024 def main(): + global acceptedXMLNamespaces parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), formatter_class=argparse.RawDescriptionHelpFormatter, @@ -3028,12 +3051,16 @@ def main(): help="produce HTML output, subsumes --links") groupP.add_argument("-l", "--links", action="store_true", help="preserve links") + groupP.add_argument("--all_links", action="store_true", + help="preserve all links (with links to all namespaces)") groupP.add_argument("-s", "--sections", action="store_true", help="preserve sections") groupP.add_argument("--lists", action="store_true", help="preserve lists") groupP.add_argument("-ns", "--namespaces", default="", metavar="ns1,ns2", help="accepted namespaces in links") + groupP.add_argument("-xns", "--xml_namespaces", default="", metavar="ns1,ns2", + help="accepted page xml namespaces -- 0 for main/articles") groupP.add_argument("--templates", help="use or create file containing templates") groupP.add_argument("--no-templates", action="store_false", @@ -3053,6 +3080,8 @@ def main(): default_process_count = max(1, cpu_count() - 1) parser.add_argument("--processes", type=int, default=default_process_count, help="Number of processes to use (default %(default)s)") + groupP.add_argument("--redirect_insert", action="store_true", + help="Insert redirect pages to output") groupS = parser.add_argument_group('Special') groupS.add_argument("-q", "--quiet", action="store_true", @@ -3068,12 +3097,14 @@ def main(): args = parser.parse_args() options.keepLinks = args.links + options.keepAllLinks = args.all_links options.keepSections = args.sections options.keepLists = args.lists options.toHTML = args.html options.write_json = args.json options.print_revision = args.revision options.min_text_length = args.min_text_length + options.redirect_insert = args.redirect_insert if args.html: options.keepLinks = True @@ -3111,6 +3142,9 @@ def main(): if args.discard_elements: options.discardElements = set(args.discard_elements.split(',')) + if args.xml_namespaces: + acceptedXMLNamespaces = set([unicode(ns) for ns in args.xml_namespaces.split(',')]) + FORMAT = '%(levelname)s: %(message)s' logging.basicConfig(format=FORMAT) From 5d6d25d993df0108f66799a9802ef86575282c38 Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 16:53:01 +0100 Subject: [PATCH 3/9] Fix formatting of README.md Fixed new lines. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 3820dfae..a63a0ba5 100644 --- a/README.md +++ b/README.md @@ -127,18 +127,23 @@ For further information, visit [the documentation](http://attardi.github.io/wiki ## Example: Howto make offline content of your MediaWiki as web pages Make dump of your mediawiki: + php5 /var/lib/mediawiki/maintenance/dumpBackup.php --current > /tmp/dump.xml Extract data from dump to one big file (param --html is necessary): + ./WikiExtractor.py -o - -xns 0,2,14,100,102,104,106,108,110,112,114,116 --links --all_links -b 200M --html --redirect_insert --keep_tables -q /tmp/dump.xml > /tmp/wiki.data Create single html pages and index.html: + cat /tmp/wiki.data | ./makehtmlfiles.pl /tmp/offline Optional you can copy images: + cd /var/lib/mediawiki/images find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \; View offline content of yout wikimedia (without skins, images, templates, ...): + links file:///tmp/offline/index.html From f5d4237fbcb1a8729915a616d5018beadee5a21d Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 16:55:15 +0100 Subject: [PATCH 4/9] Add numbered list in example usage in README.md Better reading. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a63a0ba5..d4349690 100644 --- a/README.md +++ b/README.md @@ -126,24 +126,24 @@ of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates For further information, visit [the documentation](http://attardi.github.io/wikiextractor). ## Example: Howto make offline content of your MediaWiki as web pages -Make dump of your mediawiki: +1. Make dump of your mediawiki: php5 /var/lib/mediawiki/maintenance/dumpBackup.php --current > /tmp/dump.xml -Extract data from dump to one big file (param --html is necessary): +2. Extract data from dump to one big file (param --html is necessary): ./WikiExtractor.py -o - -xns 0,2,14,100,102,104,106,108,110,112,114,116 --links --all_links -b 200M --html --redirect_insert --keep_tables -q /tmp/dump.xml > /tmp/wiki.data -Create single html pages and index.html: +3. Create single html pages and index.html: cat /tmp/wiki.data | ./makehtmlfiles.pl /tmp/offline -Optional you can copy images: +4. Optional you can copy images: cd /var/lib/mediawiki/images find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \; -View offline content of yout wikimedia (without skins, images, templates, ...): +5. View offline content of yout wikimedia (without skins, images, templates, ...): links file:///tmp/offline/index.html From defea1ac3563d6aed3131ba58756d3847ed27a9e Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 16:59:09 +0100 Subject: [PATCH 5/9] Fix typo in README.md Better reading. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4349690..16998374 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ For further information, visit [the documentation](http://attardi.github.io/wiki cd /var/lib/mediawiki/images find 0 1 2 3 4 5 6 7 8 9 a b c d e f -type f -exec cp -f {} /tmp/offline/images/ \; -5. View offline content of yout wikimedia (without skins, images, templates, ...): +5. View offline content ONLY (without skins, templates, scripts, ...) of your wikimedia: links file:///tmp/offline/index.html From 91130565d61927aeea2c2b7720c44249a104f0a7 Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 17:04:51 +0100 Subject: [PATCH 6/9] README.md: better format of usage Remove some newlines --- README.md | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 16998374..a357a37b 100644 --- a/README.md +++ b/README.md @@ -45,22 +45,18 @@ Each file will contains several documents in this [document format](http://media input Wikipedia Extractor: - Extracts and cleans text from a Wikipedia database dump and stores output in a - number of files of similar size in a given directory. + Extracts and cleans text from a Wikipedia database dump and stores output in a number of files of similar size in a given directory. Each file will contain several documents in the format: ... - If the program is invoked with the --json flag, then each file will - contain several documents formatted as json ojects, one per line, with - the following structure + If the program is invoked with the --json flag, then each file will contain several documents formatted as json ojects, one per line, with the following structure {"id": "", "revid": "", "url":"", "title": "", "text": "..."} - Template expansion requires preprocesssng first the whole dump and - collecting template definitions. + Template expansion requires preprocesssng first the whole dump and collecting template definitions. positional arguments: input XML wiki dump file @@ -72,8 +68,7 @@ Each file will contains several documents in this [document format](http://media Output: -o OUTPUT, --output OUTPUT - directory for extracted files (or '-' for dumping to - stdout) + directory for extracted files (or '-' for dumping to stdout) -b n[KMG], --bytes n[KMG] maximum bytes per output file (default 1M) -c, --compress compress output files using bzip @@ -94,34 +89,26 @@ Each file will contains several documents in this [document format](http://media --no-templates Do not expand templates -r, --revision Include the document revision id (default=False) --min_text_length MIN_TEXT_LENGTH - Minimum expanded text length required to write - document (default=0) + Minimum expanded text length required to write document (default=0) --filter_disambig_pages - Remove pages from output that contain disabmiguation - markup (default=False) + Remove pages from output that contain disabmiguation markup (default=False) -it abbr,b,big, --ignored_tags abbr,b,big - comma separated list of tags that will be dropped, - keeping their content + comma separated list of tags that will be dropped, keeping their content -de gallery,timeline,noinclude, --discard_elements gallery,timeline,noinclude - comma separated list of elements that will be removed - from the article text - --keep_tables Preserve tables in the output article text - (default=False) + comma separated list of elements that will be removed from the article text + --keep_tables Preserve tables in the output article text (default=False) --redirect_insert Insert redirect pages to output Special: -q, --quiet suppress reporting progress info --debug print debug info - -a, --article analyze a file containing a single article (debug - option) + -a, --article analyze a file containing a single article (debug option) -v, --version print program version -Saving templates to a file will speed up performing extraction the next time, -assuming template definitions have not changed. +Saving templates to a file will speed up performing extraction the next time, assuming template definitions have not changed. -Option --no-templates significantly speeds up the extractor, avoiding the cost -of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). +Option --no-templates significantly speeds up the extractor, avoiding the cost of expanding [MediaWiki templates](https://www.mediawiki.org/wiki/Help:Templates). For further information, visit [the documentation](http://attardi.github.io/wikiextractor). From bc7823191cba3241206d63806fab3ffd031a21e2 Mon Sep 17 00:00:00 2001 From: Michal Svamberg Date: Wed, 1 Mar 2017 17:08:56 +0100 Subject: [PATCH 7/9] README.md: long lines split by newline Wrap too long lines. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a357a37b..8ecb0fe6 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,16 @@ Each file will contains several documents in this [document format](http://media input Wikipedia Extractor: - Extracts and cleans text from a Wikipedia database dump and stores output in a number of files of similar size in a given directory. + Extracts and cleans text from a Wikipedia database dump and stores output in a number of files + of similar size in a given directory. Each file will contain several documents in the format: ... - If the program is invoked with the --json flag, then each file will contain several documents formatted as json ojects, one per line, with the following structure + If the program is invoked with the --json flag, then each file will contain several documents + formatted as json ojects, one per line, with the following structure {"id": "", "revid": "", "url":"", "title": "", "text": "..."} From 841576ec09a05ae423ab1815845be99c6e0704e2 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Mar 2017 20:13:18 +0100 Subject: [PATCH 8/9] Fix URI escape syntax in href attribute Make better links between pages. --- makehtmlfiles.pl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/makehtmlfiles.pl b/makehtmlfiles.pl index 504a80a9..e8ffd80b 100755 --- a/makehtmlfiles.pl +++ b/makehtmlfiles.pl @@ -59,7 +59,7 @@ print $FILE "$html_head"; print $FILE "\n\n"; # insert to index - $index{"$title"} = uri_escape("$filename"); + $index{"$title"} = "$filename"; # prepare relative path - update URI references $relative_path = $basedir; @@ -80,13 +80,14 @@ # fix URI decode_entities($line); $line =~ s#href="(.*?)"#href="$relative_path\1.$suffix"#g; - $line =~ s/(?<=href=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge ; - $line =~ s/(?<=href="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge ; + $line =~ s/(?<=href=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge; + $line =~ s/(?<=href="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge; + $line =~ s#(?<=href="http)([^"]*)#($a=$1)=~s,%2F,/,g;"$a"#ge; # fix images $line =~ s#img src="(.*?)"#img src="$relative_path../images/\u\1"#g; - $line =~ s/(?<=img src=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge ; - $line =~ s/(?<=img src="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge ; + $line =~ s/(?<=img src=")([^"]*)/($a=$1)=~s, |%20,_,g;"$a"/ge; + $line =~ s/(?<=img src="http)([^"]*)/($a=$1)=~s,%3A,:,g;"$a"/ge; print $FILE "$line"; } From 0d616bef6d875d3f1fccb799252b87d9a5643777 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 2 Mar 2017 11:14:16 +0100 Subject: [PATCH 9/9] Better syntax about conditions aftereffect of comment in https://github.com/attardi/wikiextractor/pull/110 --- WikiExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WikiExtractor.py b/WikiExtractor.py index 5357473c..fa1ab359 100755 --- a/WikiExtractor.py +++ b/WikiExtractor.py @@ -2753,7 +2753,7 @@ def pages_from(input): elif tag == 'ns': ns = m.group(3) elif tag == 'redirect': - redirect = False if options.redirect_insert else True + redirect = not options.redirect_insert elif tag == 'text': if m.lastindex == 3 and line[m.start(3)-2] == '/': # self closing #