GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — wikiteam/dumpgenerator.py at master · WikiTeam/wikiteam Expand file tree/ Copy pathdumpgenerator.pyMore file actions executable file ·2572 lines (2310 loc) · 104 KB / Copy pathdumpgenerator.pyFile metadata and controlsexecutable file ·2572 lines (2310 loc) · 104 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 #!/usr/bin/env python2 # -*- coding: utf-8 -*- # dumpgenerator.py A generator of dumps for wikis # Copyright (C) 2011-2018 WikiTeam developers # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # To learn more, read the documentation: # https://github.com/WikiTeam/wikiteam/wiki try: from kitchen.text.converters import getwriter, to_unicode except ImportError: print "Please install the kitchen module." import cookielib import cPickle import datetime import sys try: import argparse except ImportError: print "Please install the argparse module." sys.exit(1) import json try: from hashlib import md5 except ImportError: # Python 2.4 compatibility from md5 import new as md5 import os import re import subprocess try: import requests except ImportError: print "Please install or update the Requests module." sys.exit(1) try: import mwclient except ImportError: print "Please install the mwclient module if you want to use --xmlrevisions." try: from lxml import etree from lxml.builder import E except ImportError: print "Please install the lxml module if you want to use --xmlrevisions." import time import urllib try: from urlparse import urlparse, urlunparse except ImportError: from urllib.parse import urlparse, urlunparse UTF8Writer = getwriter('utf8') sys.stdout = UTF8Writer(sys.stdout) __VERSION__ = '0.4.0-alpha' # major, minor, micro: semver.org class PageMissingError(Exception): def __init__(self, title, xml): self.title = title self.xml = xml def __str__(self): return "page '%s' not found" % self.title class ExportAbortedError(Exception): def __init__(self, index): self.index = index def __str__(self): return "Export from '%s' did not return anything." % self.index def getVersion(): return(__VERSION__) def truncateFilename(other={}, filename=''): """ Truncate filenames when downloading images with large filenames """ return filename[:other['filenamelimit']] + \ md5(filename.encode('utf-8')).hexdigest() + '.' + filename.split('.')[-1] def delay(config={}, session=None): """ Add a delay if configured for that """ if config['delay'] > 0: print 'Sleeping... %d seconds...' % (config['delay']) time.sleep(config['delay']) def cleanHTML(raw=''): """ Extract only the real wiki content and remove rubbish """ """ This function is ONLY used to retrieve page titles and file names when no API is available """ """ DO NOT use this function to extract page content """ # different "tags" used by different MediaWiki versions to mark where # starts and ends content if re.search('<!-- bodytext -->', raw): raw = raw.split('<!-- bodytext -->')[1].split('<!-- /bodytext -->')[0] elif re.search('<!-- start content -->', raw): raw = raw.split( '<!-- start content -->')[1].split('<!-- end content -->')[0] elif re.search('<!-- Begin Content Area -->', raw): raw = raw.split( '<!-- Begin Content Area -->')[1].split('<!-- End Content Area -->')[0] elif re.search('<!-- content -->', raw): raw = raw.split('<!-- content -->')[1].split('<!-- mw_content -->')[0] elif re.search('<article id="WikiaMainContent" class="WikiaMainContent">', raw): raw = raw.split('<article id="WikiaMainContent" class="WikiaMainContent">')[ 1].split('</article>')[0] elif re.search('<body class=', raw): raw = raw.split('<body class=')[1].split( '<div class="printfooter">')[0] else: print raw[:250] print 'This wiki doesn\'t use marks to split content' sys.exit() return raw def handleStatusCode(response): statuscode = response.status_code if statuscode >= 200 and statuscode < 300: return print "HTTP Error %d." % statuscode if statuscode >= 300 and statuscode < 400: print "Redirect should happen automatically: please report this as a bug." print response.url elif statuscode == 400: print "Bad Request: The wiki may be malfunctioning." print "Please try again later." print response.url sys.exit(1) elif statuscode == 401 or statuscode == 403: print "Authentication required." print "Please use --user and --pass." print response.url elif statuscode == 404: print "Not found. Is Special:Export enabled for this wiki?" print response.url sys.exit(1) elif statuscode == 429 or (statuscode >= 500 and statuscode < 600): print "Server error, max retries exceeded." print "Please resume the dump later." print response.url sys.exit(1) def getNamespacesScraper(config={}, session=None): """ Hackishly gets the list of namespaces names and ids from the dropdown in the HTML of Special:AllPages """ """ Function called if no API is available """ namespaces = config['namespaces'] namespacenames = {0: ''} # main is 0, no prefix if namespaces: r = session.post( url=config['index'], params={'title': 'Special:Allpages'}, timeout=30) raw = r.text delay(config=config, session=session) # [^>]*? to include selected="selected" m = re.compile( r'<option [^>]*?value=[\'"](?P<namespaceid>\d+)[\'"][^>]*?>(?P<namespacename>[^<]+)</option>').finditer(raw) if 'all' in namespaces: namespaces = [] for i in m: namespaces.append(int(i.group("namespaceid"))) namespacenames[int(i.group("namespaceid"))] = i.group( "namespacename") else: # check if those namespaces really exist in this wiki namespaces2 = [] for i in m: if int(i.group("namespaceid")) in namespaces: namespaces2.append(int(i.group("namespaceid"))) namespacenames[int(i.group("namespaceid"))] = i.group( "namespacename") namespaces = namespaces2 else: namespaces = [0] namespaces = list(set(namespaces)) # uniques print '%d namespaces found' % (len(namespaces)) return namespaces, namespacenames def getNamespacesAPI(config={}, session=None): """ Uses the API to get the list of namespaces names and ids """ namespaces = config['namespaces'] namespacenames = {0: ''} # main is 0, no prefix if namespaces: r = session.get( url=config['api'], params={ 'action': 'query', 'meta': 'siteinfo', 'siprop': 'namespaces', 'format': 'json'}, timeout=30 ) result = getJSON(r) delay(config=config, session=session) try: nsquery = result['query']['namespaces'] except KeyError: print "Error: could not get namespaces from the API request." print "HTTP %d" % r.status_code print r.text return None if 'all' in namespaces: namespaces = [] for i in nsquery.keys(): if int(i) < 0: # -1: Special, -2: Media, excluding continue namespaces.append(int(i)) namespacenames[int(i)] = nsquery[i]['*'] else: # check if those namespaces really exist in this wiki namespaces2 = [] for i in nsquery.keys(): bi = i i = int(i) if i < 0: # -1: Special, -2: Media, excluding continue if i in namespaces: namespaces2.append(i) namespacenames[i] = nsquery[bi]['*'] namespaces = namespaces2 else: namespaces = [0] namespaces = list(set(namespaces)) # uniques print '%d namespaces found' % (len(namespaces)) return namespaces, namespacenames def getPageTitlesAPI(config={}, session=None): """ Uses the API to get the list of page titles """ titles = [] namespaces, namespacenames = getNamespacesAPI( config=config, session=session) for namespace in namespaces: if namespace in config['exnamespaces']: print ' Skipping namespace = %d' % (namespace) continue c = 0 print ' Retrieving titles in the namespace %d' % (namespace) apiurl = urlparse(config['api']) site = mwclient.Site(apiurl.netloc, apiurl.path.replace("api.php", ""), scheme=apiurl.scheme, pool=session) for page in site.allpages(namespace=namespace): title = page.name titles.append(title) c += 1 yield title if len(titles) != len(set(titles)): print 'Probably a loop, switching to next namespace' titles = list(set(titles)) delay(config=config, session=session) print ' %d titles retrieved in the namespace %d' % (c, namespace) def getPageTitlesScraper(config={}, session=None): """ Scrape the list of page titles from Special:Allpages """ titles = [] namespaces, namespacenames = getNamespacesScraper( config=config, session=session) for namespace in namespaces: print ' Retrieving titles in the namespace', namespace url = '%s?title=Special:Allpages&namespace=%s' % ( config['index'], namespace) r = session.get(url=url, timeout=30) raw = r.text raw = cleanHTML(raw) r_title = r'title="(?P<title>[^>]+)">' r_suballpages = '' r_suballpages1 = r'&from=(?P<from>[^>"]+)&to=(?P<to>[^>"]+)">' r_suballpages2 = r'Special:Allpages/(?P<from>[^>"]+)">' r_suballpages3 = r'&from=(?P<from>[^>"]+)" title="[^>]+">' if re.search(r_suballpages1, raw): r_suballpages = r_suballpages1 elif re.search(r_suballpages2, raw): r_suballpages = r_suballpages2 elif re.search(r_suballpages3, raw): r_suballpages = r_suballpages3 else: pass # perhaps no subpages # Should be enough subpages on Special:Allpages deep = 50 c = 0 oldfr = '' checked_suballpages = [] rawacum = raw while r_suballpages and re.search(r_suballpages, raw) and c < deep: # load sub-Allpages m = re.compile(r_suballpages).finditer(raw) for i in m: fr = i.group('from') currfr = fr if oldfr == currfr: # We are looping, exit the loop pass if r_suballpages == r_suballpages1: to = i.group('to') name = '%s-%s' % (fr, to) url = '%s?title=Special:Allpages&namespace=%s&from=%s&to=%s' % ( config['index'], namespace, fr, to) # do not put urllib.quote in fr or to # fix, this regexp doesn't properly save everything? or does r_title fail on this # type of subpage? (wikiindex) elif r_suballpages == r_suballpages2: # clean &namespace=\d, sometimes happens fr = fr.split('&namespace=')[0] name = fr url = '%s?title=Special:Allpages/%s&namespace=%s' % ( config['index'], name, namespace) elif r_suballpages == r_suballpages3: fr = fr.split('&namespace=')[0] name = fr url = '%s?title=Special:Allpages&from=%s&namespace=%s' % ( config['index'], name, namespace) if name not in checked_suballpages: # to avoid reload dupe subpages links checked_suballpages.append(name) delay(config=config, session=session) r = session.get(url=url, timeout=10) #print 'Fetching URL: ', url raw = r.text raw = cleanHTML(raw) rawacum += raw # merge it after removed junk print ' Reading', name, len(raw), 'bytes', \ len(re.findall(r_suballpages, raw)), 'subpages', \ len(re.findall(r_title, raw)), 'pages' delay(config=config, session=session) oldfr = currfr c += 1 c = 0 m = re.compile(r_title).finditer(rawacum) for i in m: t = undoHTMLEntities(text=i.group('title')) if not t.startswith('Special:'): if t not in titles: titles.append(t) c += 1 print ' %d titles retrieved in the namespace %d' % (c, namespace) return titles def getPageTitles(config={}, session=None): """ Get list of page titles """ # http://en.wikipedia.org/wiki/Special:AllPages # http://archiveteam.org/index.php?title=Special:AllPages # http://www.wikanda.es/wiki/Especial:Todas print 'Loading page titles from namespaces = %s' % (config['namespaces'] and ','.join([str(i) for i in config['namespaces']]) or 'None') print 'Excluding titles from namespaces = %s' % (config['exnamespaces'] and ','.join([str(i) for i in config['exnamespaces']]) or 'None') titles = [] if 'api' in config and config['api']: try: titles = getPageTitlesAPI(config=config, session=session) except: print "Error: could not get page titles from the API" titles = getPageTitlesScraper(config=config, session=session) elif 'index' in config and config['index']: titles = getPageTitlesScraper(config=config, session=session) titlesfilename = '%s-%s-titles.txt' % ( domain2prefix(config=config), config['date']) titlesfile = open('%s/%s' % (config['path'], titlesfilename), 'wt') c = 0 for title in titles: titlesfile.write(title.encode('utf-8') + "\n") c += 1 # TODO: Sort to remove dupes? In CZ, Widget:AddThis appears two times: # main namespace and widget namespace. # We can use sort -u in UNIX, but is it worth it? titlesfile.write(u'--END--\n') titlesfile.close() print 'Titles saved at...', titlesfilename print '%d page titles loaded' % (c) return titlesfilename def getImageNames(config={}, session=None): """ Get list of image names """ print 'Retrieving image filenames' images = [] if 'api' in config and config['api']: images = getImageNamesAPI(config=config, session=session) elif 'index' in config and config['index']: images = getImageNamesScraper(config=config, session=session) # images = list(set(images)) # it is a list of lists images.sort() print '%d image names loaded' % (len(images)) return images def getXMLHeader(config={}, session=None): """ Retrieve a random page to extract XML headers (namespace info, etc) """ # get the header of a random page, to attach it in the complete XML backup # similar to: <mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" # xmlns:x.... randomtitle = 'Main_Page' # previously AMF5LKE43MNFGHKSDMRTJ print config['api'] xml = '' if config['xmlrevisions'] and config['api'] and config['api'].endswith("api.php"): try: print 'Getting the XML header from the API' # Export and exportnowrap exist from MediaWiki 1.15, allpages from 1.18 r = session.get(config['api'] + '?action=query&export=1&exportnowrap=1&list=allpages&aplimit=1', timeout=10) xml = r.text # Otherwise try without exportnowrap, e.g. Wikia returns a blank page on 1.19 if not re.match(r"\s*<mediawiki", xml): r = session.get(config['api'] + '?action=query&export=1&list=allpages&aplimit=1&format=json', timeout=10) try: xml = r.json()['query']['export']['*'] except KeyError: pass if not re.match(r"\s*<mediawiki", xml): # Do without a generator, use our usual trick of a random page title r = session.get(config['api'] + '?action=query&export=1&exportnowrap=1&titles=' + randomtitle, timeout=10) xml = r.text # Again try without exportnowrap if not re.match(r"\s*<mediawiki", xml): r = session.get(config['api'] + '?action=query&export=1&format=json&titles=' + randomtitle, timeout=10) try: xml = r.json()['query']['export']['*'] except KeyError: pass except requests.exceptions.RetryError: pass else: try: xml = "".join([x for x in getXMLPage(config=config, title=randomtitle, verbose=False, session=session)]) except PageMissingError as pme: # The <page> does not exist. Not a problem, if we get the <siteinfo>. xml = pme.xml # Issue 26: Account for missing "Special" namespace. # Hope the canonical special name has not been removed. # http://albens73.fr/wiki/api.php?action=query&meta=siteinfo&siprop=namespacealiases except ExportAbortedError: try: if config['api']: print "Trying the local name for the Special namespace instead" r = session.get( url=config['api'], params={ 'action': 'query', 'meta': 'siteinfo', 'siprop': 'namespaces', 'format': 'json'}, timeout=120 ) config['export'] = json.loads(r.text)['query']['namespaces']['-1']['*'] \ + ':Export' xml = "".join([x for x in getXMLPage(config=config, title=randomtitle, verbose=False, session=session)]) except PageMissingError as pme: xml = pme.xml except ExportAbortedError: pass header = xml.split('</mediawiki>')[0] if not re.match(r"\s*<mediawiki", xml): if config['xmlrevisions']: # Try again the old way print 'Export test via the API failed. Wiki too old? Trying without xmlrevisions.' config['xmlrevisions'] = False header, config = getXMLHeader(config=config, session=session) else: print 'XML export on this wiki is broken, quitting.' logerror(u'XML export on this wiki is broken, quitting.') sys.exit() return header, config def getXMLFileDesc(config={}, title='', session=None): """ Get XML for image description page """ config['curonly'] = 1 # tricky to get only the most recent desc return("".join([x for x in getXMLPage( config=config, title=title, verbose=False, session=session)])) def getUserAgent(): """ Return a cool user-agent to hide Python user-agent """ useragents = [ # firefox #'Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0', #'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0', #'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0' 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0' ] return useragents[0] def logerror(config={}, text=''): """ Log error in file """ if text: with open('%s/errors.log' % (config['path']), 'a') as outfile: output = u'%s: %s\n' % ( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), text) outfile.write(output.encode('utf-8')) def getXMLPageCore(headers={}, params={}, config={}, session=None): """ """ # returns a XML containing params['limit'] revisions (or current only), ending in </mediawiki> # if retrieving params['limit'] revisions fails, returns a current only version # if all fail, returns the empty string xml = '' c = 0 maxseconds = 100 # max seconds to wait in a single sleeping maxretries = config['retries'] # x retries and skip increment = 20 # increment every retry while not re.search(r'</mediawiki>', xml): if c > 0 and c < maxretries: wait = increment * c < maxseconds and increment * \ c or maxseconds # incremental until maxseconds print ' In attempt %d, XML for "%s" is wrong. Waiting %d seconds and reloading...' %(c, params['pages'], wait) time.sleep(wait) # reducing server load requesting smallest chunks (if curonly then # limit = 1 from mother function) if params['limit'] > 1: params['limit'] = params['limit'] / 2 # half if c >= maxretries: print ' We have retried %d times' % (c) print ' MediaWiki error for "%s", network error or whatever...' % (params['pages']) if config['failfast']: print "Exit, it will be for another time" sys.exit() # If it's not already what we tried: our last chance, preserve only the last revision... # config['curonly'] means that the whole dump is configured to save only the last, # params['curonly'] should mean that we've already tried this # fallback, because it's set by the following if and passed to # getXMLPageCore if not config['curonly'] and not 'curonly' in params: print ' Trying to save only the last revision for this page...' params['curonly'] = 1 logerror( config=config, text=u'Error while retrieving the full history of "%s". Trying to save only the last revision for this page' % (params['pages']) ) return getXMLPageCore( headers=headers, params=params, config=config, session=session ) else: print ' Saving in the errors log, and skipping...' logerror( config=config, text=u'Error while retrieving the last revision of "%s". Skipping.' % (params['pages'])) raise ExportAbortedError(config['index']) return '' # empty xml # FIXME HANDLE HTTP Errors HERE try: r = session.post(url=config['index'], params=params, headers=headers, timeout=10) handleStatusCode(r) xml = fixBOM(r) except requests.exceptions.ConnectionError as e: print ' Connection error: %s'%(str(e[0])) xml = '' except requests.exceptions.ReadTimeout as e: print ' Read timeout: %s'%(str(e[0])) xml = '' c += 1 return xml def getXMLPage(config={}, title='', verbose=True, session=None): """ Get the full history (or current only) of a page """ # if server errors occurs while retrieving the full page history, it may return [oldest OK versions] + last version, excluding middle revisions, so it would be partially truncated # http://www.mediawiki.org/wiki/Manual_talk:Parameters_to_Special:Export#Parameters_no_longer_in_use.3F limit = 1000 truncated = False title_ = title title_ = re.sub(' ', '_', title_) # do not convert & into %26, title_ = re.sub('&', '%26', title_) try: params = {'title': config['export'], 'pages': title_, 'action': 'submit'} except KeyError: params = {'title': 'Special:Export', 'pages': title_, 'action': 'submit'} if config['curonly']: params['curonly'] = 1 params['limit'] = 1 else: params['offset'] = '1' # 1 always < 2000s params['limit'] = limit # in other case, do not set params['templates'] if 'templates' in config and config['templates']: params['templates'] = 1 xml = getXMLPageCore(params=params, config=config, session=session) if xml == "": raise ExportAbortedError(config['index']) if not "</page>" in xml: raise PageMissingError(params['title'], xml) else: # strip these sha1s sums which keep showing up in the export and # which are invalid for the XML schema (they only apply to # revisions) xml = re.sub(r'\n\s*<sha1>\w+</sha1>\s*\n', r'\n', xml) xml = re.sub(r'\n\s*<sha1/>\s*\n', r'\n', xml) yield xml.split("</page>")[0] # if complete history, check if this page history has > limit edits, if so, retrieve all using offset if available # else, warning about Special:Export truncating large page histories r_timestamp = r'<timestamp>([^<]+)</timestamp>' numberofedits = 0 numberofedits += len(re.findall(r_timestamp, xml)) # search for timestamps in xml to avoid analysing empty pages like # Special:Allpages and the random one if not config['curonly'] and re.search(r_timestamp, xml): while not truncated and params['offset']: # next chunk # get the last timestamp from the acum XML params['offset'] = re.findall(r_timestamp, xml)[-1] try: xml2 = getXMLPageCore( params=params, config=config, session=session) except MemoryError: print "The page's history exceeds our memory, halving limit." params['limit'] = params['limit'] / 2 continue # are there more edits in this next XML chunk or no <page></page>? if re.findall(r_timestamp, xml2): if re.findall(r_timestamp, xml2)[-1] == params['offset']: # again the same XML, this wiki does not support params in # Special:Export, offer complete XML up to X edits (usually # 1000) print 'ATTENTION: This wiki does not allow some parameters in Special:Export, therefore pages with large histories may be truncated' truncated = True break else: """ </namespaces> </siteinfo> <page> <title>Main Page</title> <id>15580374</id> <restrictions>edit=sysop:move=sysop</restrictions> (?) <revision> <id>418009832</id> <timestamp>2011-03-09T19:57:06Z</timestamp> <contributor> """ # offset is OK in this wiki, merge with the previous chunk # of this page history and continue try: xml2 = xml2.split("</page>")[0] yield ' <revision>' + ('<revision>'.join(xml2.split('<revision>')[1:])) except MemoryError: print "The page's history exceeds our memory, halving limit." params['limit'] = params['limit'] / 2 continue xml = xml2 numberofedits += len(re.findall(r_timestamp, xml)) else: params['offset'] = '' # no more edits in this page history yield "</page>\n" if verbose: if (numberofedits == 1): print ' %s, 1 edit' % (title.strip()) else: print ' %s, %d edits' % (title.strip(), numberofedits) def makeXmlPageFromRaw(xml): """ Discard the metadata around a <page> element in <mediawiki> string""" root = etree.XML(xml) find = etree.XPath("//*[local-name() = 'page']") # The tag will inherit the namespace, like: # <page xmlns="http://www.mediawiki.org/xml/export-0.10/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> # FIXME: pretty_print doesn't seem to work, only adds a newline return etree.tostring(find(root)[0], pretty_print=True) def cleanXML(xml=''): """ Trim redundant info from the XML however it comes """ # do not touch XML codification, leave AS IS if re.search(r'</siteinfo>\n', xml): xml = xml.split('</siteinfo>\n')[1] if re.search(r'</mediawiki>', xml): xml = xml.split('</mediawiki>')[0] return xml def generateXMLDump(config={}, titles=[], start=None, session=None): """ Generates a XML dump for a list of titles or from revision IDs """ # TODO: titles is now unused. header, config = getXMLHeader(config=config, session=session) footer = '</mediawiki>\n' # new line at the end xmlfilename = '%s-%s-%s.xml' % (domain2prefix(config=config), config['date'], config['curonly'] and 'current' or 'history') xmlfile = '' lock = True if config['xmlrevisions']: if start: print("WARNING: will try to start the download from title: %s" % start) xmlfile = open('%s/%s' % (config['path'], xmlfilename), 'a') else: print 'Retrieving the XML for every page from the beginning' xmlfile = open('%s/%s' % (config['path'], xmlfilename), 'w') xmlfile.write(header.encode('utf-8')) try: r_timestamp = r'<timestamp>([^<]+)</timestamp>' for xml in getXMLRevisions(config=config, session=session, start=start): numrevs = len(re.findall(r_timestamp, xml)) # Due to how generators work, it's expected this may be less # TODO: get the page title and reuse the usual format "X title, y edits" print " %d more revisions exported" % numrevs xml = cleanXML(xml=xml) xmlfile.write(xml.encode('utf-8')) except AttributeError as e: print(e) print "This API library version is not working" sys.exit() else: print 'Retrieving the XML for every page from "%s"' % (start and start or 'start') if start: print "Removing the last chunk of past XML dump: it is probably incomplete." for i in reverse_readline('%s/%s' % (config['path'], xmlfilename), truncate=True): pass else: # requested complete xml dump lock = False xmlfile = open('%s/%s' % (config['path'], xmlfilename), 'w') xmlfile.write(header.encode('utf-8')) xmlfile.close() xmlfile = open('%s/%s' % (config['path'], xmlfilename), 'a') c = 1 for title in readTitles(config, start): if not title: continue if title == start: # start downloading from start, included lock = False if lock: continue delay(config=config, session=session) if c % 10 == 0: print 'Downloaded %d pages' % (c) try: for xml in getXMLPage(config=config, title=title, session=session): xml = cleanXML(xml=xml) xmlfile.write(xml.encode('utf-8')) except PageMissingError: logerror( config=config, text=u'The page "%s" was missing in the wiki (probably deleted)' % title ) # here, XML is a correct <page> </page> chunk or # an empty string due to a deleted page (logged in errors log) or # an empty string due to an error while retrieving the page from server # (logged in errors log) c += 1 xmlfile.write(footer) xmlfile.close() print 'XML dump saved at...', xmlfilename def getXMLRevisions(config={}, session=None, allpages=False, start=None): # FIXME: actually figure out the various strategies for each MediaWiki version apiurl = urlparse(config['api']) # FIXME: force the protocol we asked for! Or don't verify SSL if we asked HTTP? # https://github.com/WikiTeam/wikiteam/issues/358 site = mwclient.Site(apiurl.netloc, apiurl.path.replace("api.php", ""), scheme=apiurl.scheme, pool=session) if not 'all' in config['namespaces']: namespaces = config['namespaces'] else: namespaces, namespacenames = getNamespacesAPI(config=config, session=session) try: for namespace in namespaces: print("Trying to export all revisions from namespace %s" % namespace) # arvgeneratexml exists but was deprecated in 1.26 (while arv is from 1.27?!) arvparams = { 'action': 'query', 'list': 'allrevisions', 'arvlimit': 50, 'arvnamespace': namespace } if not config['curonly']: # We have to build the XML manually... # Skip flags, presumably needed to add <minor/> which is in the schema. # Also missing: parentid and contentformat. arvparams['arvprop'] = 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|content' print("Trying to get wikitext from the allrevisions API and to build the XML") while True: try: arvrequest = site.api(http_method=config['http_method'], **arvparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" continue except requests.exceptions.ReadTimeout as err: # Hopefully temporary, just wait a bit and continue with the same request. # No point putting a limit to retries, we'd need to abort everything. # TODO: reuse the retry logic of the checkAPI phase? Or force mwclient # to use the retry adapter we use for our own requests session? print("ERROR: {}".format(str(err))) print("Sleeping for 20 seconds") time.sleep(20) continue for page in arvrequest['query']['allrevisions']: try: yield makeXmlFromPage(page) except PageMissingError as e: logerror( config=config, text=u'Error: empty revision from API. Skipping %s' % e ) if 'continue' in arvrequest: arvparams['arvcontinue'] = arvrequest['continue']['arvcontinue'] else: # End of continuation. We are done with this namespace. break else: # FIXME: this is not curonly, just different strategy to do all revisions # Just cycle through revision IDs and use the XML as is print("Trying to list the revisions and to export them one by one") # We only need the revision ID, all the rest will come from the raw export arvparams['arvprop'] = 'ids' try: arvrequest = site.api(http_method=config['http_method'], **arvparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" continue exportparams = { 'action': 'query', 'export': '1', } # Skip the namespace if it's empty if len(arvrequest['query']['allrevisions']) < 1: continue # Repeat the arvrequest with new arvparams until done while True: # Reset revision IDs from the previous batch from arv revids = [] for page in arvrequest['query']['allrevisions']: for revision in page['revisions']: revids.append(str(revision['revid'])) print " %d more revisions listed, until %s" % (len(revids), revids[-1]) # We can now get the XML for one revision at a time # FIXME: we can actually get them in batches as we used to # but need to figure out the continuation and avoid that the API # chooses to give us only the latest for each page for revid in revids: exportparams['revids'] = revid try: exportrequest = site.api(http_method=config['http_method'], **exportparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" exportrequest = site.api(http_method=config['http_method'], **exportparams) # This gives us a self-standing <mediawiki> element # but we only need the inner <page>: we can live with # duplication and non-ordering of page titles, but the # repeated header is confusing and would not even be valid xml = exportrequest['query']['export']['*'] yield makeXmlPageFromRaw(xml) if 'continue' in arvrequest: # Get the new ones arvparams['arvcontinue'] = arvrequest['continue']['arvcontinue'] try: arvrequest = site.api(http_method=config['http_method'], **arvparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" arvrequest = site.api(http_method=config['http_method'], **arvparams) except requests.exceptions.ReadTimeout as err: # As above print("ERROR: {}".format(str(err))) print("Sleeping for 20 seconds") time.sleep(20) # But avoid rewriting the same revisions arvrequest['query']['allrevisions'] = [] continue else: # End of continuation. We are done with this namespace. break except (KeyError, mwclient.errors.InvalidResponse) as e: print(e) # TODO: check whether the KeyError was really for a missing arv API print "Warning. Could not use allrevisions. Wiki too old?" if config['curonly']: # The raw XML export in the API gets a title and gives the latest revision. # We could also use the allpages API as generator but let's be consistent. print("Getting titles to export the latest revision for each") c = 0 for title in readTitles(config, start=start): # TODO: respect verbose flag, reuse output from getXMLPage print(u' {}'.format(title)) # TODO: as we're doing one page and revision at a time, we might # as well use xml format and exportnowrap=1 to use the string of, # XML as is, but need to check how well the library handles it. exportparams = { 'action': 'query', 'titles': title, 'export': '1', } try: exportrequest = site.api(http_method=config['http_method'], **exportparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" exportrequest = site.api(http_method=config['http_method'], **exportparams) xml = exportrequest['query']['export']['*'] c += 1 if c % 10 == 0: print('Downloaded {} pages'.format(c)) # Because we got the fancy XML from the JSON format, clean it: yield makeXmlPageFromRaw(xml) else: # This is the closest to what we usually do with Special:Export: # take one title at a time and try to get all revisions exported. # It differs from the allrevisions method because it actually needs # to be input the page titles; otherwise, the requests are similar. # The XML needs to be made manually because the export=1 option # refuses to return an arbitrary number of revisions (see above). print("Getting titles to export all the revisions of each") c = 0 titlelist = [] # TODO: Decide a suitable number of a batched request. Careful: # batched responses may not return all revisions. for titlelist in readTitles(config, start=start, batch=False): if type(titlelist) is not list: titlelist = [titlelist] for title in titlelist: print(u' {}'.format(title)) # Try and ask everything. At least on MediaWiki 1.16, uknown props are discarded: # "warnings":{"revisions":{"*":"Unrecognized values for parameter 'rvprop': userid, sha1, contentmodel"}}} pparams = { 'action': 'query', 'titles': '|'.join(titlelist), 'prop': 'revisions', 'rvlimit': 50, 'rvprop': 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|content', } try: prequest = site.api(http_method=config['http_method'], **pparams) except requests.exceptions.HTTPError as e: if e.response.status_code == 405 and config['http_method'] == "POST": print("POST request to the API failed, retrying with GET") config['http_method'] = "GET" prequest = site.api(http_method=config['http_method'], **exportparams) except mwclient.errors.InvalidResponse: logerror( config=config, text=u'Error: page inaccessible? Could not export page: %s' % ("; ".join(titlelist)) ) |
| You are visitor 01402503 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |