scp fork

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+__pycache__

crawl.py

@@ -1,118 +1,121 @@
-import argparse
-import sys
-import locale
-import codecs
-import os
-from wikidot import Wikidot
-from rmaint import RepoMaintainer
-
-# TODO: Files.
-# TODO: Forum and comment pages.
-# TODO: Ability to download new transactions since last dump.
-#   We'll probably check the last revision time, then query all transactions and select those with greater revision time (not equal, since we would have downloaded equals at the previous dump)
-
-rawStdout = sys.stdout
-rawStderr = sys.stderr
-sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout, 'xmlcharrefreplace')
-sys.stderr = codecs.getwriter(locale.getpreferredencoding())(sys.stderr, 'xmlcharrefreplace')
-
-parser = argparse.ArgumentParser(description='Queries Wikidot')
-parser.add_argument('site', help='URL of Wikidot site')
-# Actions
-parser.add_argument('--list-pages', action='store_true', help='List all pages on this site')
-parser.add_argument('--source', action='store_true', help='Print page source (requires --page)')
-parser.add_argument('--content', action='store_true', help='Print page content (requires --page)')
-parser.add_argument('--log', action='store_true', help='Print page revision log (requires --page)')
-parser.add_argument('--dump', type=str, help='Download page revisions to this directory')
-# Debug actions
-parser.add_argument('--list-pages-raw', action='store_true')
-parser.add_argument('--log-raw', action='store_true')
-# Action settings
-parser.add_argument('--page', type=str, help='Query only this page')
-parser.add_argument('--depth', type=int, default='10000', help='Query only last N revisions')
-parser.add_argument('--revids', action='store_true', help='Store last revision ids in the repository')
-# Common settings
-parser.add_argument('--debug', action='store_true', help='Print debug info')
-parser.add_argument('--delay', type=int, default='200', help='Delay between consequent calls to Wikidot')
-args = parser.parse_args()
-
-
-wd = Wikidot(args.site)
-wd.debug = args.debug
-wd.delay = args.delay
-
-
-def force_dirs(path):
-    try:
-        os.makedirs(path)
-    except OSError as exception:
-        if exception.errno != os.errno.EEXIST:
-            raise
-
-if args.list_pages_raw:
-	print wd.list_pages_raw(args.depth)
-
-elif args.list_pages:
-	for page in wd.list_pages(args.depth):
-		print page
-
-elif args.source:
-	if not args.page:
-		raise "Please specify --page for --source."
-
-	page_id = wd.get_page_id(args.page)
-	if not page_id:
-		raise "Page not found: "+args.page
-
-	revs = wd.get_revisions(page_id, 1) # last revision
-	print wd.get_revision_source(revs[0]['id'])
-
-elif args.content:
-	if not args.page:
-		raise "Please specify --page for --source."
-
-	page_id = wd.get_page_id(args.page)
-	if not page_id:
-		raise "Page not found: "+args.page
-
-	revs = wd.get_revisions(page_id, 1) # last revision
-	print wd.get_revision_version(revs[0]['id'])
-
-elif args.log_raw:
-	if not args.page:
-		raise "Please specify --page for --log."
-
-	page_id = wd.get_page_id(args.page)
-	if not page_id:
-		raise "Page not found: "+args.page
-
-	print wd.get_revisions_raw(page_id, args.depth)
-
-
-elif args.log:
-	if not args.page:
-		raise "Please specify --page for --log."
-
-	page_id = wd.get_page_id(args.page)
-	if not page_id:
-		raise "Page not found: "+args.page
-	for rev in wd.get_revisions(page_id, args.depth):
-		print unicode(rev)
-
-
-elif args.dump:
-	print "Downloading pages to "+args.dump
-	force_dirs(args.dump)
-
-	rm = RepoMaintainer(wd, args.dump)
-	rm.debug = args.debug
-	rm.storeRevIds = args.revids
-	rm.buildRevisionList([args.page] if args.page else None, args.depth)
-	rm.openRepo()
-
-	print "Downloading revisions..."
-	while rm.commitNext():
-		pass
-
-	rm.cleanup()
-	print "Done."
+import argparse
+import sys
+import locale
+import codecs
+import os
+from wikidot import Wikidot
+from rmaint import RepoMaintainer
+
+# TODO: Files.
+# TODO: Forum and comment pages.
+# TODO: Ability to download new transactions since last dump.
+#   We'll probably check the last revision time, then query all transactions and select those with greater revision time (not equal, since we would have downloaded equals at the previous dump)
+
+parser = argparse.ArgumentParser(description='Queries Wikidot')
+parser.add_argument('site', help='URL of Wikidot site')
+# Actions
+parser.add_argument('--list-pages', action='store_true', help='List all pages on this site')
+parser.add_argument('--max-page-count', type=int, default='10000', help='Only list/fetch up to this amount of pages')
+parser.add_argument('--source', action='store_true', help='Print page source (requires --page)')
+parser.add_argument('--content', action='store_true', help='Print page content (requires --page)')
+parser.add_argument('--log', action='store_true', help='Print page revision log (requires --page)')
+parser.add_argument('--dump', type=str, help='Download page revisions to this directory')
+# Debug actions
+parser.add_argument('--list-pages-raw', action='store_true')
+parser.add_argument('--log-raw', action='store_true')
+# Action settings
+parser.add_argument('--page', type=str, help='Query only this page')
+parser.add_argument('--depth', type=int, default='10000', help='Query only last N revisions')
+parser.add_argument('--revids', action='store_true', help='Store last revision ids in the repository', default=True)
+parser.add_argument('--skip', type=str, help='Skip the specified revision')
+parser.add_argument('--skip-pages', type=str, help='Skip the specified pages')
+parser.add_argument('--cleanup', action='store_true', help='Clean up after downloading repo')
+# Common settings
+parser.add_argument('--debug', action='store_true', help='Print debug info')
+parser.add_argument('--delay', type=int, default='200', help='Delay between consequent calls to Wikidot')
+args = parser.parse_args()
+
+
+wd = Wikidot(args.site)
+wd.debug = args.debug
+wd.delay = args.delay
+
+
+def force_dirs(path):
+    os.makedirs(path, exist_ok=True)
+
+if args.list_pages_raw:
+    print((wd.list_pages_raw(limit = args.max_pages_count)))
+
+elif args.list_pages:
+    for page in wd.list_pages(limit = args.max_pages_count):
+        print(page)
+
+elif args.source:
+    if not args.page:
+        raise Exception("Please specify --page for --source.")
+
+    page_id = wd.get_page_id(page_unix_name=args.page)
+    if not page_id:
+        raise Exception("Page not found: "+args.page)
+
+    revs = wd.get_revisions(page_id, 1) # last revision
+    print((wd.get_revision_source(revs[0]['id'])))
+
+elif args.content:
+    if not args.page:
+        raise Exception("Please specify --page for --source.")
+
+    page_id = wd.get_page_id(page_unix_name=args.page)
+    if not page_id:
+        raise Exception("Page not found: "+args.page)
+
+    revs = wd.get_revisions(page_id, 1) # last revision
+    print((wd.get_revision_version(revs[0]['id'])))
+
+elif args.log_raw:
+    if not args.page:
+        raise Exception("Please specify --page for --log.")
+
+    page_id = wd.get_page_id(page_unix_name=args.page)
+    if not page_id:
+        raise Exception("Page not found: "+args.page)
+
+    print((wd.get_revisions_raw(page_id, args.depth)))
+
+
+elif args.log:
+    if not args.page:
+        raise Exception("Please specify --page for --log.")
+
+    page_id = wd.get_page_id(page_unix_name=args.page)
+    if not page_id:
+        raise Exception("Page not found: "+args.page)
+    for rev in wd.get_revisions(page_id, args.depth):
+        print((str(rev)))
+
+
+elif args.dump:
+    print(("Downloading pages to "+args.dump))
+    force_dirs(args.dump)
+
+    rm = RepoMaintainer(wd, args.dump)
+    rm.debug = args.debug
+    rm.storeRevIds = args.revids
+    rm.max_depth = args.depth
+    rm.max_page_count = args.max_page_count
+    rm.buildRevisionList([args.page] if args.page else None)
+    rm.openRepo()
+
+    if args.skip_pages:
+        rm.pages_to_skip = args.skip_pages.split(",")
+    if args.skip:
+        rm.revs_to_skip = args.skip.split(",")
+
+    print("Downloading revisions")
+    rm.fetchAll()
+
+    if args.cleanup:
+        rm.cleanup()
+
+    print("Done.")

readme.md

@@ -1,30 +1,50 @@
-This is a Python command line client for relatively popular wiki hosting http://www.wikidot.com which lets you:
-
-* List all pages on a site
-* See all revisions of a page
-* Query page source
-
-Most interestingly, it allows you to download the whole site as a Mercurial repository, with proper commit dates and comments!
-
-##### Examples:
-
-    crawl.py http://example.wikidot.com --dump ExampleRepo
-    crawl.py http://example.wikidot.com --log --page example-page
-
-It uses internal Wikidot AJAX requests to do it's job. If you're from Wikidot, please don't break it. Thank you! We'll try to be nice and not put a load on your servers.
-
-Downloading of large sites might take a while. If anything breaks, just restart the same command, it'll continue from where it crashed.
-
-##### Useful links:
-
-Wikidot code (very old) which simplifies things a bit:
-
-* https://github.com/gabrys/wikidot/blob/master/php/modules/history/PageRevisionListModule.php
-
-The descriptions for on-site modules are heavily correlated with AJAX ones:
-
-* http://www.wikidot.com/doc-modules:listpages-module
-
-Someone else did Wikidot AJAX:
-
-* https://github.com/kerel-fs/ogn-rdb/blob/master/wikidotcrawler.py
+*This is a fork to make a permanent backup of the SCP wiki.*
+
+This is a Python command line client for relatively popular wiki hosting
+http://www.wikidot.com which lets you:
+
+* List all pages on a site
+* See all revisions of a page
+* Query page source
+
+Most interestingly, it allows you to download the whole site as a Git repository, with proper commit dates, author and comments!
+
+##### Dependencies
+
+At least:
+
+* Python 3
+* python-beautifulsoup4
+* python-gitpython
+* python-requests
+* python-tqdm
+
+##### Examples:
+
+    crawl.py http://example.wikidot.com --dump ExampleRepo
+    crawl.py http://example.wikidot.com --log --page example-page
+
+It uses internal Wikidot AJAX requests to do it's job. If you're from Wikidot, please don't break it. Thank you! We'll try to be nice and not put a load on your servers.
+
+Downloading of large sites might take a while. If anything breaks, just restart the same command, it'll continue from where it crashed.
+
+##### Useful links:
+
+Wikidot code (very old) which simplifies things a bit:
+
+* https://github.com/gabrys/wikidot/blob/master/php/modules/history/PageRevisionListModule.php
+
+The descriptions for on-site modules are heavily correlated with AJAX ones:
+
+* http://www.wikidot.com/doc-modules:listpages-module
+
+Someone else did Wikidot AJAX:
+
+* https://github.com/kerel-fs/ogn-rdb/blob/master/wikidotcrawler.py
+
+
+#### TODO
+
+ - Handle deleted images. Probably need to check the diff and check all pages for references if removed from one page.
+ - Handle tags (both added and removed).
+