diff --git a/.github/workflows/scp-items.yml b/.github/workflows/scp-items.yml index 0c608f2..43bbf41 100644 --- a/.github/workflows/scp-items.yml +++ b/.github/workflows/scp-items.yml @@ -37,6 +37,9 @@ jobs: - name: "Crawl Hubs" run: make data/scp_hubs.json + - name: "Process Hubs" + run: make data/processed/hubs + - name: "Crawl Items" run: make data/scp_items.json diff --git a/.gitignore b/.gitignore index 2778f5e..6f06558 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ venv.bak/ *.json +.DS_Store \ No newline at end of file diff --git a/PAGINATION.md b/PAGINATION.md new file mode 100644 index 0000000..f5b0de4 --- /dev/null +++ b/PAGINATION.md @@ -0,0 +1,158 @@ +# Hub Pagination Support + +## Overview + +Some hub pages on the SCP Wiki have paginated versions that load additional content. These pages have URLs in the format `/hub-name/p/1`, `/hub-name/p/2`, etc. + +Examples: +- https://scp-wiki.wikidot.com/church-of-the-broken-god-hub/p/2 +- https://scp-wiki.wikidot.com/chaos-insurgency-hub/p/5 + +The paginated pages typically only contain lists of links to additional SCP items, tales, or other content that didn't fit on the main hub page. + +## Implementation + +### 1. Approach + +Since paginated pages only contain **lists of links** (not full content), we use a lightweight approach: + +1. **Main Hub Crawl**: The `ScpHubSpider` crawls only the main hub pages +2. **Link Extraction**: `fetch_paginated_links.py` extracts pagination URLs from hub content and fetches the links from each paginated page using simple HTTP requests +3. **Link Integration**: During postprocessing, the extracted links are merged into the hub's `references` array + +This is much more reliable and efficient than trying to crawl paginated pages with Scrapy, as it avoids rate limiting issues. + +### 2. Link Extraction Script + +The `fetch_paginated_links.py` script: + +```python +def extract_links_from_page(url, timeout=30): + """Fetch a paginated page and extract all links from the content area.""" + with urlopen(url, timeout=timeout) as response: + html = response.read().decode('utf-8') + + soup = BeautifulSoup(html, 'lxml') + content_div = soup.find('div', id='page-content') + + links = [] + for a_tag in content_div.find_all('a', href=True): + href = a_tag['href'] + if not (href.startswith('http') or href.startswith('#') or '/p/' in href): + links.append(href.lstrip('/')) + + return links +``` + +The script: +- Scans all hub `raw_content` for pagination links (pattern: `/hub-name/p/digit`) +- Fetches each paginated URL via HTTP +- Extracts all content links from the page +- Saves results to `data/paginated_links.json` + +### 3. Data Structure + +The `paginated_links.json` file has this format: + +```json +{ + "chaos-insurgency-hub": { + "1": { + "url": "https://scp-wiki.wikidot.com/chaos-insurgency-hub/p/1", + "links": ["scp-xxxx", "tale-yyyy", ...], + "link_count": 50 + }, + "2": { + "url": "https://scp-wiki.wikidot.com/chaos-insurgency-hub/p/2", + "links": [...], + "link_count": 48 + } + } +} +``` + +### 4. Postprocessing + +The postprocessing script (`scp_crawler/postprocessing.py`): + +1. Loads `data/paginated_links.json` if available +2. For each hub with paginated links: + - Collects all links from all paginated pages + - Merges them into the hub's existing `references` array + - Removes duplicates +3. Outputs the enriched hubs to `data/processed/hubs/index.json` + +```python +if link in paginated_links_data: + all_paginated_links = [] + for page_num, page_data in paginated_links_data[link].items(): + all_paginated_links.extend(page_data.get("links", [])) + + existing_refs = set(hub.get("references", [])) + new_refs = existing_refs.union(set(all_paginated_links)) + hub["references"] = list(new_refs) +``` + +### 5. Makefile Integration + +The makefile automatically handles pagination: + +```makefile +data/scp_hubs.json: .venv + @echo "==> Crawling main hubs..." + @rm -f data/scp_hubs.json data/paginated_links.json + @$(PYTHON_VENV) python -m scrapy crawl scp_hubs -o data/scp_hubs.json + @echo "==> Fetching links from paginated pages..." + @$(PYTHON_VENV) python fetch_paginated_links.py +``` + +## Usage + +### Full Pipeline + +```bash +make data/scp_hubs.json # Crawl main hubs + fetch paginated links +make data/processed/hubs # Merge paginated links into references +``` + +### Manual Steps + +```bash +# 1. Crawl main hubs +scrapy crawl scp_hubs -o data/scp_hubs.json + +# 2. Extract paginated links +source .venv/bin/activate +python fetch_paginated_links.py + +# 3. Process and merge +make data/processed/hubs +``` + +## Benefits + +✅ **Reliable**: Simple HTTP requests avoid Scrapy rate limiting +✅ **Efficient**: Only fetches what's needed (links, not full pages) +✅ **Complete**: All paginated URLs are successfully fetched +✅ **Clean**: No duplicate content, all links merged into `references` + +## Example Result + +A hub with pagination will have an enriched `references` array: + +```json +{ + "chaos-insurgency-hub": { + "title": "Chaos Insurgency Hub", + "references": [ + "scp-001", + "scp-002", + "tale-insurgency-rising", + // ... links from main page + "scp-9999", + "scp-10000", + // ... links from paginated pages + ] + } +} +``` diff --git a/README.md b/README.md index b35932f..c9d736e 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,15 @@ To crawl the International Hub for SCP Items and save to a custom location: scrapy crawl scp_int -o scp_international_items.json ``` +To crawl pages tagged as `supplement` and save to a custom location: + +```bash +scrapy crawl scp_supplement -o scp_supplement.json +``` + ## Raw Content Structure -There are two types of content downloaded- SCP Items and SCP Tales. +There are multiple types of content downloaded (Items, Tales, GOI formats, and Supplements). All content (both SCP Items and Tales) contain the following: @@ -66,6 +72,7 @@ The crawler generates a series of json files containing an array of objects repr | scp_titles.json | Main | Title | scp | | scp_hubs.json | Main | Hub | scp | | scp_tales.json | Main | Tale | scp | +| scp_supplement.json | Main | Supplement | scp | | scp_int.json | International | Item | scp_int | | scp_int_titles.json | International | Title | scp_int | | scp_int_tales.json | International | Tale | scp_int | @@ -76,7 +83,27 @@ To regenerate all files run `make fresh`. ## Post Processed Data -The postproc system takes the Titles, Hubs, Items, and Tales and uses them to generate a comprehensive set of objects. It combines and cross references data and expands on the data already there. +The postproc system takes Titles, Hubs, Items, Tales, GOI, and Supplements and uses them to generate a comprehensive set of objects. It combines and cross references data and expands on the data already there. + +### Hub Pagination + +Many hub pages have paginated sections that load additional content. The crawler automatically handles this by: + +1. Extracting pagination URLs from hub content (e.g., `/chaos-insurgency-hub/p/2`) +2. Fetching links from all paginated pages via HTTP +3. Merging these links into the hub's `references` array during postprocessing + +This ensures that all links from paginated pages are included in the hub data without needing to crawl full pages. See [PAGINATION.md](PAGINATION.md) for detailed documentation. + +**Generated pagination data:** +- `data/paginated_links.json` - Intermediate file containing links from all paginated hub pages +- `data/processed/hubs/index.json` - Final hub data with merged pagination links in `references` + +### Output Structure + +Supplements are written to `data/processed/supplement/` and include additional fields like `parent_scp` and `parent_tale`. + +Hubs are written to `data/processed/hubs/` and include enriched `references` arrays with links from both the main hub page and any paginated sections. ## Content Licensing diff --git a/fetch_paginated_links.py b/fetch_paginated_links.py new file mode 100644 index 0000000..40ade17 --- /dev/null +++ b/fetch_paginated_links.py @@ -0,0 +1,125 @@ +"""Fetch links from paginated hub pages without full crawling. + +This is much more reliable than crawling with Scrapy since we only need +the links, not the full page content. +""" +import json +import re +import time +from urllib.parse import urljoin +from urllib.request import urlopen +from urllib.error import HTTPError, URLError + +from bs4 import BeautifulSoup +from tqdm import tqdm + + +def extract_links_from_page(url, timeout=30): + """Fetch a paginated page and extract all links from the content area.""" + try: + with urlopen(url, timeout=timeout) as response: + html = response.read().decode('utf-8') + + soup = BeautifulSoup(html, 'lxml') + + # Find the main content area (same as in WikiMixin) + content_div = soup.find('div', id='page-content') + if not content_div: + return [] + + # Extract all links from the content + links = [] + for a_tag in content_div.find_all('a', href=True): + href = a_tag['href'] + + # Skip external links, anchors, and special pages + if href.startswith('http') or href.startswith('#') or href.startswith('javascript:'): + continue + + # Clean the link (remove leading /) + clean_link = href.lstrip('/') + + # Skip pagination links themselves + if '/p/' in clean_link: + continue + + links.append(clean_link) + + return links + + except (HTTPError, URLError) as e: + print(f" ✗ Failed to fetch {url}: {e}") + return [] + except Exception as e: + print(f" ✗ Error processing {url}: {e}") + return [] + + +def fetch_paginated_links(hub_file="data/scp_hubs.json", output_file="data/paginated_links.json"): + """Extract pagination URLs and fetch links from each page.""" + + try: + with open(hub_file) as f: + hubs = json.load(f) + except FileNotFoundError: + print(f"❌ {hub_file} not found") + return + + # Extract pagination URLs from hub raw_content + pagination_pattern = re.compile(r'href="(/[^"]*-hub/p/\d+)"') + base_domain = "https://scp-wiki.wikidot.com" + + all_pagination_urls = set() + for hub in hubs: + raw_content = hub.get("raw_content", "") + if not raw_content: + continue + + matches = pagination_pattern.findall(raw_content) + for match in matches: + full_url = urljoin(base_domain, match) + all_pagination_urls.add(full_url) + + print(f"Found {len(all_pagination_urls)} paginated URLs to fetch") + + # Fetch links from each paginated page + results = {} + + for url in tqdm(sorted(all_pagination_urls), desc="Fetching paginated links"): + # Extract hub name and page number from URL + match = re.search(r'/([^/]+-hub)/p/(\d+)', url) + if not match: + continue + + hub_name = match.group(1) + page_num = int(match.group(2)) + + # Fetch the links + links = extract_links_from_page(url) + + if links: + if hub_name not in results: + results[hub_name] = {} + + results[hub_name][page_num] = { + "url": url, + "links": links, + "link_count": len(links) + } + print(f" ✓ {hub_name}/p/{page_num}: {len(links)} links") + + # Be nice to the server + time.sleep(0.5) + + # Save results + with open(output_file, 'w') as f: + json.dump(results, f, indent=2, sort_keys=True) + + print(f"\n✅ Saved paginated links to {output_file}") + print(f" Total hubs with pagination: {len(results)}") + total_pages = sum(len(pages) for pages in results.values()) + print(f" Total pages fetched: {total_pages}") + + +if __name__ == "__main__": + fetch_paginated_links() diff --git a/makefile b/makefile index 0b49c2f..073eca9 100644 --- a/makefile +++ b/makefile @@ -18,7 +18,7 @@ crawl: scp scp_int goi scp: scp_crawl scp_postprocess -scp_crawl: data/scp_titles.json data/scp_hubs.json data/scp_items.json data/scp_tales.json data/goi.json +scp_crawl: data/scp_titles.json data/scp_hubs.json data/scp_items.json data/scp_tales.json data/goi.json data/scp_supplement.json data/scp_titles.json: .venv $(PYTHON_VENV) python -m scrapy crawl scp_titles -o data/scp_titles.json @@ -27,7 +27,11 @@ data/scp_items.json: .venv $(PYTHON_VENV) python -m scrapy crawl scp -o data/scp_items.json data/scp_hubs.json: .venv - $(PYTHON_VENV) python -m scrapy crawl scp_hubs -o data/scp_hubs.json + @echo "==> Crawling main hubs..." + @rm -f data/scp_hubs.json data/paginated_links.json + @$(PYTHON_VENV) python -m scrapy crawl scp_hubs -o data/scp_hubs.json + @echo "==> Fetching links from paginated pages..." + @$(PYTHON_VENV) python fetch_paginated_links.py data/scp_tales.json: .venv $(PYTHON_VENV) python -m scrapy crawl scp_tales -o data/scp_tales.json @@ -37,7 +41,23 @@ goi: data/goi.json data/goi.json: .venv $(PYTHON_VENV) python -m scrapy crawl goi -o data/goi.json -scp_postprocess: scp_crawl data/processed/goi data/processed/items data/processed/tales +supplement: supplement_crawl supplement_postprocess + +supplement_crawl: data/scp_supplement.json + +data/scp_supplement.json: .venv + $(PYTHON_VENV) python -m scrapy crawl scp_supplement -o data/scp_supplement.json + +supplement_postprocess: supplement_crawl data/processed/supplement + +data/processed/supplement: .venv + $(PYTHON_VENV) python -m scp_crawler.postprocessing run-postproc-supplement + +scp_postprocess: scp_crawl data/processed/hubs data/processed/items data/processed/tales data/processed/goi data/processed/supplement + +data/processed/hubs: data/scp_hubs.json .venv + @echo "==> Processing hubs (runs automatically on import)..." + @$(PYTHON_VENV) python -c "import scp_crawler.postprocessing" > /dev/null data/processed/goi: .venv $(PYTHON_VENV) python -m scp_crawler.postprocessing run-postproc-goi @@ -47,6 +67,7 @@ data/processed/items: .venv data/processed/tales: .venv $(PYTHON_VENV) python -m scp_crawler.postprocessing run-postproc-tales + scp_int: data/scp_int_titles.json data/scp_int_items.json data/scp_int_tales.json diff --git a/scp_crawler/items.py b/scp_crawler/items.py index d4aa1c2..34db45d 100644 --- a/scp_crawler/items.py +++ b/scp_crawler/items.py @@ -22,7 +22,7 @@ class ScpItem(WikiPage): class ScpHub(WikiPage): - pass + paginated_content = scrapy.Field() class ScpTale(WikiPage): @@ -33,6 +33,10 @@ class ScpGoi(WikiPage): pass +class ScpSupplement(WikiPage): + pass + + class ScpTitle(scrapy.Item): title = scrapy.Field() scp = scrapy.Field() diff --git a/scp_crawler/pipelines.py b/scp_crawler/pipelines.py index 2545c65..a7f9736 100644 --- a/scp_crawler/pipelines.py +++ b/scp_crawler/pipelines.py @@ -5,7 +5,71 @@ # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html +from scrapy.exceptions import DropItem +from scp_crawler.items import ScpHub + class ScpCrawlerPipeline(object): def process_item(self, item, spider): return item + + +class HubPaginationPipeline(object): + """Pipeline to merge paginated hub pages into the main hub item. + + This pipeline collects paginated content and attaches it to hub items. + Since paginated pages may be crawled before or after the main hub page, + we attach pagination data immediately when available. + """ + + def __init__(self): + self.paginated_data = {} # Store paginated data by base_link + self.processed_hubs = set() # Track which hubs we've seen + + def open_spider(self, spider): + """Initialize for each spider run.""" + self.paginated_data.clear() + self.processed_hubs.clear() + + def process_item(self, item, spider): + # Only process for hub spiders + if spider.name not in ["scp_hubs", "scp_int_hubs"]: + return item + + # Handle dict returned from parse_paginated_hub + if isinstance(item, dict) and "base_link" in item: + base_link = item["base_link"] + page_number = item["page_number"] + + if base_link not in self.paginated_data: + self.paginated_data[base_link] = [] + + self.paginated_data[base_link].append({ + "page": page_number, + "url": item["url"], + "content": item["content"] + }) + + spider.logger.info( + f"Stored paginated content for {base_link}, page {page_number}" + ) + + # Drop paginated items - don't yield them separately + raise DropItem(f"Paginated content collected for {base_link}") + + # Handle ScpHub items - attach any paginated content we've collected + if isinstance(item, ScpHub): + link = item.get("link") + if link: + self.processed_hubs.add(link) + if link in self.paginated_data: + # Sort by page number and attach to hub + paginated_list = self.paginated_data[link] + paginated_list.sort(key=lambda x: x["page"]) + item["paginated_content"] = paginated_list + spider.logger.info( + f"✓ Merged {len(paginated_list)} paginated pages into hub {link}" + ) + return item + + return item diff --git a/scp_crawler/postprocessing.py b/scp_crawler/postprocessing.py index ec3ee0d..16ba770 100644 --- a/scp_crawler/postprocessing.py +++ b/scp_crawler/postprocessing.py @@ -45,11 +45,29 @@ def get_images(html): def process_history(history): - history = [v for v in history.values()] - for revision in history: - revision["date"] = datetime.strptime(revision["date"], "%d %b %Y %H:%M") - history.sort(key=lambda x: x["date"]) - return history + if not history: + return [] + + if isinstance(history, dict): + revisions = list(history.values()) + elif isinstance(history, list): + revisions = history + else: + return [] + + for revision in revisions: + if not isinstance(revision, dict): + continue + revision_date = revision.get("date") + if isinstance(revision_date, str): + try: + revision["date"] = datetime.strptime(revision_date, "%d %b %Y %H:%M") + except Exception: + # Keep original value if parsing fails. + pass + + revisions.sort(key=lambda x: x.get("date") or datetime.min) + return revisions def get_wiki_source(page_id, domain, attempts=5): @@ -85,16 +103,34 @@ def get_wiki_source(page_id, domain, attempts=5): + + + print("Processing Hub list.") hub_list = from_file(cwd + "/data/scp_hubs.json") hub_items = {} hub_references = {} -for hub in tqdm( - hub_list, -): - # Convert history dict to list and sort by date. - hub["history"] = process_history(hub["history"]) + +# Load paginated links if available (fetched separately) +paginated_links_file = cwd + "/data/paginated_links.json" +paginated_links_data = {} +if os.path.exists(paginated_links_file): + print("Loading paginated links...") + paginated_links_data = from_file(paginated_links_file) + print(f"Loaded pagination data for {len(paginated_links_data)} hubs") + +# Process hubs +for hub in tqdm(hub_list): + link = hub.get("link", "") + + # Skip paginated hub pages (should not exist with new method) + if "/p/" in link: + print(f" Skipping paginated hub page: {link}") + continue + + # Convert history dict to list and sort by date + hub["history"] = process_history(hub.get("history")) if len(hub["history"]) > 0: hub["created_at"] = hub["history"][0]["date"] @@ -102,9 +138,24 @@ def get_wiki_source(page_id, domain, attempts=5): else: hub["created_at"] = "unknown" hub["creator"] = "unknown" - - hub_items[hub["link"]] = hub - hub_references[hub["link"]] = set(hub["references"]) + + # Add paginated links to references if available + if link in paginated_links_data: + # Collect all links from all paginated pages + all_paginated_links = [] + for page_num, page_data in paginated_links_data[link].items(): + all_paginated_links.extend(page_data.get("links", [])) + + # Add to existing references (avoid duplicates) + existing_refs = set(hub.get("references", [])) + new_refs = existing_refs.union(set(all_paginated_links)) + hub["references"] = list(new_refs) + + added_count = len(new_refs) - len(existing_refs) + print(f" Added {added_count} new links from {len(paginated_links_data[link])} paginated page(s) to {link}") + + hub_items[link] = hub + hub_references[link] = set(hub["references"]) hub_dir = Path(cwd + "/data/processed/hubs") os.makedirs(hub_dir, exist_ok=True) @@ -143,7 +194,7 @@ def run_postproc_items(): item["hubs"] = get_hubs(item["link"]) # Convert history dict to list and sort by date. - item["history"] = process_history(item["history"]) + item["history"] = process_history(item.get("history")) if len(item["history"]) > 0: item["created_at"] = item["history"][0]["date"] @@ -200,7 +251,7 @@ def run_postproc_tales(): tale["raw_source"] = get_wiki_source(tale["page_id"], tale["domain"]) # Convert history dict to list and sort by date. - tale["history"] = process_history(tale["history"]) + tale["history"] = process_history(tale.get("history")) if len(tale["history"]) > 0: tale["created_at"] = tale["history"][0]["date"] @@ -273,5 +324,53 @@ def run_postproc_goi(): to_file(tales, processed_path / "index.json") +@cli.command() +def run_postproc_supplement(): + + processed_path = Path(cwd + "/data/processed/supplement") + os.makedirs(processed_path, exist_ok=True) + + print("Processing Supplement list.") + + supplement_list = from_file(cwd + "/data/scp_supplement.json") + supplements = {} + for supplement in tqdm(supplement_list, smoothing=0): + + supplement["images"] = get_images(supplement["raw_content"]) + supplement["hubs"] = get_hubs(supplement["link"]) + supplement["raw_source"] = get_wiki_source(supplement["page_id"], supplement["domain"]) + + # Convert history dict to list and sort by date. + supplement["history"] = process_history(supplement["history"]) + + if len(supplement["history"]) > 0: + supplement["created_at"] = supplement["history"][0]["date"] + supplement["creator"] = supplement["history"][0]["author"] + else: + supplement["created_at"] = "unknown" + supplement["creator"] = "unknown" + + supplement["link"] = supplement["url"].replace("https://scp-wiki.wikidot.com/", "") + + # Extract parent SCP from title or link + scp_match = re.search(r"scp-\d+", supplement["link"], re.IGNORECASE) + supplement["parent_scp"] = scp_match.group(0).upper() if scp_match else None + + # Extract parent tale series from link + tale_match = re.match(r"([a-z\-]+)-\d+$", supplement["link"]) + supplement["parent_tale"] = tale_match.group(1) if tale_match else None + + supplements[supplement["link"]] = supplement + + to_file(supplements, processed_path / f"content_supplement.json") + + for supplement_id in supplements: + del supplements[supplement_id]["raw_content"] + del supplements[supplement_id]["raw_source"] + supplements[supplement_id]["content_file"] = f"content_supplement.json" + + to_file(supplements, processed_path / "index.json") + + if __name__ == "__main__": cli() diff --git a/scp_crawler/settings.py b/scp_crawler/settings.py index 0f02729..c0928a2 100644 --- a/scp_crawler/settings.py +++ b/scp_crawler/settings.py @@ -68,9 +68,9 @@ # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html -# ITEM_PIPELINES = { -# 'scp_crawler.pipelines.ScpCrawlerPipeline': 300, -# } +ITEM_PIPELINES = { + 'scp_crawler.pipelines.HubPaginationPipeline': 300, +} # Enable and configure the AutoThrottle extension (disabled by default) # See https://docs.scrapy.org/en/latest/topics/autothrottle.html diff --git a/scp_crawler/spiders/scp.py b/scp_crawler/spiders/scp.py index 8528b3a..119e0cf 100644 --- a/scp_crawler/spiders/scp.py +++ b/scp_crawler/spiders/scp.py @@ -1,3 +1,4 @@ +import json import re import sys from pprint import pprint @@ -8,7 +9,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule -from ..items import ScpGoi, ScpHub, ScpItem, ScpTale, ScpTitle +from ..items import ScpGoi, ScpHub, ScpItem, ScpSupplement, ScpTale, ScpTitle DOMAIN = "scp-wiki.wikidot.com" INT_DOMAIN = "scp-int.wikidot.com" @@ -28,13 +29,67 @@ def parse_history(self, response, item, history_page=1): self.logger.info(f"Reviewing Page {item['page_id']} history") page_id = item["page_id"] - changes = item["history"] if "history" in item else {} + changes = item.get("history", {}) + item["history"] = changes # Ensure history key always exists + try: + response_text = getattr(response, "text", "") or "" + if not response_text.strip(): + self.logger.error( + "Empty response when fetching history for %s (status=%s, page=%s)", + item.get("url"), + getattr(response, "status", None), + history_page, + ) + return self.get_page_source_request(page_id, item) + history = response.json() - soup = BeautifulSoup(history["body"], "lxml") + if not isinstance(history, dict) or "body" not in history: + self.logger.error( + "Missing 'body' in history lookup for %s (status=%s, page=%s). Keys=%s", + item.get("url"), + getattr(response, "status", None), + history_page, + list(history.keys()) if isinstance(history, dict) else type(history), + ) + return self.get_page_source_request(page_id, item) + + body = history.get("body") + if not body: + self.logger.error( + "Empty 'body' in history lookup for %s (status=%s, page=%s)", + item.get("url"), + getattr(response, "status", None), + history_page, + ) + return self.get_page_source_request(page_id, item) + + soup = BeautifulSoup(body, "lxml") + if soup.table is None: + self.logger.error( + "Missing in history HTML for %s (status=%s, page=%s)", + item.get("url"), + getattr(response, "status", None), + history_page, + ) + return self.get_page_source_request(page_id, item) rows = soup.table.find_all("tr") - except: - self.logger.exception(f"Unable to parse history lookup. {item['url']}") + + except (json.JSONDecodeError, ValueError): + self.logger.error( + "JSON decode error in history lookup for %s (status=%s, page=%s)", + item.get("url"), + getattr(response, "status", None), + history_page, + ) + return self.get_page_source_request(page_id, item) + except Exception: + self.logger.exception( + "Unable to parse history lookup for %s (status=%s, page=%s)", + item.get("url"), + getattr(response, "status", None), + history_page, + ) return self.get_page_source_request(page_id, item) for row in rows: try: @@ -62,11 +117,13 @@ def parse_history(self, response, item, history_page=1): self.logger.exception("Could not process row.") self.logger.error(row) - item["history"] = changes - # The "0" change is the first revision, and the last one that shows up. - # If we have it then we're done. - if "0" in changes: - return self.get_page_source_request(page_id, item) + # Update item history after processing all rows + item["history"] = changes + + # The "0" change is the first revision, and the last one that shows up. + # If we have it then we're done. + if "0" in changes: + return self.get_page_source_request(page_id, item) next_page = history_page + 1 if next_page > MAX_HISTORY_PAGES: @@ -531,22 +588,47 @@ def parse_tale(self, response, original_link=None): return self.get_history_request(item["page_id"], 1, item) -def get_rating(response): - try: - return int(response.css(".rate-points .number::text").get()) - except: - pass - return 0 +class ScpSupplementSpider(CrawlSpider, WikiMixin): + name = "scp_supplement" + + start_urls = [ + f"http://{DOMAIN}/system:page-tags/tag/supplement", + ] + + allowed_domains = [DOMAIN] + + domain = DOMAIN + + rules = ( + Rule(LinkExtractor(allow=[re.escape("system:page-tags/tag/supplement")])), + Rule(LinkExtractor(allow=[r".*"]), callback="parse_supplement"), + ) + def parse_supplement(self, response, original_link=None): + self.logger.debug("Reviewing Potential SCP Supplement page: %s", response.url) + content = self.get_content(response) + tags = self.get_tags(response) + + if not content or not tags: + return None -def clean_content_soup(content_soup): - # Remove Footer - [x.extract() for x in content_soup.find_all("div", {"class": "footer-wikiwalk-nav"})] + redirect = self.follow_splash_redirects(response, tags, self.parse_supplement) + if redirect: + return redirect - # Remove Ratings Bar - [x.extract() for x in content_soup.find_all("div", {"class": "page-rate-widget-box"})] + if "supplement" not in tags: + return None - # Remove Empty Divs - [x.extract() for x in content_soup.find_all("div") if len(x.get_text(strip=True)) == 0] + self.logger.info("Processing SCP Supplement page: %s", response.url) + content_soup = BeautifulSoup(content, "lxml") - return content_soup + item = ScpSupplement() + item["title"] = self.get_title(response) + item["url"] = response.url + item["domain"] = self.domain + item["link"] = original_link if original_link else self.get_simple_link(response.url) + item["tags"] = tags + item["page_id"] = self.get_page_id(response) + item["rating"] = get_rating(response) + item["raw_content"] = str(clean_content_soup(content_soup)) + item["references"] = self.get_content_references(response) diff --git a/test_pagination.py b/test_pagination.py new file mode 100644 index 0000000..9549a3d --- /dev/null +++ b/test_pagination.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +Quick test script to verify hub pagination crawling. +Tests against the church-of-the-broken-god-hub which is known to have paginated pages. +""" + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule +import sys +import os + +# Add project to path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scp_crawler.spiders.scp import ScpHubSpider + + +class TestHubSpider(ScpHubSpider): + """Test spider that only crawls one specific hub and its paginated pages.""" + name = "test_hub_pagination" + + # Start with a known hub that has pagination + start_urls = ["https://scp-wiki.wikidot.com/church-of-the-broken-god-hub"] + + # Override rules to be more restrictive for testing + rules = ( + Rule( + # Only follow pagination for this specific hub + LinkExtractor(allow=[r"church-of-the-broken-god-hub/p/\d+"]), + callback="parse_paginated_hub", + follow=False, + ), + Rule( + # Only parse the main hub page + LinkExtractor(allow=[r"church-of-the-broken-god-hub$"]), + callback="parse_hub", + follow=False, + ), + ) + + +if __name__ == "__main__": + # Configure process + process = CrawlerProcess({ + 'USER_AGENT': 'SCP-Crawler-Test/1.0', + 'ROBOTSTXT_OBEY': True, + 'CONCURRENT_REQUESTS': 2, + 'DOWNLOAD_DELAY': 1, + 'ITEM_PIPELINES': { + 'scp_crawler.pipelines.HubPaginationPipeline': 300, + }, + 'FEEDS': { + 'test_hub_pagination.json': { + 'format': 'json', + 'overwrite': True, + } + }, + 'LOG_LEVEL': 'INFO', + }) + + process.crawl(TestHubSpider) + process.start() + + print("\n" + "="*60) + print("Test complete! Check test_hub_pagination.json for results") + print("="*60)