Added opt-in GitHub release scanning

.github/workflows/publish.yml

@@ -4,6 +4,11 @@ on:
   push:
     branches: [main]
   pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: website-${{ github.ref }}
+  cancel-in-progress: false
 
 jobs:
   build:
@@ -27,7 +32,7 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     needs: [build]
-    if: github.event_name == 'push' && github.ref_name == 'main'
+    if: github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
     permissions:
       contents: read
       pages: write
@@ -39,4 +44,4 @@ jobs:
     steps:
     - name: Deploy to GitHub Pages
       id: deployment
-      uses: actions/deploy-pages@v5
+      uses: actions/deploy-pages@v5

.github/workflows/scan-releases.yml

@@ -0,0 +1,82 @@
+name: Scan releases
+
+on:
+  schedule:
+    - cron: '37 3 * * *'
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'scan-releases.py'
+      - 'tests/**'
+      - 'homebrew_database/**'
+      - '.github/workflows/scan-releases.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-scanner-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - run: python -m unittest discover -s tests -v
+
+  scan:
+    needs: tests
+    if: github.repository == 'pspdev/homebrew' && github.ref_name == 'main' && github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: write
+      actions: write
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: main
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - name: Scan approved entries
+        id: scan
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python scan-releases.py
+      - name: Check for changes
+        id: changes
+        run: |
+          git add -- data resources/icons
+          if ! git diff --cached --quiet; then
+            echo 'changed=true' >> "$GITHUB_OUTPUT"
+          fi
+      - name: Validate with the existing website build
+        if: steps.changes.outputs.changed == 'true'
+        run: |
+          python -m pip install -r requirements.txt
+          python build.py
+      - name: Commit and push validated updates
+        if: steps.changes.outputs.changed == 'true'
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git commit -m 'Added releases from approved repositories'
+          # A concurrent change to main rejects this push. Never force or
+          # rebase generated data over a curator's edits; the next run retries.
+          git push origin HEAD:main
+      - name: Publish updated database
+        if: steps.changes.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh workflow run publish.yml --ref main
+      - name: Report scan failures
+        if: always() && steps.scan.outcome == 'failure'
+        run: |
+          echo 'Some entries could not be scanned; see the scan log. Their existing data was kept.'
+          exit 1

README.md

@@ -91,6 +91,43 @@ Here is a description for each field in a release:
 
 After creating the file, run the `add-resources-and-info.py` script once. This will add the icon and information like the md5 checksum of the `EBOOT.PBP` and the sha256 checksum and sie of the archive for each release. If this wasn't done, the build will fail.
 
+## Release updates
+
+- **Opt-in:** `"scan_releases": true` in [`data/`](data/); `source` names the GitHub repo.
+- **Daily:** add stable releases, checksums and the latest icon.
+- **Metadata:** unchanged; drafts and prereleases are skipped.
+- **ZIP:** one ZIP, or exactly one with `psp` in its name.
+  Missing or ambiguous ZIPs are skipped.
+
+```text
+daily scan -> changes? -> build check -> commit + push -> website build -> deploy
+                   no -> done
+```
+
+<details>
+<summary><b>Scanner</b> · checks, workflow, failures</summary>
+
+- **Checks:** download size, SHA-256 when GitHub supplies it, ZIP paths and CRCs,
+  one EBOOT with a valid PBP header. Records ZIP SHA-256 and EBOOT MD5.
+- **History:** keeps existing releases; `v1.0` and `1.0` count as one.
+- **Skipped:** missing or ambiguous ZIPs and invalid packages; reason in the log.
+- **Errors:** network or download verification fails → entry unchanged; run fails.
+- **Build or push fails:** no update to `main`; no force-push.
+- **Publish fails:** rerun **CI** on `main`.
+
+```sh
+python scan-releases.py --dry-run  # check
+python scan-releases.py            # update
+python -m unittest discover -s tests -v
+```
+
+[`scan-releases.py`](scan-releases.py) · [daily workflow](.github/workflows/scan-releases.yml)
+
+Requires push access to `main`. `GITHUB_TOKEN` raises the API rate limit.
+Known ZIPs are not downloaded again; differing GitHub digests are reported.
+
+</details>
+
 ## Using the homebrew database data in applications
 
 The PSP Homebrew Database supports a couple of formats for applications like homebrew stores to use. In every single format, the homebrew list is returned by last updated with the newest at the top. Below they will be listed with a description of how to use each.

add-resources-and-info.py

@@ -101,7 +101,7 @@ def main():
 
             logging.info(f"Writing {homebrew_json_path}")
             with open(homebrew_json_path, "w") as fd:
-                fd.write(json.dumps(homebrew.to_dict(), indent=2))
+                fd.write(json.dumps(homebrew.to_dict(include_ci=True), indent=2))
         except Exception:
             logging.error("Could not read %s", homebrew_json_path, exc_info=True)
     logging.info("Done, make sure to add and commit changes to the data and resources directory to git")

data/pspdx.json

@@ -27,5 +27,6 @@
   },
   "website": "https://chriopter.github.io/pspdx-catalog/",
   "source": "https://github.com/chriopter/pspdx-app",
+  "scan_releases": true,
   "license": "GPLv2.0"
-}
+}

homebrew_database/homebrew.py

@@ -59,6 +59,8 @@ class Homebrew:
     tags: list[str] = field(default_factory=list)
     languages: list[str] = field(default_factory=list)
     icon: str = None
+    # CI controls must survive add-resources-and-info.py's JSON round trip.
+    scan_releases: bool | None = None
 
     def __gt__(self, other: 'Homebrew') -> bool:
         self_last_release = self.get_last_release()
@@ -74,7 +76,7 @@ def get_last_release(self) -> Release | None:
             return None
         return sorted(self.releases, reverse=True)[0]
 
-    def to_dict(self) -> dict:
+    def to_dict(self, *, include_ci: bool = False) -> dict:
         return_dict = {
             "name": self.name,
             "id": self.id,
@@ -102,6 +104,8 @@ def to_dict(self) -> dict:
             return_dict["media"]["icon"] = self.icon
         if len(self.languages) > 0:
             return_dict["languages"] = self.languages
+        if include_ci and self.scan_releases is not None:
+            return_dict["scan_releases"] = self.scan_releases
 
         self.releases.sort(reverse=True)
         for release in self.releases:
@@ -146,6 +150,7 @@ def get_homebrew_from_json_data(id: str, data: dict) -> Homebrew:
           tags=data.get("tags", list()),
           icon=data["media"].get("icon", None),
           languages=data.get("languages", list()),
+          scan_releases=data.get("scan_releases", None),
         )
     except (KeyError, ValueError) as e:
         raise HomebrewProcessingException(f"Could not process json data for {id}") from e