feat: sync GameDB Steam presets directly

.github/workflows/sync-gamedb-steam.yml

@@ -0,0 +1,68 @@
+---
+name: Sync GameDB Steam presets
+permissions: {}
+
+env:
+  NODE_VERSION: '24'
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 7 * * *'
+
+concurrency:
+  group: gamedb-steam-sync
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+
+      # GameDB publishes its database on gh-pages.
+      - name: Checkout GameDB database
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: LizardByte/GameDB
+          ref: gh-pages
+          path: gamedb
+          sparse-checkout: |
+            /games/
+            /platforms/6.json
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      - name: Checkout PresetDB database
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: database
+          path: database-branch
+          token: ${{ secrets.GH_BOT_TOKEN }}
+
+      - name: Setup Node
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Import Steam presets
+        id: import
+        env:
+          BOT_NAME: ${{ vars.GH_BOT_NAME }}
+          BOT_EMAIL: ${{ secrets.GH_BOT_EMAIL }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          git -C database-branch config user.name "$BOT_NAME"
+          git -C database-branch config user.email "$BOT_EMAIL"
+          node src/gamedb-sync.js --gamedb gamedb --database database-branch
+
+      - name: Build updated site
+        if: always() && steps.import.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
+        run: gh workflow run build-pages.yml --ref master

README.md

@@ -19,15 +19,15 @@
   <a href="https://github.com/LizardByte/PresetDB/issues?q=is%3Aclosed+label%3Aapprove-preset"><img src="https://img.shields.io/github/issues-closed/LizardByte/PresetDB/approve-preset.svg?style=for-the-badge&label=approved&color=green" alt="Approved requests"></a>
 </div>
 
-Community maintained game and app launch presets, with [Sunshine](https://github.com/LizardByte/Sunshine) as the primary consumer. Each game or app can have many presets. Native, GOG, and app commands target a chosen host OS. Steam, Epic Games, and Microsoft Store submissions use a launch ID to generate commands for their supported launcher OSes; emulator submissions use a portable command. GitHub issue numbers provide stable IDs. Preset names use the game or app name; launch method, host OS, and optional emulator variant stay in separate fields.
+Community maintained game and app launch presets, with [Sunshine](https://github.com/LizardByte/Sunshine) as the primary consumer. Each game or app can have many presets. Native, GOG, and app commands target a chosen host OS. Steam, Epic Games, and Microsoft Store submissions use a launch ID to generate commands for their supported launcher OSes; emulator submissions use a portable command. Issue-backed presets use their GitHub issue numbers as stable IDs. Direct GameDB imports use the stable ID `steam`. Preset names use the game or app name; launch method, host OS, and optional emulator variant stay in separate fields.
 
 Each GitHub issue requests **one preset**. Game requests must identify a [GameDB](https://app.lizardbyte.dev/GameDB/) record. App requests use a separate form and require a name and official source URL for maintainer review. A bot validates the request, and an authorized reviewer enters it into the approval queue. No code or pull request is needed to contribute.
 
 ## Browse and use
 
 The [website](https://app.lizardbyte.dev/PresetDB/) shows games and apps with available presets. Each preset shows its launch method and host details as badges, its launch command, and a Sunshine application JSON export. The exported application name is the game or app name. Replace supported path placeholders with paths on your host. Review community commands before running them. Game cover images come from GameDB; app images are submitted as HTTPS URLs for the catalog. These image URLs are not used as Sunshine `image-path` values, which require local files.
 
-The published JSON API contains `index.json`, `games/<IGDB ID>.json`, and `apps/<app slug>.json`. GameDB uses the same numeric game IDs as IGDB. An entry contains all its presets. Each preset receives the originating issue number as a stable string ID, such as "4". Records use schema version 2 with a `command` field or a `commands_by_os` map and an optional `working_directory`; the website builds Sunshine JSON from these generic fields. The migration workflow updates version 1 records on the database branch; site and approval reads also convert them during the rollout.
+The published JSON API contains `index.json`, `games/<IGDB ID>.json`, and `apps/<app slug>.json`. GameDB uses the same numeric game IDs as IGDB. An entry contains all its presets. Issue-backed presets receive the originating issue number as a stable string ID, such as "4". GameDB-imported Steam presets use "steam" and have no issue history. Records use schema version 2 with a `command` field or a `commands_by_os` map and an optional `working_directory`; the website builds Sunshine JSON from these generic fields. The migration workflow updates version 1 records on the database branch; site and approval reads also convert them during the rollout.
 
 ## Database growth
 
@@ -48,7 +48,7 @@ The website publishes [catalog statistics](https://app.lizardbyte.dev/PresetDB/s
 1. Read the [preset guidelines](docs/presetGuidelines.md). For a game, copy its [IGDB game URL](https://www.igdb.com/) and [choose a game method form](https://github.com/LizardByte/PresetDB/issues/new/choose). The bot resolves the URL slug to the IGDB numeric ID and checks GameDB. For another app, [open an app preset request](https://github.com/LizardByte/PresetDB/issues/new?template=app-preset.yml) with its official URL.
 2. Fill in one launch option. Native and GOG forms ask for the host OS. Store forms ask only for the launch ID; the emulator form asks for a portable command and an optional variant name. The bot derives the issue title and preset name from the validated game or app name. App requests still ask for the host OS.
 3. The store forms share one Launch ID concept: a numeric Steam app ID, an Epic three-part launch ID, or a Microsoft Store AUMID. The bot generates OS-specific launch commands. Native, GOG, and Emulator forms ask for a Command. Validation and the website never execute commands.
-4. To replace a preset, provide its issue number and explain the change. The bot preserves the original preset ID.
+4. To replace an issue-backed preset, provide its issue number and explain the change. The bot preserves the original preset ID. Direct GameDB imports have no issue number and cannot be replaced through an issue.
 
 ProtonDB compatibility summaries are attributed to [ProtonDB contributors](https://github.com/bdefore/protondb-data) and published under the [Open Database License](https://opendatacommons.org/licenses/odbl/). The website refreshes available tiers during its Pages build.
 
@@ -67,6 +67,8 @@ Game submissions resolve the submitted slug through [IGDB authenticated API](htt
 - `auto_approved_users.json`: trusted approvers and game submitters eligible for automatic queueing.
 - `.readthedocs.yaml`: pull request preview builds using the shared LizardByte Jekyll script.
 
+The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import.
+
 On each push to master, the Migrate Database workflow checks `database/migrations.json` for completed migration IDs. Before applying pending migrations, it creates a `database-backup-<pre-migration commit SHA>` branch pointing to the untouched database. It then commits the migrated records and history file to the active `database` branch and triggers a fresh Pages build. Re-runs skip completed migrations. The backup branch is never used as the working database.
 
 For pull requests, the Pages workflow applies pending migrations in its temporary database checkout before building the preview. It does not commit or push those preview changes. The Pages workflow creates an archive from the database and site template, then calls the same [LizardByte Jekyll build workflow](https://github.com/LizardByte/LizardByte.github.io/blob/master/.github/workflows/jekyll-build.yml) used by GameDB and ThemerrDB. It deploys to `gh-pages` after changes to `master` or an approved database update.

package.json

@@ -9,7 +9,7 @@
   },
   "scripts": {
     "test": "node --test tests/*.test.js",
-    "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/build-site.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js",
+    "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/gamedb-sync.js && node --check src/build-site.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js",
     "test:ci": "node --test --experimental-test-coverage --test-reporter=junit --test-reporter=lcov --test-reporter-destination=junit.xml --test-reporter-destination=lcov.info tests/*.test.js"
   }
 }

src/build-site.js

@@ -29,13 +29,24 @@ async function protonDbRating(appId, fetcher) {
   }
 }
 
-async function addProtonDb(item, fetcher, cache) {
-  for (const preset of item.presets) {
-    if (preset.method !== 'steam' || !/^[1-9]\d{0,9}$/.test(preset.launch_id || '')) continue;
-    const appId = preset.launch_id;
-    preset.protondb_url = 'https://www.protondb.com/app/' + appId;
-    if (!cache.has(appId)) cache.set(appId, await protonDbRating(appId, fetcher));
-    preset.protondb = cache.get(appId);
+async function addProtonDb(records, fetcher) {
+  const ids = [...new Set(records.flatMap(item => item.presets
+    .filter(preset => preset.method === 'steam' && /^[1-9]\d{0,9}$/.test(preset.launch_id || ''))
+    .map(preset => preset.launch_id)))];
+  const cache = new Map();
+  let next = 0;
+  await Promise.all(Array.from({ length: Math.min(32, ids.length) }, async () => {
+    while (next < ids.length) {
+      const appId = ids[next++];
+      cache.set(appId, await protonDbRating(appId, fetcher));
+    }
+  }));
+  for (const item of records) {
+    for (const preset of item.presets) {
+      if (!cache.has(preset.launch_id)) continue;
+      preset.protondb_url = 'https://www.protondb.com/app/' + preset.launch_id;
+      preset.protondb = cache.get(preset.launch_id);
+    }
   }
 }
 
@@ -44,7 +55,6 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch)
   fs.cpSync(template, output, { recursive: true });
   const index = { schema_version: 1, games: [], apps: [] };
   const records = [];
-  const protonCache = new Map();
   for (const [folder, kind] of [['games', 'game'], ['apps', 'app']]) {
     const directory = path.join(database, folder);
     if (!fs.existsSync(directory)) continue;
@@ -57,8 +67,6 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch)
           String(item.id) !== path.basename(file, '.json')) {
         throw new Error(`Invalid database record: ${folder}/${file}`);
       }
-      await addProtonDb(item, fetcher, protonCache);
-      fs.writeFileSync(path.join(target, file), JSON.stringify(item, null, 2) + '\n');
       records.push(item);
       index[folder].push({
         id: item.id, name: item.name, preset_count: item.presets.length,
@@ -69,6 +77,11 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch)
     }
     index[folder].sort((a, b) => a.name.localeCompare(b.name) || String(a.id).localeCompare(String(b.id)));
   }
+  await addProtonDb(records, fetcher);
+  for (const item of records) {
+    const folder = item.kind === 'game' ? 'games' : 'apps';
+    fs.writeFileSync(path.join(output, folder, `${item.id}.json`), JSON.stringify(item, null, 2) + '\n');
+  }
   fs.writeFileSync(path.join(output, 'index.json'), `${JSON.stringify(index, null, 2)}\n`);
   const statistics = buildStatistics(index, records);
   fs.writeFileSync(path.join(output, 'stats.json'), `${JSON.stringify(statistics.data, null, 2)}\n`);

src/database.js

@@ -3,7 +3,7 @@
 const fs = require('node:fs');
 const path = require('node:path');
 const { PresetError } = require('./presets');
-const { normalizeRecord } = require('./record');
+const { normalizeRecord, comparePresetIds } = require('./record');
 
 function recordPath(root, preset) {
   const folder = preset.kind === 'game' ? 'games' : 'apps';
@@ -94,7 +94,7 @@ function mergePreset(root, preset, {
   };
   if (previous >= 0) record.presets[previous] = entry;
   else record.presets.push(entry);
-  record.presets.sort((a, b) => Number(a.id) - Number(b.id));
+  record.presets.sort((a, b) => comparePresetIds(a.id, b.id));
   if (write) {
     fs.mkdirSync(path.dirname(file), { recursive: true });
     fs.writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`);