From e904ceade906b7f22219c4715b346c1e6eceb204 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:58:06 -0400 Subject: [PATCH 1/3] feat: sync GameDB Steam presets directly --- .github/workflows/sync-gamedb-steam.yml | 68 +++++++++++ README.md | 8 +- package.json | 2 +- src/build-site.js | 33 ++++-- src/database.js | 4 +- src/gamedb-sync.js | 145 ++++++++++++++++++++++++ src/record.js | 15 ++- tests/gamedb-sync.test.js | 125 ++++++++++++++++++++ 8 files changed, 383 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/sync-gamedb-steam.yml create mode 100644 src/gamedb-sync.js create mode 100644 tests/gamedb-sync.test.js diff --git a/.github/workflows/sync-gamedb-steam.yml b/.github/workflows/sync-gamedb-steam.yml new file mode 100644 index 0000000000..70801abcfc --- /dev/null +++ b/.github/workflows/sync-gamedb-steam.yml @@ -0,0 +1,68 @@ +--- +name: Sync GameDB Steam presets +permissions: {} + +env: + NODE_VERSION: '24' + +on: + workflow_dispatch: + schedule: + - cron: '0 7 * * *' + +concurrency: + group: gamedb-steam-sync + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # GameDB publishes its database on gh-pages. + - name: Checkout GameDB database + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: LizardByte/GameDB + ref: gh-pages + path: gamedb + sparse-checkout: | + /games/ + /platforms/6.json + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Checkout PresetDB database + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: database + path: database-branch + token: ${{ secrets.GH_BOT_TOKEN }} + + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Import Steam presets + id: import + env: + BOT_NAME: ${{ vars.GH_BOT_NAME }} + BOT_EMAIL: ${{ secrets.GH_BOT_EMAIL }} + shell: bash + run: | + set -euo pipefail + git -C database-branch config user.name "$BOT_NAME" + git -C database-branch config user.email "$BOT_EMAIL" + node src/gamedb-sync.js --gamedb gamedb --database database-branch + + - name: Build updated site + if: always() && steps.import.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GH_BOT_TOKEN }} + run: gh workflow run build-pages.yml --ref master diff --git a/README.md b/README.md index e753c6d640..6c927b3903 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Approved requests -Community maintained game and app launch presets, with [Sunshine](https://github.com/LizardByte/Sunshine) as the primary consumer. Each game or app can have many presets. Native, GOG, and app commands target a chosen host OS. Steam, Epic Games, and Microsoft Store submissions use a launch ID to generate commands for their supported launcher OSes; emulator submissions use a portable command. GitHub issue numbers provide stable IDs. Preset names use the game or app name; launch method, host OS, and optional emulator variant stay in separate fields. +Community maintained game and app launch presets, with [Sunshine](https://github.com/LizardByte/Sunshine) as the primary consumer. Each game or app can have many presets. Native, GOG, and app commands target a chosen host OS. Steam, Epic Games, and Microsoft Store submissions use a launch ID to generate commands for their supported launcher OSes; emulator submissions use a portable command. Issue-backed presets use their GitHub issue numbers as stable IDs. Direct GameDB imports use the stable ID `steam`. Preset names use the game or app name; launch method, host OS, and optional emulator variant stay in separate fields. Each GitHub issue requests **one preset**. Game requests must identify a [GameDB](https://app.lizardbyte.dev/GameDB/) record. App requests use a separate form and require a name and official source URL for maintainer review. A bot validates the request, and an authorized reviewer enters it into the approval queue. No code or pull request is needed to contribute. @@ -27,7 +27,7 @@ Each GitHub issue requests **one preset**. Game requests must identify a [GameDB The [website](https://app.lizardbyte.dev/PresetDB/) shows games and apps with available presets. Each preset shows its launch method and host details as badges, its launch command, and a Sunshine application JSON export. The exported application name is the game or app name. Replace supported path placeholders with paths on your host. Review community commands before running them. Game cover images come from GameDB; app images are submitted as HTTPS URLs for the catalog. These image URLs are not used as Sunshine `image-path` values, which require local files. -The published JSON API contains `index.json`, `games/.json`, and `apps/.json`. GameDB uses the same numeric game IDs as IGDB. An entry contains all its presets. Each preset receives the originating issue number as a stable string ID, such as "4". Records use schema version 2 with a `command` field or a `commands_by_os` map and an optional `working_directory`; the website builds Sunshine JSON from these generic fields. The migration workflow updates version 1 records on the database branch; site and approval reads also convert them during the rollout. +The published JSON API contains `index.json`, `games/.json`, and `apps/.json`. GameDB uses the same numeric game IDs as IGDB. An entry contains all its presets. Issue-backed presets receive the originating issue number as a stable string ID, such as "4". GameDB-imported Steam presets use "steam" and have no issue history. Records use schema version 2 with a `command` field or a `commands_by_os` map and an optional `working_directory`; the website builds Sunshine JSON from these generic fields. The migration workflow updates version 1 records on the database branch; site and approval reads also convert them during the rollout. ## Database growth @@ -48,7 +48,7 @@ The website publishes [catalog statistics](https://app.lizardbyte.dev/PresetDB/s 1. Read the [preset guidelines](docs/presetGuidelines.md). For a game, copy its [IGDB game URL](https://www.igdb.com/) and [choose a game method form](https://github.com/LizardByte/PresetDB/issues/new/choose). The bot resolves the URL slug to the IGDB numeric ID and checks GameDB. For another app, [open an app preset request](https://github.com/LizardByte/PresetDB/issues/new?template=app-preset.yml) with its official URL. 2. Fill in one launch option. Native and GOG forms ask for the host OS. Store forms ask only for the launch ID; the emulator form asks for a portable command and an optional variant name. The bot derives the issue title and preset name from the validated game or app name. App requests still ask for the host OS. 3. The store forms share one Launch ID concept: a numeric Steam app ID, an Epic three-part launch ID, or a Microsoft Store AUMID. The bot generates OS-specific launch commands. Native, GOG, and Emulator forms ask for a Command. Validation and the website never execute commands. -4. To replace a preset, provide its issue number and explain the change. The bot preserves the original preset ID. +4. To replace an issue-backed preset, provide its issue number and explain the change. The bot preserves the original preset ID. Direct GameDB imports have no issue number and cannot be replaced through an issue. ProtonDB compatibility summaries are attributed to [ProtonDB contributors](https://github.com/bdefore/protondb-data) and published under the [Open Database License](https://opendatacommons.org/licenses/odbl/). The website refreshes available tiers during its Pages build. @@ -67,6 +67,8 @@ Game submissions resolve the submitted slug through [IGDB authenticated API](htt - `auto_approved_users.json`: trusted approvers and game submitters eligible for automatic queueing. - `.readthedocs.yaml`: pull request preview builds using the shared LizardByte Jekyll script. +The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import. + On each push to master, the Migrate Database workflow checks `database/migrations.json` for completed migration IDs. Before applying pending migrations, it creates a `database-backup-` branch pointing to the untouched database. It then commits the migrated records and history file to the active `database` branch and triggers a fresh Pages build. Re-runs skip completed migrations. The backup branch is never used as the working database. For pull requests, the Pages workflow applies pending migrations in its temporary database checkout before building the preview. It does not commit or push those preview changes. The Pages workflow creates an archive from the database and site template, then calls the same [LizardByte Jekyll build workflow](https://github.com/LizardByte/LizardByte.github.io/blob/master/.github/workflows/jekyll-build.yml) used by GameDB and ThemerrDB. It deploys to `gh-pages` after changes to `master` or an approved database update. diff --git a/package.json b/package.json index 457e4b10c3..9b56e8cdff 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ }, "scripts": { "test": "node --test tests/*.test.js", - "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/build-site.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js", + "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/gamedb-sync.js && node --check src/build-site.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js", "test:ci": "node --test --experimental-test-coverage --test-reporter=junit --test-reporter=lcov --test-reporter-destination=junit.xml --test-reporter-destination=lcov.info tests/*.test.js" } } diff --git a/src/build-site.js b/src/build-site.js index 1b9058fdba..689b8cfacb 100644 --- a/src/build-site.js +++ b/src/build-site.js @@ -29,13 +29,24 @@ async function protonDbRating(appId, fetcher) { } } -async function addProtonDb(item, fetcher, cache) { - for (const preset of item.presets) { - if (preset.method !== 'steam' || !/^[1-9]\d{0,9}$/.test(preset.launch_id || '')) continue; - const appId = preset.launch_id; - preset.protondb_url = 'https://www.protondb.com/app/' + appId; - if (!cache.has(appId)) cache.set(appId, await protonDbRating(appId, fetcher)); - preset.protondb = cache.get(appId); +async function addProtonDb(records, fetcher) { + const ids = [...new Set(records.flatMap(item => item.presets + .filter(preset => preset.method === 'steam' && /^[1-9]\d{0,9}$/.test(preset.launch_id || '')) + .map(preset => preset.launch_id)))]; + const cache = new Map(); + let next = 0; + await Promise.all(Array.from({ length: Math.min(32, ids.length) }, async () => { + while (next < ids.length) { + const appId = ids[next++]; + cache.set(appId, await protonDbRating(appId, fetcher)); + } + })); + for (const item of records) { + for (const preset of item.presets) { + if (!cache.has(preset.launch_id)) continue; + preset.protondb_url = 'https://www.protondb.com/app/' + preset.launch_id; + preset.protondb = cache.get(preset.launch_id); + } } } @@ -44,7 +55,6 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch) fs.cpSync(template, output, { recursive: true }); const index = { schema_version: 1, games: [], apps: [] }; const records = []; - const protonCache = new Map(); for (const [folder, kind] of [['games', 'game'], ['apps', 'app']]) { const directory = path.join(database, folder); if (!fs.existsSync(directory)) continue; @@ -57,8 +67,6 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch) String(item.id) !== path.basename(file, '.json')) { throw new Error(`Invalid database record: ${folder}/${file}`); } - await addProtonDb(item, fetcher, protonCache); - fs.writeFileSync(path.join(target, file), JSON.stringify(item, null, 2) + '\n'); records.push(item); index[folder].push({ id: item.id, name: item.name, preset_count: item.presets.length, @@ -69,6 +77,11 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch) } index[folder].sort((a, b) => a.name.localeCompare(b.name) || String(a.id).localeCompare(String(b.id))); } + await addProtonDb(records, fetcher); + for (const item of records) { + const folder = item.kind === 'game' ? 'games' : 'apps'; + fs.writeFileSync(path.join(output, folder, `${item.id}.json`), JSON.stringify(item, null, 2) + '\n'); + } fs.writeFileSync(path.join(output, 'index.json'), `${JSON.stringify(index, null, 2)}\n`); const statistics = buildStatistics(index, records); fs.writeFileSync(path.join(output, 'stats.json'), `${JSON.stringify(statistics.data, null, 2)}\n`); diff --git a/src/database.js b/src/database.js index 1fadab514b..76780ce2de 100644 --- a/src/database.js +++ b/src/database.js @@ -3,7 +3,7 @@ const fs = require('node:fs'); const path = require('node:path'); const { PresetError } = require('./presets'); -const { normalizeRecord } = require('./record'); +const { normalizeRecord, comparePresetIds } = require('./record'); function recordPath(root, preset) { const folder = preset.kind === 'game' ? 'games' : 'apps'; @@ -94,7 +94,7 @@ function mergePreset(root, preset, { }; if (previous >= 0) record.presets[previous] = entry; else record.presets.push(entry); - record.presets.sort((a, b) => Number(a.id) - Number(b.id)); + record.presets.sort((a, b) => comparePresetIds(a.id, b.id)); if (write) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`); diff --git a/src/gamedb-sync.js b/src/gamedb-sync.js new file mode 100644 index 0000000000..ff4342201b --- /dev/null +++ b/src/gamedb-sync.js @@ -0,0 +1,145 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { isDeepStrictEqual } = require('node:util'); +const { readRecord } = require('./database'); +const { generatedLaunch } = require('./presets'); +const { comparePresetIds } = require('./record'); + +const PC_PLATFORM = 6; +const STEAM_SOURCE = 1; + +function pcGameIds(index) { + if (index?.id !== PC_PLATFORM || !Array.isArray(index.games)) { + throw new Error('GameDB PC platform index is invalid'); + } + return [...new Set(index.games.map(game => game.id) + .filter(id => Number.isSafeInteger(id) && id > 0))].sort((a, b) => a - b); +} + +function steamCandidate(game) { + if (!Number.isSafeInteger(game?.id) || !Array.isArray(game.platforms) || + !game.platforms.includes(PC_PLATFORM) || typeof game.slug !== 'string' || + !/^[a-z0-9-]+$/.test(game.slug) || typeof game.name !== 'string' || + !game.name.trim() || game.name.length > 180 || /[\r\n]/.test(game.name)) return null; + const steamEntries = (Array.isArray(game.external_games) ? game.external_games : []) + .filter(external => external.external_game_source?.id === STEAM_SOURCE); + if (steamEntries.some(external => typeof external.uid !== 'string' || + !/^[1-9]\d{0,9}$/.test(external.uid) || Number(external.uid) > 4294967295)) return null; + const ids = [...new Set(steamEntries.map(external => external.uid))]; + if (ids.length !== 1) return null; + const cover = game.cover?.url; + return { + id: game.id, name: game.name, slug: game.slug, launchId: ids[0], + imageUrl: typeof cover === 'string' && cover.startsWith('//images.igdb.com/') + ? `https:${cover}` : null, + commandsByOs: generatedLaunch({ launchId: ids[0] }, 'Steam').commandsByOs + }; +} + +function mergeGame(game, existing) { + if (existing && (existing.kind !== 'game' || existing.id !== game.id)) { + throw new Error(`Game ${game.id} has an invalid PresetDB record`); + } + const record = existing ? { + ...existing, presets: existing.presets.map(preset => ({ ...preset, name: game.name })) + } : { + schema_version: 2, kind: 'game', id: game.id, name: game.name, + source_url: '', image_url: null, presets: [] + }; + record.name = game.name; + record.source_url = `https://www.igdb.com/games/${game.slug}`; + record.image_url = game.imageUrl; + record.igdb_slug = game.slug; + record.game_db_url = `https://app.lizardbyte.dev/GameDB/browse/games/?id=${game.id}`; + + const steam = record.presets.filter(preset => preset.method === 'steam'); + if (steam.length > 1) throw new Error(`Game ${game.id} has multiple Steam presets`); + if (steam.length === 0) { + record.presets.push({ + id: 'steam', name: game.name, os: null, method: 'steam', + launch_id: game.launchId, commands_by_os: game.commandsByOs, + notes: null, source: 'gamedb' + }); + } else if (steam[0].id === 'steam' && steam[0].source === 'gamedb') { + steam[0].os = null; + steam[0].launch_id = game.launchId; + steam[0].commands_by_os = game.commandsByOs; + } + // Issue-backed Steam presets are maintained by their contributors. + record.presets.sort((a, b) => comparePresetIds(a.id, b.id)); + return isDeepStrictEqual(existing, record) ? null : record; +} + +function git(checkout, args) { + return execFileSync('git', args, { cwd: checkout, encoding: 'utf8', stdio: 'pipe' }); +} + +function publishGame({ checkout, game, file, onPublished, gitCommand = git }) { + const relative = `database/games/${game.id}.json`; + for (let attempt = 1; attempt <= 5; attempt++) { + gitCommand(checkout, ['fetch', 'origin', 'database']); + gitCommand(checkout, ['reset', '--hard', 'origin/database']); + const desired = mergeGame(game, readRecord(file)); + if (!desired) return false; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(desired, null, 2)}\n`); + gitCommand(checkout, ['add', '--', relative]); + gitCommand(checkout, ['commit', '-m', `chore: sync GameDB game ${game.id}`, '--', relative]); + try { + gitCommand(checkout, ['push', 'origin', 'HEAD:database']); + onPublished?.(); + return true; + } catch (error) { + if (attempt === 5) throw error; + console.log(`Database branch moved while publishing game ${game.id}; retrying`); + } + } + return false; +} + +function run({ gameDbDir, checkout, onPublished }) { + const database = path.join(checkout, 'database'); + const index = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'platforms', '6.json'), 'utf8')); + const counts = { scanned: 0, eligible: 0, published: 0, current: 0 }; + for (const id of pcGameIds(index)) { + counts.scanned++; + const source = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'games', `${id}.json`), 'utf8')); + if (source.id !== id) throw new Error(`GameDB game ${id} has a mismatched ID`); + const game = steamCandidate(source); + if (!game) continue; + counts.eligible++; + const file = path.join(database, 'games', `${id}.json`); + if (!mergeGame(game, readRecord(file))) { counts.current++; continue; } + if (publishGame({ checkout, game, file, onPublished })) { + counts.published++; + console.log(`Published game ${id}`); + } else counts.current++; + } + console.log(`GameDB Steam sync: ${JSON.stringify(counts)}`); + return counts; +} + +module.exports = { pcGameIds, steamCandidate, mergeGame, publishGame, run }; + +if (require.main === module) { + const args = process.argv.slice(2); + function option(name) { + const index = args.indexOf(name); + if (index < 0 || !args[index + 1]) throw new Error(`Missing ${name}`); + return args[index + 1]; + } + let reported = false; + run({ + gameDbDir: path.resolve(option('--gamedb')), + checkout: path.resolve(option('--database')), + onPublished: () => { + if (!reported && process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, 'changed=true\n'); + reported = true; + } + } + }); +} diff --git a/src/record.js b/src/record.js index 5a3eebe22e..5e8050e752 100644 --- a/src/record.js +++ b/src/record.js @@ -32,4 +32,17 @@ function normalizeRecord(record) { }; } -module.exports = { normalizeRecord }; +function comparePresetIds(left, right) { + const leftNumeric = /^[1-9]\d*$/.test(left); + const rightNumeric = /^[1-9]\d*$/.test(right); + if (leftNumeric && rightNumeric) { + const a = BigInt(left); + const b = BigInt(right); + return a < b ? -1 : a > b ? 1 : 0; + } + if (leftNumeric) return -1; + if (rightNumeric) return 1; + return left.localeCompare(right); +} + +module.exports = { normalizeRecord, comparePresetIds }; diff --git a/tests/gamedb-sync.test.js b/tests/gamedb-sync.test.js new file mode 100644 index 0000000000..cae790db76 --- /dev/null +++ b/tests/gamedb-sync.test.js @@ -0,0 +1,125 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { pcGameIds, steamCandidate, mergeGame, run } = require('../src/gamedb-sync'); +const { mergePreset } = require('../src/database'); +const { buildSite } = require('../src/build-site'); + +function sourceGame(id, launchId = String(id + 1000)) { + return { + id, name: `Game ${id}`, slug: `game-${id}`, platforms: [6], + cover: { url: '//images.igdb.com/igdb/image/upload/t_thumb/cover.jpg' }, + external_games: [{ uid: launchId, external_game_source: { id: 1, name: 'Steam' } }] + }; +} + +function command(cwd, ...args) { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }).trim(); +} + +test('PC index and Steam source select one unambiguous launch ID', () => { + assert.deepEqual(pcGameIds({ id: 6, games: [{ id: 2 }, { id: 1 }, { id: 2 }] }), [1, 2]); + assert.equal(steamCandidate(sourceGame(1)).launchId, '1001'); + assert.equal(steamCandidate({ ...sourceGame(1), platforms: [3] }), null); + assert.equal(steamCandidate({ ...sourceGame(1), external_games: [ + ...sourceGame(1).external_games, { uid: 'invalid', external_game_source: { id: 1 } } + ] }), null); + assert.equal(steamCandidate({ ...sourceGame(1), external_games: [ + ...sourceGame(1).external_games, { uid: '9999', external_game_source: { id: 1 } } + ] }), null); +}); + +test('direct import has no issue identity and refreshes only imported Steam launch data', () => { + const game = steamCandidate(sourceGame(1)); + const imported = mergeGame(game, null); + assert.equal(imported.presets[0].id, 'steam'); + assert.equal(imported.presets[0].source, 'gamedb'); + assert.equal(imported.presets[0].origin_issue, undefined); + assert.equal(imported.presets[0].history, undefined); + assert.equal(mergeGame(game, imported), null); + + const updatedGame = steamCandidate({ ...sourceGame(1, '2001'), name: 'Renamed game' }); + const updated = mergeGame(updatedGame, imported); + assert.equal(updated.name, 'Renamed game'); + assert.equal(updated.presets[0].launch_id, '2001'); + assert.equal(updated.presets[0].commands_by_os.Linux, 'setsid steam steam://rungameid/2001'); + assert.equal(updated.presets[0].commands_by_os.Windows, 'steam://rungameid/2001'); + + const manual = { + ...imported, presets: [{ + id: '6', name: game.name, os: null, method: 'steam', launch_id: '9999', + commands_by_os: { Windows: 'custom command' }, origin_issue: 6, source_issue: 6, + history: [{ issue: 6, action: 'add' }] + }] + }; + const preserved = mergeGame(updatedGame, manual); + assert.equal(preserved.name, 'Renamed game'); + assert.equal(preserved.presets[0].name, 'Renamed game'); + assert.equal(preserved.presets[0].launch_id, '9999'); + assert.deepEqual(preserved.presets[0].history, manual.presets[0].history); + assert.equal(preserved.presets[0].id, '6'); + assert.equal(mergeGame(updatedGame, preserved), null); +}); + +test('sync publishes each changed game file in its own commit and skips unchanged files', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-gamedb-sync-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const remote = path.join(root, 'remote.git'); + const checkout = path.join(root, 'checkout'); + const gameDbDir = path.join(root, 'gamedb'); + fs.mkdirSync(checkout); + fs.mkdirSync(path.join(gameDbDir, 'platforms'), { recursive: true }); + fs.mkdirSync(path.join(gameDbDir, 'games')); + command(root, 'init', '--bare', remote); + command(checkout, 'init', '-b', 'database'); + command(checkout, 'remote', 'add', 'origin', remote); + command(checkout, 'config', 'user.name', 'LizardByte-bot'); + command(checkout, 'config', 'user.email', 'bot@example.com'); + fs.mkdirSync(path.join(checkout, 'database', 'games'), { recursive: true }); + fs.writeFileSync(path.join(checkout, 'database', 'games', '.gitkeep'), ''); + command(checkout, 'add', 'database/games/.gitkeep'); + command(checkout, 'commit', '-m', 'seed database'); + command(checkout, 'push', '-u', 'origin', 'database'); + + fs.writeFileSync(path.join(gameDbDir, 'platforms', '6.json'), + JSON.stringify({ id: 6, games: [{ id: 1 }, { id: 2 }] })); + for (const id of [1, 2]) { + fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id))); + } + + let published = 0; + const args = { + gameDbDir, checkout, + onPublished: () => { published++; } + }; + assert.equal(run(args).published, 2); + assert.equal(published, 2); + assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3'); + assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD'), + 'database/games/2.json'); + assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD^'), + 'database/games/1.json'); + assert.equal(run(args).published, 0); + assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3'); + const imported = JSON.parse(fs.readFileSync(path.join(checkout, 'database/games/1.json'))); + assert.throws(() => mergePreset(path.join(checkout, 'database'), { + kind: 'game', gameId: 1, gameName: 'Game 1', gameSlug: 'game-1', + gameImageUrl: imported.image_url, method: 'steam', os: null, + commandsByOs: imported.presets[0].commands_by_os, launchId: '1001', + replacementIssue: 1, replacementReason: 'Change', notes: null + }, { issueNumber: 10, approvedBy: 'reviewer' }), /No preset for issue #1/); + + const output = path.join(root, 'site'); + await buildSite(path.join(checkout, 'database'), + path.join(__dirname, '..', 'gh-pages-template'), output, + async () => ({ ok: false, status: 404 })); + const siteRecord = JSON.parse(fs.readFileSync(path.join(output, 'games/1.json'))); + assert.equal(siteRecord.presets[0].id, 'steam'); + assert.equal(siteRecord.presets[0].origin_issue, undefined); + assert.equal(siteRecord.presets[0].protondb_url, 'https://www.protondb.com/app/1001'); +}); From 281d1eaf23d9dfa4260a195f006e1cae63fa97a0 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:08:40 -0400 Subject: [PATCH 2/3] fix: address Steam sync quality checks --- src/gamedb-sync.js | 23 ++++++++++++----------- src/record.js | 4 +++- tests/gamedb-sync.test.js | 27 +++++++++++++++++++-------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/gamedb-sync.js b/src/gamedb-sync.js index ff4342201b..59b1305710 100644 --- a/src/gamedb-sync.js +++ b/src/gamedb-sync.js @@ -74,7 +74,7 @@ function mergeGame(game, existing) { } function git(checkout, args) { - return execFileSync('git', args, { cwd: checkout, encoding: 'utf8', stdio: 'pipe' }); + return execFileSync('/usr/bin/git', args, { cwd: checkout, encoding: 'utf8', stdio: 'pipe' }); } function publishGame({ checkout, game, file, onPublished, gitCommand = git }) { @@ -97,10 +97,9 @@ function publishGame({ checkout, game, file, onPublished, gitCommand = git }) { console.log(`Database branch moved while publishing game ${game.id}; retrying`); } } - return false; } -function run({ gameDbDir, checkout, onPublished }) { +function run({ gameDbDir, checkout, onPublished, gitCommand = git }) { const database = path.join(checkout, 'database'); const index = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'platforms', '6.json'), 'utf8')); const counts = { scanned: 0, eligible: 0, published: 0, current: 0 }; @@ -113,7 +112,7 @@ function run({ gameDbDir, checkout, onPublished }) { counts.eligible++; const file = path.join(database, 'games', `${id}.json`); if (!mergeGame(game, readRecord(file))) { counts.current++; continue; } - if (publishGame({ checkout, game, file, onPublished })) { + if (publishGame({ checkout, game, file, onPublished, gitCommand })) { counts.published++; console.log(`Published game ${id}`); } else counts.current++; @@ -122,24 +121,26 @@ function run({ gameDbDir, checkout, onPublished }) { return counts; } -module.exports = { pcGameIds, steamCandidate, mergeGame, publishGame, run }; - -if (require.main === module) { - const args = process.argv.slice(2); +function main(args = process.argv.slice(2), env = process.env, gitCommand = git) { function option(name) { const index = args.indexOf(name); if (index < 0 || !args[index + 1]) throw new Error(`Missing ${name}`); return args[index + 1]; } let reported = false; - run({ + return run({ gameDbDir: path.resolve(option('--gamedb')), checkout: path.resolve(option('--database')), + gitCommand, onPublished: () => { - if (!reported && process.env.GITHUB_OUTPUT) { - fs.appendFileSync(process.env.GITHUB_OUTPUT, 'changed=true\n'); + if (!reported && env.GITHUB_OUTPUT) { + fs.appendFileSync(env.GITHUB_OUTPUT, 'changed=true\n'); reported = true; } } }); } + +module.exports = { pcGameIds, steamCandidate, mergeGame, publishGame, run, main }; + +if (require.main === module) main(); diff --git a/src/record.js b/src/record.js index 5e8050e752..098a8e953f 100644 --- a/src/record.js +++ b/src/record.js @@ -38,7 +38,9 @@ function comparePresetIds(left, right) { if (leftNumeric && rightNumeric) { const a = BigInt(left); const b = BigInt(right); - return a < b ? -1 : a > b ? 1 : 0; + if (a < b) return -1; + if (a > b) return 1; + return 0; } if (leftNumeric) return -1; if (rightNumeric) return 1; diff --git a/tests/gamedb-sync.test.js b/tests/gamedb-sync.test.js index cae790db76..86d6fe51ef 100644 --- a/tests/gamedb-sync.test.js +++ b/tests/gamedb-sync.test.js @@ -6,9 +6,10 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); -const { pcGameIds, steamCandidate, mergeGame, run } = require('../src/gamedb-sync'); +const { pcGameIds, steamCandidate, mergeGame, run, main } = require('../src/gamedb-sync'); const { mergePreset } = require('../src/database'); const { buildSite } = require('../src/build-site'); +const { comparePresetIds } = require('../src/record'); function sourceGame(id, launchId = String(id + 1000)) { return { @@ -24,6 +25,7 @@ function command(cwd, ...args) { test('PC index and Steam source select one unambiguous launch ID', () => { assert.deepEqual(pcGameIds({ id: 6, games: [{ id: 2 }, { id: 1 }, { id: 2 }] }), [1, 2]); + assert.throws(() => pcGameIds({ id: 5, games: [] }), /PC platform index/); assert.equal(steamCandidate(sourceGame(1)).launchId, '1001'); assert.equal(steamCandidate({ ...sourceGame(1), platforms: [3] }), null); assert.equal(steamCandidate({ ...sourceGame(1), external_games: [ @@ -36,6 +38,7 @@ test('PC index and Steam source select one unambiguous launch ID', () => { test('direct import has no issue identity and refreshes only imported Steam launch data', () => { const game = steamCandidate(sourceGame(1)); + assert.throws(() => mergeGame(game, { kind: 'app', id: 1, presets: [] }), /invalid PresetDB record/); const imported = mergeGame(game, null); assert.equal(imported.presets[0].id, 'steam'); assert.equal(imported.presets[0].source, 'gamedb'); @@ -92,13 +95,12 @@ test('sync publishes each changed game file in its own commit and skips unchange fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id))); } - let published = 0; - const args = { - gameDbDir, checkout, - onPublished: () => { published++; } - }; - assert.equal(run(args).published, 2); - assert.equal(published, 2); + const args = { gameDbDir, checkout }; + const outputFile = path.join(root, 'github-output'); + assert.throws(() => main([], {}, () => {}), /Missing --gamedb/); + assert.equal(main(['--gamedb', gameDbDir, '--database', checkout], + { GITHUB_OUTPUT: outputFile }, (directory, gitArgs) => command(directory, ...gitArgs)).published, 2); + assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n'); assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3'); assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD'), 'database/games/2.json'); @@ -123,3 +125,12 @@ test('sync publishes each changed game file in its own commit and skips unchange assert.equal(siteRecord.presets[0].origin_issue, undefined); assert.equal(siteRecord.presets[0].protondb_url, 'https://www.protondb.com/app/1001'); }); + +test('preset ordering keeps numeric issue IDs before named imports', () => { + assert.equal(comparePresetIds('2', '10'), -1); + assert.equal(comparePresetIds('10', '2'), 1); + assert.equal(comparePresetIds('2', '2'), 0); + assert.equal(comparePresetIds('2', 'steam'), -1); + assert.equal(comparePresetIds('steam', '2'), 1); + assert.ok(comparePresetIds('gog', 'steam') < 0); +}); \ No newline at end of file From 7191c6f6a23764565099e31d9db2a3f8e78539a7 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:11:03 -0400 Subject: [PATCH 3/3] style: end importer tests with newline --- tests/gamedb-sync.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gamedb-sync.test.js b/tests/gamedb-sync.test.js index 86d6fe51ef..ead7400d99 100644 --- a/tests/gamedb-sync.test.js +++ b/tests/gamedb-sync.test.js @@ -133,4 +133,4 @@ test('preset ordering keeps numeric issue IDs before named imports', () => { assert.equal(comparePresetIds('2', 'steam'), -1); assert.equal(comparePresetIds('steam', '2'), 1); assert.ok(comparePresetIds('gog', 'steam') < 0); -}); \ No newline at end of file +});