From 81aff4882183a7d28fe46bc1a89ccd191e1e9d27 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 12 Aug 2025 15:18:45 -0400 Subject: [PATCH 01/38] test: retrieveing remote container digest --- ...ocker-manifest.service.integration.test.ts | 162 +++++++++ .../docker/docker-manifest.service.test.ts | 328 ++++++++++++++++++ .../resolvers/docker/docker-auth.service.ts | 73 ++++ .../docker/docker-manifest.service.ts | 140 ++++++++ .../graph/resolvers/docker/docker.module.ts | 4 + 5 files changed, 707 insertions(+) create mode 100644 api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts create mode 100644 api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts diff --git a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts new file mode 100644 index 0000000000..356580d65c --- /dev/null +++ b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts @@ -0,0 +1,162 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; +import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; + +describe.skipIf(!!process.env.CI)('DockerManifestService - Real Integration Tests', () => { + let service: DockerManifestService; + let dockerAuthService: DockerAuthService; + + beforeAll(() => { + dockerAuthService = new DockerAuthService(); + service = new DockerManifestService(dockerAuthService); + }, 30000); + + describe('headManifest - Real HTTP calls', () => { + it('should receive authentication challenge from Docker Hub', async () => { + const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest'; + const headers = { + Accept: 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json', + }; + + const result = await service.headManifest(manifestURL, headers); + + expect(result.statusCode).toBe(401); + expect(result.headers['www-authenticate']).toContain('Bearer'); + expect(result.headers['www-authenticate']).toContain('realm'); + }, 15000); + + it('should handle unauthorized requests gracefully', async () => { + const manifestURL = 'https://registry-1.docker.io/v2/library/hello-world/manifests/latest'; + const headers = { + Accept: 'application/vnd.docker.distribution.manifest.v2+json', + }; + + const result = await service.headManifest(manifestURL, headers); + + expect(result.statusCode).toBeGreaterThanOrEqual(200); + expect(result.statusCode).toBeLessThan(500); + }, 15000); + }); + + describe('getRemoteDigest - Real Docker Registry calls', () => { + it('should get digest for public Alpine image', async () => { + const digest = await service.getRemoteDigest('alpine:latest'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should get digest for public Nginx image', async () => { + const digest = await service.getRemoteDigest('nginx:latest'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should get digest for specific image tag', async () => { + const digest = await service.getRemoteDigest('alpine:3.18'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should handle non-existent image gracefully', async () => { + const digest = await service.getRemoteDigest('nonexistent/nonexistent:nonexistent'); + + expect(digest).toBeNull(); + }, 30000); + + it('should work with Docker Hub organization images', async () => { + const digest = await service.getRemoteDigest('library/hello-world:latest'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should handle anonymous access to public registries', async () => { + const digest = await service.getRemoteDigest('busybox:latest'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should work with images without explicit tags', async () => { + const digest = await service.getRemoteDigest('ubuntu'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should get digest with mocked empty docker auth', async () => { + const mockDockerAuthService = new DockerAuthService(); + const mockManifestService = new DockerManifestService(mockDockerAuthService); + + vi.spyOn(mockDockerAuthService, 'readDockerAuth').mockResolvedValue({}); + + const digest = await mockManifestService.getRemoteDigest('alpine:latest'); + + expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); + }, 30000); + + it('should investigate HEAD vs GET behavior with authentication', async () => { + const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest'; + const headers = { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }; + + // Test initial HEAD request (should be 401) + const initialResult = await service.headManifest(manifestURL, headers); + // console.log('Initial HEAD request - Status:', initialResult.statusCode); + + // Get Bearer token for anonymous access + const wwwAuth = (initialResult.headers?.['www-authenticate'] || '').toString(); + const token = await dockerAuthService.getBearerToken(wwwAuth, { + username: '', + password: '', + }); + // console.log('Got token:', !!token); + + if (token) { + // Test authenticated HEAD request + const authResult = await service.headManifest(manifestURL, headers, { + Authorization: `Bearer ${token}`, + }); + // console.log('Authenticated HEAD request - Status:', authResult.statusCode); + // console.log('Authenticated HEAD digest:', authResult.headers?.['docker-content-digest']); + + expect(authResult.statusCode).toBeGreaterThanOrEqual(200); + expect(authResult.statusCode).toBeLessThan(300); + } + }, 30000); + + it('should handle moderate concurrent requests', async () => { + // Start with fewer requests to identify the breaking point + const promises = Array.from({ length: 10 }, () => service.getRemoteDigest('alpine:latest')); + + const results = await Promise.all(promises); + + results.forEach((digest, index) => { + expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + }, 60000); + + it('should handle 100 requests with optimized batching', async () => { + const batchSize = 10; + const totalRequests = 100; + const results: (string | null)[] = []; + + // Process in batches with minimal delays + for (let i = 0; i < totalRequests; i += batchSize) { + const batch = Array.from({ length: Math.min(batchSize, totalRequests - i) }, () => + service.getRemoteDigest('alpine:latest') + ); + + const batchResults = await Promise.all(batch); + results.push(...batchResults); + + // Minimal delay between batches + if (i + batchSize < totalRequests) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + + expect(results).toHaveLength(totalRequests); + results.forEach((digest, index) => { + expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + }, 60000); + }); +}); diff --git a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts new file mode 100644 index 0000000000..5efec5a762 --- /dev/null +++ b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts @@ -0,0 +1,328 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; +import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; + +vi.mock('got', () => ({ + got: { + head: vi.fn(), + get: vi.fn(), + }, +})); + +vi.mock('@app/core/utils/index.js', () => ({ + docker: { + getImage: vi.fn(() => ({ + inspect: vi.fn(), + })), + getContainer: vi.fn(() => ({ + inspect: vi.fn(), + })), + }, +})); + +describe('DockerManifestService', () => { + let service: DockerManifestService; + let mockDockerAuthService: any; + let mockGot: any; + + beforeEach(async () => { + vi.clearAllMocks(); + + mockDockerAuthService = { + readDockerAuth: vi.fn(), + decodeAuth: vi.fn(), + getBearerToken: vi.fn(), + }; + + const { got } = await import('got'); + mockGot = vi.mocked(got); + mockGot.head.mockReset(); + mockGot.get.mockReset(); + + service = new DockerManifestService(mockDockerAuthService); + }); + + describe('parseImageRef - Unit Tests', () => { + it('should parse simple image name with default tag', () => { + const result = service.parseImageRef('nginx'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry-1.docker.io', + authConfigKey: 'https://index.docker.io/v1/', + repoPath: 'library/nginx', + tag: 'latest', + }); + }); + + it('should parse image name with explicit tag', () => { + const result = service.parseImageRef('nginx:1.21'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry-1.docker.io', + authConfigKey: 'https://index.docker.io/v1/', + repoPath: 'library/nginx', + tag: '1.21', + }); + }); + + it('should parse Docker Hub organization image', () => { + const result = service.parseImageRef('ubuntu/nginx:latest'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry-1.docker.io', + authConfigKey: 'https://index.docker.io/v1/', + repoPath: 'ubuntu/nginx', + tag: 'latest', + }); + }); + + it('should parse custom registry with port', () => { + const result = service.parseImageRef('registry.example.com:5000/myorg/myapp:v1.0'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry.example.com:5000', + authConfigKey: 'registry.example.com:5000', + repoPath: 'myorg/myapp', + tag: 'v1.0', + }); + }); + + it('should parse custom registry without port', () => { + const result = service.parseImageRef('registry.example.com/myorg/myapp:v1.0'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry.example.com', + authConfigKey: 'registry.example.com', + repoPath: 'myorg/myapp', + tag: 'v1.0', + }); + }); + + it('should handle docker.io explicit registry', () => { + const result = service.parseImageRef('docker.io/nginx:latest'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry-1.docker.io', + authConfigKey: 'https://index.docker.io/v1/', + repoPath: 'docker.io/nginx', + tag: 'latest', + }); + }); + + it('should handle complex tag with multiple colons', () => { + const result = service.parseImageRef('nginx:alpine-3.14.2'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry-1.docker.io', + authConfigKey: 'https://index.docker.io/v1/', + repoPath: 'library/nginx', + tag: 'alpine-3.14.2', + }); + }); + + it('should handle localhost registry', () => { + const result = service.parseImageRef('localhost:5000/myapp:latest'); + + expect(result).toEqual({ + registryBaseURL: 'https://localhost:5000', + authConfigKey: 'localhost:5000', + repoPath: 'myapp', + tag: 'latest', + }); + }); + + it('should handle deep nested repository path', () => { + const result = service.parseImageRef('registry.example.com/team/project/service:v1.0.0'); + + expect(result).toEqual({ + registryBaseURL: 'https://registry.example.com', + authConfigKey: 'registry.example.com', + repoPath: 'team/project/service', + tag: 'v1.0.0', + }); + }); + }); + + describe('headManifest - Unit Tests', () => { + it('should return response when request succeeds', async () => { + const mockResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:abc123' }, + }; + mockGot.head.mockResolvedValue(mockResponse); + + const result = await service.headManifest( + 'https://registry-1.docker.io/v2/library/nginx/manifests/latest', + { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }, + {}, + mockGot + ); + + expect(result).toEqual(mockResponse); + }); + + it('should fallback to GET when HEAD fails', async () => { + const mockGetResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:def456' }, + }; + + mockGot.head.mockRejectedValue(new Error('HEAD not supported')); + mockGot.get.mockResolvedValue(mockGetResponse); + + const result = await service.headManifest( + 'https://registry-1.docker.io/v2/library/nginx/manifests/latest', + { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }, + {}, + mockGot + ); + + expect(result).toEqual(mockGetResponse); + }); + }); + + describe('getRemoteDigest - Unit Tests', () => { + it('should return digest for public image (anonymous access)', async () => { + const mockResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:abc123def456' }, + }; + mockGot.head.mockResolvedValueOnce(mockResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBe('sha256:abc123def456'); + }); + + it('should handle array digest header', async () => { + const mockResponse = { + statusCode: 200, + headers: { 'docker-content-digest': ['sha256:abc123def456', 'sha256:other'] }, + }; + mockGot.head.mockResolvedValueOnce(mockResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBe('sha256:abc123def456'); + }); + + it('should authenticate with Bearer token when registry requires it', async () => { + const unauthorizedResponse = { + statusCode: 401, + headers: { + 'www-authenticate': + 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/nginx:pull"', + }, + }; + const authorizedResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:authenticated' }, + }; + + mockGot.head + .mockResolvedValueOnce(unauthorizedResponse) + .mockResolvedValueOnce(authorizedResponse); + + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({ + 'https://index.docker.io/v1/': { auth: 'dXNlcjpwYXNz' }, + }); + mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: 'user', password: 'pass' }); + mockDockerAuthService.getBearerToken.mockResolvedValueOnce('bearer-token-123'); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBe('sha256:authenticated'); + }); + + it('should handle private registries requiring authentication', async () => { + const authorizedResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:private-registry' }, + }; + + mockGot.head.mockResolvedValueOnce(authorizedResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({ + 'registry.example.com': { auth: 'dXNlcjpwYXNz' }, + }); + + const result = await service.getRemoteDigest('registry.example.com/private/app:v1.0'); + + expect(result).toBe('sha256:private-registry'); + }); + + it('should return null when authentication fails', async () => { + const unauthorizedResponse = { + statusCode: 401, + headers: { + 'www-authenticate': + 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io"', + }, + }; + + mockGot.head.mockResolvedValue(unauthorizedResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: '', password: '' }); + mockDockerAuthService.getBearerToken.mockResolvedValueOnce(null); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBeNull(); + }); + + it('should return null when no digest is available', async () => { + const mockResponse = { + statusCode: 200, + headers: {}, + }; + mockGot.head.mockResolvedValueOnce(mockResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBeNull(); + }); + + it('should work with custom registries', async () => { + const authorizedResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:custom-registry' }, + }; + + mockGot.head.mockResolvedValueOnce(authorizedResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('registry.example.com/myapp:v1.0'); + + expect(result).toBe('sha256:custom-registry'); + }); + + it('should handle images with no explicit tag (defaults to latest)', async () => { + const mockResponse = { + statusCode: 200, + headers: { 'docker-content-digest': 'sha256:latest-digest' }, + }; + mockGot.head.mockResolvedValueOnce(mockResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('alpine'); + + expect(result).toBe('sha256:latest-digest'); + }); + + it('should handle registry errors gracefully', async () => { + const errorResponse = { + statusCode: 500, + headers: {}, + }; + mockGot.head.mockResolvedValueOnce(errorResponse); + mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); + + const result = await service.getRemoteDigest('nginx:latest'); + + expect(result).toBeNull(); + }); + }); +}); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts new file mode 100644 index 0000000000..eb75137a45 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts @@ -0,0 +1,73 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { readFile } from 'fs/promises'; +import { join } from 'path'; + +import type { Got } from 'got'; +import { ExtendOptions, got as gotClient } from 'got'; + +export type BasicDockerCreds = { + username: string; + password: string; +}; + +export type DockerWwwAuthParts = { + realm: string; + service: string; + scope: string; +}; + +@Injectable() +export class DockerAuthService { + private readonly logger = new Logger(DockerAuthService.name); + constructor() {} + + async readDockerAuth(configPath?: string) { + try { + configPath ??= join(process.env.HOME || '/root', '.docker/config.json'); + const cfg = JSON.parse(await readFile(configPath, 'utf8')); + return cfg.auths || {}; + } catch (error) { + this.logger.debug(error, `Failed to read Docker auth from '${configPath}'`); + return {}; + } + } + + decodeAuth(auth: string): BasicDockerCreds { + try { + const [username, password] = Buffer.from(auth, 'base64').toString('utf8').split(':'); + return { username, password }; + } catch { + return { username: '', password: '' }; + } + } + + parseWWWAuth(wwwAuth: string): Partial { + // www-authenticate: Bearer realm="...",service="...",scope="repository:repo/name:pull" + const parts: Partial = {}; + const rawParts = wwwAuth.replace(/^Bearer\s+/i, '').split(',') || []; + rawParts.forEach((pair) => { + const [k, v] = pair.split('='); + parts[k.trim()] = v?.replace(/^"|"$/g, ''); + }); + return parts; + } + + async getBearerToken( + wwwAuth: string, + basicCreds: BasicDockerCreds, + got: Got = gotClient + ) { + const parts = this.parseWWWAuth(wwwAuth); + if (!parts.realm || !parts.service || !parts.scope) return null; + const { token } = await got + .get(parts.realm, { + searchParams: { service: parts.service, scope: parts.scope }, + username: basicCreds.username, + password: basicCreds.password, + timeout: { request: 15_000 }, + responseType: 'json', + }) + .json<{ token?: string }>(); + return token; + } +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts new file mode 100644 index 0000000000..ff3b875d90 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -0,0 +1,140 @@ +import { Injectable } from '@nestjs/common'; + +import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got'; + +import { docker } from '@app/core/utils/index.js'; +import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; + +/** Accept header for Docker API manifest listing */ +const ACCEPT_MANIFEST = + 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json'; + +@Injectable() +export class DockerManifestService { + constructor(private readonly dockerAuthService: DockerAuthService) {} + + parseImageRef(imageRef: string) { + // Normalize to repo:tag and extract registry/repo/name/tag + let ref = imageRef; + if (!ref.includes(':')) ref += ':latest'; + + // Registry present? + const firstSlash = ref.indexOf('/'); + const maybeRegistry = firstSlash > -1 ? ref.slice(0, firstSlash) : ''; + const hasDotOrColon = maybeRegistry.includes('.') || maybeRegistry.includes(':'); + const isDockerHub = !hasDotOrColon || maybeRegistry === 'docker.io'; + + const registry = isDockerHub ? 'registry-1.docker.io' : maybeRegistry; + const rest = isDockerHub ? ref : ref.slice(maybeRegistry.length + 1); + + const lastColon = rest.lastIndexOf(':'); + const namePart = rest.slice(0, lastColon); + const tag = rest.slice(lastColon + 1); + + // Ensure docker hub library namespace + const repoPath = isDockerHub && !namePart.includes('/') ? `library/${namePart}` : namePart; + + return { + registryBaseURL: `https://${registry}`, + authConfigKey: isDockerHub ? 'https://index.docker.io/v1/' : maybeRegistry, + repoPath, // e.g. library/nginx or org/image + tag, + }; + } + + async headManifest( + url: string, + headers: Record, + authHeader: Record = {}, + got: Got = gotClient + ) { + const opts: OptionsOfTextResponseBody = { + headers: { ...headers, ...authHeader }, + timeout: { request: 15_000 }, + throwHttpErrors: false, + }; + try { + return await got.head(url, opts); + } catch { + // Some registries don’t allow HEAD; try GET to read headers + return await got.get(url, opts); + } + } + + async getRemoteDigest(imageRef) { + const { registryBaseURL, repoPath, tag, authConfigKey } = this.parseImageRef(imageRef); + const manifestURL = `${registryBaseURL}/v2/${repoPath}/manifests/${tag}`; + + const dockerAuths = this.dockerAuthService.readDockerAuth(); + const authEntry = dockerAuths[authConfigKey]; + const basicCreds = authEntry?.auth + ? this.dockerAuthService.decodeAuth(authEntry.auth) + : { username: '', password: '' }; + + // 1) Probe without auth to learn challenge + let resp = await this.headManifest(manifestURL, { Accept: ACCEPT_MANIFEST }); + const digestHeaderRaw = resp.headers?.['docker-content-digest']; + const digestHeader = Array.isArray(digestHeaderRaw) ? digestHeaderRaw[0] : digestHeaderRaw; + if (resp.statusCode >= 200 && resp.statusCode < 300 && digestHeader) return digestHeader.trim(); + + const wwwAuth = (resp.headers?.['www-authenticate'] || '').toString(); + if (/Bearer/i.test(wwwAuth)) { + const token = await this.dockerAuthService.getBearerToken(wwwAuth, basicCreds); + if (!token) return null; + // 2) Repeat with Bearer + resp = await this.headManifest( + manifestURL, + { Accept: ACCEPT_MANIFEST }, + { Authorization: `Bearer ${token}` } + ); + } else if (/Basic/i.test(wwwAuth) && basicCreds.username && basicCreds.password) { + // 2) Repeat with Basic + const basic = + 'Basic ' + + Buffer.from(`${basicCreds.username}:${basicCreds.password}`).toString('base64'); + resp = await this.headManifest( + manifestURL, + { Accept: ACCEPT_MANIFEST }, + { Authorization: basic } + ); + } + + const digestRaw = resp.headers?.['docker-content-digest']; + const digest = Array.isArray(digestRaw) ? digestRaw[0] : digestRaw; + return digest ? digest.trim() : null; + } + + async getLocalDigest(imageRef) { + try { + const data = await docker.getImage(imageRef).inspect(); + const digests = data.RepoDigests || []; + if (digests.length === 0) return null; + // Prefer a digest matching this repo if present; else first + const pick = digests.find((d) => d.startsWith(imageRef.split(':')[0] + '@')) || digests[0]; + const at = pick.indexOf('@'); + return at >= 0 ? pick.slice(at + 1) : null; + } catch { + return null; + } + } + + async isRebuildReady(networkMode) { + if (!networkMode || !networkMode.startsWith('container:')) return false; + const target = networkMode.slice('container:'.length); + try { + await docker.getContainer(target).inspect(); + return false; + } catch { + return true; // unresolved target -> ':???' equivalent + } + } + + async isUpdateAvailable(imageRef: string) { + const [local, remote] = await Promise.all([ + this.getLocalDigest(imageRef), + this.getRemoteDigest(imageRef), + ]); + if (local && remote) return local !== remote; + return null; // unknown + } +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index 824b84198e..f038b6e93d 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; +import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; @@ -12,6 +14,8 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser DockerService, DockerConfigService, DockerOrganizerService, + DockerAuthService, + DockerManifestService, // DockerEventService, // Resolvers From a9e62b4c3bded7d21299a557a74ff0264b95b68d Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 12 Aug 2025 15:34:00 -0400 Subject: [PATCH 02/38] don't skip ci --- .../docker/docker-manifest.service.integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts index 356580d65c..0d2ce094d1 100644 --- a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts +++ b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts @@ -3,7 +3,7 @@ import { beforeAll, describe, expect, it, vi } from 'vitest'; import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; -describe.skipIf(!!process.env.CI)('DockerManifestService - Real Integration Tests', () => { +describe('DockerManifestService - Integration Tests', () => { let service: DockerManifestService; let dockerAuthService: DockerAuthService; From 9655c006fef9154094f64f3ad5e379b1ebb660af Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 12 Aug 2025 16:29:38 -0400 Subject: [PATCH 03/38] use php-loader to load containers (untested) --- .../resolvers/docker/docker-php.service.ts | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts new file mode 100644 index 0000000000..ef8f268cbd --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -0,0 +1,72 @@ +import { Injectable } from '@nestjs/common'; + +import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; + +type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; +type ExplicitStatusItem = { + name: string; + updateStatus: 'up to date' | 'update available' | 'rebuild ready' | 'unknown'; +}; + +@Injectable() +export class DockerPhpService { + constructor() {} + + async refreshDigests( + dockerUpdatePath = '/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerUpdate.php' + ) { + // Triggers getAllInfo(true) → recomputes local/remote digests + try { + await phpLoader({ + file: dockerUpdatePath, + method: 'GET', + }); + } catch { + // ignore; offline may keep remote as 'undef' + } + } + + private parseStatusesFromDockerPush(js: string): ExplicitStatusItem[] { + const items: ExplicitStatusItem[] = []; + const re = /docker\.push\(\{[^}]*name:'([^']+)'[^}]*update:(\d)[^}]*\}\);/g; + for (const m of js.matchAll(re)) { + const name = m[1]; + const updateStatus = Number(m[2]) as StatusItem['updateStatus']; + items.push({ name, updateStatus: this.updateStatusToString(updateStatus) }); + } + return items; + } + + private updateStatusToString(updateStatus: 0): 'up to date'; + private updateStatusToString(updateStatus: 1): 'update available'; + private updateStatusToString(updateStatus: 2): 'rebuild ready'; + private updateStatusToString(updateStatus: 3): 'unknown'; + // prettier-ignore + private updateStatusToString(updateStatus: StatusItem['updateStatus']): ExplicitStatusItem['updateStatus']; + private updateStatusToString( + updateStatus: StatusItem['updateStatus'] + ): ExplicitStatusItem['updateStatus'] { + switch (updateStatus) { + case 0: + return 'up to date'; + case 1: + return 'update available'; + case 2: + return 'rebuild ready'; + default: + return 'unknown'; + } + } + + async getContainerUpdateStatuses( + dockerContainersPath = '/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerContainers.php' + ): Promise { + const stdout = await phpLoader({ + file: dockerContainersPath, + method: 'GET', + }); + const parts = stdout.split('\0'); // [html, "docker.push(...)", busyFlag] + const js = parts[1] || ''; + return this.parseStatusesFromDockerPush(js); + } +} From 21466f1b887691e60f1408b21a6056e38f67d03a Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 13 Aug 2025 10:14:26 -0400 Subject: [PATCH 04/38] add container status query to gql --- .../graph/resolvers/docker/docker.module.spec.ts | 2 ++ .../graph/resolvers/docker/docker.module.ts | 2 ++ .../resolvers/docker/docker.resolver.spec.ts | 9 ++++++++- .../graph/resolvers/docker/docker.resolver.ts | 16 ++++++++++++++-- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts index 97e3b28e49..8bfadaff3a 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from 'vitest'; import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerEventService } from '@app/unraid-api/graph/resolvers/docker/docker-event.service.js'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { DockerModule } from '@app/unraid-api/graph/resolvers/docker/docker.module.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; @@ -61,6 +62,7 @@ describe('DockerModule', () => { DockerResolver, { provide: DockerService, useValue: {} }, { provide: DockerOrganizerService, useValue: {} }, + { provide: DockerPhpService, useValue: { getContainerUpdateStatuses: vi.fn() } }, ], }).compile(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index f038b6e93d..d92e6d1030 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -4,6 +4,7 @@ import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; @@ -16,6 +17,7 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser DockerOrganizerService, DockerAuthService, DockerManifestService, + DockerPhpService, // DockerEventService, // Resolvers diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts index 01f2508b2d..6b08425740 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts @@ -4,6 +4,7 @@ import { Test } from '@nestjs/testing'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { ContainerState, DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; @@ -26,7 +27,13 @@ describe('DockerResolver', () => { { provide: DockerOrganizerService, useValue: { - getResolvedOrganizer: vi.fn(), + resolveOrganizer: vi.fn(), + }, + }, + { + provide: DockerPhpService, + useValue: { + getContainerUpdateStatuses: vi.fn(), }, }, ], diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts index 5948cc6e75..5dcf3038f8 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts @@ -2,8 +2,10 @@ import { Args, Mutation, Query, ResolveField, Resolver } from '@nestjs/graphql'; import { AuthAction, Resource } from '@unraid/shared/graphql.model.js'; import { UsePermissions } from '@unraid/shared/use-permissions.directive.js'; +import { GraphQLJSON } from 'graphql-scalars'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { Docker, DockerContainer, @@ -11,13 +13,14 @@ import { } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; import { DEFAULT_ORGANIZER_ROOT_ID } from '@app/unraid-api/organizer/organizer.js'; -import { OrganizerV1, ResolvedOrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; +import { ResolvedOrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; @Resolver(() => Docker) export class DockerResolver { constructor( private readonly dockerService: DockerService, - private readonly dockerOrganizerService: DockerOrganizerService + private readonly dockerOrganizerService: DockerOrganizerService, + private readonly dockerPhpService: DockerPhpService ) {} @UsePermissions({ @@ -123,4 +126,13 @@ export class DockerResolver { }); return this.dockerOrganizerService.resolveOrganizer(organizer); } + + @UsePermissions({ + action: AuthAction.READ_ANY, + resource: Resource.DOCKER, + }) + @ResolveField(() => GraphQLJSON) + public async containerUpdateStatuses() { + return this.dockerPhpService.getContainerUpdateStatuses(); + } } From 17d151a020c006f941c1e9e30314c5ab9be1de1d Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 13 Aug 2025 11:28:25 -0400 Subject: [PATCH 05/38] fix: copy wrapper.php into api build --- api/vite.config.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/api/vite.config.ts b/api/vite.config.ts index 02706e4ca3..c16742c1ca 100644 --- a/api/vite.config.ts +++ b/api/vite.config.ts @@ -70,6 +70,24 @@ export default defineConfig(({ mode }): ViteUserConfig => { }, }, }), + // Copy PHP files to assets directory + { + name: 'copy-php-files', + async generateBundle() { + const { readFileSync } = await import('fs'); + const { join, basename } = await import('path'); + + const phpFiles = ['src/core/utils/plugins/wrapper.php']; + phpFiles.forEach((file) => { + const content = readFileSync(file); + this.emitFile({ + type: 'asset', + fileName: join('assets', basename(file)), + source: content, + }); + }); + }, + }, ], define: { // Allows vite to preserve process.env variables and not hardcode them From 95560c91555689f8f76c7836ffeeec6bdaf09526 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 13 Aug 2025 13:44:13 -0400 Subject: [PATCH 06/38] add docker container resolver --- api/generated-schema.graphql | 3 ++ .../docker/docker-container.resolver.ts | 46 +++++++++++++++++++ .../docker/docker-manifest.service.ts | 11 +++-- .../graph/resolvers/docker/docker.module.ts | 2 + 4 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts diff --git a/api/generated-schema.graphql b/api/generated-schema.graphql index 90017c2dd2..945504d74a 100644 --- a/api/generated-schema.graphql +++ b/api/generated-schema.graphql @@ -1083,6 +1083,8 @@ type DockerContainer implements Node { networkSettings: JSON mounts: [JSON!] autoStart: Boolean! + isUpdateAvailable: Boolean + isRebuildReady: Boolean } enum ContainerState { @@ -1113,6 +1115,7 @@ type Docker implements Node { containers(skipCache: Boolean! = false): [DockerContainer!]! networks(skipCache: Boolean! = false): [DockerNetwork!]! organizer: ResolvedOrganizerV1! + containerUpdateStatuses: JSON! } type ResolvedOrganizerView { diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts new file mode 100644 index 0000000000..2999aedf48 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -0,0 +1,46 @@ +import { Args, Mutation, Parent, Query, ResolveField, Resolver } from '@nestjs/graphql'; + +import { Resource } from '@unraid/shared/graphql.model.js'; +import { + AuthActionVerb, + AuthPossession, + UsePermissions, +} from '@unraid/shared/use-permissions.directive.js'; +import { GraphQLJSON } from 'graphql-scalars'; + +import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; +import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { + Docker, + DockerContainer, + DockerNetwork, +} from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; +import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DEFAULT_ORGANIZER_ROOT_ID } from '@app/unraid-api/organizer/organizer.js'; +import { OrganizerV1, ResolvedOrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; + +@Resolver(() => DockerContainer) +export class DockerContainerResolver { + constructor(private readonly dockerManifestService: DockerManifestService) {} + + @UsePermissions({ + action: AuthActionVerb.READ, + resource: Resource.DOCKER, + possession: AuthPossession.ANY, + }) + @ResolveField(() => Boolean, { nullable: true }) + public async isUpdateAvailable(@Parent() container: DockerContainer) { + return this.dockerManifestService.isUpdateAvailable(container.image); + } + + @UsePermissions({ + action: AuthActionVerb.READ, + resource: Resource.DOCKER, + possession: AuthPossession.ANY, + }) + @ResolveField(() => Boolean, { nullable: true }) + public async isRebuildReady(@Parent() container: DockerContainer) { + return this.dockerManifestService.isRebuildReady(container.hostConfig?.networkMode); + } +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index ff3b875d90..bc71fd061b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got'; @@ -11,6 +11,7 @@ const ACCEPT_MANIFEST = @Injectable() export class DockerManifestService { + private readonly logger = new Logger(DockerManifestService.name); constructor(private readonly dockerAuthService: DockerAuthService) {} parseImageRef(imageRef: string) { @@ -104,21 +105,21 @@ export class DockerManifestService { return digest ? digest.trim() : null; } - async getLocalDigest(imageRef) { + async getLocalDigest(imageRef: string) { try { const data = await docker.getImage(imageRef).inspect(); const digests = data.RepoDigests || []; if (digests.length === 0) return null; // Prefer a digest matching this repo if present; else first const pick = digests.find((d) => d.startsWith(imageRef.split(':')[0] + '@')) || digests[0]; - const at = pick.indexOf('@'); - return at >= 0 ? pick.slice(at + 1) : null; + const [, shaDigestString] = pick.split('@'); + return shaDigestString ?? null; } catch { return null; } } - async isRebuildReady(networkMode) { + async isRebuildReady(networkMode?: string) { if (!networkMode || !networkMode.startsWith('container:')) return false; const target = networkMode.slice('container:'.length); try { diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index d92e6d1030..e1243fb682 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; +import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/docker-container.resolver.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; @@ -23,6 +24,7 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser // Resolvers DockerResolver, DockerMutationsResolver, + DockerContainerResolver, ], exports: [DockerService], }) From 19fa436287ddbcfd76824b81399235a052519569 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 13 Aug 2025 14:39:02 -0400 Subject: [PATCH 07/38] feat: check cached update status --- .../docker/docker-container.resolver.ts | 25 ++++++++++++++++-- .../docker/docker-manifest.service.ts | 26 +++++++++++++++++++ .../resolvers/docker/docker-php.service.ts | 2 ++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index 2999aedf48..cafd0788b8 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -1,3 +1,4 @@ +import { Logger } from '@nestjs/common'; import { Args, Mutation, Parent, Query, ResolveField, Resolver } from '@nestjs/graphql'; import { Resource } from '@unraid/shared/graphql.model.js'; @@ -8,6 +9,7 @@ import { } from '@unraid/shared/use-permissions.directive.js'; import { GraphQLJSON } from 'graphql-scalars'; +import { AppError } from '@app/core/errors/app-error.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; @@ -22,7 +24,11 @@ import { OrganizerV1, ResolvedOrganizerV1 } from '@app/unraid-api/organizer/orga @Resolver(() => DockerContainer) export class DockerContainerResolver { - constructor(private readonly dockerManifestService: DockerManifestService) {} + private readonly logger = new Logger(DockerContainerResolver.name); + constructor( + private readonly dockerManifestService: DockerManifestService, + private readonly dockerPhpService: DockerPhpService + ) {} @UsePermissions({ action: AuthActionVerb.READ, @@ -31,7 +37,12 @@ export class DockerContainerResolver { }) @ResolveField(() => Boolean, { nullable: true }) public async isUpdateAvailable(@Parent() container: DockerContainer) { - return this.dockerManifestService.isUpdateAvailable(container.image); + try { + return await this.dockerManifestService.isUpdateAvailableCached(container.image); + } catch (error) { + this.logger.error(error); + throw new AppError('Failed to read cached update status. See graphql-api.log for details.'); + } } @UsePermissions({ @@ -43,4 +54,14 @@ export class DockerContainerResolver { public async isRebuildReady(@Parent() container: DockerContainer) { return this.dockerManifestService.isRebuildReady(container.hostConfig?.networkMode); } + + @UsePermissions({ + action: AuthActionVerb.UPDATE, + resource: Resource.DOCKER, + possession: AuthPossession.ANY, + }) + @Mutation(() => Boolean) + public async refreshDigests() { + return this.dockerPhpService.refreshDigests(); + } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index bc71fd061b..a39cc7ee12 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -1,4 +1,5 @@ import { Injectable, Logger } from '@nestjs/common'; +import { readFile } from 'fs/promises'; import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got'; @@ -9,6 +10,15 @@ import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker const ACCEPT_MANIFEST = 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json'; +export type CachedStatusEntry = { + /** sha256 digest - "sha256:..." */ + local: string; + /** sha256 digest - "sha256:..." */ + remote: string; + /** whether update is available (true), not available (false), or unknown (null) */ + status: 'true' | 'false' | null; +}; + @Injectable() export class DockerManifestService { private readonly logger = new Logger(DockerManifestService.name); @@ -138,4 +148,20 @@ export class DockerManifestService { if (local && remote) return local !== remote; return null; // unknown } + + async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { + const cache = await readFile(cacheFile, 'utf8'); + const cacheData = JSON.parse(cache); + return cacheData as Record; + } + + async isUpdateAvailableCached(imageRef: string, cacheData?: Record) { + let taggedRef = imageRef; + if (!taggedRef.includes(':')) taggedRef += ':latest'; + + cacheData ??= await this.readCachedUpdateStatus(); + const containerData = cacheData[taggedRef]; + if (!containerData) return null; + return containerData.status?.toLowerCase() === 'true'; + } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index ef8f268cbd..1a717ff819 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -21,8 +21,10 @@ export class DockerPhpService { file: dockerUpdatePath, method: 'GET', }); + return true; } catch { // ignore; offline may keep remote as 'undef' + return false; } } From bb6dac29137c7a4b311524ca5b75ee4edd8f2dee Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 10:27:07 -0400 Subject: [PATCH 08/38] chore: add AsyncMutex to @unraid/shared/util/processing.ts --- .../src/util/__tests__/processing.test.ts | 311 ++++++++++++++++++ packages/unraid-shared/src/util/processing.ts | 38 +++ 2 files changed, 349 insertions(+) create mode 100644 packages/unraid-shared/src/util/__tests__/processing.test.ts diff --git a/packages/unraid-shared/src/util/__tests__/processing.test.ts b/packages/unraid-shared/src/util/__tests__/processing.test.ts new file mode 100644 index 0000000000..e1336e95b3 --- /dev/null +++ b/packages/unraid-shared/src/util/__tests__/processing.test.ts @@ -0,0 +1,311 @@ +import { describe, it, expect, vi } from 'vitest'; +import { AsyncMutex } from '../processing.js'; + +describe('AsyncMutex', () => { + + describe('constructor-based operation', () => { + it('should execute the default operation when do() is called without parameters', async () => { + const mockOperation = vi.fn().mockResolvedValue('result'); + const mutex = new AsyncMutex(mockOperation); + + const result = await mutex.do(); + + expect(result).toBe('result'); + expect(mockOperation).toHaveBeenCalledTimes(1); + }); + + it('should return the same promise when multiple calls are made concurrently', async () => { + let resolveOperation: (value: string) => void; + const operationPromise = new Promise((resolve) => { + resolveOperation = resolve; + }); + const mockOperation = vi.fn().mockReturnValue(operationPromise); + const mutex = new AsyncMutex(mockOperation); + + const promise1 = mutex.do(); + const promise2 = mutex.do(); + const promise3 = mutex.do(); + + expect(mockOperation).toHaveBeenCalledTimes(1); + expect(promise1).toBe(promise2); + expect(promise2).toBe(promise3); + + resolveOperation!('result'); + const [result1, result2, result3] = await Promise.all([promise1, promise2, promise3]); + + expect(result1).toBe('result'); + expect(result2).toBe('result'); + expect(result3).toBe('result'); + }); + + it('should allow new operations after the first completes', async () => { + const mockOperation = vi.fn() + .mockResolvedValueOnce('first') + .mockResolvedValueOnce('second'); + const mutex = new AsyncMutex(mockOperation); + + const result1 = await mutex.do(); + expect(result1).toBe('first'); + expect(mockOperation).toHaveBeenCalledTimes(1); + + const result2 = await mutex.do(); + expect(result2).toBe('second'); + expect(mockOperation).toHaveBeenCalledTimes(2); + }); + + it('should handle errors in the default operation', async () => { + const error = new Error('Operation failed'); + const mockOperation = vi.fn().mockRejectedValue(error); + const mutex = new AsyncMutex(mockOperation); + + await expect(mutex.do()).rejects.toThrow(error); + expect(mockOperation).toHaveBeenCalledTimes(1); + + const secondOperation = vi.fn().mockResolvedValue('success'); + const mutex2 = new AsyncMutex(secondOperation); + const result = await mutex2.do(); + expect(result).toBe('success'); + }); + }); + + describe('per-call operation', () => { + it('should execute the provided operation', async () => { + const mutex = new AsyncMutex(); + const mockOperation = vi.fn().mockResolvedValue(42); + + const result = await mutex.do(mockOperation); + + expect(result).toBe(42); + expect(mockOperation).toHaveBeenCalledTimes(1); + }); + + it('should return the same promise for concurrent calls with same operation type', async () => { + const mutex = new AsyncMutex(); + let resolveOperation: (value: string) => void; + const operationPromise = new Promise((resolve) => { + resolveOperation = resolve; + }); + const mockOperation = vi.fn().mockReturnValue(operationPromise); + + const promise1 = mutex.do(mockOperation); + const promise2 = mutex.do(mockOperation); + const promise3 = mutex.do(mockOperation); + + expect(mockOperation).toHaveBeenCalledTimes(1); + expect(promise1).toBe(promise2); + expect(promise2).toBe(promise3); + + resolveOperation!('shared-result'); + const [result1, result2, result3] = await Promise.all([promise1, promise2, promise3]); + + expect(result1).toBe('shared-result'); + expect(result2).toBe('shared-result'); + expect(result3).toBe('shared-result'); + }); + + it('should allow different operations with different types', async () => { + const mutex = new AsyncMutex(); + + const stringOp = vi.fn().mockResolvedValue('string-result'); + const numberOp = vi.fn().mockResolvedValue(123); + + const stringResult = await mutex.do(stringOp); + const numberResult = await mutex.do(numberOp); + + expect(stringResult).toBe('string-result'); + expect(numberResult).toBe(123); + expect(stringOp).toHaveBeenCalledTimes(1); + expect(numberOp).toHaveBeenCalledTimes(1); + }); + + it('should handle errors in per-call operations', async () => { + const mutex = new AsyncMutex(); + const error = new Error('Operation failed'); + const failingOp = vi.fn().mockRejectedValue(error); + + await expect(mutex.do(failingOp)).rejects.toThrow(error); + expect(failingOp).toHaveBeenCalledTimes(1); + + const successOp = vi.fn().mockResolvedValue('success'); + const result = await mutex.do(successOp); + expect(result).toBe('success'); + expect(successOp).toHaveBeenCalledTimes(1); + }); + + it('should throw an error when no operation is provided and no default is set', async () => { + const mutex = new AsyncMutex(); + + await expect(mutex.do()).rejects.toThrow('No operation provided and no default operation set'); + }); + }); + + describe('mixed usage', () => { + it('should allow overriding default operation with per-call operation', async () => { + const defaultOp = vi.fn().mockResolvedValue('default'); + const mutex = new AsyncMutex(defaultOp); + + const customOp = vi.fn().mockResolvedValue('custom'); + + const customResult = await mutex.do(customOp); + expect(customResult).toBe('custom'); + expect(customOp).toHaveBeenCalledTimes(1); + expect(defaultOp).not.toHaveBeenCalled(); + + const defaultResult = await mutex.do(); + expect(defaultResult).toBe('default'); + expect(defaultOp).toHaveBeenCalledTimes(1); + }); + + it('should share lock between default and custom operations', async () => { + let resolveDefault: (value: string) => void; + const defaultPromise = new Promise((resolve) => { + resolveDefault = resolve; + }); + const defaultOp = vi.fn().mockReturnValue(defaultPromise); + const mutex = new AsyncMutex(defaultOp); + + const customOp = vi.fn().mockResolvedValue('custom'); + + const defaultCall = mutex.do(); + const customCall = mutex.do(customOp); + + expect(defaultOp).toHaveBeenCalledTimes(1); + expect(customOp).not.toHaveBeenCalled(); + expect(customCall).toBe(defaultCall); + + resolveDefault!('default'); + const [defaultResult, customResult] = await Promise.all([defaultCall, customCall]); + + expect(defaultResult).toBe('default'); + expect(customResult).toBe('default'); + }); + }); + + describe('timing and concurrency', () => { + it('should handle sequential slow operations', async () => { + const mutex = new AsyncMutex(); + let callCount = 0; + + const slowOp = vi.fn().mockImplementation(() => { + return new Promise((resolve) => { + const currentCall = ++callCount; + setTimeout(() => resolve(`result-${currentCall}`), 100); + }); + }); + + const result1 = await mutex.do(slowOp); + expect(result1).toBe('result-1'); + + const result2 = await mutex.do(slowOp); + expect(result2).toBe('result-2'); + + expect(slowOp).toHaveBeenCalledTimes(2); + }); + + it('should deduplicate concurrent slow operations', async () => { + const mutex = new AsyncMutex(); + let resolveOperation: (value: string) => void; + + const slowOp = vi.fn().mockImplementation(() => { + return new Promise((resolve) => { + resolveOperation = resolve; + }); + }); + + const promises = [ + mutex.do(slowOp), + mutex.do(slowOp), + mutex.do(slowOp), + mutex.do(slowOp), + mutex.do(slowOp) + ]; + + expect(slowOp).toHaveBeenCalledTimes(1); + + resolveOperation!('shared-slow-result'); + const results = await Promise.all(promises); + + expect(results).toEqual([ + 'shared-slow-result', + 'shared-slow-result', + 'shared-slow-result', + 'shared-slow-result', + 'shared-slow-result' + ]); + }); + + it('should properly clean up after operation completes', async () => { + const mutex = new AsyncMutex(); + const op1 = vi.fn().mockResolvedValue('first'); + const op2 = vi.fn().mockResolvedValue('second'); + + await mutex.do(op1); + expect(op1).toHaveBeenCalledTimes(1); + + await mutex.do(op2); + expect(op2).toHaveBeenCalledTimes(1); + }); + + it('should handle multiple rapid sequences of operations', async () => { + const mutex = new AsyncMutex(); + const results: string[] = []; + + for (let i = 0; i < 5; i++) { + const op = vi.fn().mockResolvedValue(`result-${i}`); + const result = await mutex.do(op); + results.push(result as string); + } + + expect(results).toEqual(['result-0', 'result-1', 'result-2', 'result-3', 'result-4']); + }); + }); + + describe('edge cases', () => { + it('should handle operations that return undefined', async () => { + const mutex = new AsyncMutex(); + const op = vi.fn().mockResolvedValue(undefined); + + const result = await mutex.do(op); + expect(result).toBeUndefined(); + expect(op).toHaveBeenCalledTimes(1); + }); + + it('should handle operations that return null', async () => { + const mutex = new AsyncMutex(); + const op = vi.fn().mockResolvedValue(null); + + const result = await mutex.do(op); + expect(result).toBeNull(); + expect(op).toHaveBeenCalledTimes(1); + }); + + it('should handle nested operations correctly', async () => { + const mutex = new AsyncMutex(); + + const innerOp = vi.fn().mockResolvedValue('inner'); + const outerOp = vi.fn().mockImplementation(async () => { + return 'outer'; + }); + + const result = await mutex.do(outerOp); + expect(result).toBe('outer'); + expect(outerOp).toHaveBeenCalledTimes(1); + }); + + it('should maintain type safety with generic operations', async () => { + const mutex = new AsyncMutex(); + + const stringOp = vi.fn().mockResolvedValue('string'); + const numberOp = vi.fn().mockResolvedValue(42); + const booleanOp = vi.fn().mockResolvedValue(true); + + const stringResult: string = await mutex.do(stringOp); + const numberResult: number = await mutex.do(numberOp); + const booleanResult: boolean = await mutex.do(booleanOp); + + expect(stringResult).toBe('string'); + expect(numberResult).toBe(42); + expect(booleanResult).toBe(true); + }); + }); +}); \ No newline at end of file diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index 37b6e6991e..14d10e95b3 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -31,3 +31,41 @@ export function makeSafeRunner(onError: (error: unknown) => void) { } }; } + +type AsyncOperation = () => Promise; + +export class AsyncMutex { + private currentOperation: Promise | null = null; + private defaultOperation?: AsyncOperation; + + constructor(operation?: AsyncOperation) { + this.defaultOperation = operation; + } + + do(): Promise; + do(operation: AsyncOperation): Promise; + do(operation?: AsyncOperation): Promise { + if (!operation && !this.defaultOperation) { + return Promise.reject(new Error('No operation provided and no default operation set')); + } + + if (this.currentOperation) { + return this.currentOperation; + } + + const op = (operation || this.defaultOperation) as AsyncOperation; + + const promise = this.executeOperation(op).finally(() => { + if (this.currentOperation === promise) { + this.currentOperation = null; + } + }); + + this.currentOperation = promise; + return promise; + } + + private async executeOperation(operation: AsyncOperation): Promise { + return operation(); + } +} From 261d6c55ed55fea24ce6690b99a504233960b0d0 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 10:53:56 -0400 Subject: [PATCH 09/38] productionize & simplify docker digest computation --- api/generated-schema.graphql | 1 + ...ocker-manifest.service.integration.test.ts | 162 --------- .../docker/docker-manifest.service.test.ts | 328 ------------------ .../resolvers/docker/container-status.job.ts | 22 ++ .../docker/docker-container.resolver.ts | 2 +- .../docker/docker-manifest.service.ts | 136 +------- .../resolvers/docker/docker-php.service.ts | 34 +- 7 files changed, 67 insertions(+), 618 deletions(-) delete mode 100644 api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts delete mode 100644 api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/container-status.job.ts diff --git a/api/generated-schema.graphql b/api/generated-schema.graphql index 945504d74a..41219934e4 100644 --- a/api/generated-schema.graphql +++ b/api/generated-schema.graphql @@ -2416,6 +2416,7 @@ type Mutation { setDockerFolderChildren(folderId: String, childrenIds: [String!]!): ResolvedOrganizerV1! deleteDockerEntries(entryIds: [String!]!): ResolvedOrganizerV1! moveDockerEntriesToFolder(sourceEntryIds: [String!]!, destinationFolderId: String!): ResolvedOrganizerV1! + refreshDockerDigests: Boolean! """Initiates a flash drive backup using a configured remote.""" initiateFlashBackup(input: InitiateFlashBackupInput!): FlashBackupStatus! diff --git a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts deleted file mode 100644 index 0d2ce094d1..0000000000 --- a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.integration.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { beforeAll, describe, expect, it, vi } from 'vitest'; - -import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; -import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; - -describe('DockerManifestService - Integration Tests', () => { - let service: DockerManifestService; - let dockerAuthService: DockerAuthService; - - beforeAll(() => { - dockerAuthService = new DockerAuthService(); - service = new DockerManifestService(dockerAuthService); - }, 30000); - - describe('headManifest - Real HTTP calls', () => { - it('should receive authentication challenge from Docker Hub', async () => { - const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest'; - const headers = { - Accept: 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json', - }; - - const result = await service.headManifest(manifestURL, headers); - - expect(result.statusCode).toBe(401); - expect(result.headers['www-authenticate']).toContain('Bearer'); - expect(result.headers['www-authenticate']).toContain('realm'); - }, 15000); - - it('should handle unauthorized requests gracefully', async () => { - const manifestURL = 'https://registry-1.docker.io/v2/library/hello-world/manifests/latest'; - const headers = { - Accept: 'application/vnd.docker.distribution.manifest.v2+json', - }; - - const result = await service.headManifest(manifestURL, headers); - - expect(result.statusCode).toBeGreaterThanOrEqual(200); - expect(result.statusCode).toBeLessThan(500); - }, 15000); - }); - - describe('getRemoteDigest - Real Docker Registry calls', () => { - it('should get digest for public Alpine image', async () => { - const digest = await service.getRemoteDigest('alpine:latest'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should get digest for public Nginx image', async () => { - const digest = await service.getRemoteDigest('nginx:latest'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should get digest for specific image tag', async () => { - const digest = await service.getRemoteDigest('alpine:3.18'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should handle non-existent image gracefully', async () => { - const digest = await service.getRemoteDigest('nonexistent/nonexistent:nonexistent'); - - expect(digest).toBeNull(); - }, 30000); - - it('should work with Docker Hub organization images', async () => { - const digest = await service.getRemoteDigest('library/hello-world:latest'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should handle anonymous access to public registries', async () => { - const digest = await service.getRemoteDigest('busybox:latest'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should work with images without explicit tags', async () => { - const digest = await service.getRemoteDigest('ubuntu'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should get digest with mocked empty docker auth', async () => { - const mockDockerAuthService = new DockerAuthService(); - const mockManifestService = new DockerManifestService(mockDockerAuthService); - - vi.spyOn(mockDockerAuthService, 'readDockerAuth').mockResolvedValue({}); - - const digest = await mockManifestService.getRemoteDigest('alpine:latest'); - - expect(digest).toMatch(/^sha256:[a-f0-9]{64}$/); - }, 30000); - - it('should investigate HEAD vs GET behavior with authentication', async () => { - const manifestURL = 'https://registry-1.docker.io/v2/library/alpine/manifests/latest'; - const headers = { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }; - - // Test initial HEAD request (should be 401) - const initialResult = await service.headManifest(manifestURL, headers); - // console.log('Initial HEAD request - Status:', initialResult.statusCode); - - // Get Bearer token for anonymous access - const wwwAuth = (initialResult.headers?.['www-authenticate'] || '').toString(); - const token = await dockerAuthService.getBearerToken(wwwAuth, { - username: '', - password: '', - }); - // console.log('Got token:', !!token); - - if (token) { - // Test authenticated HEAD request - const authResult = await service.headManifest(manifestURL, headers, { - Authorization: `Bearer ${token}`, - }); - // console.log('Authenticated HEAD request - Status:', authResult.statusCode); - // console.log('Authenticated HEAD digest:', authResult.headers?.['docker-content-digest']); - - expect(authResult.statusCode).toBeGreaterThanOrEqual(200); - expect(authResult.statusCode).toBeLessThan(300); - } - }, 30000); - - it('should handle moderate concurrent requests', async () => { - // Start with fewer requests to identify the breaking point - const promises = Array.from({ length: 10 }, () => service.getRemoteDigest('alpine:latest')); - - const results = await Promise.all(promises); - - results.forEach((digest, index) => { - expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/); - }); - }, 60000); - - it('should handle 100 requests with optimized batching', async () => { - const batchSize = 10; - const totalRequests = 100; - const results: (string | null)[] = []; - - // Process in batches with minimal delays - for (let i = 0; i < totalRequests; i += batchSize) { - const batch = Array.from({ length: Math.min(batchSize, totalRequests - i) }, () => - service.getRemoteDigest('alpine:latest') - ); - - const batchResults = await Promise.all(batch); - results.push(...batchResults); - - // Minimal delay between batches - if (i + batchSize < totalRequests) { - await new Promise((resolve) => setTimeout(resolve, 50)); - } - } - - expect(results).toHaveLength(totalRequests); - results.forEach((digest, index) => { - expect(digest, `Request ${index + 1} failed`).toMatch(/^sha256:[a-f0-9]{64}$/); - }); - }, 60000); - }); -}); diff --git a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts b/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts deleted file mode 100644 index 5efec5a762..0000000000 --- a/api/src/__test__/unraid-api/graph/resolvers/docker/docker-manifest.service.test.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; -import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; - -vi.mock('got', () => ({ - got: { - head: vi.fn(), - get: vi.fn(), - }, -})); - -vi.mock('@app/core/utils/index.js', () => ({ - docker: { - getImage: vi.fn(() => ({ - inspect: vi.fn(), - })), - getContainer: vi.fn(() => ({ - inspect: vi.fn(), - })), - }, -})); - -describe('DockerManifestService', () => { - let service: DockerManifestService; - let mockDockerAuthService: any; - let mockGot: any; - - beforeEach(async () => { - vi.clearAllMocks(); - - mockDockerAuthService = { - readDockerAuth: vi.fn(), - decodeAuth: vi.fn(), - getBearerToken: vi.fn(), - }; - - const { got } = await import('got'); - mockGot = vi.mocked(got); - mockGot.head.mockReset(); - mockGot.get.mockReset(); - - service = new DockerManifestService(mockDockerAuthService); - }); - - describe('parseImageRef - Unit Tests', () => { - it('should parse simple image name with default tag', () => { - const result = service.parseImageRef('nginx'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry-1.docker.io', - authConfigKey: 'https://index.docker.io/v1/', - repoPath: 'library/nginx', - tag: 'latest', - }); - }); - - it('should parse image name with explicit tag', () => { - const result = service.parseImageRef('nginx:1.21'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry-1.docker.io', - authConfigKey: 'https://index.docker.io/v1/', - repoPath: 'library/nginx', - tag: '1.21', - }); - }); - - it('should parse Docker Hub organization image', () => { - const result = service.parseImageRef('ubuntu/nginx:latest'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry-1.docker.io', - authConfigKey: 'https://index.docker.io/v1/', - repoPath: 'ubuntu/nginx', - tag: 'latest', - }); - }); - - it('should parse custom registry with port', () => { - const result = service.parseImageRef('registry.example.com:5000/myorg/myapp:v1.0'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry.example.com:5000', - authConfigKey: 'registry.example.com:5000', - repoPath: 'myorg/myapp', - tag: 'v1.0', - }); - }); - - it('should parse custom registry without port', () => { - const result = service.parseImageRef('registry.example.com/myorg/myapp:v1.0'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry.example.com', - authConfigKey: 'registry.example.com', - repoPath: 'myorg/myapp', - tag: 'v1.0', - }); - }); - - it('should handle docker.io explicit registry', () => { - const result = service.parseImageRef('docker.io/nginx:latest'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry-1.docker.io', - authConfigKey: 'https://index.docker.io/v1/', - repoPath: 'docker.io/nginx', - tag: 'latest', - }); - }); - - it('should handle complex tag with multiple colons', () => { - const result = service.parseImageRef('nginx:alpine-3.14.2'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry-1.docker.io', - authConfigKey: 'https://index.docker.io/v1/', - repoPath: 'library/nginx', - tag: 'alpine-3.14.2', - }); - }); - - it('should handle localhost registry', () => { - const result = service.parseImageRef('localhost:5000/myapp:latest'); - - expect(result).toEqual({ - registryBaseURL: 'https://localhost:5000', - authConfigKey: 'localhost:5000', - repoPath: 'myapp', - tag: 'latest', - }); - }); - - it('should handle deep nested repository path', () => { - const result = service.parseImageRef('registry.example.com/team/project/service:v1.0.0'); - - expect(result).toEqual({ - registryBaseURL: 'https://registry.example.com', - authConfigKey: 'registry.example.com', - repoPath: 'team/project/service', - tag: 'v1.0.0', - }); - }); - }); - - describe('headManifest - Unit Tests', () => { - it('should return response when request succeeds', async () => { - const mockResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:abc123' }, - }; - mockGot.head.mockResolvedValue(mockResponse); - - const result = await service.headManifest( - 'https://registry-1.docker.io/v2/library/nginx/manifests/latest', - { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }, - {}, - mockGot - ); - - expect(result).toEqual(mockResponse); - }); - - it('should fallback to GET when HEAD fails', async () => { - const mockGetResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:def456' }, - }; - - mockGot.head.mockRejectedValue(new Error('HEAD not supported')); - mockGot.get.mockResolvedValue(mockGetResponse); - - const result = await service.headManifest( - 'https://registry-1.docker.io/v2/library/nginx/manifests/latest', - { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }, - {}, - mockGot - ); - - expect(result).toEqual(mockGetResponse); - }); - }); - - describe('getRemoteDigest - Unit Tests', () => { - it('should return digest for public image (anonymous access)', async () => { - const mockResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:abc123def456' }, - }; - mockGot.head.mockResolvedValueOnce(mockResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBe('sha256:abc123def456'); - }); - - it('should handle array digest header', async () => { - const mockResponse = { - statusCode: 200, - headers: { 'docker-content-digest': ['sha256:abc123def456', 'sha256:other'] }, - }; - mockGot.head.mockResolvedValueOnce(mockResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBe('sha256:abc123def456'); - }); - - it('should authenticate with Bearer token when registry requires it', async () => { - const unauthorizedResponse = { - statusCode: 401, - headers: { - 'www-authenticate': - 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/nginx:pull"', - }, - }; - const authorizedResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:authenticated' }, - }; - - mockGot.head - .mockResolvedValueOnce(unauthorizedResponse) - .mockResolvedValueOnce(authorizedResponse); - - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({ - 'https://index.docker.io/v1/': { auth: 'dXNlcjpwYXNz' }, - }); - mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: 'user', password: 'pass' }); - mockDockerAuthService.getBearerToken.mockResolvedValueOnce('bearer-token-123'); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBe('sha256:authenticated'); - }); - - it('should handle private registries requiring authentication', async () => { - const authorizedResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:private-registry' }, - }; - - mockGot.head.mockResolvedValueOnce(authorizedResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({ - 'registry.example.com': { auth: 'dXNlcjpwYXNz' }, - }); - - const result = await service.getRemoteDigest('registry.example.com/private/app:v1.0'); - - expect(result).toBe('sha256:private-registry'); - }); - - it('should return null when authentication fails', async () => { - const unauthorizedResponse = { - statusCode: 401, - headers: { - 'www-authenticate': - 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io"', - }, - }; - - mockGot.head.mockResolvedValue(unauthorizedResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - mockDockerAuthService.decodeAuth.mockReturnValueOnce({ username: '', password: '' }); - mockDockerAuthService.getBearerToken.mockResolvedValueOnce(null); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBeNull(); - }); - - it('should return null when no digest is available', async () => { - const mockResponse = { - statusCode: 200, - headers: {}, - }; - mockGot.head.mockResolvedValueOnce(mockResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBeNull(); - }); - - it('should work with custom registries', async () => { - const authorizedResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:custom-registry' }, - }; - - mockGot.head.mockResolvedValueOnce(authorizedResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('registry.example.com/myapp:v1.0'); - - expect(result).toBe('sha256:custom-registry'); - }); - - it('should handle images with no explicit tag (defaults to latest)', async () => { - const mockResponse = { - statusCode: 200, - headers: { 'docker-content-digest': 'sha256:latest-digest' }, - }; - mockGot.head.mockResolvedValueOnce(mockResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('alpine'); - - expect(result).toBe('sha256:latest-digest'); - }); - - it('should handle registry errors gracefully', async () => { - const errorResponse = { - statusCode: 500, - headers: {}, - }; - mockGot.head.mockResolvedValueOnce(errorResponse); - mockDockerAuthService.readDockerAuth.mockResolvedValueOnce({}); - - const result = await service.getRemoteDigest('nginx:latest'); - - expect(result).toBeNull(); - }); - }); -}); diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts new file mode 100644 index 0000000000..e87e0dbb32 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; +import { Cron, CronExpression, Timeout } from '@nestjs/schedule'; + +import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; + +@Injectable() +export class ContainerStatusJob { + constructor(private readonly dockerPhpService: DockerPhpService) {} + + @Cron(CronExpression.EVERY_DAY_AT_6AM) + async refreshContainerDigests() { + await this.dockerPhpService.refreshDigests(); + } + + /** + * Refresh container digests 5 seconds after application start. + */ + @Timeout(5_000) + async refreshContainerDigestsAfterStartup() { + await this.dockerPhpService.refreshDigests(); + } +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index cafd0788b8..c0cb2fa039 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -61,7 +61,7 @@ export class DockerContainerResolver { possession: AuthPossession.ANY, }) @Mutation(() => Boolean) - public async refreshDigests() { + public async refreshDockerDigests() { return this.dockerPhpService.refreshDigests(); } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index a39cc7ee12..e3006dc5dc 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -24,131 +24,6 @@ export class DockerManifestService { private readonly logger = new Logger(DockerManifestService.name); constructor(private readonly dockerAuthService: DockerAuthService) {} - parseImageRef(imageRef: string) { - // Normalize to repo:tag and extract registry/repo/name/tag - let ref = imageRef; - if (!ref.includes(':')) ref += ':latest'; - - // Registry present? - const firstSlash = ref.indexOf('/'); - const maybeRegistry = firstSlash > -1 ? ref.slice(0, firstSlash) : ''; - const hasDotOrColon = maybeRegistry.includes('.') || maybeRegistry.includes(':'); - const isDockerHub = !hasDotOrColon || maybeRegistry === 'docker.io'; - - const registry = isDockerHub ? 'registry-1.docker.io' : maybeRegistry; - const rest = isDockerHub ? ref : ref.slice(maybeRegistry.length + 1); - - const lastColon = rest.lastIndexOf(':'); - const namePart = rest.slice(0, lastColon); - const tag = rest.slice(lastColon + 1); - - // Ensure docker hub library namespace - const repoPath = isDockerHub && !namePart.includes('/') ? `library/${namePart}` : namePart; - - return { - registryBaseURL: `https://${registry}`, - authConfigKey: isDockerHub ? 'https://index.docker.io/v1/' : maybeRegistry, - repoPath, // e.g. library/nginx or org/image - tag, - }; - } - - async headManifest( - url: string, - headers: Record, - authHeader: Record = {}, - got: Got = gotClient - ) { - const opts: OptionsOfTextResponseBody = { - headers: { ...headers, ...authHeader }, - timeout: { request: 15_000 }, - throwHttpErrors: false, - }; - try { - return await got.head(url, opts); - } catch { - // Some registries don’t allow HEAD; try GET to read headers - return await got.get(url, opts); - } - } - - async getRemoteDigest(imageRef) { - const { registryBaseURL, repoPath, tag, authConfigKey } = this.parseImageRef(imageRef); - const manifestURL = `${registryBaseURL}/v2/${repoPath}/manifests/${tag}`; - - const dockerAuths = this.dockerAuthService.readDockerAuth(); - const authEntry = dockerAuths[authConfigKey]; - const basicCreds = authEntry?.auth - ? this.dockerAuthService.decodeAuth(authEntry.auth) - : { username: '', password: '' }; - - // 1) Probe without auth to learn challenge - let resp = await this.headManifest(manifestURL, { Accept: ACCEPT_MANIFEST }); - const digestHeaderRaw = resp.headers?.['docker-content-digest']; - const digestHeader = Array.isArray(digestHeaderRaw) ? digestHeaderRaw[0] : digestHeaderRaw; - if (resp.statusCode >= 200 && resp.statusCode < 300 && digestHeader) return digestHeader.trim(); - - const wwwAuth = (resp.headers?.['www-authenticate'] || '').toString(); - if (/Bearer/i.test(wwwAuth)) { - const token = await this.dockerAuthService.getBearerToken(wwwAuth, basicCreds); - if (!token) return null; - // 2) Repeat with Bearer - resp = await this.headManifest( - manifestURL, - { Accept: ACCEPT_MANIFEST }, - { Authorization: `Bearer ${token}` } - ); - } else if (/Basic/i.test(wwwAuth) && basicCreds.username && basicCreds.password) { - // 2) Repeat with Basic - const basic = - 'Basic ' + - Buffer.from(`${basicCreds.username}:${basicCreds.password}`).toString('base64'); - resp = await this.headManifest( - manifestURL, - { Accept: ACCEPT_MANIFEST }, - { Authorization: basic } - ); - } - - const digestRaw = resp.headers?.['docker-content-digest']; - const digest = Array.isArray(digestRaw) ? digestRaw[0] : digestRaw; - return digest ? digest.trim() : null; - } - - async getLocalDigest(imageRef: string) { - try { - const data = await docker.getImage(imageRef).inspect(); - const digests = data.RepoDigests || []; - if (digests.length === 0) return null; - // Prefer a digest matching this repo if present; else first - const pick = digests.find((d) => d.startsWith(imageRef.split(':')[0] + '@')) || digests[0]; - const [, shaDigestString] = pick.split('@'); - return shaDigestString ?? null; - } catch { - return null; - } - } - - async isRebuildReady(networkMode?: string) { - if (!networkMode || !networkMode.startsWith('container:')) return false; - const target = networkMode.slice('container:'.length); - try { - await docker.getContainer(target).inspect(); - return false; - } catch { - return true; // unresolved target -> ':???' equivalent - } - } - - async isUpdateAvailable(imageRef: string) { - const [local, remote] = await Promise.all([ - this.getLocalDigest(imageRef), - this.getRemoteDigest(imageRef), - ]); - if (local && remote) return local !== remote; - return null; // unknown - } - async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { const cache = await readFile(cacheFile, 'utf8'); const cacheData = JSON.parse(cache); @@ -164,4 +39,15 @@ export class DockerManifestService { if (!containerData) return null; return containerData.status?.toLowerCase() === 'true'; } + + async isRebuildReady(networkMode?: string) { + if (!networkMode || !networkMode.startsWith('container:')) return false; + const target = networkMode.slice('container:'.length); + try { + await docker.getContainer(target).inspect(); + return false; + } catch { + return true; // unresolved target -> ':???' equivalent + } + } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 1a717ff819..558658af1c 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -1,5 +1,7 @@ import { Injectable } from '@nestjs/common'; +import { AsyncMutex } from '@unraid/shared/util/processing.js'; + import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; @@ -12,10 +14,34 @@ type ExplicitStatusItem = { export class DockerPhpService { constructor() {} - async refreshDigests( + /**---------------------- + * Refresh Container Digests + *------------------------**/ + + private readonly refreshDigestsMutex = new AsyncMutex(() => { + return this.refreshDigestsViaPhp(); + }); + + /** + * Recomputes local/remote docker container digests and writes them to /var/lib/docker/unraid-update-status.json + * @param mutex - Optional mutex to use for the operation. If not provided, a default mutex will be used. + * @param dockerUpdatePath - Optional path to the DockerUpdate.php file. If not provided, the default path will be used. + * @returns True if the digests were refreshed, false if the operation failed + */ + async refreshDigests(mutex = this.refreshDigestsMutex, dockerUpdatePath?: string) { + return mutex.do(() => { + return this.refreshDigestsViaPhp(dockerUpdatePath); + }); + } + + /** + * Recomputes local/remote digests by triggering `DockerTemplates->getAllInfo(true)` via DockerUpdate.php + * @param dockerUpdatePath - Path to the DockerUpdate.php file + * @returns True if the digests were refreshed, false if the file is not found or the operation failed + */ + private async refreshDigestsViaPhp( dockerUpdatePath = '/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerUpdate.php' ) { - // Triggers getAllInfo(true) → recomputes local/remote digests try { await phpLoader({ file: dockerUpdatePath, @@ -28,6 +54,10 @@ export class DockerPhpService { } } + /**---------------------- + * Parse Container Statuses + *------------------------**/ + private parseStatusesFromDockerPush(js: string): ExplicitStatusItem[] { const items: ExplicitStatusItem[] = []; const re = /docker\.push\(\{[^}]*name:'([^']+)'[^}]*update:(\d)[^}]*\}\);/g; From f704143ed3880ac62bb94e9ac00d2613003e3a9d Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 11:00:26 -0400 Subject: [PATCH 10/38] rm redundant executeOperation function --- packages/unraid-shared/src/util/processing.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index 14d10e95b3..1dbd796f43 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -55,7 +55,7 @@ export class AsyncMutex { const op = (operation || this.defaultOperation) as AsyncOperation; - const promise = this.executeOperation(op).finally(() => { + const promise = op().finally(() => { if (this.currentOperation === promise) { this.currentOperation = null; } @@ -64,8 +64,4 @@ export class AsyncMutex { this.currentOperation = promise; return promise; } - - private async executeOperation(operation: AsyncOperation): Promise { - return operation(); - } } From f9ebcb615575de2ebc74bf6951bd8e60e843d03c Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 12:58:58 -0400 Subject: [PATCH 11/38] add ContainerStatusJob to docker.module --- api/src/unraid-api/graph/resolvers/docker/docker.module.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index e1243fb682..7a1aace790 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; +import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js'; import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/docker-container.resolver.js'; @@ -21,6 +22,9 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser DockerPhpService, // DockerEventService, + // Jobs + ContainerStatusJob, + // Resolvers DockerResolver, DockerMutationsResolver, From aa04064949e27ed5eefd2b029b2b3ae3a1020ecb Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 13:00:07 -0400 Subject: [PATCH 12/38] rm fast-xml-parser not needed yet --- pnpm-lock.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18069905dd..82a4b89d32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13424,8 +13424,8 @@ snapshots: '@jridgewell/source-map@0.3.6': dependencies: - '@jridgewell/gen-mapping': 0.3.8 - '@jridgewell/trace-mapping': 0.3.25 + '@jridgewell/gen-mapping': 0.3.12 + '@jridgewell/trace-mapping': 0.3.29 '@jridgewell/sourcemap-codec@1.5.0': {} From cf96f14a4b180df2f7abc81ea22d8fcd9564118a Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 13:02:43 -0400 Subject: [PATCH 13/38] rm docker-auth.service --- .../resolvers/docker/docker-auth.service.ts | 73 ------------------- .../docker/docker-manifest.service.ts | 4 +- .../graph/resolvers/docker/docker.module.ts | 2 - 3 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts deleted file mode 100644 index eb75137a45..0000000000 --- a/api/src/unraid-api/graph/resolvers/docker/docker-auth.service.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { readFile } from 'fs/promises'; -import { join } from 'path'; - -import type { Got } from 'got'; -import { ExtendOptions, got as gotClient } from 'got'; - -export type BasicDockerCreds = { - username: string; - password: string; -}; - -export type DockerWwwAuthParts = { - realm: string; - service: string; - scope: string; -}; - -@Injectable() -export class DockerAuthService { - private readonly logger = new Logger(DockerAuthService.name); - constructor() {} - - async readDockerAuth(configPath?: string) { - try { - configPath ??= join(process.env.HOME || '/root', '.docker/config.json'); - const cfg = JSON.parse(await readFile(configPath, 'utf8')); - return cfg.auths || {}; - } catch (error) { - this.logger.debug(error, `Failed to read Docker auth from '${configPath}'`); - return {}; - } - } - - decodeAuth(auth: string): BasicDockerCreds { - try { - const [username, password] = Buffer.from(auth, 'base64').toString('utf8').split(':'); - return { username, password }; - } catch { - return { username: '', password: '' }; - } - } - - parseWWWAuth(wwwAuth: string): Partial { - // www-authenticate: Bearer realm="...",service="...",scope="repository:repo/name:pull" - const parts: Partial = {}; - const rawParts = wwwAuth.replace(/^Bearer\s+/i, '').split(',') || []; - rawParts.forEach((pair) => { - const [k, v] = pair.split('='); - parts[k.trim()] = v?.replace(/^"|"$/g, ''); - }); - return parts; - } - - async getBearerToken( - wwwAuth: string, - basicCreds: BasicDockerCreds, - got: Got = gotClient - ) { - const parts = this.parseWWWAuth(wwwAuth); - if (!parts.realm || !parts.service || !parts.scope) return null; - const { token } = await got - .get(parts.realm, { - searchParams: { service: parts.service, scope: parts.scope }, - username: basicCreds.username, - password: basicCreds.password, - timeout: { request: 15_000 }, - responseType: 'json', - }) - .json<{ token?: string }>(); - return token; - } -} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index e3006dc5dc..edc33770a9 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -4,7 +4,6 @@ import { readFile } from 'fs/promises'; import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got'; import { docker } from '@app/core/utils/index.js'; -import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; /** Accept header for Docker API manifest listing */ const ACCEPT_MANIFEST = @@ -21,8 +20,7 @@ export type CachedStatusEntry = { @Injectable() export class DockerManifestService { - private readonly logger = new Logger(DockerManifestService.name); - constructor(private readonly dockerAuthService: DockerAuthService) {} + constructor() {} async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { const cache = await readFile(cacheFile, 'utf8'); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index 7a1aace790..222d91d11d 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -1,7 +1,6 @@ import { Module } from '@nestjs/common'; import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js'; -import { DockerAuthService } from '@app/unraid-api/graph/resolvers/docker/docker-auth.service.js'; import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/docker-container.resolver.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; @@ -17,7 +16,6 @@ import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.ser DockerService, DockerConfigService, DockerOrganizerService, - DockerAuthService, DockerManifestService, DockerPhpService, // DockerEventService, From 5f728c06f7fff33c0d494a4a3151b979bbcdab1b Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 13:21:02 -0400 Subject: [PATCH 14/38] refactor: improve code organization --- .../resolvers/docker/container-status.job.ts | 8 ++-- .../docker/docker-container.resolver.ts | 7 +-- .../docker/docker-manifest.service.ts | 46 +++++++++---------- .../resolvers/docker/docker-php.service.ts | 33 +++++++------ 4 files changed, 44 insertions(+), 50 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts index e87e0dbb32..0903ed8cba 100644 --- a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -1,15 +1,15 @@ import { Injectable } from '@nestjs/common'; import { Cron, CronExpression, Timeout } from '@nestjs/schedule'; -import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; @Injectable() export class ContainerStatusJob { - constructor(private readonly dockerPhpService: DockerPhpService) {} + constructor(private readonly dockerManifestService: DockerManifestService) {} @Cron(CronExpression.EVERY_DAY_AT_6AM) async refreshContainerDigests() { - await this.dockerPhpService.refreshDigests(); + await this.dockerManifestService.refreshDigests(); } /** @@ -17,6 +17,6 @@ export class ContainerStatusJob { */ @Timeout(5_000) async refreshContainerDigestsAfterStartup() { - await this.dockerPhpService.refreshDigests(); + await this.dockerManifestService.refreshDigests(); } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index c0cb2fa039..2a4f054e5b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -25,10 +25,7 @@ import { OrganizerV1, ResolvedOrganizerV1 } from '@app/unraid-api/organizer/orga @Resolver(() => DockerContainer) export class DockerContainerResolver { private readonly logger = new Logger(DockerContainerResolver.name); - constructor( - private readonly dockerManifestService: DockerManifestService, - private readonly dockerPhpService: DockerPhpService - ) {} + constructor(private readonly dockerManifestService: DockerManifestService) {} @UsePermissions({ action: AuthActionVerb.READ, @@ -62,6 +59,6 @@ export class DockerContainerResolver { }) @Mutation(() => Boolean) public async refreshDockerDigests() { - return this.dockerPhpService.refreshDigests(); + return this.dockerManifestService.refreshDigests(); } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index edc33770a9..87ffcf9c68 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -1,38 +1,36 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { readFile } from 'fs/promises'; - -import { ExtendOptions, Got, got as gotClient, OptionsOfTextResponseBody } from 'got'; +import { Injectable } from '@nestjs/common'; import { docker } from '@app/core/utils/index.js'; - -/** Accept header for Docker API manifest listing */ -const ACCEPT_MANIFEST = - 'application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json'; - -export type CachedStatusEntry = { - /** sha256 digest - "sha256:..." */ - local: string; - /** sha256 digest - "sha256:..." */ - remote: string; - /** whether update is available (true), not available (false), or unknown (null) */ - status: 'true' | 'false' | null; -}; +import { + CachedStatusEntry, + DockerPhpService, +} from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; @Injectable() export class DockerManifestService { - constructor() {} - - async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { - const cache = await readFile(cacheFile, 'utf8'); - const cacheData = JSON.parse(cache); - return cacheData as Record; + constructor(private readonly dockerPhpService: DockerPhpService) {} + + private readonly refreshDigestsMutex = new AsyncMutex(() => { + return this.dockerPhpService.refreshDigestsViaPhp(); + }); + + /** + * Recomputes local/remote docker container digests and writes them to /var/lib/docker/unraid-update-status.json + * @param mutex - Optional mutex to use for the operation. If not provided, a default mutex will be used. + * @param dockerUpdatePath - Optional path to the DockerUpdate.php file. If not provided, the default path will be used. + * @returns True if the digests were refreshed, false if the operation failed + */ + async refreshDigests(mutex = this.refreshDigestsMutex, dockerUpdatePath?: string) { + return mutex.do(() => { + return this.dockerPhpService.refreshDigestsViaPhp(dockerUpdatePath); + }); } async isUpdateAvailableCached(imageRef: string, cacheData?: Record) { let taggedRef = imageRef; if (!taggedRef.includes(':')) taggedRef += ':latest'; - cacheData ??= await this.readCachedUpdateStatus(); + cacheData ??= await this.dockerPhpService.readCachedUpdateStatus(); const containerData = cacheData[taggedRef]; if (!containerData) return null; return containerData.status?.toLowerCase() === 'true'; diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 558658af1c..1089d8c60b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { readFile } from 'fs/promises'; import { AsyncMutex } from '@unraid/shared/util/processing.js'; @@ -9,37 +10,35 @@ type ExplicitStatusItem = { name: string; updateStatus: 'up to date' | 'update available' | 'rebuild ready' | 'unknown'; }; +export type CachedStatusEntry = { + /** sha256 digest - "sha256:..." */ + local: string; + /** sha256 digest - "sha256:..." */ + remote: string; + /** whether update is available (true), not available (false), or unknown (null) */ + status: 'true' | 'false' | null; +}; @Injectable() export class DockerPhpService { constructor() {} + async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { + const cache = await readFile(cacheFile, 'utf8'); + const cacheData = JSON.parse(cache); + return cacheData as Record; + } + /**---------------------- * Refresh Container Digests *------------------------**/ - private readonly refreshDigestsMutex = new AsyncMutex(() => { - return this.refreshDigestsViaPhp(); - }); - - /** - * Recomputes local/remote docker container digests and writes them to /var/lib/docker/unraid-update-status.json - * @param mutex - Optional mutex to use for the operation. If not provided, a default mutex will be used. - * @param dockerUpdatePath - Optional path to the DockerUpdate.php file. If not provided, the default path will be used. - * @returns True if the digests were refreshed, false if the operation failed - */ - async refreshDigests(mutex = this.refreshDigestsMutex, dockerUpdatePath?: string) { - return mutex.do(() => { - return this.refreshDigestsViaPhp(dockerUpdatePath); - }); - } - /** * Recomputes local/remote digests by triggering `DockerTemplates->getAllInfo(true)` via DockerUpdate.php * @param dockerUpdatePath - Path to the DockerUpdate.php file * @returns True if the digests were refreshed, false if the file is not found or the operation failed */ - private async refreshDigestsViaPhp( + async refreshDigestsViaPhp( dockerUpdatePath = '/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerUpdate.php' ) { try { From 88b08754eab7648b40799911e1064278a3e32a33 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 20 Aug 2025 13:53:32 -0400 Subject: [PATCH 15/38] fix AsynxMutex import --- .../graph/resolvers/docker/docker-manifest.service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index 87ffcf9c68..03a40430c3 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -1,5 +1,7 @@ import { Injectable } from '@nestjs/common'; +import { AsyncMutex } from '@unraid/shared/util/processing.js'; + import { docker } from '@app/core/utils/index.js'; import { CachedStatusEntry, From 90aecc3df3b913d14cc22624141dd8c83a339606 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 2 Sep 2025 16:40:37 -0400 Subject: [PATCH 16/38] use enum for update status --- .../resolvers/docker/docker-php.service.ts | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 1089d8c60b..d30b08959e 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -1,15 +1,27 @@ import { Injectable } from '@nestjs/common'; import { readFile } from 'fs/promises'; -import { AsyncMutex } from '@unraid/shared/util/processing.js'; - import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; + +/** Note that these values propogate down to api consumers, so be aware of breaking changes. */ +enum UpdateStatus { + UP_TO_DATE = 'UP_TO_DATE', + UPDATE_AVAILABLE = 'UPDATE_AVAILABLE', + REBUILD_READY = 'REBUILD_READY', + UNKNOWN = 'UNKNOWN', +} + type ExplicitStatusItem = { name: string; - updateStatus: 'up to date' | 'update available' | 'rebuild ready' | 'unknown'; + updateStatus: UpdateStatus; }; + +/** + * These types reflect the structure of the /var/lib/docker/unraid-update-status.json file, + * which is not controlled by the Unraid API. + */ export type CachedStatusEntry = { /** sha256 digest - "sha256:..." */ local: string; @@ -68,10 +80,10 @@ export class DockerPhpService { return items; } - private updateStatusToString(updateStatus: 0): 'up to date'; - private updateStatusToString(updateStatus: 1): 'update available'; - private updateStatusToString(updateStatus: 2): 'rebuild ready'; - private updateStatusToString(updateStatus: 3): 'unknown'; + private updateStatusToString(updateStatus: 0): UpdateStatus.UP_TO_DATE; + private updateStatusToString(updateStatus: 1): UpdateStatus.UPDATE_AVAILABLE; + private updateStatusToString(updateStatus: 2): UpdateStatus.REBUILD_READY; + private updateStatusToString(updateStatus: 3): UpdateStatus.UNKNOWN; // prettier-ignore private updateStatusToString(updateStatus: StatusItem['updateStatus']): ExplicitStatusItem['updateStatus']; private updateStatusToString( @@ -79,13 +91,13 @@ export class DockerPhpService { ): ExplicitStatusItem['updateStatus'] { switch (updateStatus) { case 0: - return 'up to date'; + return UpdateStatus.UP_TO_DATE; case 1: - return 'update available'; + return UpdateStatus.UPDATE_AVAILABLE; case 2: - return 'rebuild ready'; + return UpdateStatus.REBUILD_READY; default: - return 'unknown'; + return UpdateStatus.UNKNOWN; } } From 240e104ccac82ff83e6cf860e756160ea3d7df6e Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 2 Sep 2025 16:48:54 -0400 Subject: [PATCH 17/38] refactor: docker config service -> docker organizer config service --- .../docker/docker-container.resolver.ts | 14 ++------------ .../graph/resolvers/docker/docker.module.spec.ts | 7 +++---- .../graph/resolvers/docker/docker.module.ts | 6 +++--- .../resolvers/docker/docker.resolver.spec.ts | 2 +- .../graph/resolvers/docker/docker.resolver.ts | 2 +- .../docker-organizer-config.service.ts} | 2 +- .../docker-organizer.service.spec.ts | 16 ++++++++-------- .../{ => organizer}/docker-organizer.service.ts | 4 ++-- 8 files changed, 21 insertions(+), 32 deletions(-) rename api/src/unraid-api/graph/resolvers/docker/{docker-config.service.ts => organizer/docker-organizer-config.service.ts} (95%) rename api/src/unraid-api/graph/resolvers/docker/{ => organizer}/docker-organizer.service.spec.ts (98%) rename api/src/unraid-api/graph/resolvers/docker/{ => organizer}/docker-organizer.service.ts (97%) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index 2a4f054e5b..c04e63e2d3 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common'; -import { Args, Mutation, Parent, Query, ResolveField, Resolver } from '@nestjs/graphql'; +import { Mutation, Parent, ResolveField, Resolver } from '@nestjs/graphql'; import { Resource } from '@unraid/shared/graphql.model.js'; import { @@ -7,20 +7,10 @@ import { AuthPossession, UsePermissions, } from '@unraid/shared/use-permissions.directive.js'; -import { GraphQLJSON } from 'graphql-scalars'; import { AppError } from '@app/core/errors/app-error.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; -import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; -import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; -import { - Docker, - DockerContainer, - DockerNetwork, -} from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; -import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; -import { DEFAULT_ORGANIZER_ROOT_ID } from '@app/unraid-api/organizer/organizer.js'; -import { OrganizerV1, ResolvedOrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; +import { DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; @Resolver(() => DockerContainer) export class DockerContainerResolver { diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts index 8bfadaff3a..c7d9e1269d 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts @@ -1,16 +1,15 @@ -import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { describe, expect, it, vi } from 'vitest'; -import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerEventService } from '@app/unraid-api/graph/resolvers/docker/docker-event.service.js'; -import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { DockerModule } from '@app/unraid-api/graph/resolvers/docker/docker.module.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerConfigService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.js'; +import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; describe('DockerModule', () => { it('should compile the module', async () => { @@ -19,7 +18,7 @@ describe('DockerModule', () => { }) .overrideProvider(DockerService) .useValue({ getDockerClient: vi.fn() }) - .overrideProvider(DockerConfigService) + .overrideProvider(DockerOrganizerConfigService) .useValue({ getConfig: vi.fn() }) .compile(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index 222d91d11d..f8007e670e 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -1,20 +1,20 @@ import { Module } from '@nestjs/common'; import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js'; -import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/docker-container.resolver.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; -import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { DockerMutationsResolver } from '@app/unraid-api/graph/resolvers/docker/docker.mutations.resolver.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerConfigService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.js'; +import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; @Module({ providers: [ // Services DockerService, - DockerConfigService, + DockerOrganizerConfigService, DockerOrganizerService, DockerManifestService, DockerPhpService, diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts index 6b08425740..a5cf4aeec8 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.spec.ts @@ -3,11 +3,11 @@ import { Test } from '@nestjs/testing'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { ContainerState, DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerResolver } from '@app/unraid-api/graph/resolvers/docker/docker.resolver.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; describe('DockerResolver', () => { let resolver: DockerResolver; diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts index 5dcf3038f8..1d6180a0ec 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts @@ -4,7 +4,6 @@ import { AuthAction, Resource } from '@unraid/shared/graphql.model.js'; import { UsePermissions } from '@unraid/shared/use-permissions.directive.js'; import { GraphQLJSON } from 'graphql-scalars'; -import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { Docker, @@ -12,6 +11,7 @@ import { DockerNetwork, } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; import { DEFAULT_ORGANIZER_ROOT_ID } from '@app/unraid-api/organizer/organizer.js'; import { ResolvedOrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts similarity index 95% rename from api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts rename to api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts index 402f3a81fb..3223a5249f 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts @@ -13,7 +13,7 @@ import { OrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; import { validateOrganizerIntegrity } from '@app/unraid-api/organizer/organizer.validation.js'; @Injectable() -export class DockerConfigService extends ConfigFilePersister { +export class DockerOrganizerConfigService extends ConfigFilePersister { constructor(configService: ConfigService) { super(configService); } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.spec.ts b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.spec.ts similarity index 98% rename from api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.spec.ts rename to api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.spec.ts index b7159438c0..ecb0bb1a71 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.spec.ts @@ -2,17 +2,17 @@ import { Test } from '@nestjs/testing'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; -import { - containerToResource, - DockerOrganizerService, -} from '@app/unraid-api/graph/resolvers/docker/docker-organizer.service.js'; import { ContainerPortType, ContainerState, DockerContainer, } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerConfigService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.js'; +import { + containerToResource, + DockerOrganizerService, +} from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; import { OrganizerV1 } from '@app/unraid-api/organizer/organizer.model.js'; describe('containerToResource', () => { @@ -138,7 +138,7 @@ describe('containerToResource', () => { describe('DockerOrganizerService', () => { let service: DockerOrganizerService; - let configService: DockerConfigService; + let configService: DockerOrganizerConfigService; let dockerService: DockerService; const mockOrganizer: OrganizerV1 = { @@ -178,7 +178,7 @@ describe('DockerOrganizerService', () => { providers: [ DockerOrganizerService, { - provide: DockerConfigService, + provide: DockerOrganizerConfigService, useValue: { getConfig: vi.fn().mockImplementation(() => structuredClone(mockOrganizer)), validate: vi.fn().mockImplementation((config) => Promise.resolve(config)), @@ -220,7 +220,7 @@ describe('DockerOrganizerService', () => { }).compile(); service = moduleRef.get(DockerOrganizerService); - configService = moduleRef.get(DockerConfigService); + configService = moduleRef.get(DockerOrganizerConfigService); dockerService = moduleRef.get(DockerService); }); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.ts b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.ts similarity index 97% rename from api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.ts rename to api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.ts index 8f2f7ec2bf..41dff8257d 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-organizer.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.ts @@ -3,9 +3,9 @@ import { Injectable, Logger } from '@nestjs/common'; import type { ContainerListOptions } from 'dockerode'; import { AppError } from '@app/core/errors/app-error.js'; -import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; import { DockerService } from '@app/unraid-api/graph/resolvers/docker/docker.service.js'; +import { DockerOrganizerConfigService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.js'; import { addMissingResourcesToView, createFolderInView, @@ -47,7 +47,7 @@ export function containerListToResourcesObject(containers: DockerContainer[]): O export class DockerOrganizerService { private readonly logger = new Logger(DockerOrganizerService.name); constructor( - private readonly dockerConfigService: DockerConfigService, + private readonly dockerConfigService: DockerOrganizerConfigService, private readonly dockerService: DockerService ) {} From 99a2103d16582aaf2cffe0648ae005538fee0bbe Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 10:05:41 -0400 Subject: [PATCH 18/38] revert to cron 4.3.0 for compat with @nest/schedule@6.0.0 --- api/package.json | 2 +- pnpm-lock.yaml | 22 ++++------------------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/api/package.json b/api/package.json index 3569d9adfe..ba82c31b53 100644 --- a/api/package.json +++ b/api/package.json @@ -94,7 +94,7 @@ "command-exists": "1.2.9", "convert": "5.12.0", "cookie": "1.0.2", - "cron": "4.3.3", + "cron": "4.3.0", "cross-fetch": "4.1.0", "diff": "8.0.2", "dockerode": "4.0.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82a4b89d32..4c2726729e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -164,8 +164,8 @@ importers: specifier: 1.0.2 version: 1.0.2 cron: - specifier: 4.3.3 - version: 4.3.3 + specifier: 4.3.0 + version: 4.3.0 cross-fetch: specifier: 4.1.0 version: 4.1.0 @@ -4201,9 +4201,6 @@ packages: '@types/luxon@3.6.2': resolution: {integrity: sha512-R/BdP7OxEMc44l2Ex5lSXHoIXTB2JLNa3y2QISIbr58U/YcsffyQrYW//hZSdrfxrjRZj3GcUoxMPGdO8gSYuw==} - '@types/luxon@3.7.1': - resolution: {integrity: sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==} - '@types/mdx@2.0.13': resolution: {integrity: sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==} @@ -5868,10 +5865,6 @@ packages: resolution: {integrity: sha512-ciiYNLfSlF9MrDqnbMdRWFiA6oizSF7kA1osPP9lRzNu0Uu+AWog1UKy7SkckiDY2irrNjeO6qLyKnXC8oxmrw==} engines: {node: '>=18.x'} - cron@4.3.3: - resolution: {integrity: sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==} - engines: {node: '>=18.x'} - croner@4.1.97: resolution: {integrity: sha512-/f6gpQuxDaqXu+1kwQYSckUglPaOrHdbIlBAu0YuW8/Cdb45XwXYNUBXg3r/9Mo6n540Kn/smKcZWko5x99KrQ==} @@ -13424,8 +13417,8 @@ snapshots: '@jridgewell/source-map@0.3.6': dependencies: - '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/gen-mapping': 0.3.8 + '@jridgewell/trace-mapping': 0.3.25 '@jridgewell/sourcemap-codec@1.5.0': {} @@ -14806,8 +14799,6 @@ snapshots: '@types/luxon@3.6.2': {} - '@types/luxon@3.7.1': {} - '@types/mdx@2.0.13': {} '@types/methods@1.1.4': {} @@ -16673,11 +16664,6 @@ snapshots: '@types/luxon': 3.6.2 luxon: 3.6.1 - cron@4.3.3: - dependencies: - '@types/luxon': 3.7.1 - luxon: 3.7.1 - croner@4.1.97: {} cross-fetch@3.2.0: From 05bbe841757df5ff7670e8352d56328141368efb Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 10:35:12 -0400 Subject: [PATCH 19/38] feat: make cron schedule configurable for container manifest refresh --- api/src/unraid-api/app/app.module.ts | 3 +- api/src/unraid-api/cron/cron.module.ts | 4 +- api/src/unraid-api/cron/job.module.ts | 12 ++++++ .../resolvers/docker/container-status.job.ts | 32 +++++++++++---- .../resolvers/docker/docker-config.model.ts | 7 ++++ .../resolvers/docker/docker-config.service.ts | 40 +++++++++++++++++++ .../docker/docker-container.resolver.ts | 12 ++---- .../resolvers/docker/docker.module.spec.ts | 4 ++ .../graph/resolvers/docker/docker.module.ts | 4 ++ 9 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 api/src/unraid-api/cron/job.module.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts diff --git a/api/src/unraid-api/app/app.module.ts b/api/src/unraid-api/app/app.module.ts index e8bc4a71b6..8617b0c3d8 100644 --- a/api/src/unraid-api/app/app.module.ts +++ b/api/src/unraid-api/app/app.module.ts @@ -14,6 +14,7 @@ import { AuthModule } from '@app/unraid-api/auth/auth.module.js'; import { AuthenticationGuard } from '@app/unraid-api/auth/authentication.guard.js'; import { LegacyConfigModule } from '@app/unraid-api/config/legacy-config.module.js'; import { CronModule } from '@app/unraid-api/cron/cron.module.js'; +import { JobModule } from '@app/unraid-api/cron/job.module.js'; import { GraphModule } from '@app/unraid-api/graph/graph.module.js'; import { GlobalDepsModule } from '@app/unraid-api/plugin/global-deps.module.js'; import { RestModule } from '@app/unraid-api/rest/rest.module.js'; @@ -24,7 +25,7 @@ import { UnraidFileModifierModule } from '@app/unraid-api/unraid-file-modifier/u GlobalDepsModule, LegacyConfigModule, PubSubModule, - ScheduleModule.forRoot(), + JobModule, LoggerModule.forRoot({ pinoHttp: { logger: apiLogger, diff --git a/api/src/unraid-api/cron/cron.module.ts b/api/src/unraid-api/cron/cron.module.ts index 86b0b625f6..bada7aae57 100644 --- a/api/src/unraid-api/cron/cron.module.ts +++ b/api/src/unraid-api/cron/cron.module.ts @@ -1,11 +1,11 @@ import { Module } from '@nestjs/common'; -import { ScheduleModule } from '@nestjs/schedule'; +import { JobModule } from '@app/unraid-api/cron/job.module.js'; import { LogRotateService } from '@app/unraid-api/cron/log-rotate.service.js'; import { WriteFlashFileService } from '@app/unraid-api/cron/write-flash-file.service.js'; @Module({ - imports: [], + imports: [JobModule], providers: [WriteFlashFileService, LogRotateService], }) export class CronModule {} diff --git a/api/src/unraid-api/cron/job.module.ts b/api/src/unraid-api/cron/job.module.ts new file mode 100644 index 0000000000..b3372630c0 --- /dev/null +++ b/api/src/unraid-api/cron/job.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { ScheduleModule } from '@nestjs/schedule'; + +/** + * Sets up common dependencies for initializing jobs (e.g. scheduler registry, cron jobs). + * + * Simplifies testing setup & application dependency tree by ensuring `forRoot` is called only once. + */ +@Module({ + imports: [ScheduleModule.forRoot()], +}) +export class JobModule {} diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts index 0903ed8cba..831c226605 100644 --- a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -1,15 +1,33 @@ -import { Injectable } from '@nestjs/common'; -import { Cron, CronExpression, Timeout } from '@nestjs/schedule'; +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { SchedulerRegistry, Timeout } from '@nestjs/schedule'; +import { CronJob } from 'cron'; + +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; @Injectable() -export class ContainerStatusJob { - constructor(private readonly dockerManifestService: DockerManifestService) {} +export class ContainerStatusJob implements OnApplicationBootstrap { + private readonly logger = new Logger(ContainerStatusJob.name); + constructor( + private readonly dockerManifestService: DockerManifestService, + private readonly schedulerRegistry: SchedulerRegistry, + private readonly dockerConfigService: DockerConfigService + ) {} - @Cron(CronExpression.EVERY_DAY_AT_6AM) - async refreshContainerDigests() { - await this.dockerManifestService.refreshDigests(); + onApplicationBootstrap() { + const cronExpression = this.dockerConfigService.getConfig().updateCheckCronSchedule; + const cronJob = CronJob.from({ + cronTime: cronExpression, + onTick: () => { + this.dockerManifestService.refreshDigests(); + }, + start: true, + }); + this.schedulerRegistry.addCronJob(ContainerStatusJob.name, cronJob); + this.logger.verbose( + `Initialized cron job for refreshing container update status: ${ContainerStatusJob.name}` + ); } /** diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts new file mode 100644 index 0000000000..e7a47ae660 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.model.ts @@ -0,0 +1,7 @@ +import { Field, ObjectType } from '@nestjs/graphql'; + +@ObjectType() +export class DockerConfig { + @Field(() => String) + updateCheckCronSchedule!: string; +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts new file mode 100644 index 0000000000..a335de391a --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts @@ -0,0 +1,40 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { CronExpression } from '@nestjs/schedule'; + +import { ConfigFilePersister } from '@unraid/shared/services/config-file.js'; +import { validateCronExpression } from 'cron'; + +import { AppError } from '@app/core/errors/app-error.js'; +import { DockerConfig } from '@app/unraid-api/graph/resolvers/docker/docker-config.model.js'; +import { validateObject } from '@app/unraid-api/graph/resolvers/validation.utils.js'; + +@Injectable() +export class DockerConfigService extends ConfigFilePersister { + constructor(configService: ConfigService) { + super(configService); + } + + configKey(): string { + return 'docker'; + } + + fileName(): string { + return 'docker.config.json'; + } + + defaultConfig(): DockerConfig { + return { + updateCheckCronSchedule: CronExpression.EVERY_DAY_AT_6AM, + }; + } + + async validate(config: object): Promise { + const dockerConfig = await validateObject(DockerConfig, config); + const cronExpression = validateCronExpression(dockerConfig.updateCheckCronSchedule); + if (!cronExpression.valid) { + throw new AppError(`Cron expression not supported: ${dockerConfig.updateCheckCronSchedule}`); + } + return dockerConfig; + } +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index c04e63e2d3..b6598d01b1 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -3,8 +3,7 @@ import { Mutation, Parent, ResolveField, Resolver } from '@nestjs/graphql'; import { Resource } from '@unraid/shared/graphql.model.js'; import { - AuthActionVerb, - AuthPossession, + AuthAction, UsePermissions, } from '@unraid/shared/use-permissions.directive.js'; @@ -18,9 +17,8 @@ export class DockerContainerResolver { constructor(private readonly dockerManifestService: DockerManifestService) {} @UsePermissions({ - action: AuthActionVerb.READ, + action: AuthAction.READ_ANY, resource: Resource.DOCKER, - possession: AuthPossession.ANY, }) @ResolveField(() => Boolean, { nullable: true }) public async isUpdateAvailable(@Parent() container: DockerContainer) { @@ -33,9 +31,8 @@ export class DockerContainerResolver { } @UsePermissions({ - action: AuthActionVerb.READ, + action: AuthAction.READ_ANY, resource: Resource.DOCKER, - possession: AuthPossession.ANY, }) @ResolveField(() => Boolean, { nullable: true }) public async isRebuildReady(@Parent() container: DockerContainer) { @@ -43,9 +40,8 @@ export class DockerContainerResolver { } @UsePermissions({ - action: AuthActionVerb.UPDATE, + action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, - possession: AuthPossession.ANY, }) @Mutation(() => Boolean) public async refreshDockerDigests() { diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts index c7d9e1269d..7856ba19db 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts @@ -1,7 +1,9 @@ +import { ConfigModule } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { describe, expect, it, vi } from 'vitest'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerEventService } from '@app/unraid-api/graph/resolvers/docker/docker-event.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { DockerModule } from '@app/unraid-api/graph/resolvers/docker/docker.module.js'; @@ -20,6 +22,8 @@ describe('DockerModule', () => { .useValue({ getDockerClient: vi.fn() }) .overrideProvider(DockerOrganizerConfigService) .useValue({ getConfig: vi.fn() }) + .overrideProvider(DockerConfigService) + .useValue({ getConfig: vi.fn() }) .compile(); expect(module).toBeDefined(); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts index f8007e670e..22095f518d 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; +import { JobModule } from '@app/unraid-api/cron/job.module.js'; import { ContainerStatusJob } from '@app/unraid-api/graph/resolvers/docker/container-status.job.js'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; import { DockerContainerResolver } from '@app/unraid-api/graph/resolvers/docker/docker-container.resolver.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; @@ -11,6 +13,7 @@ import { DockerOrganizerConfigService } from '@app/unraid-api/graph/resolvers/do import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/organizer/docker-organizer.service.js'; @Module({ + imports: [JobModule], providers: [ // Services DockerService, @@ -18,6 +21,7 @@ import { DockerOrganizerService } from '@app/unraid-api/graph/resolvers/docker/o DockerOrganizerService, DockerManifestService, DockerPhpService, + DockerConfigService, // DockerEventService, // Jobs From fc5fb1a1bad19f0fc1ef2e2e7d4f86699d88a346 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 10:55:19 -0400 Subject: [PATCH 20/38] watch php files --- api/vite.config.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/vite.config.ts b/api/vite.config.ts index c16742c1ca..bddf826b48 100644 --- a/api/vite.config.ts +++ b/api/vite.config.ts @@ -1,3 +1,6 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; + import type { ViteUserConfig } from 'vitest/config'; import { viteCommonjs } from '@originjs/vite-plugin-commonjs'; import nodeResolve from '@rollup/plugin-node-resolve'; @@ -73,12 +76,17 @@ export default defineConfig(({ mode }): ViteUserConfig => { // Copy PHP files to assets directory { name: 'copy-php-files', + buildStart() { + const phpFiles = ['src/core/utils/plugins/wrapper.php']; + phpFiles.forEach((file) => this.addWatchFile(file)); + }, async generateBundle() { - const { readFileSync } = await import('fs'); - const { join, basename } = await import('path'); - const phpFiles = ['src/core/utils/plugins/wrapper.php']; phpFiles.forEach((file) => { + if (!existsSync(file)) { + this.warn(`[copy-php-files] PHP file ${file} does not exist`); + return; + } const content = readFileSync(file); this.emitFile({ type: 'asset', From d1c98495c9125774b66fe2924b11b24f67991a92 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 11:03:13 -0400 Subject: [PATCH 21/38] code cleanup --- .../resolvers/docker/container-status.job.ts | 4 +++- .../resolvers/docker/docker-php.service.ts | 20 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts index 831c226605..dca0093109 100644 --- a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -20,7 +20,9 @@ export class ContainerStatusJob implements OnApplicationBootstrap { const cronJob = CronJob.from({ cronTime: cronExpression, onTick: () => { - this.dockerManifestService.refreshDigests(); + this.dockerManifestService.refreshDigests().catch((error) => { + this.logger.warn(error, 'Failed to refresh container update status'); + }); }, start: true, }); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index d30b08959e..7210741134 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { readFile } from 'fs/promises'; import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; @@ -33,12 +33,24 @@ export type CachedStatusEntry = { @Injectable() export class DockerPhpService { + private readonly logger = new Logger(DockerPhpService.name); constructor() {} + /** + * Reads JSON from a file containing cached update status. + * If the file does not exist, an empty object is returned. + * @param cacheFile + * @returns + */ async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { - const cache = await readFile(cacheFile, 'utf8'); - const cacheData = JSON.parse(cache); - return cacheData as Record; + try { + const cache = await readFile(cacheFile, 'utf8'); + const cacheData = JSON.parse(cache); + return cacheData as Record; + } catch (error) { + this.logger.warn(error, 'Failed to read cached update status'); + return {}; + } } /**---------------------- From 7a6806835cf264050caed24828e52cc58f6ecef2 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 11:23:44 -0400 Subject: [PATCH 22/38] add docs --- .../resolvers/docker/container-status.job.ts | 3 + .../docker/docker-manifest.service.ts | 11 +++ .../resolvers/docker/docker-php.service.ts | 5 ++ packages/unraid-shared/src/util/processing.ts | 76 +++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts index dca0093109..16fa04e495 100644 --- a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -15,6 +15,9 @@ export class ContainerStatusJob implements OnApplicationBootstrap { private readonly dockerConfigService: DockerConfigService ) {} + /** + * Initialize cron job for refreshing the update status for all containers on a user-configurable schedule. + */ onApplicationBootstrap() { const cronExpression = this.dockerConfigService.getConfig().updateCheckCronSchedule; const cronJob = CronJob.from({ diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts index 03a40430c3..b14fe8606b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-manifest.service.ts @@ -28,6 +28,12 @@ export class DockerManifestService { }); } + /** + * Checks if an update is available for a given container image. + * @param imageRef - The image reference to check, e.g. "unraid/baseimage:latest". If no tag is provided, "latest" is assumed, following the webgui's implementation. + * @param cacheData read from /var/lib/docker/unraid-update-status.json by default + * @returns True if an update is available, false if not, or null if the status is unknown + */ async isUpdateAvailableCached(imageRef: string, cacheData?: Record) { let taggedRef = imageRef; if (!taggedRef.includes(':')) taggedRef += ':latest'; @@ -38,6 +44,11 @@ export class DockerManifestService { return containerData.status?.toLowerCase() === 'true'; } + /** + * Checks if a container is rebuild ready. + * @param networkMode - The network mode of the container, e.g. "container:unraid/baseimage:latest". + * @returns True if the container is rebuild ready, false if not + */ async isRebuildReady(networkMode?: string) { if (!networkMode || !networkMode.startsWith('container:')) return false; const target = networkMode.slice('container:'.length); diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 7210741134..8bbcb317c4 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -113,6 +113,11 @@ export class DockerPhpService { } } + /** + * Gets the update statuses for all containers by triggering `DockerTemplates->getAllInfo(true)` via DockerContainers.php + * @param dockerContainersPath - Path to the DockerContainers.php file + * @returns The update statuses for all containers + */ async getContainerUpdateStatuses( dockerContainersPath = '/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerContainers.php' ): Promise { diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index 1dbd796f43..5764bfc37d 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -34,15 +34,91 @@ export function makeSafeRunner(onError: (error: unknown) => void) { type AsyncOperation = () => Promise; +/** + * A mutex for asynchronous operations that ensures only one operation runs at a time. + * + * When multiple callers attempt to execute operations simultaneously, they will all + * receive the same promise from the currently running operation, effectively deduplicating + * concurrent calls. This is useful for expensive operations like API calls, file operations, + * or database queries that should not be executed multiple times concurrently. + * + * @template T - The default return type for operations when using a default operation + * + * @example + * // Basic usage with explicit operations + * const mutex = new AsyncMutex(); + * + * // Multiple concurrent calls will deduplicate + * const [result1, result2, result3] = await Promise.all([ + * mutex.do(() => fetch('/api/data')), + * mutex.do(() => fetch('/api/data')), // Same request, will get same promise + * mutex.do(() => fetch('/api/data')) // Same request, will get same promise + * ]); + * // Only one fetch actually happens + * + * @example + * // Usage with a default operation + * const dataLoader = new AsyncMutex(() => + * fetch('/api/expensive-data').then(res => res.json()) + * ); + * + * // Multiple components can call this without duplication + * const data1 = await dataLoader.do(); // Executes the fetch + * const data2 = await dataLoader.do(); // Gets the same promise result + */ export class AsyncMutex { private currentOperation: Promise | null = null; private defaultOperation?: AsyncOperation; + /** + * Creates a new AsyncMutex instance. + * + * @param operation - Optional default operation to execute when calling `do()` without arguments. + * This is useful when you have a specific operation that should be deduplicated. + * + * @example + * // Without default operation + * const mutex = new AsyncMutex(); + * await mutex.do(() => someAsyncWork()); + * + * @example + * // With default operation + * const dataMutex = new AsyncMutex(() => loadExpensiveData()); + * await dataMutex.do(); // Executes loadExpensiveData() + */ constructor(operation?: AsyncOperation) { this.defaultOperation = operation; } + /** + * Executes the default operation if one was provided in the constructor. + * @returns Promise that resolves with the result of the default operation + * @throws Error if no default operation was set in the constructor + */ do(): Promise; + /** + * Executes the provided operation, ensuring only one runs at a time. + * + * If an operation is already running, all subsequent calls will receive + * the same promise from the currently running operation. This effectively + * deduplicates concurrent calls to the same expensive operation. + * + * @param operation - Optional operation to execute. If not provided, uses the default operation. + * @returns Promise that resolves with the result of the operation + * @throws Error if no operation is provided and no default operation was set + * + * @example + * const mutex = new AsyncMutex(); + * + * // These will all return the same promise + * const promise1 = mutex.do(() => fetch('/api/data')); + * const promise2 = mutex.do(() => fetch('/api/other')); // Still gets first promise! + * const promise3 = mutex.do(() => fetch('/api/another')); // Still gets first promise! + * + * // After the first operation completes, new operations can run + * await promise1; + * const newPromise = mutex.do(() => fetch('/api/new')); // This will execute + */ do(operation: AsyncOperation): Promise; do(operation?: AsyncOperation): Promise { if (!operation && !this.defaultOperation) { From 49189d9bb7922a7411fa2363052d27c6e0031882 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Wed, 3 Sep 2025 11:37:25 -0400 Subject: [PATCH 23/38] fix details in AsyncMutex util docs --- .../docker/docker-container.resolver.ts | 5 +---- packages/unraid-shared/src/util/processing.ts | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index b6598d01b1..8e23491316 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -2,10 +2,7 @@ import { Logger } from '@nestjs/common'; import { Mutation, Parent, ResolveField, Resolver } from '@nestjs/graphql'; import { Resource } from '@unraid/shared/graphql.model.js'; -import { - AuthAction, - UsePermissions, -} from '@unraid/shared/use-permissions.directive.js'; +import { AuthAction, UsePermissions } from '@unraid/shared/use-permissions.directive.js'; import { AppError } from '@app/core/errors/app-error.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index 5764bfc37d..43291ac1d0 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -62,9 +62,8 @@ type AsyncOperation = () => Promise; * fetch('/api/expensive-data').then(res => res.json()) * ); * - * // Multiple components can call this without duplication * const data1 = await dataLoader.do(); // Executes the fetch - * const data2 = await dataLoader.do(); // Gets the same promise result + * const data2 = await dataLoader.do(); // If first promise is finished, a new fetch is executed */ export class AsyncMutex { private currentOperation: Promise | null = null; @@ -77,12 +76,20 @@ export class AsyncMutex { * This is useful when you have a specific operation that should be deduplicated. * * @example - * // Without default operation + * // Without default operation (shared mutex) * const mutex = new AsyncMutex(); - * await mutex.do(() => someAsyncWork()); + * const promise1 = mutex.do(() => someAsyncWork()); + * const promise2 = mutex.do(() => someOtherAsyncWork()); + * + * // Both promises will be the same + * expect(await promise1).toBe(await promise2); + * + * // After the first operation completes, new operations can run + * await promise1; + * const newPromise = mutex.do(() => someOtherAsyncWork()); // This will execute * * @example - * // With default operation + * // With default operation (deduplicating a specific operation) * const dataMutex = new AsyncMutex(() => loadExpensiveData()); * await dataMutex.do(); // Executes loadExpensiveData() */ From 473608eba3cda24440a0afb01584898c335f0234 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 8 Sep 2025 14:48:16 -0400 Subject: [PATCH 24/38] feat: feature flag system --- api/docs/developer/feature-flags.md | 247 ++++++++++++++ .../decorators/omit-if.decorator.spec.ts | 172 ++++++++++ .../decorators/omit-if.decorator.ts | 80 +++++ .../use-feature-flag.decorator.spec.ts | 317 ++++++++++++++++++ .../decorators/use-feature-flag.decorator.ts | 22 ++ api/src/unraid-api/graph/graph.module.ts | 8 +- .../unraid-api/utils/feature-flag.helper.ts | 28 ++ 7 files changed, 873 insertions(+), 1 deletion(-) create mode 100644 api/docs/developer/feature-flags.md create mode 100644 api/src/unraid-api/decorators/omit-if.decorator.spec.ts create mode 100644 api/src/unraid-api/decorators/omit-if.decorator.ts create mode 100644 api/src/unraid-api/decorators/use-feature-flag.decorator.spec.ts create mode 100644 api/src/unraid-api/decorators/use-feature-flag.decorator.ts create mode 100644 api/src/unraid-api/utils/feature-flag.helper.ts diff --git a/api/docs/developer/feature-flags.md b/api/docs/developer/feature-flags.md new file mode 100644 index 0000000000..53d9425b8c --- /dev/null +++ b/api/docs/developer/feature-flags.md @@ -0,0 +1,247 @@ +# Feature Flags + +Feature flags allow you to conditionally enable or disable functionality in the Unraid API. This is useful for gradually rolling out new features, A/B testing, or keeping experimental code behind flags during development. + +## Setting Up Feature Flags + +### 1. Define the Feature Flag + +Feature flags are defined as environment variables and collected in `src/consts.ts`: + +```typescript +// src/environment.ts +export const ENABLE_MY_NEW_FEATURE = process.env.ENABLE_MY_NEW_FEATURE === 'true'; + +// src/consts.ts +export const FeatureFlags = Object.freeze({ + ENABLE_NEXT_DOCKER_RELEASE, + ENABLE_MY_NEW_FEATURE, // Add your new flag here +}); +``` + +### 2. Set the Environment Variable + +Set the environment variable when running the API: + +```bash +ENABLE_MY_NEW_FEATURE=true unraid-api start +``` + +Or add it to your `.env` file: + +```env +ENABLE_MY_NEW_FEATURE=true +``` + +## Using Feature Flags in GraphQL + +### Method 1: @UseFeatureFlag Decorator (Schema-Level) + +The `@UseFeatureFlag` decorator conditionally includes or excludes GraphQL fields, queries, and mutations from the schema based on feature flags. When a feature flag is disabled, the field won't appear in the GraphQL schema at all. + +```typescript +import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; +import { Query, Mutation, ResolveField } from '@nestjs/graphql'; + +@Resolver() +export class MyResolver { + + // Conditionally include a query + @UseFeatureFlag('ENABLE_MY_NEW_FEATURE') + @Query(() => String) + async experimentalQuery() { + return 'This query only exists when ENABLE_MY_NEW_FEATURE is true'; + } + + // Conditionally include a mutation + @UseFeatureFlag('ENABLE_MY_NEW_FEATURE') + @Mutation(() => Boolean) + async experimentalMutation() { + return true; + } + + // Conditionally include a field resolver + @UseFeatureFlag('ENABLE_MY_NEW_FEATURE') + @ResolveField(() => String) + async experimentalField() { + return 'This field only exists when the flag is enabled'; + } +} +``` + +**Benefits:** +- Clean schema - disabled features don't appear in GraphQL introspection +- No runtime overhead for disabled features +- Clear feature boundaries + +**Use when:** +- You want to completely hide features from the GraphQL schema +- The feature is experimental or in beta +- You're doing a gradual rollout + +### Method 2: checkFeatureFlag Function (Runtime) + +The `checkFeatureFlag` function provides runtime feature flag checking within resolver methods. It throws a `ForbiddenException` if the feature is disabled. + +```typescript +import { checkFeatureFlag } from '@app/unraid-api/utils/feature-flag.helper.js'; +import { FeatureFlags } from '@app/consts.js'; +import { Query, ResolveField } from '@nestjs/graphql'; + +@Resolver() +export class MyResolver { + + @Query(() => String) + async myQuery( + @Args('useNewAlgorithm', { nullable: true }) useNewAlgorithm?: boolean + ) { + // Conditionally use new logic based on feature flag + if (useNewAlgorithm) { + checkFeatureFlag(FeatureFlags, 'ENABLE_MY_NEW_FEATURE'); + return this.newAlgorithm(); + } + + return this.oldAlgorithm(); + } + + @ResolveField(() => String) + async dataField() { + // Check flag at the start of the method + checkFeatureFlag(FeatureFlags, 'ENABLE_MY_NEW_FEATURE'); + + // Feature-specific logic here + return this.computeExperimentalData(); + } +} +``` + +**Benefits:** +- More granular control within methods +- Can conditionally execute parts of a method +- Useful for A/B testing scenarios +- Good for gradual migration strategies + +**Use when:** +- You need conditional logic within a method +- The field should exist but behavior changes based on the flag +- You're migrating from old to new implementation gradually + +## Feature Flag Patterns + +### Pattern 1: Complete Feature Toggle + +Hide an entire feature behind a flag: + +```typescript +@UseFeatureFlag('ENABLE_DOCKER_TEMPLATES') +@Resolver(() => DockerTemplate) +export class DockerTemplateResolver { + // All resolvers in this class are toggled by the flag +} +``` + +### Pattern 2: Gradual Migration + +Migrate from old to new implementation: + +```typescript +@Query(() => [Container]) +async getContainers(@Args('version') version?: string) { + if (version === 'v2') { + checkFeatureFlag(FeatureFlags, 'ENABLE_CONTAINERS_V2'); + return this.getContainersV2(); + } + + return this.getContainersV1(); +} +``` + +### Pattern 3: Beta Features + +Mark features as beta: + +```typescript +@UseFeatureFlag('ENABLE_BETA_FEATURES') +@ResolveField(() => BetaMetrics, { + description: 'BETA: Advanced metrics (requires ENABLE_BETA_FEATURES flag)' +}) +async betaMetrics() { + return this.computeBetaMetrics(); +} +``` + +### Pattern 4: Performance Optimizations + +Toggle expensive operations: + +```typescript +@ResolveField(() => Statistics) +async statistics() { + const basicStats = await this.getBasicStats(); + + try { + checkFeatureFlag(FeatureFlags, 'ENABLE_ADVANCED_ANALYTICS'); + const advancedStats = await this.getAdvancedStats(); + return { ...basicStats, ...advancedStats }; + } catch { + // Feature disabled, return only basic stats + return basicStats; + } +} +``` + +## Testing with Feature Flags + +When writing tests for feature-flagged code, create a mock to control feature flag values: + +```typescript +import { vi } from 'vitest'; + +// Mock the entire consts module +vi.mock('@app/consts.js', async () => { + const actual = await vi.importActual('@app/consts.js'); + return { + ...actual, + FeatureFlags: { + ENABLE_MY_NEW_FEATURE: true, // Set your test value + ENABLE_NEXT_DOCKER_RELEASE: false, + } + }; +}); + +describe('MyResolver', () => { + it('should execute new logic when feature is enabled', async () => { + // Test new behavior with mocked flag + }); +}); +``` + +## Best Practices + +1. **Naming Convention**: Use `ENABLE_` prefix for boolean feature flags +2. **Environment Variables**: Always use uppercase with underscores +3. **Documentation**: Document what each feature flag controls +4. **Cleanup**: Remove feature flags once features are stable and fully rolled out +5. **Default State**: New features should default to `false` (disabled) +6. **Granularity**: Keep feature flags focused on a single feature or capability +7. **Testing**: Always test both enabled and disabled states + +## Common Use Cases + +- **Experimental Features**: Hide unstable features in production +- **Gradual Rollouts**: Enable features for specific environments first +- **A/B Testing**: Toggle between different implementations +- **Performance**: Disable expensive operations when not needed +- **Breaking Changes**: Provide migration path with both old and new behavior +- **Debug Features**: Enable additional logging or debugging tools + +## Checking Active Feature Flags + +To see which feature flags are currently active: + +```typescript +// Log all feature flags on startup +console.log('Active Feature Flags:', FeatureFlags); +``` + +Or check via GraphQL introspection to see which fields are available based on current flags. diff --git a/api/src/unraid-api/decorators/omit-if.decorator.spec.ts b/api/src/unraid-api/decorators/omit-if.decorator.spec.ts new file mode 100644 index 0000000000..eb936390ce --- /dev/null +++ b/api/src/unraid-api/decorators/omit-if.decorator.spec.ts @@ -0,0 +1,172 @@ +import { Reflector } from '@nestjs/core'; +import { Field, Mutation, ObjectType, Query, ResolveField, Resolver } from '@nestjs/graphql'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { OMIT_IF_METADATA_KEY, OmitIf } from '@app/unraid-api/decorators/omit-if.decorator.js'; + +describe('OmitIf Decorator', () => { + let reflector: Reflector; + + beforeEach(() => { + reflector = new Reflector(); + }); + + describe('OmitIf', () => { + it('should set metadata when condition is true', () => { + class TestResolver { + @OmitIf(true) + testMethod() { + return 'test'; + } + } + + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testMethod); + expect(metadata).toBe(true); + }); + + it('should not set metadata when condition is false', () => { + class TestResolver { + @OmitIf(false) + testMethod() { + return 'test'; + } + } + + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testMethod); + expect(metadata).toBeUndefined(); + }); + + it('should evaluate function conditions', () => { + const mockCondition = vi.fn(() => true); + + class TestResolver { + @OmitIf(mockCondition) + testMethod() { + return 'test'; + } + } + + expect(mockCondition).toHaveBeenCalledOnce(); + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testMethod); + expect(metadata).toBe(true); + }); + + it('should evaluate function conditions that return false', () => { + const mockCondition = vi.fn(() => false); + + class TestResolver { + @OmitIf(mockCondition) + testMethod() { + return 'test'; + } + } + + expect(mockCondition).toHaveBeenCalledOnce(); + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testMethod); + expect(metadata).toBeUndefined(); + }); + + it('should work with environment variables', () => { + const originalEnv = process.env.NODE_ENV; + process.env.NODE_ENV = 'production'; + + class TestResolver { + @OmitIf(process.env.NODE_ENV === 'production') + testMethod() { + return 'test'; + } + } + + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testMethod); + expect(metadata).toBe(true); + + process.env.NODE_ENV = originalEnv; + }); + }); + + describe('Integration with NestJS GraphQL decorators', () => { + it('should work with @Query decorator', () => { + @Resolver() + class TestResolver { + @OmitIf(true) + @Query(() => String) + omittedQuery() { + return 'test'; + } + + @OmitIf(false) + @Query(() => String) + includedQuery() { + return 'test'; + } + } + + const instance = new TestResolver(); + const omittedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.omittedQuery); + const includedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.includedQuery); + + expect(omittedMetadata).toBe(true); + expect(includedMetadata).toBeUndefined(); + }); + + it('should work with @Mutation decorator', () => { + @Resolver() + class TestResolver { + @OmitIf(true) + @Mutation(() => String) + omittedMutation() { + return 'test'; + } + + @OmitIf(false) + @Mutation(() => String) + includedMutation() { + return 'test'; + } + } + + const instance = new TestResolver(); + const omittedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.omittedMutation); + const includedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.includedMutation); + + expect(omittedMetadata).toBe(true); + expect(includedMetadata).toBeUndefined(); + }); + + it('should work with @ResolveField decorator', () => { + @ObjectType() + class TestType { + @Field() + id: string = ''; + } + + @Resolver(() => TestType) + class TestResolver { + @OmitIf(true) + @ResolveField(() => String) + omittedField() { + return 'test'; + } + + @OmitIf(false) + @ResolveField(() => String) + includedField() { + return 'test'; + } + } + + const instance = new TestResolver(); + const omittedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.omittedField); + const includedMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.includedField); + + expect(omittedMetadata).toBe(true); + expect(includedMetadata).toBeUndefined(); + }); + }); +}); diff --git a/api/src/unraid-api/decorators/omit-if.decorator.ts b/api/src/unraid-api/decorators/omit-if.decorator.ts new file mode 100644 index 0000000000..696355d7c1 --- /dev/null +++ b/api/src/unraid-api/decorators/omit-if.decorator.ts @@ -0,0 +1,80 @@ +import { SetMetadata } from '@nestjs/common'; +import { Extensions } from '@nestjs/graphql'; + +import { MapperKind, mapSchema } from '@graphql-tools/utils'; +import { GraphQLFieldConfig, GraphQLSchema } from 'graphql'; + +export const OMIT_IF_METADATA_KEY = 'omitIf'; + +/** + * Decorator that conditionally omits a GraphQL field/query/mutation based on a condition. + * The field will only be omitted from the schema when the condition evaluates to true. + * + * @param condition - If the condition evaluates to true, the field will be omitted from the schema + * @returns A decorator that wraps the target field/query/mutation + * + * @example + * ```typescript + * @OmitIf(process.env.NODE_ENV === 'production') + * @Query(() => String) + * async debugQuery() { + * return 'This query is omitted in production'; + * } + * ``` + */ +export function OmitIf(condition: boolean | (() => boolean)): MethodDecorator & PropertyDecorator { + const shouldOmit = typeof condition === 'function' ? condition() : condition; + + return (target: any, propertyKey?: string | symbol, descriptor?: PropertyDescriptor) => { + if (shouldOmit) { + SetMetadata(OMIT_IF_METADATA_KEY, true)( + target, + propertyKey as string, + descriptor as PropertyDescriptor + ); + Extensions({ omitIf: true })( + target, + propertyKey as string, + descriptor as PropertyDescriptor + ); + } + + return descriptor; + }; +} + +/** + * Schema transformer that omits fields/queries/mutations based on the OmitIf decorator. + * @param schema - The GraphQL schema to transform + * @returns The transformed GraphQL schema + */ +export function omitIfSchemaTransformer(schema: GraphQLSchema): GraphQLSchema { + return mapSchema(schema, { + [MapperKind.OBJECT_FIELD]: ( + fieldConfig: GraphQLFieldConfig, + fieldName: string, + typeName: string + ) => { + const extensions = fieldConfig.extensions || {}; + + if (extensions.omitIf === true) { + return null; + } + + return fieldConfig; + }, + [MapperKind.ROOT_FIELD]: ( + fieldConfig: GraphQLFieldConfig, + fieldName: string, + typeName: string + ) => { + const extensions = fieldConfig.extensions || {}; + + if (extensions.omitIf === true) { + return null; + } + + return fieldConfig; + }, + }); +} diff --git a/api/src/unraid-api/decorators/use-feature-flag.decorator.spec.ts b/api/src/unraid-api/decorators/use-feature-flag.decorator.spec.ts new file mode 100644 index 0000000000..5143652a79 --- /dev/null +++ b/api/src/unraid-api/decorators/use-feature-flag.decorator.spec.ts @@ -0,0 +1,317 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck +// fixme: types don't sync with mocks, and there's no override to simplify testing. + +import { Reflector } from '@nestjs/core'; +import { Mutation, Query, ResolveField, Resolver } from '@nestjs/graphql'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { OMIT_IF_METADATA_KEY } from '@app/unraid-api/decorators/omit-if.decorator.js'; +import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; + +// Mock the FeatureFlags +vi.mock('@app/consts.js', () => ({ + FeatureFlags: Object.freeze({ + ENABLE_NEXT_DOCKER_RELEASE: false, + ENABLE_EXPERIMENTAL_FEATURE: true, + ENABLE_DEBUG_MODE: false, + ENABLE_BETA_FEATURES: true, + }), +})); + +describe('UseFeatureFlag Decorator', () => { + let reflector: Reflector; + + beforeEach(() => { + reflector = new Reflector(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe('Basic functionality', () => { + it('should omit field when feature flag is false', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Query(() => String) + testQuery() { + return 'test'; + } + } + + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testQuery); + expect(metadata).toBe(true); // Should be omitted because flag is false + }); + + it('should include field when feature flag is true', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_EXPERIMENTAL_FEATURE') + @Query(() => String) + testQuery() { + return 'test'; + } + } + + const instance = new TestResolver(); + const metadata = reflector.get(OMIT_IF_METADATA_KEY, instance.testQuery); + expect(metadata).toBeUndefined(); // Should not be omitted because flag is true + }); + }); + + describe('With different decorator types', () => { + it('should work with @Query decorator', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_DEBUG_MODE') + @Query(() => String) + debugQuery() { + return 'debug'; + } + + @UseFeatureFlag('ENABLE_BETA_FEATURES') + @Query(() => String) + betaQuery() { + return 'beta'; + } + } + + const instance = new TestResolver(); + const debugMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.debugQuery); + const betaMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.betaQuery); + + expect(debugMetadata).toBe(true); // ENABLE_DEBUG_MODE is false + expect(betaMetadata).toBeUndefined(); // ENABLE_BETA_FEATURES is true + }); + + it('should work with @Mutation decorator', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Mutation(() => String) + dockerMutation() { + return 'docker'; + } + + @UseFeatureFlag('ENABLE_EXPERIMENTAL_FEATURE') + @Mutation(() => String) + experimentalMutation() { + return 'experimental'; + } + } + + const instance = new TestResolver(); + const dockerMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.dockerMutation); + const experimentalMetadata = reflector.get( + OMIT_IF_METADATA_KEY, + instance.experimentalMutation + ); + + expect(dockerMetadata).toBe(true); // ENABLE_NEXT_DOCKER_RELEASE is false + expect(experimentalMetadata).toBeUndefined(); // ENABLE_EXPERIMENTAL_FEATURE is true + }); + + it('should work with @ResolveField decorator', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_DEBUG_MODE') + @ResolveField(() => String) + debugField() { + return 'debug'; + } + + @UseFeatureFlag('ENABLE_BETA_FEATURES') + @ResolveField(() => String) + betaField() { + return 'beta'; + } + } + + const instance = new TestResolver(); + const debugMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.debugField); + const betaMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.betaField); + + expect(debugMetadata).toBe(true); // ENABLE_DEBUG_MODE is false + expect(betaMetadata).toBeUndefined(); // ENABLE_BETA_FEATURES is true + }); + }); + + describe('Multiple decorators on same class', () => { + it('should handle multiple feature flags independently', () => { + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Query(() => String) + dockerQuery() { + return 'docker'; + } + + @UseFeatureFlag('ENABLE_EXPERIMENTAL_FEATURE') + @Query(() => String) + experimentalQuery() { + return 'experimental'; + } + + @UseFeatureFlag('ENABLE_DEBUG_MODE') + @Query(() => String) + debugQuery() { + return 'debug'; + } + + @UseFeatureFlag('ENABLE_BETA_FEATURES') + @Query(() => String) + betaQuery() { + return 'beta'; + } + } + + const instance = new TestResolver(); + + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.dockerQuery)).toBe(true); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.experimentalQuery)).toBeUndefined(); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.debugQuery)).toBe(true); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.betaQuery)).toBeUndefined(); + }); + }); + + describe('Type safety', () => { + it('should only accept valid feature flag keys', () => { + // This test verifies TypeScript compile-time type safety + // The following would cause a TypeScript error if uncommented: + // @UseFeatureFlag('INVALID_FLAG') + + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Query(() => String) + validQuery() { + return 'valid'; + } + } + + const instance = new TestResolver(); + expect(instance.validQuery).toBeDefined(); + }); + }); + + describe('Integration scenarios', () => { + it('should work correctly with other decorators', () => { + const customDecorator = ( + target: any, + propertyKey: string | symbol, + descriptor: PropertyDescriptor + ) => { + Reflect.defineMetadata('custom', true, target, propertyKey); + return descriptor; + }; + + @Resolver() + class TestResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @customDecorator + @Query(() => String) + multiDecoratorQuery() { + return 'multi'; + } + } + + const instance = new TestResolver(); + const omitMetadata = reflector.get(OMIT_IF_METADATA_KEY, instance.multiDecoratorQuery); + const customMetadata = Reflect.getMetadata('custom', instance, 'multiDecoratorQuery'); + + expect(omitMetadata).toBe(true); + expect(customMetadata).toBe(true); + }); + + it('should maintain correct decorator order', () => { + const orderTracker: string[] = []; + + const trackingDecorator = (name: string) => { + return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { + orderTracker.push(name); + return descriptor; + }; + }; + + @Resolver() + class TestResolver { + @trackingDecorator('first') + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @trackingDecorator('last') + @Query(() => String) + orderedQuery() { + return 'ordered'; + } + } + + // Decorators are applied bottom-up + expect(orderTracker).toEqual(['last', 'first']); + }); + }); + + describe('Real-world usage patterns', () => { + it('should work with Docker resolver pattern', () => { + @Resolver() + class DockerResolver { + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Mutation(() => String) + async createDockerFolder(name: string) { + return `Created folder: ${name}`; + } + + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Mutation(() => String) + async deleteDockerEntries(entryIds: string[]) { + return `Deleted entries: ${entryIds.join(', ')}`; + } + + @Query(() => String) + async getDockerInfo() { + return 'Docker info'; + } + } + + const instance = new DockerResolver(); + + // Feature flag is false, so these should be omitted + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.createDockerFolder)).toBe(true); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.deleteDockerEntries)).toBe(true); + + // No feature flag, so this should not be omitted + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.getDockerInfo)).toBeUndefined(); + }); + + it('should handle mixed feature flags in same resolver', () => { + @Resolver() + class MixedResolver { + @UseFeatureFlag('ENABLE_EXPERIMENTAL_FEATURE') + @Query(() => String) + experimentalQuery() { + return 'experimental'; + } + + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + @Query(() => String) + dockerQuery() { + return 'docker'; + } + + @UseFeatureFlag('ENABLE_BETA_FEATURES') + @Mutation(() => String) + betaMutation() { + return 'beta'; + } + } + + const instance = new MixedResolver(); + + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.experimentalQuery)).toBeUndefined(); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.dockerQuery)).toBe(true); + expect(reflector.get(OMIT_IF_METADATA_KEY, instance.betaMutation)).toBeUndefined(); + }); + }); +}); diff --git a/api/src/unraid-api/decorators/use-feature-flag.decorator.ts b/api/src/unraid-api/decorators/use-feature-flag.decorator.ts new file mode 100644 index 0000000000..96910bfe5a --- /dev/null +++ b/api/src/unraid-api/decorators/use-feature-flag.decorator.ts @@ -0,0 +1,22 @@ +import { FeatureFlags } from '@app/consts.js'; +import { OmitIf } from '@app/unraid-api/decorators/omit-if.decorator.js'; + +/** + * Decorator that conditionally includes a GraphQL field/query/mutation based on a feature flag. + * The field will only be included in the schema when the feature flag is enabled. + * + * @param flagKey - The key of the feature flag in FeatureFlags + * @returns A decorator that wraps OmitIf + * + * @example + * ```typescript + * @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') + * @Mutation(() => String) + * async experimentalMutation() { + * return 'This mutation is only available when ENABLE_NEXT_DOCKER_RELEASE is true'; + * } + * ``` + */ +export function UseFeatureFlag(flagKey: keyof typeof FeatureFlags): MethodDecorator & PropertyDecorator { + return OmitIf(!FeatureFlags[flagKey]); +} diff --git a/api/src/unraid-api/graph/graph.module.ts b/api/src/unraid-api/graph/graph.module.ts index 276610f16e..9cd9368810 100644 --- a/api/src/unraid-api/graph/graph.module.ts +++ b/api/src/unraid-api/graph/graph.module.ts @@ -12,6 +12,7 @@ import { NoUnusedVariablesRule } from 'graphql'; import { ENVIRONMENT } from '@app/environment.js'; import { ApiConfigModule } from '@app/unraid-api/config/api-config.module.js'; +import { omitIfSchemaTransformer } from '@app/unraid-api/decorators/omit-if.decorator.js'; // Import enum registrations to ensure they're registered with GraphQL import '@app/unraid-api/graph/auth/auth-action.enum.js'; @@ -64,7 +65,12 @@ import { PluginModule } from '@app/unraid-api/plugin/plugin.module.js'; }, // Only add transform when not in test environment to avoid GraphQL version conflicts transformSchema: - process.env.NODE_ENV === 'test' ? undefined : usePermissionsSchemaTransformer, + process.env.NODE_ENV === 'test' + ? undefined + : (schema) => { + const schemaWithPermissions = usePermissionsSchemaTransformer(schema); + return omitIfSchemaTransformer(schemaWithPermissions); + }, validationRules: [NoUnusedVariablesRule], }; }, diff --git a/api/src/unraid-api/utils/feature-flag.helper.ts b/api/src/unraid-api/utils/feature-flag.helper.ts new file mode 100644 index 0000000000..24c8e868a7 --- /dev/null +++ b/api/src/unraid-api/utils/feature-flag.helper.ts @@ -0,0 +1,28 @@ +import { ForbiddenException } from '@nestjs/common'; + +/** + * Checks if a feature flag is enabled and throws an exception if disabled. + * Use this at the beginning of resolver methods for immediate feature flag checks. + * + * @example + * ```typescript + * @ResolveField(() => String) + * async organizer() { + * checkFeatureFlag(FeatureFlags, 'ENABLE_NEXT_DOCKER_RELEASE'); + * return this.dockerOrganizerService.resolveOrganizer(); + * } + * ``` + * + * @param flags - The feature flag object containing boolean/truthy values + * @param key - The key within the feature flag object to check + * @throws ForbiddenException if the feature flag is disabled + */ +export function checkFeatureFlag>(flags: T, key: keyof T): void { + const isEnabled = Boolean(flags[key]); + + if (!isEnabled) { + throw new ForbiddenException( + `Feature "${String(key)}" is currently disabled. This functionality is not available at this time.` + ); + } +} From c128d8e3fc1fd9e51d785ebfa2c9d34b36c84aa0 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 8 Sep 2025 14:49:06 -0400 Subject: [PATCH 25/38] feat: guard new docker features behind `ENABLE_NEXT_DOCKER_RELEASE` env --- api/.gitignore | 3 +++ api/generated-schema.graphql | 6 ++++++ api/src/consts.ts | 13 ++++++++++++- api/src/environment.ts | 3 +++ .../graph/resolvers/docker/container-status.job.ts | 2 ++ .../graph/resolvers/docker/docker-config.service.ts | 5 +++++ .../resolvers/docker/docker-container.resolver.ts | 4 ++++ .../graph/resolvers/docker/docker.resolver.ts | 7 +++++++ .../organizer/docker-organizer-config.service.ts | 5 +++++ 9 files changed, 47 insertions(+), 1 deletion(-) diff --git a/api/.gitignore b/api/.gitignore index 3d895b2eb4..77fdfdbeee 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -93,3 +93,6 @@ dev/local-session # local OIDC config for testing - contains secrets dev/configs/oidc.local.json + +# local api keys +dev/keys/* diff --git a/api/generated-schema.graphql b/api/generated-schema.graphql index 41219934e4..bf56181a24 100644 --- a/api/generated-schema.graphql +++ b/api/generated-schema.graphql @@ -139,6 +139,9 @@ type ArrayDisk implements Node { """ata | nvme | usb | (others)""" transport: String color: ArrayDiskFsColor + + """Whether the disk is currently spinning""" + isSpinning: Boolean } interface Node { @@ -346,6 +349,9 @@ type Disk implements Node { """The partitions on the disk""" partitions: [DiskPartition!]! + + """Whether the disk is spinning or not""" + isSpinning: Boolean! } """The type of interface the disk uses to connect to the system""" diff --git a/api/src/consts.ts b/api/src/consts.ts index b4bc015c23..92ff199e93 100644 --- a/api/src/consts.ts +++ b/api/src/consts.ts @@ -2,7 +2,7 @@ import { join } from 'path'; import type { JSONWebKeySet } from 'jose'; -import { PORT } from '@app/environment.js'; +import { ENABLE_NEXT_DOCKER_RELEASE, PORT } from '@app/environment.js'; export const getInternalApiAddress = (isHttp = true, nginxPort = 80) => { const envPort = PORT; @@ -79,3 +79,14 @@ export const KEYSERVER_VALIDATION_ENDPOINT = 'https://keys.lime-technology.com/v /** Set the max retries for the GraphQL Client */ export const MAX_RETRIES_FOR_LINEAR_BACKOFF = 100; + +/** + * Feature flags are used to conditionally enable or disable functionality in the Unraid API. + * + * Keys are human readable feature flag names -- will be used to construct error messages. + * + * Values are boolean/truthy values. + */ +export const FeatureFlags = Object.freeze({ + ENABLE_NEXT_DOCKER_RELEASE, +}); diff --git a/api/src/environment.ts b/api/src/environment.ts index 1aebbefb49..b1d3c2bad3 100644 --- a/api/src/environment.ts +++ b/api/src/environment.ts @@ -110,3 +110,6 @@ export const PATHS_CONFIG_MODULES = export const PATHS_LOCAL_SESSION_FILE = process.env.PATHS_LOCAL_SESSION_FILE ?? '/var/run/unraid-api/local-session'; + +/** feature flag for the upcoming docker release */ +export const ENABLE_NEXT_DOCKER_RELEASE = process.env.ENABLE_NEXT_DOCKER_RELEASE === 'true'; diff --git a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts index 16fa04e495..2131585d49 100644 --- a/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts +++ b/api/src/unraid-api/graph/resolvers/docker/container-status.job.ts @@ -19,6 +19,7 @@ export class ContainerStatusJob implements OnApplicationBootstrap { * Initialize cron job for refreshing the update status for all containers on a user-configurable schedule. */ onApplicationBootstrap() { + if (!this.dockerConfigService.enabled()) return; const cronExpression = this.dockerConfigService.getConfig().updateCheckCronSchedule; const cronJob = CronJob.from({ cronTime: cronExpression, @@ -40,6 +41,7 @@ export class ContainerStatusJob implements OnApplicationBootstrap { */ @Timeout(5_000) async refreshContainerDigestsAfterStartup() { + if (!this.dockerConfigService.enabled()) return; await this.dockerManifestService.refreshDigests(); } } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts index a335de391a..eb3eecd581 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts @@ -6,6 +6,7 @@ import { ConfigFilePersister } from '@unraid/shared/services/config-file.js'; import { validateCronExpression } from 'cron'; import { AppError } from '@app/core/errors/app-error.js'; +import { ENABLE_NEXT_DOCKER_RELEASE } from '@app/environment.js'; import { DockerConfig } from '@app/unraid-api/graph/resolvers/docker/docker-config.model.js'; import { validateObject } from '@app/unraid-api/graph/resolvers/validation.utils.js'; @@ -15,6 +16,10 @@ export class DockerConfigService extends ConfigFilePersister { super(configService); } + enabled(): boolean { + return ENABLE_NEXT_DOCKER_RELEASE; + } + configKey(): string { return 'docker'; } diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts index 8e23491316..4528b24658 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-container.resolver.ts @@ -5,6 +5,7 @@ import { Resource } from '@unraid/shared/graphql.model.js'; import { AuthAction, UsePermissions } from '@unraid/shared/use-permissions.directive.js'; import { AppError } from '@app/core/errors/app-error.js'; +import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; import { DockerManifestService } from '@app/unraid-api/graph/resolvers/docker/docker-manifest.service.js'; import { DockerContainer } from '@app/unraid-api/graph/resolvers/docker/docker.model.js'; @@ -13,6 +14,7 @@ export class DockerContainerResolver { private readonly logger = new Logger(DockerContainerResolver.name); constructor(private readonly dockerManifestService: DockerManifestService) {} + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.READ_ANY, resource: Resource.DOCKER, @@ -27,6 +29,7 @@ export class DockerContainerResolver { } } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.READ_ANY, resource: Resource.DOCKER, @@ -36,6 +39,7 @@ export class DockerContainerResolver { return this.dockerManifestService.isRebuildReady(container.hostConfig?.networkMode); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts index 1d6180a0ec..00906e61ca 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts @@ -4,6 +4,7 @@ import { AuthAction, Resource } from '@unraid/shared/graphql.model.js'; import { UsePermissions } from '@unraid/shared/use-permissions.directive.js'; import { GraphQLJSON } from 'graphql-scalars'; +import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; import { Docker, @@ -56,6 +57,7 @@ export class DockerResolver { return this.dockerService.getNetworks({ skipCache }); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.READ_ANY, resource: Resource.DOCKER, @@ -65,6 +67,7 @@ export class DockerResolver { return this.dockerOrganizerService.resolveOrganizer(); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, @@ -83,6 +86,7 @@ export class DockerResolver { return this.dockerOrganizerService.resolveOrganizer(organizer); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, @@ -99,6 +103,7 @@ export class DockerResolver { return this.dockerOrganizerService.resolveOrganizer(organizer); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, @@ -111,6 +116,7 @@ export class DockerResolver { return this.dockerOrganizerService.resolveOrganizer(organizer); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.UPDATE_ANY, resource: Resource.DOCKER, @@ -127,6 +133,7 @@ export class DockerResolver { return this.dockerOrganizerService.resolveOrganizer(organizer); } + @UseFeatureFlag('ENABLE_NEXT_DOCKER_RELEASE') @UsePermissions({ action: AuthAction.READ_ANY, resource: Resource.DOCKER, diff --git a/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts index 3223a5249f..288edcf53c 100644 --- a/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts @@ -4,6 +4,7 @@ import { ConfigService } from '@nestjs/config'; import { ConfigFilePersister } from '@unraid/shared/services/config-file.js'; import { AppError } from '@app/core/errors/app-error.js'; +import { ENABLE_NEXT_DOCKER_RELEASE } from '@app/environment.js'; import { validateObject } from '@app/unraid-api/graph/resolvers/validation.utils.js'; import { DEFAULT_ORGANIZER_ROOT_ID, @@ -18,6 +19,10 @@ export class DockerOrganizerConfigService extends ConfigFilePersister Date: Mon, 8 Sep 2025 14:52:51 -0400 Subject: [PATCH 26/38] export schedule module from job module --- api/src/unraid-api/cron/job.module.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/unraid-api/cron/job.module.ts b/api/src/unraid-api/cron/job.module.ts index b3372630c0..22dc8fd9e5 100644 --- a/api/src/unraid-api/cron/job.module.ts +++ b/api/src/unraid-api/cron/job.module.ts @@ -8,5 +8,6 @@ import { ScheduleModule } from '@nestjs/schedule'; */ @Module({ imports: [ScheduleModule.forRoot()], + exports: [ScheduleModule], }) export class JobModule {} From 28a1ec552e1d17b8b2aef5d46004c6af8e19f055 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 8 Sep 2025 15:13:48 -0400 Subject: [PATCH 27/38] test: docker config validation --- .../docker/docker-config.service.spec.ts | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-config.service.spec.ts diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.spec.ts new file mode 100644 index 0000000000..cad15ceae0 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.spec.ts @@ -0,0 +1,195 @@ +import { ConfigService } from '@nestjs/config'; +import { CronExpression } from '@nestjs/schedule'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ValidationError } from 'class-validator'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { AppError } from '@app/core/errors/app-error.js'; +import { DockerConfigService } from '@app/unraid-api/graph/resolvers/docker/docker-config.service.js'; + +vi.mock('cron', () => ({ + validateCronExpression: vi.fn(), +})); + +vi.mock('@app/unraid-api/graph/resolvers/validation.utils.js', () => ({ + validateObject: vi.fn(), +})); + +describe('DockerConfigService - validate', () => { + let service: DockerConfigService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DockerConfigService, + { + provide: ConfigService, + useValue: { + get: vi.fn(), + }, + }, + ], + }).compile(); + + service = module.get(DockerConfigService); + vi.clearAllMocks(); + }); + + describe('validate', () => { + it('should validate and return docker config for valid cron expression', async () => { + const inputConfig = { updateCheckCronSchedule: '0 6 * * *' }; + const validatedConfig = { updateCheckCronSchedule: '0 6 * * *' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: true }); + + const result = await service.validate(inputConfig); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith('0 6 * * *'); + expect(result).toBe(validatedConfig); + }); + + it('should validate and return docker config for predefined cron expression', async () => { + const inputConfig = { updateCheckCronSchedule: CronExpression.EVERY_DAY_AT_6AM }; + const validatedConfig = { updateCheckCronSchedule: CronExpression.EVERY_DAY_AT_6AM }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: true }); + + const result = await service.validate(inputConfig); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith(CronExpression.EVERY_DAY_AT_6AM); + expect(result).toBe(validatedConfig); + }); + + it('should throw AppError for invalid cron expression', async () => { + const inputConfig = { updateCheckCronSchedule: 'invalid-cron' }; + const validatedConfig = { updateCheckCronSchedule: 'invalid-cron' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: false }); + + await expect(service.validate(inputConfig)).rejects.toThrow( + new AppError('Cron expression not supported: invalid-cron') + ); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith('invalid-cron'); + }); + + it('should throw AppError for empty cron expression', async () => { + const inputConfig = { updateCheckCronSchedule: '' }; + const validatedConfig = { updateCheckCronSchedule: '' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: false }); + + await expect(service.validate(inputConfig)).rejects.toThrow( + new AppError('Cron expression not supported: ') + ); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith(''); + }); + + it('should throw AppError for malformed cron expression', async () => { + const inputConfig = { updateCheckCronSchedule: '* * * *' }; + const validatedConfig = { updateCheckCronSchedule: '* * * *' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: false }); + + await expect(service.validate(inputConfig)).rejects.toThrow( + new AppError('Cron expression not supported: * * * *') + ); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith('* * * *'); + }); + + it('should propagate validation errors from validateObject', async () => { + const inputConfig = { updateCheckCronSchedule: '0 6 * * *' }; + const validationError = new ValidationError(); + validationError.property = 'updateCheckCronSchedule'; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + + vi.mocked(validateObject).mockRejectedValue(validationError); + + await expect(service.validate(inputConfig)).rejects.toThrow(); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + }); + + it('should handle complex valid cron expressions', async () => { + const inputConfig = { updateCheckCronSchedule: '0 0,12 * * 1-5' }; + const validatedConfig = { updateCheckCronSchedule: '0 0,12 * * 1-5' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: true }); + + const result = await service.validate(inputConfig); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith('0 0,12 * * 1-5'); + expect(result).toBe(validatedConfig); + }); + + it('should handle input with extra properties', async () => { + const inputConfig = { + updateCheckCronSchedule: '0 6 * * *', + extraProperty: 'should be ignored', + }; + const validatedConfig = { updateCheckCronSchedule: '0 6 * * *' }; + + const { validateObject } = await import( + '@app/unraid-api/graph/resolvers/validation.utils.js' + ); + const { validateCronExpression } = await import('cron'); + + vi.mocked(validateObject).mockResolvedValue(validatedConfig); + vi.mocked(validateCronExpression).mockReturnValue({ valid: true }); + + const result = await service.validate(inputConfig); + + expect(validateObject).toHaveBeenCalledWith(expect.any(Function), inputConfig); + expect(validateCronExpression).toHaveBeenCalledWith('0 6 * * *'); + expect(result).toBe(validatedConfig); + }); + }); +}); From e76dd65b4120c5cee2cfa29d1fe237a2322ee911 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Mon, 8 Sep 2025 15:29:02 -0400 Subject: [PATCH 28/38] enable `ENABLE_NEXT_DOCKER_RELEASE` during development --- api/.env.development | 1 + 1 file changed, 1 insertion(+) diff --git a/api/.env.development b/api/.env.development index 7c42ec26ce..3261fcf359 100644 --- a/api/.env.development +++ b/api/.env.development @@ -31,3 +31,4 @@ BYPASS_CORS_CHECKS=true CHOKIDAR_USEPOLLING=true LOG_TRANSPORT=console LOG_LEVEL=trace +ENABLE_NEXT_DOCKER_RELEASE=true From 74835d1938aab119e9148fcca8a0919b4c66e4c5 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 10:10:49 -0400 Subject: [PATCH 29/38] add validation for docker digest cache file --- .../graph/resolvers/docker/docker-php.service.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 8bbcb317c4..b7c516c704 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -46,13 +46,26 @@ export class DockerPhpService { try { const cache = await readFile(cacheFile, 'utf8'); const cacheData = JSON.parse(cache); - return cacheData as Record; + if (this.validateCacheData(cacheData)) return cacheData; + this.logger.warn(cacheData, 'Invalid cached update status'); + return {}; } catch (error) { this.logger.warn(error, 'Failed to read cached update status'); return {}; } } + private validateCacheData(cacheData: unknown): cacheData is Record { + return ( + typeof cacheData === 'object' && + cacheData !== null && + Object.values(cacheData).every( + (entry) => + typeof entry === 'object' && entry !== null && 'local' in entry && 'remote' in entry + ) + ); + } + /**---------------------- * Refresh Container Digests *------------------------**/ From 20986b217b654d32762b79d95418788357eafd96 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 10:20:36 -0400 Subject: [PATCH 30/38] use zod for validation instead --- .../resolvers/docker/docker-php.service.ts | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index b7c516c704..4ff39c1e01 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -1,6 +1,8 @@ import { Injectable, Logger } from '@nestjs/common'; import { readFile } from 'fs/promises'; +import { z } from 'zod'; + import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; @@ -22,14 +24,16 @@ type ExplicitStatusItem = { * These types reflect the structure of the /var/lib/docker/unraid-update-status.json file, * which is not controlled by the Unraid API. */ -export type CachedStatusEntry = { +const CachedStatusEntrySchema = z.object({ /** sha256 digest - "sha256:..." */ - local: string; + local: z.string(), /** sha256 digest - "sha256:..." */ - remote: string; + remote: z.string(), /** whether update is available (true), not available (false), or unknown (null) */ - status: 'true' | 'false' | null; -}; + status: z.enum(['true', 'false']).nullable(), +}); +const CachedStatusSchema = z.record(z.string(), CachedStatusEntrySchema); +export type CachedStatusEntry = z.infer; @Injectable() export class DockerPhpService { @@ -42,11 +46,14 @@ export class DockerPhpService { * @param cacheFile * @returns */ - async readCachedUpdateStatus(cacheFile = '/var/lib/docker/unraid-update-status.json') { + async readCachedUpdateStatus( + cacheFile = '/var/lib/docker/unraid-update-status.json' + ): Promise> { try { const cache = await readFile(cacheFile, 'utf8'); const cacheData = JSON.parse(cache); - if (this.validateCacheData(cacheData)) return cacheData; + const { success, data } = CachedStatusSchema.safeParse(cacheData); + if (success) return data; this.logger.warn(cacheData, 'Invalid cached update status'); return {}; } catch (error) { @@ -55,17 +62,6 @@ export class DockerPhpService { } } - private validateCacheData(cacheData: unknown): cacheData is Record { - return ( - typeof cacheData === 'object' && - cacheData !== null && - Object.values(cacheData).every( - (entry) => - typeof entry === 'object' && entry !== null && 'local' in entry && 'remote' in entry - ) - ); - } - /**---------------------- * Refresh Container Digests *------------------------**/ From 4fa89b95dcc9d933f013d1fa6c1d5f3cfc2abf07 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 10:28:58 -0400 Subject: [PATCH 31/38] refactor: explicit status item type for gql --- .../resolvers/docker/docker-php.service.ts | 17 +++---------- .../docker/docker-update-status.model.ts | 25 +++++++++++++++++++ .../resolvers/docker/docker.module.spec.ts | 1 - .../graph/resolvers/docker/docker.resolver.ts | 4 +-- 4 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 api/src/unraid-api/graph/resolvers/docker/docker-update-status.model.ts diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index 4ff39c1e01..ad97dad73c 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -4,22 +4,13 @@ import { readFile } from 'fs/promises'; import { z } from 'zod'; import { phpLoader } from '@app/core/utils/plugins/php-loader.js'; +import { + ExplicitStatusItem, + UpdateStatus, +} from '@app/unraid-api/graph/resolvers/docker/docker-update-status.model.js'; type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; -/** Note that these values propogate down to api consumers, so be aware of breaking changes. */ -enum UpdateStatus { - UP_TO_DATE = 'UP_TO_DATE', - UPDATE_AVAILABLE = 'UPDATE_AVAILABLE', - REBUILD_READY = 'REBUILD_READY', - UNKNOWN = 'UNKNOWN', -} - -type ExplicitStatusItem = { - name: string; - updateStatus: UpdateStatus; -}; - /** * These types reflect the structure of the /var/lib/docker/unraid-update-status.json file, * which is not controlled by the Unraid API. diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-update-status.model.ts b/api/src/unraid-api/graph/resolvers/docker/docker-update-status.model.ts new file mode 100644 index 0000000000..a6276edd15 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/docker-update-status.model.ts @@ -0,0 +1,25 @@ +import { Field, ObjectType, registerEnumType } from '@nestjs/graphql'; + +/** + * Note that these values propagate down to API consumers, so be aware of breaking changes. + */ +export enum UpdateStatus { + UP_TO_DATE = 'UP_TO_DATE', + UPDATE_AVAILABLE = 'UPDATE_AVAILABLE', + REBUILD_READY = 'REBUILD_READY', + UNKNOWN = 'UNKNOWN', +} + +registerEnumType(UpdateStatus, { + name: 'UpdateStatus', + description: 'Update status of a container.', +}); + +@ObjectType() +export class ExplicitStatusItem { + @Field(() => String) + name!: string; + + @Field(() => UpdateStatus) + updateStatus!: UpdateStatus; +} diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts index 7856ba19db..af5500d91b 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.module.spec.ts @@ -1,4 +1,3 @@ -import { ConfigModule } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { describe, expect, it, vi } from 'vitest'; diff --git a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts index 00906e61ca..65a7276d47 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker.resolver.ts @@ -2,10 +2,10 @@ import { Args, Mutation, Query, ResolveField, Resolver } from '@nestjs/graphql'; import { AuthAction, Resource } from '@unraid/shared/graphql.model.js'; import { UsePermissions } from '@unraid/shared/use-permissions.directive.js'; -import { GraphQLJSON } from 'graphql-scalars'; import { UseFeatureFlag } from '@app/unraid-api/decorators/use-feature-flag.decorator.js'; import { DockerPhpService } from '@app/unraid-api/graph/resolvers/docker/docker-php.service.js'; +import { ExplicitStatusItem } from '@app/unraid-api/graph/resolvers/docker/docker-update-status.model.js'; import { Docker, DockerContainer, @@ -138,7 +138,7 @@ export class DockerResolver { action: AuthAction.READ_ANY, resource: Resource.DOCKER, }) - @ResolveField(() => GraphQLJSON) + @ResolveField(() => [ExplicitStatusItem]) public async containerUpdateStatuses() { return this.dockerPhpService.getContainerUpdateStatuses(); } From 5b33e90ed56a166806a001fa87836a1357847cd4 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 10:30:15 -0400 Subject: [PATCH 32/38] use FeatureFlags const instead of direct env var --- .../graph/resolvers/docker/docker-config.service.ts | 4 ++-- .../docker/organizer/docker-organizer-config.service.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts index eb3eecd581..1ed27212f8 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-config.service.ts @@ -5,8 +5,8 @@ import { CronExpression } from '@nestjs/schedule'; import { ConfigFilePersister } from '@unraid/shared/services/config-file.js'; import { validateCronExpression } from 'cron'; +import { FeatureFlags } from '@app/consts.js'; import { AppError } from '@app/core/errors/app-error.js'; -import { ENABLE_NEXT_DOCKER_RELEASE } from '@app/environment.js'; import { DockerConfig } from '@app/unraid-api/graph/resolvers/docker/docker-config.model.js'; import { validateObject } from '@app/unraid-api/graph/resolvers/validation.utils.js'; @@ -17,7 +17,7 @@ export class DockerConfigService extends ConfigFilePersister { } enabled(): boolean { - return ENABLE_NEXT_DOCKER_RELEASE; + return FeatureFlags.ENABLE_NEXT_DOCKER_RELEASE; } configKey(): string { diff --git a/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts index 288edcf53c..f627ee88d7 100644 --- a/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/organizer/docker-organizer-config.service.ts @@ -3,8 +3,8 @@ import { ConfigService } from '@nestjs/config'; import { ConfigFilePersister } from '@unraid/shared/services/config-file.js'; +import { FeatureFlags } from '@app/consts.js'; import { AppError } from '@app/core/errors/app-error.js'; -import { ENABLE_NEXT_DOCKER_RELEASE } from '@app/environment.js'; import { validateObject } from '@app/unraid-api/graph/resolvers/validation.utils.js'; import { DEFAULT_ORGANIZER_ROOT_ID, @@ -20,7 +20,7 @@ export class DockerOrganizerConfigService extends ConfigFilePersister Date: Tue, 9 Sep 2025 10:42:33 -0400 Subject: [PATCH 33/38] fix: uncaught synchronous exception in AsyncMutex --- packages/unraid-shared/src/util/processing.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index 43291ac1d0..c3f37a8145 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -137,8 +137,7 @@ export class AsyncMutex { } const op = (operation || this.defaultOperation) as AsyncOperation; - - const promise = op().finally(() => { + const promise = Promise.resolve().then(op).finally(() => { if (this.currentOperation === promise) { this.currentOperation = null; } From 88cc6164facc37f067f50b9d642260852534c8d6 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 14:38:51 -0400 Subject: [PATCH 34/38] replace `target: any` with `object` in omit-if decorator --- api/src/unraid-api/decorators/omit-if.decorator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/unraid-api/decorators/omit-if.decorator.ts b/api/src/unraid-api/decorators/omit-if.decorator.ts index 696355d7c1..3fc0cf345f 100644 --- a/api/src/unraid-api/decorators/omit-if.decorator.ts +++ b/api/src/unraid-api/decorators/omit-if.decorator.ts @@ -25,7 +25,7 @@ export const OMIT_IF_METADATA_KEY = 'omitIf'; export function OmitIf(condition: boolean | (() => boolean)): MethodDecorator & PropertyDecorator { const shouldOmit = typeof condition === 'function' ? condition() : condition; - return (target: any, propertyKey?: string | symbol, descriptor?: PropertyDescriptor) => { + return (target: object, propertyKey?: string | symbol, descriptor?: PropertyDescriptor) => { if (shouldOmit) { SetMetadata(OMIT_IF_METADATA_KEY, true)( target, From bcd3bbaa84d9dce966bde2f4255da33a748ddb20 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 14:42:24 -0400 Subject: [PATCH 35/38] unit test docker-push regex in docker-php.service.ts --- .../resolvers/docker/docker-php.service.ts | 14 +- .../docker/utils/docker-push-parser.test.ts | 124 ++++++++++++++++++ .../docker/utils/docker-push-parser.ts | 24 ++++ 3 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.test.ts create mode 100644 api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.ts diff --git a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts index ad97dad73c..46edc3f6da 100644 --- a/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts +++ b/api/src/unraid-api/graph/resolvers/docker/docker-php.service.ts @@ -8,6 +8,7 @@ import { ExplicitStatusItem, UpdateStatus, } from '@app/unraid-api/graph/resolvers/docker/docker-update-status.model.js'; +import { parseDockerPushCalls } from '@app/unraid-api/graph/resolvers/docker/utils/docker-push-parser.js'; type StatusItem = { name: string; updateStatus: 0 | 1 | 2 | 3 }; @@ -82,14 +83,11 @@ export class DockerPhpService { *------------------------**/ private parseStatusesFromDockerPush(js: string): ExplicitStatusItem[] { - const items: ExplicitStatusItem[] = []; - const re = /docker\.push\(\{[^}]*name:'([^']+)'[^}]*update:(\d)[^}]*\}\);/g; - for (const m of js.matchAll(re)) { - const name = m[1]; - const updateStatus = Number(m[2]) as StatusItem['updateStatus']; - items.push({ name, updateStatus: this.updateStatusToString(updateStatus) }); - } - return items; + const matches = parseDockerPushCalls(js); + return matches.map(({ name, updateStatus }) => ({ + name, + updateStatus: this.updateStatusToString(updateStatus as StatusItem['updateStatus']), + })); } private updateStatusToString(updateStatus: 0): UpdateStatus.UP_TO_DATE; diff --git a/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.test.ts b/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.test.ts new file mode 100644 index 0000000000..f878396915 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest'; + +import type { DockerPushMatch } from '@app/unraid-api/graph/resolvers/docker/utils/docker-push-parser.js'; +import { parseDockerPushCalls } from '@app/unraid-api/graph/resolvers/docker/utils/docker-push-parser.js'; + +describe('parseDockerPushCalls', () => { + it('should extract name and update status from valid docker.push call', () => { + const jsCode = "docker.push({name:'nginx',update:1});"; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'nginx', updateStatus: 1 }]); + }); + + it('should handle multiple docker.push calls in same string', () => { + const jsCode = ` + docker.push({name:'nginx',update:1}); + docker.push({name:'mysql',update:0}); + docker.push({name:'redis',update:2}); + `; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([ + { name: 'nginx', updateStatus: 1 }, + { name: 'mysql', updateStatus: 0 }, + { name: 'redis', updateStatus: 2 }, + ]); + }); + + it('should handle docker.push calls with additional properties', () => { + const jsCode = + "docker.push({id:'123',name:'nginx',version:'latest',update:3,status:'running'});"; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'nginx', updateStatus: 3 }]); + }); + + it('should handle different property order', () => { + const jsCode = "docker.push({update:2,name:'postgres',id:'456'});"; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'postgres', updateStatus: 2 }]); + }); + + it('should handle container names with special characters', () => { + const jsCode = "docker.push({name:'my-app_v2.0',update:1});"; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'my-app_v2.0', updateStatus: 1 }]); + }); + + it('should handle whitespace variations', () => { + const jsCode = "docker.push({ name: 'nginx' , update: 1 });"; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'nginx', updateStatus: 1 }]); + }); + + it('should return empty array for empty string', () => { + const result = parseDockerPushCalls(''); + expect(result).toEqual([]); + }); + + it('should return empty array when no docker.push calls found', () => { + const jsCode = "console.log('no docker calls here');"; + const result = parseDockerPushCalls(jsCode); + expect(result).toEqual([]); + }); + + it('should ignore malformed docker.push calls', () => { + const jsCode = ` + docker.push({name:'valid',update:1}); + docker.push({name:'missing-update'}); + docker.push({update:2}); + docker.push({name:'another-valid',update:0}); + `; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([ + { name: 'valid', updateStatus: 1 }, + { name: 'another-valid', updateStatus: 0 }, + ]); + }); + + it('should handle all valid update status values', () => { + const jsCode = ` + docker.push({name:'container0',update:0}); + docker.push({name:'container1',update:1}); + docker.push({name:'container2',update:2}); + docker.push({name:'container3',update:3}); + `; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([ + { name: 'container0', updateStatus: 0 }, + { name: 'container1', updateStatus: 1 }, + { name: 'container2', updateStatus: 2 }, + { name: 'container3', updateStatus: 3 }, + ]); + }); + + it('should handle real-world example with HTML and multiple containers', () => { + const jsCode = ` +
some html
+ docker.push({id:'abc123',name:'plex',version:'1.32',update:1,autostart:true}); + docker.push({id:'def456',name:'nextcloud',version:'latest',update:0,ports:'80:8080'}); + + docker.push({id:'ghi789',name:'homeassistant',update:2}); + `; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([ + { name: 'plex', updateStatus: 1 }, + { name: 'nextcloud', updateStatus: 0 }, + { name: 'homeassistant', updateStatus: 2 }, + ]); + }); + + it('should handle nested braces in other properties', () => { + const jsCode = 'docker.push({config:\'{"nested":"value"}\',name:\'test\',update:1});'; + const result = parseDockerPushCalls(jsCode); + + expect(result).toEqual([{ name: 'test', updateStatus: 1 }]); + }); +}); diff --git a/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.ts b/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.ts new file mode 100644 index 0000000000..bc96b1b0bd --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/docker/utils/docker-push-parser.ts @@ -0,0 +1,24 @@ +export interface DockerPushMatch { + name: string; + updateStatus: number; +} + +export function parseDockerPushCalls(jsCode: string): DockerPushMatch[] { + const dockerPushRegex = /docker\.push\(\{[^}]*(?:(?:[^{}]|{[^}]*})*)\}\);/g; + const matches: DockerPushMatch[] = []; + + for (const match of jsCode.matchAll(dockerPushRegex)) { + const objectContent = match[0]; + + const nameMatch = objectContent.match(/name\s*:\s*'([^']+)'/); + const updateMatch = objectContent.match(/update\s*:\s*(\d)/); + + if (nameMatch && updateMatch) { + const name = nameMatch[1]; + const updateStatus = Number(updateMatch[1]); + matches.push({ name, updateStatus }); + } + } + + return matches; +} From 783e818a8d573a10ffcd1f0787107bf1e8525455 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 14:43:16 -0400 Subject: [PATCH 36/38] update generated schema --- api/generated-schema.graphql | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/api/generated-schema.graphql b/api/generated-schema.graphql index bf56181a24..581523aef4 100644 --- a/api/generated-schema.graphql +++ b/api/generated-schema.graphql @@ -1050,6 +1050,19 @@ enum ThemeName { white } +type ExplicitStatusItem { + name: String! + updateStatus: UpdateStatus! +} + +"""Update status of a container.""" +enum UpdateStatus { + UP_TO_DATE + UPDATE_AVAILABLE + REBUILD_READY + UNKNOWN +} + type ContainerPort { ip: String privatePort: Port @@ -1121,7 +1134,7 @@ type Docker implements Node { containers(skipCache: Boolean! = false): [DockerContainer!]! networks(skipCache: Boolean! = false): [DockerNetwork!]! organizer: ResolvedOrganizerV1! - containerUpdateStatuses: JSON! + containerUpdateStatuses: [ExplicitStatusItem!]! } type ResolvedOrganizerView { From 2d0135e997878820d7ad344ffd4da5c24d896e1d Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 15:33:30 -0400 Subject: [PATCH 37/38] fix processing.ts tests --- packages/unraid-shared/src/util/processing.ts | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index c3f37a8145..ebd7fb2b09 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -36,18 +36,18 @@ type AsyncOperation = () => Promise; /** * A mutex for asynchronous operations that ensures only one operation runs at a time. - * + * * When multiple callers attempt to execute operations simultaneously, they will all * receive the same promise from the currently running operation, effectively deduplicating * concurrent calls. This is useful for expensive operations like API calls, file operations, * or database queries that should not be executed multiple times concurrently. - * + * * @template T - The default return type for operations when using a default operation - * + * * @example * // Basic usage with explicit operations * const mutex = new AsyncMutex(); - * + * * // Multiple concurrent calls will deduplicate * const [result1, result2, result3] = await Promise.all([ * mutex.do(() => fetch('/api/data')), @@ -55,13 +55,13 @@ type AsyncOperation = () => Promise; * mutex.do(() => fetch('/api/data')) // Same request, will get same promise * ]); * // Only one fetch actually happens - * + * * @example * // Usage with a default operation - * const dataLoader = new AsyncMutex(() => + * const dataLoader = new AsyncMutex(() => * fetch('/api/expensive-data').then(res => res.json()) * ); - * + * * const data1 = await dataLoader.do(); // Executes the fetch * const data2 = await dataLoader.do(); // If first promise is finished, a new fetch is executed */ @@ -71,23 +71,23 @@ export class AsyncMutex { /** * Creates a new AsyncMutex instance. - * + * * @param operation - Optional default operation to execute when calling `do()` without arguments. * This is useful when you have a specific operation that should be deduplicated. - * + * * @example * // Without default operation (shared mutex) * const mutex = new AsyncMutex(); * const promise1 = mutex.do(() => someAsyncWork()); * const promise2 = mutex.do(() => someOtherAsyncWork()); - * + * * // Both promises will be the same * expect(await promise1).toBe(await promise2); - * + * * // After the first operation completes, new operations can run * await promise1; * const newPromise = mutex.do(() => someOtherAsyncWork()); // This will execute - * + * * @example * // With default operation (deduplicating a specific operation) * const dataMutex = new AsyncMutex(() => loadExpensiveData()); @@ -105,23 +105,23 @@ export class AsyncMutex { do(): Promise; /** * Executes the provided operation, ensuring only one runs at a time. - * + * * If an operation is already running, all subsequent calls will receive * the same promise from the currently running operation. This effectively * deduplicates concurrent calls to the same expensive operation. - * + * * @param operation - Optional operation to execute. If not provided, uses the default operation. * @returns Promise that resolves with the result of the operation * @throws Error if no operation is provided and no default operation was set - * + * * @example * const mutex = new AsyncMutex(); - * + * * // These will all return the same promise * const promise1 = mutex.do(() => fetch('/api/data')); * const promise2 = mutex.do(() => fetch('/api/other')); // Still gets first promise! * const promise3 = mutex.do(() => fetch('/api/another')); // Still gets first promise! - * + * * // After the first operation completes, new operations can run * await promise1; * const newPromise = mutex.do(() => fetch('/api/new')); // This will execute @@ -129,7 +129,9 @@ export class AsyncMutex { do(operation: AsyncOperation): Promise; do(operation?: AsyncOperation): Promise { if (!operation && !this.defaultOperation) { - return Promise.reject(new Error('No operation provided and no default operation set')); + return Promise.reject( + new Error("No operation provided and no default operation set") + ); } if (this.currentOperation) { @@ -137,12 +139,19 @@ export class AsyncMutex { } const op = (operation || this.defaultOperation) as AsyncOperation; - const promise = Promise.resolve().then(op).finally(() => { + const safeOp = () => { + try { + return op(); + } catch (error) { + return Promise.reject(error); + } + }; + const promise = safeOp().finally(() => { if (this.currentOperation === promise) { this.currentOperation = null; } }); - + this.currentOperation = promise; return promise; } From 3c4b007543cfe9ffad3e6c4108e550c07e140737 Mon Sep 17 00:00:00 2001 From: Pujit Mehrotra Date: Tue, 9 Sep 2025 16:19:15 -0400 Subject: [PATCH 38/38] simplify asyncmutex definition -- don't support generic operations anymore --- .../src/util/__tests__/processing.test.ts | 18 +------------ packages/unraid-shared/src/util/processing.ts | 25 ++++++------------- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/packages/unraid-shared/src/util/__tests__/processing.test.ts b/packages/unraid-shared/src/util/__tests__/processing.test.ts index e1336e95b3..9f7d7a0b4e 100644 --- a/packages/unraid-shared/src/util/__tests__/processing.test.ts +++ b/packages/unraid-shared/src/util/__tests__/processing.test.ts @@ -291,21 +291,5 @@ describe('AsyncMutex', () => { expect(result).toBe('outer'); expect(outerOp).toHaveBeenCalledTimes(1); }); - - it('should maintain type safety with generic operations', async () => { - const mutex = new AsyncMutex(); - - const stringOp = vi.fn().mockResolvedValue('string'); - const numberOp = vi.fn().mockResolvedValue(42); - const booleanOp = vi.fn().mockResolvedValue(true); - - const stringResult: string = await mutex.do(stringOp); - const numberResult: number = await mutex.do(numberOp); - const booleanResult: boolean = await mutex.do(booleanOp); - - expect(stringResult).toBe('string'); - expect(numberResult).toBe(42); - expect(booleanResult).toBe(true); - }); }); -}); \ No newline at end of file +}); diff --git a/packages/unraid-shared/src/util/processing.ts b/packages/unraid-shared/src/util/processing.ts index ebd7fb2b09..f884f04b7a 100644 --- a/packages/unraid-shared/src/util/processing.ts +++ b/packages/unraid-shared/src/util/processing.ts @@ -66,7 +66,7 @@ type AsyncOperation = () => Promise; * const data2 = await dataLoader.do(); // If first promise is finished, a new fetch is executed */ export class AsyncMutex { - private currentOperation: Promise | null = null; + private currentOperation: Promise | null = null; private defaultOperation?: AsyncOperation; /** @@ -97,12 +97,6 @@ export class AsyncMutex { this.defaultOperation = operation; } - /** - * Executes the default operation if one was provided in the constructor. - * @returns Promise that resolves with the result of the default operation - * @throws Error if no default operation was set in the constructor - */ - do(): Promise; /** * Executes the provided operation, ensuring only one runs at a time. * @@ -126,19 +120,16 @@ export class AsyncMutex { * await promise1; * const newPromise = mutex.do(() => fetch('/api/new')); // This will execute */ - do(operation: AsyncOperation): Promise; - do(operation?: AsyncOperation): Promise { - if (!operation && !this.defaultOperation) { + do(operation?: AsyncOperation): Promise { + if (this.currentOperation) { + return this.currentOperation; + } + const op = operation ?? this.defaultOperation; + if (!op) { return Promise.reject( new Error("No operation provided and no default operation set") ); } - - if (this.currentOperation) { - return this.currentOperation; - } - - const op = (operation || this.defaultOperation) as AsyncOperation; const safeOp = () => { try { return op(); @@ -146,12 +137,12 @@ export class AsyncMutex { return Promise.reject(error); } }; + const promise = safeOp().finally(() => { if (this.currentOperation === promise) { this.currentOperation = null; } }); - this.currentOperation = promise; return promise; }