Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions js/.changeset/issue-148-detached-docker-oom-status.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'start-command': patch
---

Treat detached Docker sessions with `oomKilled` as terminal in status output, using Docker's exit code when available and 137 as the OOM fallback.
68 changes: 53 additions & 15 deletions js/src/lib/status-formatter.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,23 @@ function inspectDockerState(sessionName) {
};
}

function isDetachedDockerRecord(record) {
const opts = record.options || {};
return (
opts.isolated === 'docker' &&
opts.isolationMode === 'detached' &&
Boolean(opts.sessionName)
);
}

function readDockerState(record) {
const opts = record.options || {};
if (opts.isolated !== 'docker' || !opts.sessionName) {
return null;
}
return inspectDockerState(opts.sessionName);
}

/**
* Best-effort terminal exit code reported by the isolation backend itself
* (currently docker via `docker inspect .State.ExitCode`). Returns null when
Expand All @@ -71,21 +88,23 @@ function inspectDockerState(sessionName) {
* @returns {number|null}
*/
function readBackendExitCode(record) {
const opts = record.options || {};
if (opts.isolated !== 'docker' || !opts.sessionName) {
return null;
}
const state = inspectDockerState(opts.sessionName);
const state = readDockerState(record);
return state && !state.running ? state.exitCode : null;
}

function readDockerOomKilled(record) {
const opts = record.options || {};
if (opts.isolated !== 'docker' || !opts.sessionName) {
return null;
function resolveOomExitCode(footerExit, dockerState) {
if (footerExit !== null && footerExit !== undefined) {
return footerExit;
}
if (
dockerState &&
dockerState.exitCode !== null &&
dockerState.exitCode !== undefined &&
(!dockerState.running || dockerState.exitCode !== 0)
) {
return dockerState.exitCode;
}
const state = inspectDockerState(opts.sessionName);
return state ? state.oomKilled : null;
return 137;
}

/**
Expand Down Expand Up @@ -172,8 +191,15 @@ function readExitCodeFromLog(logPath) {
* @returns {Object} Possibly updated execution record
*/
function enrichDetachedStatus(record) {
const alive = isDetachedSessionAlive(record);
const footerExit = readExitCodeFromLog(record.logPath);
const dockerState = isDetachedDockerRecord(record)
? readDockerState(record)
: null;
const alive = isDetachedDockerRecord(record)
? dockerState === null
? null
: dockerState.running
: isDetachedSessionAlive(record);

// Create a shallow copy to avoid mutating the original
const cloneRecord = () => {
Expand All @@ -182,6 +208,19 @@ function enrichDetachedStatus(record) {
return enriched;
};

if (record.oomKilled === true || dockerState?.oomKilled === true) {
const enriched = cloneRecord();
enriched.oomKilled = true;
enriched.status = 'executed';
if (enriched.exitCode === null || enriched.exitCode === undefined) {
enriched.exitCode = resolveOomExitCode(footerExit, dockerState);
}
if (!enriched.endTime) {
enriched.endTime = new Date().toISOString();
}
return enriched;
}

if (alive === null) {
// Liveness is unknown: the backend could not be probed (e.g. a detached
// docker container that is not visible yet on a slow Docker-in-Docker host,
Expand All @@ -204,9 +243,8 @@ function enrichDetachedStatus(record) {
}

const enriched = cloneRecord();
const oomKilled = readDockerOomKilled(enriched);
if (oomKilled !== null) {
enriched.oomKilled = oomKilled;
if (dockerState?.oomKilled !== null && dockerState?.oomKilled !== undefined) {
enriched.oomKilled = dockerState.oomKilled;
}

if (alive && enriched.status === 'executed') {
Expand Down
59 changes: 59 additions & 0 deletions js/test/session-name-status.js
Original file line number Diff line number Diff line change
Expand Up @@ -677,6 +677,65 @@ describe('Issue #144: detached docker OOMKilled status signal', () => {
});
});

describe('Issue #148: detached docker OOMKilled terminal status', () => {
let store;

beforeEach(() => {
cleanupTestDir();
store = new ExecutionStore({
appFolder: TEST_APP_FOLDER,
useLinks: false,
});
});

afterEach(() => {
cleanupTestDir();
});

function saveDockerRecord() {
const record = new ExecutionRecord({
command: 'sh -c "allocate memory"',
logPath: '/tmp/issue-148.log',
options: {
sessionName: 'issue148-oom',
isolated: 'docker',
isolationMode: 'detached',
},
});
store.save(record);
return record;
}

it('treats oomKilled as terminal even while Docker still reports running', () => {
const record = saveDockerRecord();

withFakeDockerInspect('true 137 true', () => {
const result = queryStatus(store, record.uuid, 'json');
expect(result.success).toBe(true);
const parsed = JSON.parse(result.output);
expect(parsed.status).toBe('executed');
expect(parsed.exitCode).toBe(137);
expect(parsed.oomKilled).toBe(true);
expect(parsed.endTime).toBeTruthy();
expect(parsed.currentTime).toBeUndefined();
});
});

it('uses 137 when oomKilled is terminal but Docker has no terminal exit code yet', () => {
const record = saveDockerRecord();

withFakeDockerInspect('true 0 true', () => {
const result = queryStatus(store, record.uuid, 'json');
expect(result.success).toBe(true);
const parsed = JSON.parse(result.output);
expect(parsed.status).toBe('executed');
expect(parsed.exitCode).toBe(137);
expect(parsed.oomKilled).toBe(true);
expect(parsed.endTime).toBeTruthy();
});
});
});

describe('Issue #105: attachCurrentTime for executing status', () => {
it('should add currentTime to serialization when status is executing', () => {
const record = new ExecutionRecord({
Expand Down
2 changes: 1 addition & 1 deletion rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions rust/changelog.d/issue-148-detached-docker-oom-status.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
bump: patch
---

Treat detached Docker sessions with OOMKilled as terminal in status output, using Docker's exit code when available and 137 as the OOM fallback.
Loading
Loading