From 2c184ab20ac24d72973892d9f3d0c7a7bdbda3aa Mon Sep 17 00:00:00 2001 From: BigDataDZ <1437100870@qq.com> Date: Sun, 18 Oct 2026 17:08:57 +0800 Subject: [PATCH 1/3] fix: preserve complete date ranges in conversation time slots --- src/turnContext.js | 5 +- tests/postgresql.query.http.test.js | 2 +- tests/queryDateRange.http.test.js | 86 +++++++++++++++++++++++++++++ tests/turnContext.test.js | 31 +++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tests/queryDateRange.http.test.js diff --git a/src/turnContext.js b/src/turnContext.js index 6e755a2..82d9ef9 100644 --- a/src/turnContext.js +++ b/src/turnContext.js @@ -21,7 +21,10 @@ const SLOT_LABELS = { dimensions: '维度', analysisMode: '分析方式', }; -const TIME_PATTERN = /(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|(?:今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年)|(?:第[一二三四1-4]季度|Q[1-4])|\d{4}年(?:\d{1,2}月)?|\d{1,2}月/gi; +const EXPLICIT_DATE_PATTERN = String.raw`(?:\d{4}年)?\d{1,2}月\d{1,2}[日号]|\d{4}[-/.]\d{1,2}[-/.]\d{1,2}`; +// Match complete day ranges before their year/month prefixes so inheritance keeps day precision. +const EXPLICIT_DATE_RANGE_PATTERN = String.raw`(?:${EXPLICIT_DATE_PATTERN})(?:\s*(?:到|至|~|—|－|-)\s*(?:${EXPLICIT_DATE_PATTERN}|\d{1,2}[日号]))?`; +const TIME_PATTERN = new RegExp(String.raw`${EXPLICIT_DATE_RANGE_PATTERN}|(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|(?:今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年)|(?:第[一二三四1-4]季度|Q[1-4])|\d{4}年(?:\d{1,2}月)?|\d{1,2}月`, 'gi'); const TIME_HINT_PATTERN = /(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年|第[一二三四1-4]季度|Q[1-4]|\d{4}年|\d{1,2}月/i; const FILTER_PATTERN = /(?:只看|仅看|限定|过滤|筛选|排除)([^，。；,;]{1,24})/g; const TIME_GRAINS = new Set(['日', '天', '周', '月', '季度', '年']); diff --git a/tests/postgresql.query.http.test.js b/tests/postgresql.query.http.test.js index 32f4b11..d7ee04a 100644 --- a/tests/postgresql.query.http.test.js +++ b/tests/postgresql.query.http.test.js @@ -122,7 +122,7 @@ test('live PostgreSQL: authenticated Agent contract, row policy, audit and same- }); assert.equal(streamed.status, 200); const events = (await streamed.text()).trim().split('\n').map((line) => JSON.parse(line)); - assert.ok(!events.some((event) => event.type === 'stream_error'), JSON.stringify(events)); + assert.deepEqual(events.filter((event) => event.type === 'stream_error'), []); const success = { response: streamed, data: events.find((event) => event.type === 'final')?.answer }; assert.equal(success.response.status, 200, JSON.stringify(success.data)); assert.ok(success.data); diff --git a/tests/queryDateRange.http.test.js b/tests/queryDateRange.http.test.js new file mode 100644 index 0000000..36b7196 --- /dev/null +++ b/tests/queryDateRange.http.test.js @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { withSkillServer } from './fixtures/skillServer.js'; + +test('authenticated streaming contracts keep explicit date ranges intact and apply row policy', async () => { + await withSkillServer(async ({ application, login, baseUrl }) => { + const cookie = await login('east_manager'); + const user = application.database.getUserByUsername('east_manager'); + const source = application.database.saveDataSource({ + code: 'range_pg', name: 'Range PostgreSQL', dbType: 'POSTGRESQL', + host: 'localhost', port: 5432, databaseName: 'test', username: 'test', + encryptedPassword: '', options: {}, + }); + const dataset = application.database.saveBusinessDataset({ + code: 'range_sales', name: 'Range sales', datasourceId: source.id, + schemaName: 'public', primaryTable: 'sales', + config: { autoLatestDateRange: false, policyFieldMap: { region: 'region' } }, + }); + application.database.replaceDatasetFields(dataset.id, [ + { fieldName: 'day', dataType: 'date', semanticType: 'DATE', role: 'TIME', + aggregator: 'NONE', allowedOperators: ['BETWEEN'] }, + { fieldName: 'region', dataType: 'text', semanticType: 'STRING', role: 'DIMENSION', + aggregator: 'NONE', allowedOperators: ['IN'] }, + { fieldName: 'amount', dataType: 'numeric', semanticType: 'NUMBER', role: 'METRIC', + aggregator: 'SUM', allowedOperators: ['>', 'BETWEEN'] }, + ]); + application.database.grantDatasetAccess(user.id, dataset.id, true); + application.database.saveTheme({ ...application.database.getTheme(1), indicatorIds: [], + businessDatasetIds: [dataset.id], primaryBusinessDatasetId: dataset.id, + semanticPolicy: {}, systemPrompt: 'amount 为销售额，使用 SUM；day 为日期，region 为区域。', + }, 1); + application.database.savePlatformSetting('supersonic.enabled', false); + const statements = []; + // Keep auth, context, compiler, SQL generation and audit real; replace only database I/O. + application.businessDatasets.getPoolById = () => ({ query: async (config) => { + statements.push(config); + return [[{ amount: 40 }], [{ name: 'amount' }]]; + } }); + application.agent.harnessFactory = { forTheme: () => ({ + mode: 'local-rule', model: 'range-acceptance', capabilities: {}, + run: async ({ executeTool }) => { + await executeTool('list_business_datasets', {}); + await executeTool('get_business_dataset_definition', { datasetId: dataset.id }); + const compiled = await executeTool('compile_query_contract', { + sourceType: 'BUSINESS_DATASET', datasetId: dataset.id, + conditions: [ + { id: 'time', sourceText: '2026年1月1日至2026年3月31日', kind: 'TIME', status: 'RESOLVED' }, + { id: 'region', sourceText: 'west', kind: 'FILTER', status: 'RESOLVED' }, + { id: 'metric', sourceText: 'amount', kind: 'METRIC', status: 'RESOLVED' }, + ], + metricFields: [{ field: 'amount', sourceText: 'amount', aggregator: 'SUM', conditionIds: ['metric'] }], + dimensionFields: [], + filterFields: [{ field: 'region', operator: 'IN', value: ['west'], sourceText: 'west', conditionIds: ['region'] }], + timeWindows: [{ sourceText: '2026年1月1日至2026年3月31日', dateMode: 'BETWEEN', + startDate: '2026-01-01', endDate: '2026-03-31', conditionIds: ['time'] }], + limit: 20, + }); + assert.equal(compiled.valid, true, JSON.stringify(compiled.issues)); + await executeTool('execute_query_contract', { contractId: compiled.contractId }); + return { content: '受控查询已完成。', trace: [] }; + }, + }) }; + const response = await fetch(`${baseUrl}/api/chat/query/stream`, { + method: 'POST', headers: { Cookie: cookie, 'Content-Type': 'application/json' }, + body: JSON.stringify({ themeId: 1, question: '2026年1月1日至2026年3月31日 west amount' }), + }); + assert.equal(response.status, 200); + const events = (await response.text()).trim().split('\n').map(line => JSON.parse(line)); + assert.deepEqual(events.filter(event => event.type === 'stream_error'), []); + const answer = events.find(event => event.type === 'final')?.answer; + assert.ok(answer); + assert.deepEqual(answer.data.rows, [{ amount: 40 }]); + const plan = application.database.getQueryPlan(answer.semanticParse.planId, user.id); + assert.equal(plan.status, 'EXECUTED'); + assert.equal(plan.plan.timeWindows[0].startDate, '2026-01-01'); + assert.equal(plan.plan.timeWindows[0].endDate, '2026-03-31'); + const audit = application.database.listDatasetQueryLogs(100).filter(log => log.datasetId === dataset.id); + assert.equal(audit.length, 1); + const statement = statements.find(item => item.sql === audit[0].sqlText); + assert.ok(statement); + assert.ok(statement.values.includes('2026-01-01')); + assert.ok(statement.values.includes('2026-03-31')); + assert.ok(statement.values.flat(Infinity).includes('华东')); + assert.ok(!statement.values.flat(Infinity).includes('west')); + }); +}); diff --git a/tests/turnContext.test.js b/tests/turnContext.test.js index d172709..ba26a8c 100644 --- a/tests/turnContext.test.js +++ b/tests/turnContext.test.js @@ -5,6 +5,7 @@ import { TURN_TYPES, completeTaskState, createTurnContext, + extractExplicitSlots, } from '../src/turnContext.js'; function slot(value, source = 'explicit') { @@ -28,6 +29,36 @@ function completedHistory(context, content = '查询已完成') { ]; } +for (const time of [ + '2026年1月1日至2026年3月31日', + '2026-01-01 至 2026-03-31', + '2026年1月1日至3月31日', + '2026年1月1日', +]) { + test(`explicit time slots preserve the complete date expression: ${time}`, () => { + assert.deepEqual(extractExplicitSlots(`${time}销售额`).time.map(item => item.sourceText), [time]); + }); +} + +test('two distinct explicit date ranges remain separate time slots', () => { + const question = '对比2026年1月1日至2026年1月31日与2026年3月1日至2026年3月31日销售额'; + assert.deepEqual(extractExplicitSlots(question).time.map(item => item.sourceText), [ + '2026年1月1日至2026年1月31日', '2026年3月1日至2026年3月31日', + ]); +}); + +test('followups inherit full date ranges and filters while a time revision replaces only time', () => { + const first = createTurnContext({ question: '2026年1月1日至2026年3月31日销售额，只看华东' }); + const second = createTurnContext({ question: '按渠道拆开', history: completedHistory(first), + turnType: TURN_TYPES.CONTINUATION }); + assert.deepEqual(second.inheritedSlots.time.map(item => item.sourceText), ['2026年1月1日至2026年3月31日']); + assert.deepEqual(second.inheritedSlots.filters.map(item => item.sourceText), ['华东']); + const revised = createTurnContext({ question: '改查2026年4月1日至2026年4月15日', + history: completedHistory(second), turnType: TURN_TYPES.REVISION }); + assert.deepEqual(revised.slots.time.map(item => item.sourceText), ['2026年4月1日至2026年4月15日']); + assert.deepEqual(revised.inheritedSlots.filters.map(item => item.sourceText), ['华东']); +}); + test('continuations keep the task id, advance the turn id and inherit field by field', () => { const first = createTurnContext({ question: '查询本月含税销售金额', From 2d3ec8a04521c3e454d1b6923efd9d8dd35ee91f Mon Sep 17 00:00:00 2001 From: BigDataDZ <1437100870@qq.com> Date: Sun, 18 Oct 2026 17:10:44 +0800 Subject: [PATCH 2/3] fix: keep ISO date clarifications out of filter slots --- src/turnContext.js | 2 +- tests/turnContext.test.js | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/turnContext.js b/src/turnContext.js index 82d9ef9..f81955f 100644 --- a/src/turnContext.js +++ b/src/turnContext.js @@ -192,7 +192,7 @@ export function inferClarificationSlots(question) { return {}; } const slots = {}; - if (TIME_HINT_PATTERN.test(source)) { + if (TIME_HINT_PATTERN.test(source) || extractExplicitSlots(source).time?.length) { return slots; } if (turnPatterns().fieldLabel?.test(source)) { diff --git a/tests/turnContext.test.js b/tests/turnContext.test.js index ba26a8c..73ee6af 100644 --- a/tests/turnContext.test.js +++ b/tests/turnContext.test.js @@ -47,6 +47,16 @@ test('two distinct explicit date ranges remain separate time slots', () => { ]); }); +test('an ISO date clarification fills time without replacing the inherited region filter', () => { + const first = createTurnContext({ question: '销售额，只看华东' }); + const clarified = createTurnContext({ question: '2026-01-01', history: completedHistory(first), + turnType: TURN_TYPES.CLARIFICATION_ANSWER }); + assert.deepEqual(clarified.slots.time.map(item => item.sourceText), ['2026-01-01']); + assert.equal(clarified.explicitSlots.filters, undefined); + assert.deepEqual(clarified.inheritedSlots.filters.map(item => item.sourceText), ['华东']); + assert.deepEqual(clarified.slots.filters.map(item => item.sourceText), ['华东']); +}); + test('followups inherit full date ranges and filters while a time revision replaces only time', () => { const first = createTurnContext({ question: '2026年1月1日至2026年3月31日销售额，只看华东' }); const second = createTurnContext({ question: '按渠道拆开', history: completedHistory(first), From 93cd3fc41ec7a26efccca526af7ec643509a6030 Mon Sep 17 00:00:00 2001 From: BigDataDZ <1437100870@qq.com> Date: Sun, 18 Oct 2026 17:54:19 +0800 Subject: [PATCH 3/3] fix: validate uploads before creating artifacts --- src/uploads.js | 102 +++++++++++++++++++-------------------- tests/localFiles.test.js | 36 ++++++++++++++ 2 files changed, 87 insertions(+), 51 deletions(-) diff --git a/src/uploads.js b/src/uploads.js index bfe9978..aed744b 100644 --- a/src/uploads.js +++ b/src/uploads.js @@ -95,7 +95,7 @@ function base64ByteLength(text) { return Math.floor((value.length * 3) / 4) - padding; } -// 上传服务：校验 -> 落原始文件产物 -> 解析成表格产物 -> 落识别过程文件。 +// 上传服务：校验并解析 -> 落原始文件与表格产物 -> 落识别过程文件。 export class UploadService { constructor({ database, workspace, config = {} }) { this.database = database; @@ -324,31 +324,6 @@ export class UploadService { incomingBytes: buffer.length, }); - const sha256 = createHash('sha256').update(buffer).digest('hex'); - const source = { - type: 'UPLOAD', - name: safeName, - extension, - size: buffer.length, - mimeType: MIME_TYPES[extension] ?? 'text/plain; charset=utf-8', - sha256, - uploadedAt: new Date().toISOString(), - }; - const fileArtifact = this.workspace.createFileArtifact({ - workspaceId: workspaceRecord.id, - userId, - sessionId: session.id, - title: `${safeName}（原始文件）`, - content: buffer.toString('base64'), - encoding: 'base64', - format: extension, - mimeType: source.mimeType, - purpose: '用户上传的本地文件', - size: buffer.length, - source, - }); - - let tableArtifact = null; let parsed = null; const warnings = []; if (SPREADSHEET_EXTENSIONS.has(extension)) { @@ -380,31 +355,56 @@ export class UploadService { if (!parseable && !parsed.unparsed) { throw uploadError('未能从文件中识别出列，请检查文件是否有表头行'); } - if (parseable) { - tableArtifact = this.workspace.createTableArtifact({ - workspaceId: workspaceRecord.id, - userId, - sessionId: session.id, - title: `${safeName}（解析表）`, - metadata: { - source: { type: 'UPLOAD_FILE', id: fileArtifact.id, name: safeName }, - derivedFromArtifactId: fileArtifact.id, - parentArtifactIds: [fileArtifact.id], - inputArtifactIds: [fileArtifact.id], - lineageQuestion: `上传本地文件：${safeName}`, - format: extension, - encoding: parsed.encoding, - delimiter: parsed.delimiter, - sheetName: parsed.sheetName ?? null, - rowCount: parsed.rows.length, - totalRowCount: parsed.totalRows ?? parsed.rows.length, - columnCount: parsed.columns.length, - truncated: Boolean(parsed.truncated), - warnings, - }, - payload: { data: { columns: parsed.columns, rows: parsed.rows } }, - }); - } + } + + const sha256 = createHash('sha256').update(buffer).digest('hex'); + const source = { + type: 'UPLOAD', + name: safeName, + extension, + size: buffer.length, + mimeType: MIME_TYPES[extension] ?? 'text/plain; charset=utf-8', + sha256, + uploadedAt: new Date().toISOString(), + }; + const fileArtifact = this.workspace.createFileArtifact({ + workspaceId: workspaceRecord.id, + userId, + sessionId: session.id, + title: `${safeName}（原始文件）`, + content: buffer.toString('base64'), + encoding: 'base64', + format: extension, + mimeType: source.mimeType, + purpose: '用户上传的本地文件', + size: buffer.length, + source, + }); + let tableArtifact = null; + if (parsed && !parsed.unparsed && parsed.columns.length > 0) { + tableArtifact = this.workspace.createTableArtifact({ + workspaceId: workspaceRecord.id, + userId, + sessionId: session.id, + title: `${safeName}（解析表）`, + metadata: { + source: { type: 'UPLOAD_FILE', id: fileArtifact.id, name: safeName }, + derivedFromArtifactId: fileArtifact.id, + parentArtifactIds: [fileArtifact.id], + inputArtifactIds: [fileArtifact.id], + lineageQuestion: `上传本地文件：${safeName}`, + format: extension, + encoding: parsed.encoding, + delimiter: parsed.delimiter, + sheetName: parsed.sheetName ?? null, + rowCount: parsed.rows.length, + totalRowCount: parsed.totalRows ?? parsed.rows.length, + columnCount: parsed.columns.length, + truncated: Boolean(parsed.truncated), + warnings, + }, + payload: { data: { columns: parsed.columns, rows: parsed.rows } }, + }); } const profile = tableArtifact ? this.profileTable(parsed) : null; diff --git a/tests/localFiles.test.js b/tests/localFiles.test.js index 702e72e..bf24ae9 100644 --- a/tests/localFiles.test.js +++ b/tests/localFiles.test.js @@ -402,6 +402,38 @@ test('upload rejects unsupported, oversized, binary and session-less requests', } }); +for (const rejected of [ + { name: 'damaged.xlsx', content: Buffer.from([0x50, 0x4B, 0x03, 0x04]), message: /有效的 xlsx 工作簿/ }, + { name: 'blank.csv', content: Buffer.from(' \n\t\n'), message: /识别出列/ }, + { name: 'invalid.json', content: Buffer.from('{'), message: /JSON 文件解析失败/ }, + { name: 'empty.json', content: Buffer.from('[{}]'), message: /识别出列/ }, + { name: 'blank.xls', content: Buffer.from(' \n'), message: /识别出列/ }, +]) { + test(`rejected ${rejected.name} leaves no artifacts or versions and preserves upload quota`, () => { + const harness = createHarness({ maxFilesPerSession: 1, allowedExtensions: ['csv', 'xls', 'xlsx', 'json'] }); + try { + const base = { userId: harness.user.id, sessionId: harness.session.id }; + assert.throws( + () => harness.uploads.upload({ + ...base, name: rejected.name, contentBase64: rejected.content.toString('base64'), + }), + (error) => error.statusCode === 400 && rejected.message.test(error.message), + ); + assert.equal(harness.database.countRows('workspace_artifacts'), 0); + assert.equal(harness.database.countRows('artifact_versions'), 0); + assert.deepEqual(harness.database.listAuditLogs(50).map((entry) => entry.action), ['FILE_REJECTED']); + + const valid = harness.uploads.upload({ ...base, name: 'valid.csv', contentBase64: toBase64('a\n1\n') }); + assert.equal(valid.table.rowCount, 1); + assert.equal(harness.database.countRows('workspace_artifacts'), 3); + assert.equal(harness.database.countRows('artifact_versions'), 3); + } finally { + harness.database.close(); + fs.rmSync(harness.directory, { recursive: true, force: true }); + } + }); +} + test('upload enforces the per-session quota', () => { const harness = createHarness({ maxFilesPerSession: 1 }); try { @@ -646,6 +678,10 @@ test('legacy binary xls is stored and flagged instead of being guessed', () => { assert.equal(binary.table, null); assert.ok(binary.file.artifactId); assert.ok(binary.warnings.length > 0); + assert.equal(harness.database.countRows('workspace_artifacts'), 2); + assert.equal(harness.database.countRows('artifact_versions'), 2); + const stored = harness.workspace.getArtifact({ artifactId: binary.file.artifactId, userId: harness.user.id }); + assert.equal(stored.payload.content, ole.toString('base64')); // 文本伪装成 .xls（把 CSV 另存为 xls 是常见情况）按文本表格解析 const textXls = harness.uploads.upload({