Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/turnContext.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ const SLOT_LABELS = {
dimensions: '维度',
analysisMode: '分析方式',
};
const TIME_PATTERN = /(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|(?:今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年)|(?:第[一二三四1-4]季度|Q[1-4])|\d{4}年(?:\d{1,2}月)?|\d{1,2}月/gi;
const EXPLICIT_DATE_PATTERN = String.raw`(?:\d{4}年)?\d{1,2}月\d{1,2}[日号]|\d{4}[-/.]\d{1,2}[-/.]\d{1,2}`;
// Match complete day ranges before their year/month prefixes so inheritance keeps day precision.
const EXPLICIT_DATE_RANGE_PATTERN = String.raw`(?:${EXPLICIT_DATE_PATTERN})(?:\s*(?:到|至|~|—|-|-)\s*(?:${EXPLICIT_DATE_PATTERN}|\d{1,2}[日号]))?`;
const TIME_PATTERN = new RegExp(String.raw`${EXPLICIT_DATE_RANGE_PATTERN}|(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|(?:今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年)|(?:第[一二三四1-4]季度|Q[1-4])|\d{4}年(?:\d{1,2}月)?|\d{1,2}月`, 'gi');
const TIME_HINT_PATTERN = /(?:近|最近|过去)\s*(?:\d+|[一二两三四五六七八九十]+)\s*个?(?:天|日|周|星期|月|季度|年)|(?:本|这|上|下)(?:个)?(?:周|星期|月|季度|年)|今天|今日|昨天|昨日|前天|明天|后天|今年|去年|明年|第[一二三四1-4]季度|Q[1-4]|\d{4}年|\d{1,2}月/i;
const FILTER_PATTERN = /(?:只看|仅看|限定|过滤|筛选|排除)([^,。;,;]{1,24})/g;
const TIME_GRAINS = new Set(['日', '天', '周', '月', '季度', '年']);
Expand Down Expand Up @@ -189,7 +192,7 @@ export function inferClarificationSlots(question) {
return {};
}
const slots = {};
if (TIME_HINT_PATTERN.test(source)) {
if (TIME_HINT_PATTERN.test(source) || extractExplicitSlots(source).time?.length) {
return slots;
}
if (turnPatterns().fieldLabel?.test(source)) {
Expand Down
102 changes: 51 additions & 51 deletions src/uploads.js
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ function base64ByteLength(text) {
return Math.floor((value.length * 3) / 4) - padding;
}

// 上传服务:校验 -> 落原始文件产物 -> 解析成表格产物 -> 落识别过程文件。
// 上传服务:校验并解析 -> 落原始文件与表格产物 -> 落识别过程文件。
export class UploadService {
constructor({ database, workspace, config = {} }) {
this.database = database;
Expand Down Expand Up @@ -324,31 +324,6 @@ export class UploadService {
incomingBytes: buffer.length,
});

const sha256 = createHash('sha256').update(buffer).digest('hex');
const source = {
type: 'UPLOAD',
name: safeName,
extension,
size: buffer.length,
mimeType: MIME_TYPES[extension] ?? 'text/plain; charset=utf-8',
sha256,
uploadedAt: new Date().toISOString(),
};
const fileArtifact = this.workspace.createFileArtifact({
workspaceId: workspaceRecord.id,
userId,
sessionId: session.id,
title: `${safeName}(原始文件)`,
content: buffer.toString('base64'),
encoding: 'base64',
format: extension,
mimeType: source.mimeType,
purpose: '用户上传的本地文件',
size: buffer.length,
source,
});

let tableArtifact = null;
let parsed = null;
const warnings = [];
if (SPREADSHEET_EXTENSIONS.has(extension)) {
Expand Down Expand Up @@ -380,31 +355,56 @@ export class UploadService {
if (!parseable && !parsed.unparsed) {
throw uploadError('未能从文件中识别出列,请检查文件是否有表头行');
}
if (parseable) {
tableArtifact = this.workspace.createTableArtifact({
workspaceId: workspaceRecord.id,
userId,
sessionId: session.id,
title: `${safeName}(解析表)`,
metadata: {
source: { type: 'UPLOAD_FILE', id: fileArtifact.id, name: safeName },
derivedFromArtifactId: fileArtifact.id,
parentArtifactIds: [fileArtifact.id],
inputArtifactIds: [fileArtifact.id],
lineageQuestion: `上传本地文件:${safeName}`,
format: extension,
encoding: parsed.encoding,
delimiter: parsed.delimiter,
sheetName: parsed.sheetName ?? null,
rowCount: parsed.rows.length,
totalRowCount: parsed.totalRows ?? parsed.rows.length,
columnCount: parsed.columns.length,
truncated: Boolean(parsed.truncated),
warnings,
},
payload: { data: { columns: parsed.columns, rows: parsed.rows } },
});
}
}

const sha256 = createHash('sha256').update(buffer).digest('hex');
const source = {
type: 'UPLOAD',
name: safeName,
extension,
size: buffer.length,
mimeType: MIME_TYPES[extension] ?? 'text/plain; charset=utf-8',
sha256,
uploadedAt: new Date().toISOString(),
};
const fileArtifact = this.workspace.createFileArtifact({
workspaceId: workspaceRecord.id,
userId,
sessionId: session.id,
title: `${safeName}(原始文件)`,
content: buffer.toString('base64'),
encoding: 'base64',
format: extension,
mimeType: source.mimeType,
purpose: '用户上传的本地文件',
size: buffer.length,
source,
});
let tableArtifact = null;
if (parsed && !parsed.unparsed && parsed.columns.length > 0) {
tableArtifact = this.workspace.createTableArtifact({
workspaceId: workspaceRecord.id,
userId,
sessionId: session.id,
title: `${safeName}(解析表)`,
metadata: {
source: { type: 'UPLOAD_FILE', id: fileArtifact.id, name: safeName },
derivedFromArtifactId: fileArtifact.id,
parentArtifactIds: [fileArtifact.id],
inputArtifactIds: [fileArtifact.id],
lineageQuestion: `上传本地文件:${safeName}`,
format: extension,
encoding: parsed.encoding,
delimiter: parsed.delimiter,
sheetName: parsed.sheetName ?? null,
rowCount: parsed.rows.length,
totalRowCount: parsed.totalRows ?? parsed.rows.length,
columnCount: parsed.columns.length,
truncated: Boolean(parsed.truncated),
warnings,
},
payload: { data: { columns: parsed.columns, rows: parsed.rows } },
});
}

const profile = tableArtifact ? this.profileTable(parsed) : null;
Expand Down
36 changes: 36 additions & 0 deletions tests/localFiles.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,38 @@ test('upload rejects unsupported, oversized, binary and session-less requests',
}
});

for (const rejected of [
{ name: 'damaged.xlsx', content: Buffer.from([0x50, 0x4B, 0x03, 0x04]), message: /有效的 xlsx 工作簿/ },
{ name: 'blank.csv', content: Buffer.from(' \n\t\n'), message: /识别出列/ },
{ name: 'invalid.json', content: Buffer.from('{'), message: /JSON 文件解析失败/ },
{ name: 'empty.json', content: Buffer.from('[{}]'), message: /识别出列/ },
{ name: 'blank.xls', content: Buffer.from(' \n'), message: /识别出列/ },
]) {
test(`rejected ${rejected.name} leaves no artifacts or versions and preserves upload quota`, () => {
const harness = createHarness({ maxFilesPerSession: 1, allowedExtensions: ['csv', 'xls', 'xlsx', 'json'] });
try {
const base = { userId: harness.user.id, sessionId: harness.session.id };
assert.throws(
() => harness.uploads.upload({
...base, name: rejected.name, contentBase64: rejected.content.toString('base64'),
}),
(error) => error.statusCode === 400 && rejected.message.test(error.message),
);
assert.equal(harness.database.countRows('workspace_artifacts'), 0);
assert.equal(harness.database.countRows('artifact_versions'), 0);
assert.deepEqual(harness.database.listAuditLogs(50).map((entry) => entry.action), ['FILE_REJECTED']);

const valid = harness.uploads.upload({ ...base, name: 'valid.csv', contentBase64: toBase64('a\n1\n') });
assert.equal(valid.table.rowCount, 1);
assert.equal(harness.database.countRows('workspace_artifacts'), 3);
assert.equal(harness.database.countRows('artifact_versions'), 3);
} finally {
harness.database.close();
fs.rmSync(harness.directory, { recursive: true, force: true });
}
});
}

test('upload enforces the per-session quota', () => {
const harness = createHarness({ maxFilesPerSession: 1 });
try {
Expand Down Expand Up @@ -646,6 +678,10 @@ test('legacy binary xls is stored and flagged instead of being guessed', () => {
assert.equal(binary.table, null);
assert.ok(binary.file.artifactId);
assert.ok(binary.warnings.length > 0);
assert.equal(harness.database.countRows('workspace_artifacts'), 2);
assert.equal(harness.database.countRows('artifact_versions'), 2);
const stored = harness.workspace.getArtifact({ artifactId: binary.file.artifactId, userId: harness.user.id });
assert.equal(stored.payload.content, ole.toString('base64'));

// 文本伪装成 .xls(把 CSV 另存为 xls 是常见情况)按文本表格解析
const textXls = harness.uploads.upload({
Expand Down
2 changes: 1 addition & 1 deletion tests/postgresql.query.http.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ test('live PostgreSQL: authenticated Agent contract, row policy, audit and same-
});
assert.equal(streamed.status, 200);
const events = (await streamed.text()).trim().split('\n').map((line) => JSON.parse(line));
assert.ok(!events.some((event) => event.type === 'stream_error'), JSON.stringify(events));
assert.deepEqual(events.filter((event) => event.type === 'stream_error'), []);
const success = { response: streamed, data: events.find((event) => event.type === 'final')?.answer };
assert.equal(success.response.status, 200, JSON.stringify(success.data));
assert.ok(success.data);
Expand Down
86 changes: 86 additions & 0 deletions tests/queryDateRange.http.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { withSkillServer } from './fixtures/skillServer.js';

test('authenticated streaming contracts keep explicit date ranges intact and apply row policy', async () => {
await withSkillServer(async ({ application, login, baseUrl }) => {
const cookie = await login('east_manager');
const user = application.database.getUserByUsername('east_manager');
const source = application.database.saveDataSource({
code: 'range_pg', name: 'Range PostgreSQL', dbType: 'POSTGRESQL',
host: 'localhost', port: 5432, databaseName: 'test', username: 'test',
encryptedPassword: '', options: {},
});
const dataset = application.database.saveBusinessDataset({
code: 'range_sales', name: 'Range sales', datasourceId: source.id,
schemaName: 'public', primaryTable: 'sales',
config: { autoLatestDateRange: false, policyFieldMap: { region: 'region' } },
});
application.database.replaceDatasetFields(dataset.id, [
{ fieldName: 'day', dataType: 'date', semanticType: 'DATE', role: 'TIME',
aggregator: 'NONE', allowedOperators: ['BETWEEN'] },
{ fieldName: 'region', dataType: 'text', semanticType: 'STRING', role: 'DIMENSION',
aggregator: 'NONE', allowedOperators: ['IN'] },
{ fieldName: 'amount', dataType: 'numeric', semanticType: 'NUMBER', role: 'METRIC',
aggregator: 'SUM', allowedOperators: ['>', 'BETWEEN'] },
]);
application.database.grantDatasetAccess(user.id, dataset.id, true);
application.database.saveTheme({ ...application.database.getTheme(1), indicatorIds: [],
businessDatasetIds: [dataset.id], primaryBusinessDatasetId: dataset.id,
semanticPolicy: {}, systemPrompt: 'amount 为销售额,使用 SUM;day 为日期,region 为区域。',
}, 1);
application.database.savePlatformSetting('supersonic.enabled', false);
const statements = [];
// Keep auth, context, compiler, SQL generation and audit real; replace only database I/O.
application.businessDatasets.getPoolById = () => ({ query: async (config) => {
statements.push(config);
return [[{ amount: 40 }], [{ name: 'amount' }]];
} });
application.agent.harnessFactory = { forTheme: () => ({
mode: 'local-rule', model: 'range-acceptance', capabilities: {},
run: async ({ executeTool }) => {
await executeTool('list_business_datasets', {});
await executeTool('get_business_dataset_definition', { datasetId: dataset.id });
const compiled = await executeTool('compile_query_contract', {
sourceType: 'BUSINESS_DATASET', datasetId: dataset.id,
conditions: [
{ id: 'time', sourceText: '2026年1月1日至2026年3月31日', kind: 'TIME', status: 'RESOLVED' },
{ id: 'region', sourceText: 'west', kind: 'FILTER', status: 'RESOLVED' },
{ id: 'metric', sourceText: 'amount', kind: 'METRIC', status: 'RESOLVED' },
],
metricFields: [{ field: 'amount', sourceText: 'amount', aggregator: 'SUM', conditionIds: ['metric'] }],
dimensionFields: [],
filterFields: [{ field: 'region', operator: 'IN', value: ['west'], sourceText: 'west', conditionIds: ['region'] }],
timeWindows: [{ sourceText: '2026年1月1日至2026年3月31日', dateMode: 'BETWEEN',
startDate: '2026-01-01', endDate: '2026-03-31', conditionIds: ['time'] }],
limit: 20,
});
assert.equal(compiled.valid, true, JSON.stringify(compiled.issues));
await executeTool('execute_query_contract', { contractId: compiled.contractId });
return { content: '受控查询已完成。', trace: [] };
},
}) };
const response = await fetch(`${baseUrl}/api/chat/query/stream`, {
method: 'POST', headers: { Cookie: cookie, 'Content-Type': 'application/json' },
body: JSON.stringify({ themeId: 1, question: '2026年1月1日至2026年3月31日 west amount' }),
});
assert.equal(response.status, 200);
const events = (await response.text()).trim().split('\n').map(line => JSON.parse(line));
assert.deepEqual(events.filter(event => event.type === 'stream_error'), []);
const answer = events.find(event => event.type === 'final')?.answer;
assert.ok(answer);
assert.deepEqual(answer.data.rows, [{ amount: 40 }]);
const plan = application.database.getQueryPlan(answer.semanticParse.planId, user.id);
assert.equal(plan.status, 'EXECUTED');
assert.equal(plan.plan.timeWindows[0].startDate, '2026-01-01');
assert.equal(plan.plan.timeWindows[0].endDate, '2026-03-31');
const audit = application.database.listDatasetQueryLogs(100).filter(log => log.datasetId === dataset.id);
assert.equal(audit.length, 1);
const statement = statements.find(item => item.sql === audit[0].sqlText);
assert.ok(statement);
assert.ok(statement.values.includes('2026-01-01'));
assert.ok(statement.values.includes('2026-03-31'));
assert.ok(statement.values.flat(Infinity).includes('华东'));
assert.ok(!statement.values.flat(Infinity).includes('west'));
});
});
41 changes: 41 additions & 0 deletions tests/turnContext.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
TURN_TYPES,
completeTaskState,
createTurnContext,
extractExplicitSlots,
} from '../src/turnContext.js';

function slot(value, source = 'explicit') {
Expand All @@ -28,6 +29,46 @@ function completedHistory(context, content = '查询已完成') {
];
}

for (const time of [
'2026年1月1日至2026年3月31日',
'2026-01-01 至 2026-03-31',
'2026年1月1日至3月31日',
'2026年1月1日',
]) {
test(`explicit time slots preserve the complete date expression: ${time}`, () => {
assert.deepEqual(extractExplicitSlots(`${time}销售额`).time.map(item => item.sourceText), [time]);
});
}

test('two distinct explicit date ranges remain separate time slots', () => {
const question = '对比2026年1月1日至2026年1月31日与2026年3月1日至2026年3月31日销售额';
assert.deepEqual(extractExplicitSlots(question).time.map(item => item.sourceText), [
'2026年1月1日至2026年1月31日', '2026年3月1日至2026年3月31日',
]);
});

test('an ISO date clarification fills time without replacing the inherited region filter', () => {
const first = createTurnContext({ question: '销售额,只看华东' });
const clarified = createTurnContext({ question: '2026-01-01', history: completedHistory(first),
turnType: TURN_TYPES.CLARIFICATION_ANSWER });
assert.deepEqual(clarified.slots.time.map(item => item.sourceText), ['2026-01-01']);
assert.equal(clarified.explicitSlots.filters, undefined);
assert.deepEqual(clarified.inheritedSlots.filters.map(item => item.sourceText), ['华东']);
assert.deepEqual(clarified.slots.filters.map(item => item.sourceText), ['华东']);
});

test('followups inherit full date ranges and filters while a time revision replaces only time', () => {
const first = createTurnContext({ question: '2026年1月1日至2026年3月31日销售额,只看华东' });
const second = createTurnContext({ question: '按渠道拆开', history: completedHistory(first),
turnType: TURN_TYPES.CONTINUATION });
assert.deepEqual(second.inheritedSlots.time.map(item => item.sourceText), ['2026年1月1日至2026年3月31日']);
assert.deepEqual(second.inheritedSlots.filters.map(item => item.sourceText), ['华东']);
const revised = createTurnContext({ question: '改查2026年4月1日至2026年4月15日',
history: completedHistory(second), turnType: TURN_TYPES.REVISION });
assert.deepEqual(revised.slots.time.map(item => item.sourceText), ['2026年4月1日至2026年4月15日']);
assert.deepEqual(revised.inheritedSlots.filters.map(item => item.sourceText), ['华东']);
});

test('continuations keep the task id, advance the turn id and inherit field by field', () => {
const first = createTurnContext({
question: '查询本月含税销售金额',
Expand Down
Loading