Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions apps/desktop/src/main/core/local-data-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ export class LocalDataService {
this.evidence = new EvidenceService(
this.database,
(hash) => this.objectPath(hash),
(sourceId, relativePath) => this.resolveLocalItemPath(sourceId, relativePath),
(sourceId) => this.notifyChanged(sourceId, true),
)
this.highRiskImports?.setAutoResolver((batch, accepted) => this.resolveAutoScanBatch(batch, accepted))
Expand Down Expand Up @@ -748,7 +749,7 @@ export class LocalDataService {
}

async previewFile(dataSourceId: string, fileId: string): Promise<MarkdownPreview> {
this.requireSource(dataSourceId)
const source = this.requireSource(dataSourceId)
const row = this.database.prepare(`
SELECT relative_path, extension, modified_at, content_hash, state
FROM source_items WHERE id = ? AND data_source_id = ?
Expand All @@ -762,7 +763,13 @@ export class LocalDataService {
if (!row) throw new Error('文件记录不存在。')
if (row.state !== 'present' || !row.content_hash) throw new Error('文件当前不可预览。')
if (!['.md', '.mdx', '.markdown'].includes(row.extension.toLowerCase())) throw new Error('仅支持 Markdown 文件预览。')
const content = await readFile(this.objectPath(row.content_hash), 'utf8')
// 本地文件夹来源不落对象库,直接读原文件。
const contentPath = source.kind === 'local-folder'
? this.resolveLocalItemPath(dataSourceId, row.relative_path)
: this.objectPath(row.content_hash)
const content = await readFile(contentPath, 'utf8').catch((error: NodeJS.ErrnoException) => {
throw error.code === 'ENOENT' ? new Error('源文件已不存在,请恢复文件后重新扫描数据源。') : error
})
if (Buffer.byteLength(content, 'utf8') > 2 * 1024 * 1024) throw new Error('Markdown 文件过大,无法预览。')
return { fileName: basename(row.relative_path), relativePath: row.relative_path, modifiedAt: row.modified_at, content }
}
Expand Down Expand Up @@ -1585,7 +1592,7 @@ export class LocalDataService {
deferExport ? 'pending' : 'normal',
new Date().toISOString(),
)
if (!shouldExport && !deferExport && isLocalParseableExtension(item.extension)) {
if (!deferExport && isLocalParseableExtension(item.extension)) {
this.evidence.enqueueVersion(versionId, item.extension)
}
if (shouldExport && this.fileExports) {
Expand All @@ -1603,7 +1610,7 @@ export class LocalDataService {
): Promise<HighRiskImportResolution> {
const findVersion = this.database.prepare(`
SELECT source_versions.id, source_versions.object_hash, source_versions.source_item_id,
source_items.remote_id, source_items.relative_path,
source_items.remote_id, source_items.relative_path, source_items.extension,
(
SELECT COUNT(*)
FROM source_versions AS accepted_version
Expand Down Expand Up @@ -1642,12 +1649,16 @@ export class LocalDataService {
source_item_id: string
remote_id: string
relative_path: string
extension: string
accepted_version_count: number
} | undefined
if (!version) continue
if (accepted) {
updatePolicy.run('approved', versionId)
enqueue.run(versionId, new Date().toISOString())
if (isLocalParseableExtension(version.extension)) {
this.evidence.enqueueVersion(versionId, version.extension)
}
} else {
addIgnored.run(batch.sourceId, version.remote_id, version.relative_path, new Date().toISOString())
if (Number(version.accepted_version_count) === 0) {
Expand Down Expand Up @@ -1836,6 +1847,13 @@ export class LocalDataService {
)
}

private resolveLocalItemPath(sourceId: string, relativePath: string): string {
const source = this.requireSource(sourceId)
const connector = this.connectors.get(source.kind)
if (!connector.resolveLocalPath) throw new Error('该数据源没有本机文件可读。')
return connector.resolveLocalPath(this.toConnection(source), relativePath)
}

private requireSource(id: string): SourceRow {
const source = this.database
.prepare('SELECT * FROM data_sources WHERE id = ?')
Expand Down
25 changes: 22 additions & 3 deletions apps/desktop/src/main/evidence/evidence-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ interface PendingJobRow {
object_hash: string
extension: string
data_source_id: string
source_kind: string
relative_path: string
}

interface EvidenceDocumentRow {
Expand Down Expand Up @@ -83,6 +85,7 @@ export class EvidenceService {
constructor(
private readonly database: DatabaseSync,
private readonly objectPath: (hash: string) => string,
private readonly resolveLocalPath: (dataSourceId: string, relativePath: string) => string,
private readonly onUpdated: (dataSourceId: string) => void,
) {}

Expand Down Expand Up @@ -157,6 +160,12 @@ export class EvidenceService {
SET status = 'pending', error_message = '应用在解析完成前退出', started_at = NULL
WHERE status = 'running'
`).run()
// 失败任务在重启后重试,让修复过的解析路径可以救回历史失败记录。
this.database.prepare(`
UPDATE evidence_parse_jobs
SET status = 'pending', error_message = NULL, started_at = NULL
WHERE status = 'failed' AND attempt_count < 5
`).run()
this.database.prepare(`
INSERT OR IGNORE INTO evidence_parse_jobs (
source_version_id, parser, status, queued_at
Expand Down Expand Up @@ -327,10 +336,12 @@ export class EvidenceService {
while (!this.stopping) {
const job = this.database.prepare(`
SELECT evidence_parse_jobs.source_version_id, source_versions.object_hash,
source_items.extension, source_items.data_source_id
source_items.extension, source_items.data_source_id,
data_sources.kind AS source_kind, source_items.relative_path
FROM evidence_parse_jobs
JOIN source_versions ON source_versions.id = evidence_parse_jobs.source_version_id
JOIN source_items ON source_items.id = source_versions.source_item_id
JOIN data_sources ON data_sources.id = source_items.data_source_id
WHERE evidence_parse_jobs.status = 'pending'
ORDER BY evidence_parse_jobs.queued_at
LIMIT 1
Expand All @@ -349,7 +360,11 @@ export class EvidenceService {
`).run(new Date().toISOString(), job.source_version_id)

try {
const buffer = await readFile(this.objectPath(job.object_hash))
// 本地文件夹来源不落对象库,直接读原文件;连接器来源读扫描时存的对象。
const contentPath = job.source_kind === 'local-folder'
? this.resolveLocalPath(job.data_source_id, job.relative_path)
: this.objectPath(job.object_hash)
const buffer = await readFile(contentPath)
const text = new TextDecoder('utf-8', { fatal: true }).decode(buffer).replace(/^\uFEFF/, '')
const blocks = MARKDOWN_EXTENSIONS.has(job.extension.toLowerCase())
? parseMarkdown(text)
Expand All @@ -362,7 +377,11 @@ export class EvidenceService {
`).run(new Date().toISOString(), job.source_version_id)
this.onUpdated(job.data_source_id)
} catch (error) {
const message = error instanceof Error ? error.message : '文档解析失败'
const isMissingFile = typeof error === 'object' && error !== null && 'code' in error &&
(error as NodeJS.ErrnoException).code === 'ENOENT'
const message = isMissingFile
? '源文件已不存在,请恢复文件后重新扫描数据源。'
: error instanceof Error ? error.message : '文档解析失败'
this.database.prepare(`
UPDATE evidence_parse_jobs
SET status = 'failed', error_message = ?, parsed_at = ?
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ export function toImportedRealityEvent(
const normalizedCaptureDevice = captureDevice && typeof captureDevice === 'object' && !Array.isArray(captureDevice)
&& typeof (captureDevice as Record<string, unknown>).id === 'string'
&& typeof (captureDevice as Record<string, unknown>).name === 'string'
&& ['desktop', 'iphone', 'watch'].includes(String((captureDevice as Record<string, unknown>).kind))
&& ['desktop', 'iphone', 'apple_watch'].includes(String((captureDevice as Record<string, unknown>).kind))
? captureDevice as ImportRealityEventInput['captureDevice']
: { id: 'synced-iphone', name: 'iPhone', kind: 'iphone' as const }
const audioSource = sourceMetadata.audioSource === 'system' ? 'system' : 'microphone'
Expand Down Expand Up @@ -572,7 +572,8 @@ export class PrivateTranscriptionSyncService {
const input = toImportedRealityEvent(source, summary)
if (!input) continue
activeEventIds.add(input.id)
const fingerprint = `${source.revision}:${source.updatedAt}:${summary?.revision ?? 0}:${summary?.updatedAt ?? ''}:${summary ? 'valid' : 'missing'}`
// fingerprint 含 captureDevice:云端下发补上来源后(哪怕 revision 没变)也要重新物化,纠正旧的 iPhone 兜底数据。
const fingerprint = `${source.revision}:${source.updatedAt}:${summary?.revision ?? 0}:${summary?.updatedAt ?? ''}:${summary ? 'valid' : 'missing'}:${input.captureDevice.id}:${input.captureDevice.kind}`
if (materialized[input.id] === fingerprint) continue
await this.reality.importEvent(input)
materialized[input.id] = fingerprint
Expand Down
Loading
Loading