From fe684cf197cd97b7744c7344fb71f33fe9d07262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Qu=E1=BB=91c=20=C4=90=E1=BA=A1t?= Date: Thu, 14 May 2026 14:54:48 +0700 Subject: [PATCH 1/2] fix(import): apply backpressure in SQL parser so no statements are dropped --- CHANGELOG.md | 2 + .../Core/Plugins/SqlFileImportSource.swift | 2 +- .../Core/Utilities/SQL/SQLFileParser.swift | 339 ++++++++++-------- .../Utilities/SQL/SQLFileParserTests.swift | 41 ++- 4 files changed, 238 insertions(+), 146 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b735ab99..7bb4c4d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- SQL import dropped statements when the database executed them slower than the file was parsed, so a re-imported export could fail with errors like "relation does not exist". The parser now waits for each statement to be consumed before reading more. (#1264) +- SQL import ignored the database dialect, so PostgreSQL dumps with dollar-quoted function bodies were split at semicolons inside the body. (#1264) - iOS: connections, groups, and tags no longer silently disappear after a TestFlight or App Store update. Persistence files are now stored with `.completeFileProtectionUntilFirstUserAuthentication` so they stay readable across background sync runs, load failures are no longer swallowed, and the sync engine refuses to overwrite local data when the load was not actually empty. ### Removed diff --git a/TablePro/Core/Plugins/SqlFileImportSource.swift b/TablePro/Core/Plugins/SqlFileImportSource.swift index de4bb10a8..c3c100041 100644 --- a/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -50,7 +50,7 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { let fileURL = try await resolveURL() - return parser.parseFile(url: fileURL, encoding: encoding) + return parser.parseFile(url: fileURL, encoding: encoding, dialect: dialect) } func cleanup() { diff --git a/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 62977c968..d4590d6a8 100644 --- a/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -138,6 +138,7 @@ final class SQLFileParser: Sendable { var isSingleCharDelimiter = true var dollarTag: String = "" var backslashEscapesActive = false + var collected: [(statement: String, lineNumber: Int)] = [] } private static func trimmedStatement(_ ctx: ParserContext) -> String { @@ -211,8 +212,7 @@ final class SQLFileParser: Sendable { nextChar: unichar?, i: inout Int, nsBuffer: NSString, - bufLen: Int, - continuation: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.Continuation + bufLen: Int ) -> StepResult { processDelimiterChange(&ctx, char: char) @@ -280,13 +280,13 @@ final class SQLFileParser: Sendable { } if ctx.isSingleCharDelimiter && char == kSemicolon { - yieldAndReset(&ctx, continuation: continuation) + yieldAndReset(&ctx) return StepResult(advanced: false, deferred: false) } if !ctx.isSingleCharDelimiter && matchesDelimiter(at: i, delimiter: ctx.currentDelimiter, in: nsBuffer, bufLen: bufLen) { - yieldAndReset(&ctx, continuation: continuation) + yieldAndReset(&ctx) i += ctx.currentDelimiter.length return StepResult(advanced: true, deferred: false) } @@ -335,13 +335,10 @@ final class SQLFileParser: Sendable { return nil } - private static func yieldAndReset( - _ ctx: inout ParserContext, - continuation: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.Continuation - ) { + private static func yieldAndReset(_ ctx: inout ParserContext) { if ctx.hasStatementContent { let text = trimmedStatement(ctx) - continuation.yield((text, ctx.statementStartLine)) + ctx.collected.append((text, ctx.statementStartLine)) } resetStatement(&ctx) } @@ -503,149 +500,203 @@ final class SQLFileParser: Sendable { dialect: SqlDialect = .generic, countOnly: Bool = false ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - AsyncThrowingStream(bufferingPolicy: .bufferingNewest(8)) { continuation in - let task = Task.detached { + let session = ParseSession(url: url, encoding: encoding, dialect: dialect, countOnly: countOnly) + return AsyncThrowingStream(unfolding: { + try await session.next() + }) + } + + private final class ParseSession: @unchecked Sendable { + private let url: URL + private let encoding: String.Encoding + private let dialect: SqlDialect + private let chunkSize = 65_536 + + private var fileHandle: FileHandle? + private var ctx: ParserContext + private let nsBuffer = NSMutableString() + private var pendingTail = Data() + private var emitIndex = 0 + private var finished = false + + init(url: URL, encoding: String.Encoding, dialect: SqlDialect, countOnly: Bool) { + self.url = url + self.encoding = encoding + self.dialect = dialect + self.ctx = ParserContext( + dialect: dialect, + currentStatement: countOnly ? nil : NSMutableString() + ) + } + + deinit { + closeFile() + } + + func next() async throws -> (statement: String, lineNumber: Int)? { + while true { + if emitIndex < ctx.collected.count { + let item = ctx.collected[emitIndex] + emitIndex += 1 + return item + } + ctx.collected.removeAll(keepingCapacity: true) + emitIndex = 0 + + if finished { + return nil + } + if Task.isCancelled { + finished = true + closeFile() + return nil + } + do { - let fileHandle = try FileHandle(forReadingFrom: url) - defer { - do { - try fileHandle.close() - } catch { - Self.logger.warning("Failed to close file handle for \(url.path): \(error)") - } - } + try advanceOneChunk() + } catch { + finished = true + closeFile() + SQLFileParser.logger.error("SQL file parsing failed: \(error.localizedDescription)") + throw error + } + } + } - var ctx = ParserContext( - dialect: dialect, - currentStatement: countOnly ? nil : NSMutableString() - ) - let nsBuffer = NSMutableString() - let chunkSize = 65_536 - var pendingTail = Data() - - while true { - guard !Task.isCancelled else { - continuation.finish() - return - } - let rawData = fileHandle.readData(ofLength: chunkSize) - if rawData.isEmpty && pendingTail.isEmpty { break } - - let isFinalChunk = rawData.isEmpty - guard let chunk = Self.decodeChunkOrCarryTail( - rawData: rawData, pendingTail: &pendingTail, encoding: encoding - ) else { - Self.logger.error("Failed to decode chunk with encoding \(encoding.description)") - continuation.finish(throwing: DecompressionError.fileReadFailed( - "Failed to decode file with \(encoding.description) encoding" - )) - return - } - - if isFinalChunk && !pendingTail.isEmpty { - Self.logger.error("Trailing bytes did not form a valid \(encoding.description) sequence at end of file") - continuation.finish(throwing: DecompressionError.fileReadFailed( - "Trailing bytes did not form a valid \(encoding.description) sequence at end of file" - )) - return - } - - nsBuffer.append(chunk) - let bufLen = nsBuffer.length - var i = 0 - - while i < bufLen { - let char = nsBuffer.character(at: i) - let nextChar: unichar? = (i + 1 < bufLen) ? nsBuffer.character(at: i + 1) : nil - - if nextChar == nil && Self.needsLookahead( - char, - state: ctx.state, - dialect: dialect, - delimiter: ctx.currentDelimiter, - isSingleCharDelimiter: ctx.isSingleCharDelimiter - ) { - break - } - - if char == Self.kNewline { ctx.currentLine += 1 } - var didManuallyAdvance = false - var shouldDefer = false - - switch ctx.state { - case .normal: - let result = Self.processNormalChar( - &ctx, char: char, nextChar: nextChar, - i: &i, nsBuffer: nsBuffer, bufLen: bufLen, - continuation: continuation) - didManuallyAdvance = result.advanced - shouldDefer = result.deferred - - case .inSingleLineComment: - if char == Self.kNewline { - ctx.state = .normal - } - - case .inMultiLineComment: - didManuallyAdvance = Self.processMultiLineComment( - &ctx, char: char, nextChar: nextChar, i: &i) - - case .inSingleQuotedString: - let result = Self.processQuotedString( - &ctx, quoteChar: Self.kSingleQuote, - i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - didManuallyAdvance = result.advanced - shouldDefer = result.deferred - - case .inDoubleQuotedString: - let result = Self.processQuotedString( - &ctx, quoteChar: Self.kDoubleQuote, - i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - didManuallyAdvance = result.advanced - shouldDefer = result.deferred - - case .inBacktickQuotedString: - let result = Self.processQuotedString( - &ctx, quoteChar: Self.kBacktick, - i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - didManuallyAdvance = result.advanced - shouldDefer = result.deferred - - case .inDollarQuote: - let result = Self.processDollarQuote( - &ctx, i: &i, - nsBuffer: nsBuffer, bufLen: bufLen) - didManuallyAdvance = result.advanced - shouldDefer = result.deferred - } - - if shouldDefer { break } - if !didManuallyAdvance { i += 1 } - } - - if i < bufLen { - nsBuffer.deleteCharacters(in: NSRange(location: 0, length: i)) - } else { - nsBuffer.setString("") - } - } + private func advanceOneChunk() throws { + let handle = try openFileIfNeeded() + let rawData = handle.readData(ofLength: chunkSize) - if ctx.hasStatementContent { - let text = Self.trimmedStatement(ctx) - if Self.extractDelimiterChange(text) == nil { - continuation.yield((text, ctx.statementStartLine)) - } + if rawData.isEmpty && pendingTail.isEmpty { + emitTrailingStatement() + finished = true + closeFile() + return + } + + let isFinalChunk = rawData.isEmpty + guard let chunk = SQLFileParser.decodeChunkOrCarryTail( + rawData: rawData, pendingTail: &pendingTail, encoding: encoding + ) else { + throw DecompressionError.fileReadFailed( + "Failed to decode file with \(encoding.description) encoding" + ) + } + + if isFinalChunk && !pendingTail.isEmpty { + throw DecompressionError.fileReadFailed( + "Trailing bytes did not form a valid \(encoding.description) sequence at end of file" + ) + } + + nsBuffer.append(chunk) + processBuffer() + } + + private func processBuffer() { + let bufLen = nsBuffer.length + var i = 0 + + while i < bufLen { + let char = nsBuffer.character(at: i) + let nextChar: unichar? = (i + 1 < bufLen) ? nsBuffer.character(at: i + 1) : nil + + if nextChar == nil && SQLFileParser.needsLookahead( + char, + state: ctx.state, + dialect: dialect, + delimiter: ctx.currentDelimiter, + isSingleCharDelimiter: ctx.isSingleCharDelimiter + ) { + break + } + + if char == SQLFileParser.kNewline { ctx.currentLine += 1 } + var didManuallyAdvance = false + var shouldDefer = false + + switch ctx.state { + case .normal: + let result = SQLFileParser.processNormalChar( + &ctx, char: char, nextChar: nextChar, + i: &i, nsBuffer: nsBuffer, bufLen: bufLen) + didManuallyAdvance = result.advanced + shouldDefer = result.deferred + + case .inSingleLineComment: + if char == SQLFileParser.kNewline { + ctx.state = .normal } - continuation.finish() - } catch { - Self.logger.error("SQL file parsing failed: \(error.localizedDescription)") - continuation.finish(throwing: error) + case .inMultiLineComment: + didManuallyAdvance = SQLFileParser.processMultiLineComment( + &ctx, char: char, nextChar: nextChar, i: &i) + + case .inSingleQuotedString: + let result = SQLFileParser.processQuotedString( + &ctx, quoteChar: SQLFileParser.kSingleQuote, + i: &i, nsBuffer: nsBuffer, bufLen: bufLen) + didManuallyAdvance = result.advanced + shouldDefer = result.deferred + + case .inDoubleQuotedString: + let result = SQLFileParser.processQuotedString( + &ctx, quoteChar: SQLFileParser.kDoubleQuote, + i: &i, nsBuffer: nsBuffer, bufLen: bufLen) + didManuallyAdvance = result.advanced + shouldDefer = result.deferred + + case .inBacktickQuotedString: + let result = SQLFileParser.processQuotedString( + &ctx, quoteChar: SQLFileParser.kBacktick, + i: &i, nsBuffer: nsBuffer, bufLen: bufLen) + didManuallyAdvance = result.advanced + shouldDefer = result.deferred + + case .inDollarQuote: + let result = SQLFileParser.processDollarQuote( + &ctx, i: &i, + nsBuffer: nsBuffer, bufLen: bufLen) + didManuallyAdvance = result.advanced + shouldDefer = result.deferred } + + if shouldDefer { break } + if !didManuallyAdvance { i += 1 } + } + + if i < bufLen { + nsBuffer.deleteCharacters(in: NSRange(location: 0, length: i)) + } else { + nsBuffer.setString("") + } + } + + private func emitTrailingStatement() { + guard ctx.hasStatementContent else { return } + let text = SQLFileParser.trimmedStatement(ctx) + if SQLFileParser.extractDelimiterChange(text) == nil { + ctx.collected.append((text, ctx.statementStartLine)) } + } + + private func openFileIfNeeded() throws -> FileHandle { + if let fileHandle { + return fileHandle + } + let handle = try FileHandle(forReadingFrom: url) + fileHandle = handle + return handle + } - continuation.onTermination = { @Sendable _ in - task.cancel() + private func closeFile() { + guard let handle = fileHandle else { return } + fileHandle = nil + do { + try handle.close() + } catch { + SQLFileParser.logger.warning( + "Failed to close file handle for \(self.url.path): \(error.localizedDescription)") } } } diff --git a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 0da1380f7..f15167f18 100644 --- a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -212,7 +212,7 @@ struct SQLFileParserTests { @Test("Large multi-row INSERT yields correct statement count and content") func large_multi_row_insert_correctness() async throws { - let rows = (1...5_000).map { " ($0, 'row\($0)')" }.joined(separator: ",\n") + let rows = (1...5_000).map { " (\($0), 'row\($0)')" }.joined(separator: ",\n") let sql = "INSERT INTO t (id, label) VALUES\n\(rows);\nSELECT 100;" let stmts = try await Self.parse(sql, dialect: .postgres) #expect(stmts.count == 2) @@ -222,6 +222,45 @@ struct SQLFileParserTests { #expect(stmts[1] == "SELECT 100") } + @Test("Slow consumer receives every statement in order with no drops") + func slow_consumer_no_dropped_statements() async throws { + let statementCount = 200 + let sql = (1...statementCount) + .map { "CREATE INDEX idx_\($0) ON t USING btree (c\($0));" } + .joined(separator: "\n") + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + var received: [String] = [] + let parser = SQLFileParser() + for try await (stmt, _) in parser.parseFile(url: url, encoding: .utf8, dialect: .postgres) { + received.append(stmt) + try await Task.sleep(nanoseconds: 100_000) + } + + #expect(received.count == statementCount) + #expect(received.first == "CREATE INDEX idx_1 ON t USING btree (c1)") + #expect(received.last == "CREATE INDEX idx_200 ON t USING btree (c200)") + } + + @Test("Statement count matches across a file larger than one read chunk") + func count_statements_across_chunk_boundary() async throws { + let statementCount = 4_000 + let sql = (1...statementCount) + .map { "INSERT INTO t (id) VALUES (\($0));" } + .joined(separator: "\n") + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + let parser = SQLFileParser() + let count = try await parser.countStatements(url: url, encoding: .utf8, dialect: .postgres) + #expect(count == statementCount) + } + @Test("Dialect.from maps known database type ids") func dialect_from_database_type_id() { #expect(SqlDialect.from(databaseTypeId: "PostgreSQL") == .postgres) From d9e1f72002913ad1fb90959b64bf8d7ba4c60ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Qu=E1=BB=91c=20=C4=90=E1=BA=A1t?= Date: Thu, 14 May 2026 14:55:00 +0700 Subject: [PATCH 2/2] fix(plugin-sql-export): emit correct DROP for views and materialized views --- CHANGELOG.md | 1 + Plugins/SQLExportPlugin/SQLExportPlugin.swift | 12 +++++++++++- TablePro/Core/Services/Export/ExportService.swift | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb4c4d2b..18d9f6bcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SQL import dropped statements when the database executed them slower than the file was parsed, so a re-imported export could fail with errors like "relation does not exist". The parser now waits for each statement to be consumed before reading more. (#1264) - SQL import ignored the database dialect, so PostgreSQL dumps with dollar-quoted function bodies were split at semicolons inside the body. (#1264) +- SQL export emitted `DROP TABLE` for views, materialized views, and foreign tables, so re-importing failed with "is not a table". It now emits `DROP VIEW`, `DROP MATERIALIZED VIEW`, or `DROP FOREIGN TABLE` to match the object. (#1264) - iOS: connections, groups, and tags no longer silently disappear after a TestFlight or App Store update. Persistence files are now stored with `.completeFileProtectionUntilFirstUserAuthentication` so they stay readable across background sync runs, load failures are no longer swallowed, and the sync engine refuses to overwrite local data when the load was not actually empty. ### Removed diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index db7f8b484..00a1d515a 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -230,11 +230,21 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin { guard !dropTargets.isEmpty else { return } for table in dropTargets { let tableRef = dataSource.quoteIdentifier(table.name) - try fileHandle.write(contentsOf: "DROP TABLE IF EXISTS \(tableRef) CASCADE;\n".toUTF8Data()) + let keyword = dropStatementKeyword(for: table.tableType) + try fileHandle.write(contentsOf: "\(keyword) IF EXISTS \(tableRef) CASCADE;\n".toUTF8Data()) } try fileHandle.write(contentsOf: "\n".toUTF8Data()) } + private func dropStatementKeyword(for tableType: String) -> String { + switch tableType { + case "view": return "DROP VIEW" + case "materialized view": return "DROP MATERIALIZED VIEW" + case "foreign table": return "DROP FOREIGN TABLE" + default: return "DROP TABLE" + } + } + private func writeDependentTypesAndSequences( tables: [PluginExportTable], dataSource: any PluginExportDataSource, diff --git a/TablePro/Core/Services/Export/ExportService.swift b/TablePro/Core/Services/Export/ExportService.swift index 90c9b095f..4906a6b44 100644 --- a/TablePro/Core/Services/Export/ExportService.swift +++ b/TablePro/Core/Services/Export/ExportService.swift @@ -153,7 +153,7 @@ final class ExportService { PluginExportTable( name: table.name, databaseName: table.databaseName, - tableType: table.type == .view ? "view" : "table", + tableType: table.type.rawValue.lowercased(), optionValues: table.optionValues ) }