From c1b6b66492286d424cc23514c390d9bd05f904b1 Mon Sep 17 00:00:00 2001 From: kimikr <176592950+kimikr@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:05:23 +0000 Subject: [PATCH 1/2] swb: default .strings input encoding to UTF-8 on Linux CopyStringsFile can only auto-detect the encoding of a .strings file on Darwin, so on Linux any .strings or .stringsdict without an explicit text encoding failed with "no --inputencoding specified and could not detect encoding from input file". This includes every table compiled from a String Catalog. Default STRINGS_FILE_INPUT_ENCODING to UTF-8 when neither the project nor the file reference declares an encoding. UTF-16 files without a declared encoding still fail, as before, just with a different message. --- Sources/XToolSupport/BuildCoordinator.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/XToolSupport/BuildCoordinator.swift b/Sources/XToolSupport/BuildCoordinator.swift index 4308bb5f..86761e95 100644 --- a/Sources/XToolSupport/BuildCoordinator.swift +++ b/Sources/XToolSupport/BuildCoordinator.swift @@ -67,6 +67,10 @@ public actor BuildCoordinator { #else // on Linux, host targets (eg macros) should not receive Darwin codesigning flags overrides["CODE_SIGN_IDENTITY[__destination_platform=YES]"] = "-" + // CopyStringsFile can only detect a .strings encoding on Darwin, so files without + // an explicit text encoding (e.g. those compiled from String Catalogs) need one. + // A value set by the project or the file reference still takes precedence. + overrides["STRINGS_FILE_INPUT_ENCODING"] = "$(inherited:default=$(InputFileTextEncoding:default=utf-8))" #endif overrides["AD_HOC_CODE_SIGNING_ALLOWED"] = "YES" // when LINKER_DRIVER=swiftc, LC_BUILD_VERSION.sdk is not set correctly by default. From 7cec035189d2e61c9a6902354c27cc80c306015b Mon Sep 17 00:00:00 2001 From: kimikr <176592950+kimikr@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:05:23 +0000 Subject: [PATCH 2/2] swb: emulate xcstringstool on Linux String Catalogs (.xcstrings) are compiled by xcstringstool, which only ships with Xcode, so on Linux build planning failed with "Could not read xcstrings file" for any target containing one. Add an emulated xcstringstool that supports the invocations made by XCStringsCompilerSpec: - compile [--dry-run] [-l ]... --output-directory writes .lproj/.strings and .stringsdict as XML plists (accepted by CopyStringsFile). Plural variations, substitutions and device variations become stringsdict rules; format strings with substitutions are rewritten to use only positional specifiers. Plural rules without an `other` case (e.g. not translated yet) are left out, since the runtime needs it. Translations in the `new` state are skipped unless --force-build-all-strings is passed; source language strings are always compiled, since their values may come from code. Unlike xcstringstool, the source language always gets a table, so Bundle doesn't pick another language when every source string is `new`. - generate-symbols writes an empty file, since symbol generation isn't supported yet. swift-build enables it for every localized package, but only manually managed strings get symbols, so it only warns when the catalog has some. --- Sources/XToolSupport/EmulatedTools+Live.swift | 1 + .../EmulatedTools+XCStrings.swift | 173 +++++++ .../XToolSupport/StringCatalogCompiler.swift | 381 +++++++++++++++ Tests/XToolTests/XCStringsToolTests.swift | 445 ++++++++++++++++++ 4 files changed, 1000 insertions(+) create mode 100644 Sources/XToolSupport/EmulatedTools+XCStrings.swift create mode 100644 Sources/XToolSupport/StringCatalogCompiler.swift create mode 100644 Tests/XToolTests/XCStringsToolTests.swift diff --git a/Sources/XToolSupport/EmulatedTools+Live.swift b/Sources/XToolSupport/EmulatedTools+Live.swift index 8988d70b..b46276c8 100644 --- a/Sources/XToolSupport/EmulatedTools+Live.swift +++ b/Sources/XToolSupport/EmulatedTools+Live.swift @@ -19,6 +19,7 @@ extension EmulatedTools: DependencyKey { ChmodTool(), CodesignTool(), DerqTool(), + XCStringsTool(), ] #endif return tools diff --git a/Sources/XToolSupport/EmulatedTools+XCStrings.swift b/Sources/XToolSupport/EmulatedTools+XCStrings.swift new file mode 100644 index 00000000..b218f752 --- /dev/null +++ b/Sources/XToolSupport/EmulatedTools+XCStrings.swift @@ -0,0 +1,173 @@ +import Foundation + +/// Emulates `xcstringstool`, which swift-build uses to compile String Catalogs +/// (`.xcstrings`) but which only ships with Xcode. +/// +/// Supports the two invocations swift-build makes (see `XCStringsCompilerSpec`): +/// +/// - `xcstringstool compile [--dry-run] [-l ]... [--force-build-all-strings] +/// --output-directory ` +/// During planning, `--dry-run` prints the absolute paths of the files that will be +/// produced, one per line; the real run writes them. +/// - `xcstringstool generate-symbols --language swift --output-directory ` +/// (with `STRING_CATALOG_GENERATE_SYMBOLS = YES`, the default for localized packages). +/// Symbol generation isn't supported yet, so this writes an empty source file, and a +/// warning if the catalog has manually managed strings (the only ones with symbols). +struct XCStringsTool: EmulatedTool { + var name: String { "xcstringstool" } + + struct UsageError: Error, CustomStringConvertible { + var description: String + } + + func execute( + commandLine: [String], + workingDirectory: String?, + environment: [String: String] + ) async throws -> EmulatedToolResult { + let directory = URL( + fileURLWithPath: workingDirectory ?? FileManager.default.currentDirectoryPath, + isDirectory: true + ) + var input: String? + do { + let arguments = try Arguments(Array(commandLine.dropFirst())) + input = arguments.input + let output = try run(arguments, in: directory) + return .result(status: .exit(0), stdout: Data(output.stdout.utf8), stderr: Data(output.stderr.utf8)) + } catch { + // swift-build scrapes `[: ]error: ` lines from stderr. + let prefix = input.map { "\($0): " } ?? "xcstringstool: " + return .result(status: .exit(1), stdout: Data(), stderr: Data("\(prefix)error: \(error)\n".utf8)) + } + } + + private func run(_ arguments: Arguments, in directory: URL) throws -> (stdout: String, stderr: String) { + let input = URL(fileURLWithPath: arguments.input, relativeTo: directory).standardizedFileURL + let outputDirectory = URL( + fileURLWithPath: arguments.outputDirectory, + isDirectory: true, + relativeTo: directory + ).standardizedFileURL + let tableName = input.deletingPathExtension().lastPathComponent + + switch arguments.command { + case .compile: + var compiler = StringCatalogCompiler( + languages: arguments.languages.isEmpty ? nil : Set(arguments.languages), + forceBuildAllStrings: arguments.forceBuildAllStrings + ) + let outputs = try compiler.compile(catalog: Data(contentsOf: input), tableName: tableName) + let urls = outputs.map { outputDirectory.appending(path: $0.path) } + if arguments.dryRun { + return (urls.map { "\($0.path)\n" }.joined(), "") + } + for (output, url) in zip(outputs, urls) { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try output.contents.write(to: url, options: .atomic) + } + let warnings = compiler.warnings.map { "\(input.path): warning: \($0)\n" }.joined() + return ("", warnings) + + case .generateSymbols: + let url = outputDirectory.appending(path: "GeneratedStringSymbols_\(tableName).swift") + try FileManager.default.createDirectory(at: outputDirectory, withIntermediateDirectories: true) + try Data("// String Catalog symbol generation is not supported by xtool yet.\n".utf8) + .write(to: url, options: .atomic) + // swift-build turns symbol generation on for every localized package, but only + // manually managed strings get symbols, so most catalogs lose nothing. + guard try StringCatalogCompiler.hasManualStrings(catalog: Data(contentsOf: input)) else { + return ("", "") + } + return ("", """ + \(input.path): warning: symbols for manually managed strings are not generated \ + on this platform\n + """) + } + } + + struct Arguments { + enum Command { + case compile + case generateSymbols + } + + var command: Command + var dryRun = false + var forceBuildAllStrings = false + var languages: [String] = [] + var outputDirectory: String + var input: String + + init(_ arguments: [String]) throws { + guard let first = arguments.first else { + throw UsageError(description: "missing subcommand") + } + switch first { + case "compile": command = .compile + case "generate-symbols": command = .generateSymbols + default: throw UsageError(description: "unsupported subcommand '\(first)'") + } + + var options = Options() + var remaining = arguments.dropFirst()[...] + while let argument = remaining.popFirst() { + try options.parse(argument, command: command, remaining: &remaining) + } + dryRun = options.dryRun + forceBuildAllStrings = options.forceBuildAllStrings + languages = options.languages + guard let outputDirectory = options.outputDirectory else { + throw UsageError(description: "missing --output-directory") + } + guard options.inputs.count == 1, let input = options.inputs.first else { + throw UsageError(description: "expected exactly one .xcstrings file, got \(options.inputs.count)") + } + self.outputDirectory = outputDirectory + self.input = input + } + } + + private struct Options { + var dryRun = false + var forceBuildAllStrings = false + var languages: [String] = [] + var outputDirectory: String? + var inputs: [String] = [] + + mutating func parse( + _ argument: String, + command: Arguments.Command, + remaining: inout ArraySlice + ) throws { + func value() throws -> String { + guard let value = remaining.popFirst() else { + throw UsageError(description: "missing value for \(argument)") + } + return value + } + switch argument { + case "--dry-run": + dryRun = true + case "-l": + languages.append(try value()) + case "--language" where command == .generateSymbols: + let language = try value() + guard language == "swift" else { + throw UsageError(description: "unsupported symbol language '\(language)'") + } + case "--output-directory": + outputDirectory = try value() + case "--force-build-all-strings": + forceBuildAllStrings = true + case _ where argument.hasPrefix("-"): + throw UsageError(description: "unsupported option '\(argument)'") + default: + inputs.append(argument) + } + } + } +} diff --git a/Sources/XToolSupport/StringCatalogCompiler.swift b/Sources/XToolSupport/StringCatalogCompiler.swift new file mode 100644 index 00000000..9788df99 --- /dev/null +++ b/Sources/XToolSupport/StringCatalogCompiler.swift @@ -0,0 +1,381 @@ +import Foundation + +/// Compiles a String Catalog (`.xcstrings`) into the `.strings` and `.stringsdict` +/// tables that iOS reads at runtime, like `xcstringstool compile` does. +/// +/// For each language this produces `.lproj/
.strings` with the plain +/// strings and `.lproj/
.stringsdict` with the strings that vary by +/// plural or device, or that use substitutions. Both are written as XML property +/// lists, which swift-build's CopyStringsFile step accepts as input. +struct StringCatalogCompiler { + struct Output: Equatable { + /// Path relative to the output directory, e.g. `ko.lproj/Localizable.strings`. + var path: String + var contents: Data + } + + struct CompileError: Error, CustomStringConvertible { + var description: String + } + + /// Only compile these languages (`-l`). `nil` compiles every language in the catalog. + var languages: Set? + + /// Also compile translations whose state is `new` (`--force-build-all-strings`). These + /// are skipped by default, since they haven't been translated yet (and are often empty). + var forceBuildAllStrings: Bool + + /// Whether `new` strings are compiled for the language being compiled. Source language + /// strings are always `new` unless reviewed, and their values may come from code + /// (`defaultValue:`), so they're compiled too. + private var compilesNewStrings = false + + /// Non-fatal problems found while compiling, e.g. variations that had to be simplified. + private(set) var warnings: [String] = [] + + init(languages: Set? = nil, forceBuildAllStrings: Bool = false) { + self.languages = languages + self.forceBuildAllStrings = forceBuildAllStrings + } + + /// Whether the catalog has strings that symbols would be generated for. Only + /// manually managed strings get symbols; strings extracted from code don't. + static func hasManualStrings(catalog data: Data) throws -> Bool { + try decode(data).strings?.values.contains { $0.extractionState == "manual" } ?? false + } + + private static func decode(_ data: Data) throws -> Catalog { + do { + return try JSONDecoder().decode(Catalog.self, from: data) + } catch { + throw CompileError(description: "The data couldn't be read: \(error)") + } + } + + mutating func compile(catalog data: Data, tableName: String) throws -> [Output] { + let catalog = try Self.decode(data) + + var plain: [String: [String: String]] = [:] + var rich: [String: [String: Any]] = [:] + + for (key, entry) in catalog.strings ?? [:] { + var localizations = entry.localizations ?? [:] + // The source language must always get a table, even if it only maps keys to + // themselves: Bundle picks the best language among the .lproj folders it finds, + // so without it an English user would be shown whichever translation exists. + // (xcstringstool leaves the table out when every source string is `new`.) + if localizations[catalog.sourceLanguage] == nil { + localizations[catalog.sourceLanguage] = Localization( + stringUnit: StringUnit(state: nil, value: key) + ) + } + for (language, localization) in localizations { + if entry.shouldTranslate == false && language != catalog.sourceLanguage { continue } + if let languages, !languages.contains(language) { continue } + try Self.validate(language: language) + compilesNewStrings = forceBuildAllStrings || language == catalog.sourceLanguage + + let (value, item) = entries( + key: key, + localization: localization, + isSourceLanguage: language == catalog.sourceLanguage + ) + if let value { plain[language, default: [:]][key] = value } + if let item { rich[language, default: [:]][key] = item } + } + } + + var outputs: [Output] = [] + for language in Set(plain.keys).union(rich.keys).sorted() { + if let strings = plain[language] { + outputs.append(Output( + path: "\(language).lproj/\(tableName).strings", + contents: try Self.plist(strings) + )) + } + if let stringsdict = rich[language] { + outputs.append(Output( + path: "\(language).lproj/\(tableName).stringsdict", + contents: try Self.plist(stringsdict) + )) + } + } + return outputs + } + + /// The `.strings` value and the `.stringsdict` entry (either may be nil) for a localization. + private mutating func entries( + key: String, + localization: Localization, + isSourceLanguage: Bool + ) -> (value: String?, entry: [String: Any]?) { + guard localization.substitutions == nil, localization.variations == nil else { + return stringsdictEntry(key: key, localization: localization) + } + return (value(of: localization.stringUnit) ?? (isSourceLanguage ? key : nil), nil) + } + + // MARK: - stringsdict + + private static let pluralVariable = "__plural" + + /// The stringsdict entry for a localization with variations or substitutions, and the + /// plain `.strings` value to fall back on, if any. + private mutating func stringsdictEntry( + key: String, + localization: Localization + ) -> (value: String?, entry: [String: Any]?) { + guard let device = localization.variations?.device else { + return (nil, formatEntry(key: key, localization: localization)) + } + // A device variation whose cases are all plain strings maps directly onto + // NSStringDeviceSpecificRuleType. `other` isn't a device there: devices without + // their own case use the .strings value. Nested variations can't be expressed in + // a stringsdict, so fall back to the iPhone (or generic) case. + if device.values.allSatisfy(\.isPlain), localization.substitutions == nil { + var cases = device.compactMapValues { value(of: $0.stringUnit) } + let fallback = cases.removeValue(forKey: "other") + return (fallback, cases.isEmpty ? nil : ["NSStringDeviceSpecificRuleType": cases]) + } + guard let chosen = device["iphone"] ?? device["other"] ?? device.min(by: { $0.key < $1.key })?.value + else { return (nil, nil) } + warnings.append("'\(key)' varies by device and by plural or substitution; only its iPhone variant is compiled") + let substitutions = (localization.substitutions ?? [:]) + .merging(chosen.substitutions ?? [:]) { _, new in new } + return (nil, formatEntry(key: key, localization: Localization( + stringUnit: chosen.stringUnit, + variations: chosen.variations, + substitutions: substitutions.isEmpty ? nil : substitutions + ))) + } + + /// An entry whose whole string varies by plural, or whose string has substitutions. + private mutating func formatEntry(key: String, localization: Localization) -> [String: Any]? { + var item: [String: Any] = [:] + if let plural = localization.variations?.plural { + // The format key is a single variable whose cases are complete strings, + // selected by the first numeric argument. + guard let forms = pluralForms(plural, key: key) else { return nil } + // The count is the first numeric argument of the key (the source format, when it + // is one). Otherwise look through the forms, since one may leave the number out. + let formats = [key, forms["other"]].compactMap { $0 } + forms.sorted { $0.key < $1.key }.map(\.value) + let argument = formats.lazy.compactMap { FormatSpecifier.parse($0).first { $0.isNumeric } }.first + item["NSStringLocalizedFormatKey"] = Self.variableReference( + Self.pluralVariable, + position: argument?.position ?? 1 + ) + item[Self.pluralVariable] = Self.pluralRule(valueType: argument?.valueType ?? "lld", forms: forms) + } else if let value = value(of: localization.stringUnit) { + item["NSStringLocalizedFormatKey"] = value + } else { + return nil + } + + for (name, substitution) in localization.substitutions ?? [:] { + guard let plural = substitution.variations?.plural else { + warnings.append("'\(key)': substitution '\(name)' has no plural variation and was left out") + continue + } + let position = substitution.argNum ?? 1 + let valueType = substitution.formatSpecifier ?? "lld" + // `%arg` stands for the substituted argument itself. + guard let forms = pluralForms(plural, key: key) else { return nil } + item[name] = Self.pluralRule(valueType: valueType, forms: forms.mapValues { + $0.replacingOccurrences(of: "%arg", with: "%\(position)$\(valueType)") + }) + } + if localization.variations?.plural == nil, let substitutions = localization.substitutions, + let format = item["NSStringLocalizedFormatKey"] as? String { + // Bind every argument explicitly: the variables may not appear in argument order, + // and the forms above are positional, which mustn't be mixed with sequential ones. + item["NSStringLocalizedFormatKey"] = FormatSpecifier.positional( + format, + variables: substitutions.mapValues { $0.argNum ?? 1 } + ) + } + return item + } + + /// The language becomes a directory name in the output directory, so it mustn't be a path. + private static func validate(language: String) throws { + guard !language.isEmpty, language.allSatisfy({ $0.isASCII && ($0.isLetter || $0.isNumber || "-_".contains($0)) }) + else { throw CompileError(description: "invalid language identifier '\(language)'") } + } + + /// The cases of a plural variation. A stringsdict can't nest further variations or + /// substitutions inside a plural case, so such cases are left out. Returns nil without + /// an `other` case (e.g. not translated yet), which every plural rule needs. + private mutating func pluralForms(_ plural: [String: Localization], key: String) -> [String: String]? { + if plural.values.contains(where: { !$0.isPlain }) { + warnings.append("'\(key)': plural cases that vary further or have substitutions were left out") + } + let forms = plural.filter { $0.value.isPlain }.compactMapValues { value(of: $0.stringUnit) } + return forms["other"] == nil ? nil : forms + } + + private func value(of unit: StringUnit?) -> String? { + guard let unit, compilesNewStrings || unit.state != "new" else { return nil } + return unit.value + } + + private static func pluralRule(valueType: String, forms: [String: String]) -> [String: String] { + forms.merging([ + "NSStringFormatSpecTypeKey": "NSStringPluralRuleType", + "NSStringFormatValueTypeKey": valueType, + ]) { _, new in new } + } + + private static func variableReference(_ name: String, position: Int) -> String { + position == 1 ? "%#@\(name)@" : "%\(position)$#@\(name)@" + } + + private static func plist(_ object: Any) throws -> Data { + try PropertyListSerialization.data(fromPropertyList: object, format: .xml, options: 0) + } +} + +// MARK: - Format specifiers + +struct FormatSpecifier: Equatable { + /// 1-based argument position, explicit (`%2$d`) or implied by order. + var position: Int + /// Length modifier and conversion, e.g. `lld`, `@`, `f`. + var valueType: String + + var isNumeric: Bool { + guard let conversion = valueType.last else { return false } + return "dDiuUoOxXfFeEgGaA".contains(conversion) + } + + /// The printf-style specifiers in `format`, ignoring `%%` and `%#@variable@`. + static func parse(_ format: String) -> [FormatSpecifier] { + var next = 1 + return tokens(in: Array(format)).compactMap { token in + guard let valueType = token.valueType else { return nil } + let position = token.position ?? next + next = position + 1 + return FormatSpecifier(position: position, valueType: valueType) + } + } + + /// Rewrites `format` so that every specifier names its argument (`%@` → `%1$@`). + /// `%#@variable@` references take the argument in `variables`; the remaining + /// sequential specifiers take the arguments left over, in order. + static func positional(_ format: String, variables: [String: Int]) -> String { + let chars = Array(format) + let tokens = tokens(in: chars) + let explicit = tokens.map { token in token.position ?? token.variable.flatMap { variables[$0] } } + var taken = Set(explicit.compactMap { $0 }) + var next = 1 + var result = "" + var index = 0 + for (token, position) in zip(tokens, explicit) { + let position = position ?? { + while taken.contains(next) { next += 1 } + taken.insert(next) + return next + }() + result += String(chars[index.. + /// Set for `%#@variable@` references. + var variable: String? + /// Set for ordinary specifiers: length modifier and conversion. + var valueType: String? + } + + private static func tokens(in chars: [Character]) -> [Token] { + func skip(_ j: inout Int, while predicate: (Character) -> Bool) { + while j < chars.count, predicate(chars[j]) { j += 1 } + } + let isDigit: (Character) -> Bool = { $0.isASCII && $0.isNumber } + var tokens: [Token] = [] + var index = 0 + while index < chars.count { + guard chars[index] == "%" else { index += 1; continue } + let start = index + var j = index + 1 + guard j < chars.count else { break } + if chars[j] == "%" { index = j + 1; continue } + + var position: Int? + var k = j + skip(&k, while: isDigit) + if k > j, k < chars.count, chars[k] == "$" { + position = Int(String(chars[j.. bodyStart, chars[j - 1] == "#" { + // %#@variable@ + guard let end = chars[(j + 1)...].firstIndex(of: "@") else { index = j + 1; continue } + let variable = String(chars[(j + 1)..? = nil, + forceBuildAllStrings: Bool = false + ) throws -> [String: Data] { + var compiler = StringCatalogCompiler(languages: languages, forceBuildAllStrings: forceBuildAllStrings) + let outputs = try compiler.compile(catalog: Data(json.utf8), tableName: "Localizable") + return Dictionary(uniqueKeysWithValues: outputs.map { ($0.path, $0.contents) }) + } + + private func dictionary(_ data: Data?) throws -> [String: Any] { + let data = try #require(data) + return try #require(try PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any]) + } + + private func strings(_ data: Data?) throws -> [String: String] { + try #require(try dictionary(data) as? [String: String]) + } +}