From a83218ca64d07523c78325c64dac18976c8f00dd Mon Sep 17 00:00:00 2001 From: Etherlink Intern Date: Tue, 2 Jun 2026 19:26:54 +0800 Subject: [PATCH] Document transitive dependency license policy --- Docs/ImportButtonIntegration.md | 422 ++++++++++++++++++ Docs/NativeConverterBackends.md | 67 +++ LICENSE | 21 + README.md | 50 ++- .../Converters/ImageOCRConverter.swift | 63 +++ Sources/SwiftMarkItDown/DocumentFormat.swift | 19 + Sources/SwiftMarkItDown/MarkItDown.swift | 3 +- .../SwiftMarkItDownTests.swift | 110 +++++ 8 files changed, 744 insertions(+), 11 deletions(-) create mode 100644 Docs/ImportButtonIntegration.md create mode 100644 Docs/NativeConverterBackends.md create mode 100644 LICENSE create mode 100644 Sources/SwiftMarkItDown/Converters/ImageOCRConverter.swift diff --git a/Docs/ImportButtonIntegration.md b/Docs/ImportButtonIntegration.md new file mode 100644 index 0000000..92af9c9 --- /dev/null +++ b/Docs/ImportButtonIntegration.md @@ -0,0 +1,422 @@ +# Integrating SwiftMarkItDown with Import Buttons and Share Sheet Imports + +Use this guide when an iOS, iPadOS, or macOS app wants to accept user-selected documents or images and convert them to Markdown with `SwiftMarkItDown`. The same conversion core works for: + +- an in-app **Import** button backed by the system document picker, +- photo-library OCR imports, +- inbound iOS/iPadOS Share Sheet actions through a Share Extension, +- and drag/drop or other app-specific import surfaces that can produce `Data` plus filename or content-type hints. + +The library exposes a small synchronous API: + +```swift +let request = ConversionRequest( + data: data, + fileName: fileName, + contentType: contentType +) +let document = try MarkItDown().convert(request) +let markdown = document.markdown +``` + +Image OCR is automatic for supported image formats when the app is running on Apple platforms with Vision, CoreGraphics, and ImageIO available. On platforms without those frameworks, image formats are still recognized but conversion returns `unsupportedFormat`. + +## Supported import inputs + +The default converter pipeline can handle these inputs from an import button or Share Extension: + +| Category | Extensions | MIME / UTType examples | Notes | +| --- | --- | --- | --- | +| Plain text | `txt`, `text` | `text/plain`, `.plainText`, `.text`, `.utf8PlainText` | Decoded as text and cleaned up. | +| Markdown | `md`, `markdown` | `text/markdown`, `text/x-markdown` | Treated as text-like input and cleaned up. | +| HTML | `html`, `htm` | `text/html`, `application/xhtml+xml`, `.html` | Converted to Markdown for common tags. | +| CSV | `csv` | `text/csv`, `application/csv`, `.commaSeparatedText` | Converted to GitHub-Flavored Markdown tables. | +| JSON | `json` | `application/json`, `text/json`, `.json` | Converted to nested Markdown bullets. | +| Images | `png`, `jpg`, `jpeg`, `heic`, `heif`, `tif`, `tiff`, `gif` | `image/png`, `image/jpeg`, `image/heic`, `image/heif`, `image/tiff`, `image/gif`, `.image` | Uses Apple Vision OCR where available. GIF inputs are decoded as an image source; OCR is performed on the decoded first image. | + +PDF, DOCX, PPTX, and XLSX are recognized by `DocumentFormat`, but still return `unsupportedFormat` until their converter modules are implemented. + +## Reusable conversion helper + +Use a small helper to keep file access, content type inference, and background conversion out of your SwiftUI views and Share Extension controllers. + +```swift +import Foundation +import SwiftMarkItDown +import UniformTypeIdentifiers + +enum SwiftMarkItDownImporter { + static func convertData( + _ data: Data, + fileName: String? = nil, + contentType: String? = nil, + formatHint: DocumentFormat? = nil + ) async throws -> MarkdownDocument { + let request = ConversionRequest( + data: data, + fileName: fileName, + contentType: contentType, + formatHint: formatHint + ) + + return try await Task.detached(priority: .userInitiated) { + try MarkItDown().convert(request) + }.value + } + + static func convertFile(_ url: URL) async throws -> MarkdownDocument { + try await Task.detached(priority: .userInitiated) { + let didStartAccessing = url.startAccessingSecurityScopedResource() + defer { + if didStartAccessing { + url.stopAccessingSecurityScopedResource() + } + } + + let data = try Data(contentsOf: url) + let contentType = UTType(filenameExtension: url.pathExtension)?.preferredMIMEType + + let request = ConversionRequest( + data: data, + fileName: url.lastPathComponent, + contentType: contentType + ) + return try MarkItDown().convert(request) + }.value + } +} +``` + +The security-scoped-resource calls are important for URLs returned by the document picker and for some URLs delivered by extensions. + +## SwiftUI document import button + +For most apps, start with SwiftUI's `fileImporter`. It opens the system document picker, reads the selected file into `Data`, and hands that payload to `SwiftMarkItDown`. + +```swift +import SwiftMarkItDown +import SwiftUI +import UniformTypeIdentifiers + +struct ImportMarkdownButton: View { + @State private var isImporting = false + @State private var markdown = "" + @State private var errorMessage: String? + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + Button("Import") { + isImporting = true + } + + if let errorMessage { + Text(errorMessage) + .foregroundStyle(.red) + } + + ScrollView { + Text(markdown) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) + } + } + .fileImporter( + isPresented: $isImporting, + allowedContentTypes: SwiftMarkItDownImportTypes.allowedDocumentTypes, + allowsMultipleSelection: false + ) { result in + Task { + await importSelection(result) + } + } + } + + @MainActor + private func importSelection(_ result: Result<[URL], Error>) async { + do { + guard let url = try result.get().first else { return } + let converted = try await SwiftMarkItDownImporter.convertFile(url) + markdown = converted.markdown + errorMessage = nil + } catch { + errorMessage = error.localizedDescription + } + } +} +``` + +## Allowed document types + +Expose the formats your app wants the system picker to show. This list includes current converters plus image OCR inputs. + +```swift +import UniformTypeIdentifiers + +enum SwiftMarkItDownImportTypes { + static let allowedDocumentTypes: [UTType] = [ + .plainText, + .text, + .utf8PlainText, + .html, + .commaSeparatedText, + .json, + .png, + .jpeg, + .heic, + .tiff, + .gif + ] +} +``` + +If your app wants to display future/reserved file types in the picker, add their UTTypes and handle `ConversionError.unsupportedFormat` in the error UI. + +## Photo-library import button + +If your app's import action should specifically pick a photo instead of opening the document picker, use `PhotosPicker` and pass the selected image data to `SwiftMarkItDown` with an image filename or format hint. + +```swift +import PhotosUI +import SwiftMarkItDown +import SwiftUI + +struct PhotoOCRImportButton: View { + @State private var selectedItem: PhotosPickerItem? + @State private var markdown = "" + @State private var errorMessage: String? + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + PhotosPicker("Import Photo", selection: $selectedItem, matching: .images) + + if let errorMessage { + Text(errorMessage) + .foregroundStyle(.red) + } + + Text(markdown) + .textSelection(.enabled) + } + .onChange(of: selectedItem) { _, item in + Task { + await importPhoto(item) + } + } + } + + @MainActor + private func importPhoto(_ item: PhotosPickerItem?) async { + do { + guard let data = try await item?.loadTransferable(type: Data.self) else { return } + let document = try await SwiftMarkItDownImporter.convertData( + data, + fileName: "photo.jpeg", + contentType: "image/jpeg" + ) + + markdown = document.markdown + errorMessage = nil + } catch { + errorMessage = error.localizedDescription + } + } +} +``` + +## Inbound Share Sheet import + +To let users send documents or images into your app from Files, Photos, Mail, Safari, or another app's Share Sheet, add an iOS/iPadOS **Share Extension** target to the consuming app. The extension receives one or more `NSItemProvider` attachments, loads supported file or data representations, converts each attachment with `SwiftMarkItDown`, then stores or forwards the Markdown to the containing app. Link the Share Extension target against `SwiftMarkItDown` the same way you link the main app target, and use an app group if the extension needs to hand converted Markdown back to the containing app. + +A typical Share Extension flow is: + +1. Configure the extension activation rule for text, web content, files, and images. +2. Iterate through `extensionContext.inputItems`. +3. Prefer `loadFileRepresentation(forTypeIdentifier:)` for document-like attachments so you can preserve filenames and file extensions. +4. Fall back to `loadDataRepresentation(forTypeIdentifier:)` for in-memory text/image payloads. +5. Convert each payload off the main actor. +6. Save the Markdown to an app-group container, post it to your app backend, or open the containing app with a URL scheme/deep link. + +### Share Extension activation rule + +In the Share Extension target's `Info.plist`, allow the same categories that `SwiftMarkItDown` can convert today. Keep the rule as narrow as your product needs. + +```xml +NSExtension + + NSExtensionPointIdentifier + com.apple.share-services + NSExtensionAttributes + + NSExtensionActivationRule + + NSExtensionActivationSupportsText + + NSExtensionActivationSupportsWebURLWithMaxCount + 1 + NSExtensionActivationSupportsWebPageWithMaxCount + 1 + NSExtensionActivationSupportsFileWithMaxCount + 10 + NSExtensionActivationSupportsImageWithMaxCount + 10 + + + +``` + +### Share Extension conversion example + +The exact UI is up to the host app, but the import core can be isolated in a coordinator like this: + +```swift +import Foundation +import SwiftMarkItDown +import UniformTypeIdentifiers + +final class ShareSheetImportCoordinator { + private let supportedTypes: [UTType] = [ + .plainText, + .text, + .utf8PlainText, + .html, + .commaSeparatedText, + .json, + .png, + .jpeg, + .heic, + .tiff, + .gif, + .image, + .fileURL + ] + + func convertSharedItems(from extensionContext: NSExtensionContext) async -> [Result] { + let providers = extensionContext.inputItems + .compactMap { $0 as? NSExtensionItem } + .flatMap { $0.attachments ?? [] } + + return await withTaskGroup(of: Result.self) { group in + for provider in providers { + group.addTask { + do { + return .success(try await self.convert(provider)) + } catch { + return .failure(error) + } + } + } + + var results: [Result] = [] + for await result in group { + results.append(result) + } + return results + } + } + + private func convert(_ provider: NSItemProvider) async throws -> MarkdownDocument { + guard let type = supportedTypes.first(where: { provider.hasItemConformingToTypeIdentifier($0.identifier) }) else { + throw ConversionError.unsupportedFormat(.unknown) + } + + if type == .fileURL { + let url = try await provider.loadURL(typeIdentifier: type.identifier) + return try await SwiftMarkItDownImporter.convertFile(url) + } + + if let url = try? await provider.loadFile(typeIdentifier: type.identifier) { + return try await SwiftMarkItDownImporter.convertFile(url) + } + + let data = try await provider.loadData(typeIdentifier: type.identifier) + return try await SwiftMarkItDownImporter.convertData( + data, + fileName: suggestedFileName(for: provider, type: type), + contentType: type.preferredMIMEType + ) + } + + private func suggestedFileName(for provider: NSItemProvider, type: UTType) -> String? { + guard let extensionName = type.preferredFilenameExtension else { + return provider.suggestedName + } + + let baseName = provider.suggestedName ?? "shared-item" + if baseName.lowercased().hasSuffix(".\(extensionName.lowercased())") { + return baseName + } + return "\(baseName).\(extensionName)" + } +} + +private extension NSItemProvider { + func loadURL(typeIdentifier: String) async throws -> URL { + try await withCheckedThrowingContinuation { continuation in + loadItem(forTypeIdentifier: typeIdentifier, options: nil) { item, error in + if let error { + continuation.resume(throwing: error) + } else if let url = item as? URL { + continuation.resume(returning: url) + } else if let data = item as? Data, + let url = URL(dataRepresentation: data, relativeTo: nil) { + continuation.resume(returning: url) + } else { + continuation.resume(throwing: ConversionError.malformedInput("The shared file URL could not be loaded.")) + } + } + } + } + + func loadFile(typeIdentifier: String) async throws -> URL { + try await withCheckedThrowingContinuation { continuation in + loadFileRepresentation(forTypeIdentifier: typeIdentifier) { url, error in + if let error { + continuation.resume(throwing: error) + } else if let url { + continuation.resume(returning: url) + } else { + continuation.resume(throwing: ConversionError.malformedInput("The shared file could not be loaded.")) + } + } + } + } + + func loadData(typeIdentifier: String) async throws -> Data { + try await withCheckedThrowingContinuation { continuation in + loadDataRepresentation(forTypeIdentifier: typeIdentifier) { data, error in + if let error { + continuation.resume(throwing: error) + } else if let data { + continuation.resume(returning: data) + } else { + continuation.resume(throwing: ConversionError.malformedInput("The shared data could not be loaded.")) + } + } + } + } +} +``` + +`loadFileRepresentation` can hand your extension a temporary file URL. If you need the original data after the completion handler returns, copy that file into your extension's temporary directory or an app-group container before returning from the callback. + +## Error handling recommendations + +Handle these cases in the app's import UI or Share Extension UI: + +- `unsupportedFormat`: show a friendly message that the selected file type is recognized but not implemented on this platform or in this release. +- `malformedInput`: show a message that the file could not be decoded, parsed, or recognized. +- Empty OCR output: keep the import successful but tell the user that no readable text was detected. +- Large files or images: run conversion off the main actor, as shown above, and show progress or a spinner. +- Multiple shared attachments: convert each attachment independently, and present partial successes instead of failing the entire share operation. + +## Where to send the Markdown + +After `MarkItDown().convert(...)` returns, use `document.markdown` wherever your app already stores imported content: + +- populate an editor buffer, +- attach it to a note, +- save it to a local Markdown file, +- write it into an app-group container for the containing app, +- open the containing app with a deep link that references the converted item, +- send it to a share sheet, +- or pass it into your app's search/indexing pipeline. diff --git a/Docs/NativeConverterBackends.md b/Docs/NativeConverterBackends.md new file mode 100644 index 0000000..1f1aefe --- /dev/null +++ b/Docs/NativeConverterBackends.md @@ -0,0 +1,67 @@ +# Native and FOSS Converter Backend Plan + +This document records the likely native/FOSS backends for the reserved document formats in `DocumentFormat`. These are **not package dependencies yet**; the current release still returns `unsupportedFormat` for PDF, DOCX, PPTX, and XLSX. The goal is to keep the app's import UI honest while making the implementation path explicit. + +## Short answer: are these simple to add? + +They are straightforward to integrate as Swift Package / Apple-framework building blocks, but they are not all equally "drop-in" as Markdown converters: + +| Format | Proposed backend | Integration effort | Why | +| --- | --- | --- | --- | +| PDF | Apple's PDFKit | Low for embedded text; medium when OCR fallback is included. | PDFKit is built into Apple platforms and can extract text from many PDFs, but scanned/image-only PDFs still need page rendering plus Vision OCR. | +| DOCX | ZIPFoundation + OOXML parsing | Medium. | DOCX is a ZIP of XML parts, but useful Markdown needs document body parsing, relationships, styles, numbering, tables, hyperlinks, and images. | +| PPTX | ZIPFoundation + OOXML parsing | Medium-high. | PPTX uses the same OpenXML ZIP structure, but slide ordering, shapes, notes, and layout-driven reading order make Markdown extraction more involved than DOCX. | +| XLSX | CoreXLSX + its transitive ZIPFoundation/XMLCoder graph | Medium-low for worksheet tables; medium for richer workbooks. | CoreXLSX already parses XLSX structure in Swift, but Markdown output still needs shared strings, sheet selection, empty-cell handling, formulas, merged cells, and table shaping decisions. Its dependency graph must be acknowledged alongside the direct package. | + +## Backend source acknowledgements + +When one of these backends is implemented, the PR that adds it should also add the dependency/framework acknowledgement to this table and to any required license notice files. + +| Backend | Source | License / status | Intended use | +| --- | --- | --- | --- | +| Apple PDFKit | | Apple system framework; no SwiftPM dependency. | PDF text extraction and optional page rendering for OCR fallback on Apple platforms. | +| ZIPFoundation | | MIT-licensed Swift package. | ZIP container access for DOCX and PPTX OpenXML parts. | +| CoreXLSX | | Apache-2.0-licensed Swift package. | Read-only parsing of XLSX workbooks and worksheets. | +| XMLCoder | | MIT-licensed Swift package; transitive dependency of CoreXLSX. | XML decoding support used by CoreXLSX when mapping XLSX XML parts into Swift models. | +| Office Open XML structure | | Published standard. | Format reference for DOCX/PPTX/XLSX XML parts and relationships. | + +> Verification note: as of the current dependency review, CoreXLSX declares both XMLCoder and ZIPFoundation as SwiftPM dependencies. That means an XLSX converter PR must include acknowledgements for CoreXLSX itself **and** for each transitive dependency that ships in the resolved dependency graph. + +## Recommended implementation order + +1. **PDF text extraction with PDFKit** + - Add a `PDFConverter` behind `#if canImport(PDFKit)`. + - Extract embedded page text first. + - If a page has no embedded text, optionally render the page and reuse the Vision OCR path already used by image ingestion. + - Keep non-Apple platforms returning `unsupportedFormat` unless a separate cross-platform PDF backend is added. + +2. **Shared OpenXML ZIP infrastructure** + - Add ZIPFoundation as a SwiftPM dependency only when DOCX or PPTX work starts. + - Build a small internal helper for reading XML parts, relationships, content types, and document metadata from OpenXML packages. + - Use this helper for DOCX first, then PPTX. + +3. **DOCX converter** + - Parse `word/document.xml` in document order. + - Resolve relationships for hyperlinks and images. + - Map paragraphs, headings, lists, tables, emphasis, and links to Markdown. + - Add fixtures that cover common Word exports rather than only hand-written XML. + +4. **PPTX converter** + - Parse slide order from `ppt/presentation.xml` and slide relationship parts. + - Extract text from shapes, grouped shapes, speaker notes, and tables. + - Use simple slide-section Markdown first; improve layout ordering later. + +5. **XLSX converter with CoreXLSX** + - Add CoreXLSX as a SwiftPM dependency when XLSX implementation begins. + - Re-run SwiftPM dependency resolution and record the full resolved graph, including CoreXLSX transitive packages such as XMLCoder and ZIPFoundation. + - Convert each selected worksheet to Markdown tables. + - Decide how to handle formulas, empty rows/columns, merged cells, dates, and multiple sheets. + +## Dependency policy + +- Do not add ZIPFoundation or CoreXLSX to `Package.swift` until a converter actually uses them. +- Prefer conditional compilation for Apple-only frameworks such as PDFKit and Vision. +- Keep unsupported formats visible in `DocumentFormat` so apps can show useful import affordances and friendly `unsupportedFormat` errors. +- Add license acknowledgements in the same PR that introduces any FOSS dependency. +- Verify and acknowledge the **full resolved SwiftPM dependency graph**, not just direct dependencies. For example, a CoreXLSX-based converter must also account for its transitive XMLCoder and ZIPFoundation packages. +- Refresh dependency acknowledgements whenever `Package.resolved` changes so newly added, removed, or upgraded transitive packages are not missed. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..556c6c2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 SwiftMarkItDown contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index bb463a5..721c9d5 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,37 @@ SwiftMarkItDown is the start of a native Swift/iOS document-to-Markdown pipeline ## What works today -The current MVP is intentionally small and deterministic so it can run fully on-device: +The current MVP is intentionally small and deterministic. These formats have converters in the default `MarkItDown` pipeline: + +| Input family | Extensions / aliases | Content-type hints | Conversion behavior | Platform availability | +| --- | --- | --- | --- | --- | +| Plain text | `txt`, `text` | `text/plain` | Decodes text and normalizes blank lines. | All package platforms. | +| Markdown | `md`, `markdown` | `text/markdown`, `text/x-markdown` | Treats Markdown as text-like input and normalizes blank lines. | All package platforms. | +| HTML | `html`, `htm` | `text/html`, `application/xhtml+xml` | Converts common headings, inline emphasis, links, code, paragraphs, and list items; ignores document ``, script, and style content. | All package platforms. | +| CSV | `csv` | `text/csv`, `application/csv` | Converts rows to GitHub-Flavored Markdown tables, including quoted fields and escaped pipes. | All package platforms. | +| JSON | `json` | `application/json`, `text/json` | Converts objects and arrays to nested Markdown bullets with stable key ordering. | All package platforms. | +| Images | `png`, `jpg`, `jpeg`, `heic`, `heif`, `tif`, `tiff`, `gif` | `image/png`, `image/jpeg`, `image/heic`, `image/heif`, `image/tiff`, `image/gif` | Uses Apple Vision OCR and returns recognized text lines as Markdown text. GIF OCR uses the decoded first image. | Apple platforms that provide Vision, CoreGraphics, and ImageIO. Other platforms recognize the formats but return `unsupportedFormat`. | + +The package also includes: -- `txt` and `md` passthrough with text decoding and blank-line cleanup. -- `html` to Markdown for common headings, inline emphasis, links, code, paragraphs, and list items, with document ``, script, and style content ignored. -- `csv` to GitHub-Flavored Markdown tables, including quoted fields and escaped pipes. -- `json` to nested Markdown bullets with stable key ordering. - A CLI wrapper for local/manual conversion checks. - A SwiftUI iOS demo app for editing sample input and converting it to Markdown in the simulator. -PDF, DOCX, PPTX, and XLSX are represented in the format model but still return `unsupportedFormat` until their native converter modules are implemented. +PDF, DOCX, PPTX, and XLSX are represented in the format model for future native converters, but they still return `unsupportedFormat` today. + + +## Planned native/FOSS converter backends + +PDF, DOCX, PPTX, and XLSX are intentionally reserved in `DocumentFormat`, but they are not converter-backed yet. The planned backend direction is: + +| Reserved format | Planned backend | Current status | +| --- | --- | --- | +| PDF | Apple's PDFKit, with optional Vision OCR fallback for scanned pages | Recognized, returns `unsupportedFormat`. | +| DOCX | ZIPFoundation plus targeted Office Open XML parsing | Recognized, returns `unsupportedFormat`. | +| PPTX | ZIPFoundation plus targeted Office Open XML parsing | Recognized, returns `unsupportedFormat`. | +| XLSX | CoreXLSX for workbook parsing, with its resolved transitive dependencies acknowledged too | Recognized, returns `unsupportedFormat`. | + +See [Docs/NativeConverterBackends.md](Docs/NativeConverterBackends.md) for the implementation plan, integration effort, source links, and acknowledgement policy. That policy requires checking the full SwiftPM dependency graph, not just direct packages; for example, a future CoreXLSX converter must also account for transitive dependencies such as XMLCoder and ZIPFoundation. ## Repository layout @@ -44,6 +65,10 @@ let document = try MarkItDown().convert(request) print(document.markdown) ``` +## App import integration + +See [Docs/ImportButtonIntegration.md](Docs/ImportButtonIntegration.md) for SwiftUI `fileImporter`, `PhotosPicker`, and inbound Share Extension examples that wire an app's import surfaces into `SwiftMarkItDown`, including image OCR inputs. + ## CLI usage ```bash @@ -61,10 +86,15 @@ Scripts/smoke-test.sh GitHub Actions runs the same checks on pushes to `main`, pull requests, and manual workflow dispatches. +## License + +SwiftMarkItDown is available under the [MIT License](LICENSE). + ## Roadmap 1. Expand the text/HTML/CSV/JSON converters with richer Markdown normalization and metadata extraction. -2. Add a ZIP/OpenXML package reader as shared infrastructure for DOCX, PPTX, and XLSX. -3. Implement DOCX paragraph, heading, table, hyperlink, and image-reference extraction. -4. Add PDFKit/Vision-backed PDF text and OCR extraction for Apple platforms behind conditional compilation. -5. Evolve the demo into a more complete iOS MVP with document picker import, share/export flows, progress reporting, and a pluggable backend escape hatch for heavyweight conversions. +2. Improve OCR layout reconstruction for headings, lists, tables, and multi-column scans. +3. Add PDFKit-backed PDF text extraction, with Vision OCR fallback for scanned pages on Apple platforms. +4. Add ZIPFoundation-backed OpenXML package infrastructure for DOCX and PPTX. +5. Add DOCX/PPTX converters on top of targeted Office Open XML parsing, then add XLSX conversion with CoreXLSX. +6. Evolve the demo into a more complete iOS MVP with document picker import, inbound Share Extension handling, share/export flows, progress reporting, and a pluggable backend escape hatch for heavyweight conversions. diff --git a/Sources/SwiftMarkItDown/Converters/ImageOCRConverter.swift b/Sources/SwiftMarkItDown/Converters/ImageOCRConverter.swift new file mode 100644 index 0000000..bc9e566 --- /dev/null +++ b/Sources/SwiftMarkItDown/Converters/ImageOCRConverter.swift @@ -0,0 +1,63 @@ +import Foundation + +#if canImport(Vision) && canImport(CoreGraphics) && canImport(ImageIO) +import CoreGraphics +import ImageIO +import Vision +#endif + +/// Uses Apple Vision text recognition to extract Markdown-ready text from images. +public struct ImageOCRConverter: DocumentConverter { + public let supportedFormats: Set = DocumentFormat.imageFormats + + public init() {} + + public func convert(_ request: ConversionRequest, format: DocumentFormat) throws -> MarkdownDocument { + #if canImport(Vision) && canImport(CoreGraphics) && canImport(ImageIO) + guard let imageSource = CGImageSourceCreateWithData(request.data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(imageSource, 0, nil) else { + throw ConversionError.malformedInput("The input could not be decoded as an image.") + } + + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + request.minimumTextHeight = 0.01 + + let handler = VNImageRequestHandler(cgImage: image, options: [:]) + try handler.perform([request]) + + let lines = (request.results ?? []) + .compactMap { observation -> RecognizedTextLine? in + guard let candidate = observation.topCandidates(1).first else { return nil } + let text = candidate.string.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return nil } + return RecognizedTextLine(text: text, bounds: observation.boundingBox, confidence: candidate.confidence) + } + .sorted { lhs, rhs in + let verticalDelta = abs(lhs.bounds.midY - rhs.bounds.midY) + if verticalDelta > 0.02 { + return lhs.bounds.midY > rhs.bounds.midY + } + return lhs.bounds.minX < rhs.bounds.minX + } + + let markdown = lines.map(\.text).joined(separator: "\n").smid_trimmedBlankLines + return MarkdownDocument( + markdown: markdown, + sourceFormat: format, + metadata: ["recognizedTextLineCount": String(lines.count)] + ) + #else + throw ConversionError.unsupportedFormat(format) + #endif + } +} + +#if canImport(Vision) && canImport(CoreGraphics) && canImport(ImageIO) +private struct RecognizedTextLine { + let text: String + let bounds: CGRect + let confidence: VNConfidence +} +#endif diff --git a/Sources/SwiftMarkItDown/DocumentFormat.swift b/Sources/SwiftMarkItDown/DocumentFormat.swift index 7c9eb98..d84ce06 100644 --- a/Sources/SwiftMarkItDown/DocumentFormat.swift +++ b/Sources/SwiftMarkItDown/DocumentFormat.swift @@ -7,6 +7,11 @@ public enum DocumentFormat: String, CaseIterable, Sendable { case html = "html" case csv = "csv" case json = "json" + case png = "png" + case jpeg = "jpg" + case heic = "heic" + case tiff = "tiff" + case gif = "gif" case pdf = "pdf" case docx = "docx" case pptx = "pptx" @@ -29,6 +34,11 @@ public enum DocumentFormat: String, CaseIterable, Sendable { case "htm", "html": return .html case "csv": return .csv case "json": return .json + case "png": return .png + case "jpg", "jpeg": return .jpeg + case "heic", "heif": return .heic + case "tif", "tiff": return .tiff + case "gif": return .gif case "pdf": return .pdf case "docx": return .docx case "pptx": return .pptx @@ -44,6 +54,11 @@ public enum DocumentFormat: String, CaseIterable, Sendable { case "text/html", "application/xhtml+xml": .html case "text/csv", "application/csv": .csv case "application/json", "text/json": .json + case "image/png": .png + case "image/jpeg", "image/jpg": .jpeg + case "image/heic", "image/heif": .heic + case "image/tiff": .tiff + case "image/gif": .gif case "application/pdf": .pdf case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": .docx case "application/vnd.openxmlformats-officedocument.presentationml.presentation": .pptx @@ -52,3 +67,7 @@ public enum DocumentFormat: String, CaseIterable, Sendable { } } } + +public extension DocumentFormat { + static let imageFormats: Set = [.png, .jpeg, .heic, .tiff, .gif] +} diff --git a/Sources/SwiftMarkItDown/MarkItDown.swift b/Sources/SwiftMarkItDown/MarkItDown.swift index 8047697..d38f208 100644 --- a/Sources/SwiftMarkItDown/MarkItDown.swift +++ b/Sources/SwiftMarkItDown/MarkItDown.swift @@ -13,7 +13,8 @@ public struct MarkItDown: Sendable { PlainTextConverter(), HTMLConverter(), CSVConverter(), - JSONConverter() + JSONConverter(), + ImageOCRConverter() ] } diff --git a/Tests/SwiftMarkItDownTests/SwiftMarkItDownTests.swift b/Tests/SwiftMarkItDownTests/SwiftMarkItDownTests.swift index fec71d2..bd4cecf 100644 --- a/Tests/SwiftMarkItDownTests/SwiftMarkItDownTests.swift +++ b/Tests/SwiftMarkItDownTests/SwiftMarkItDownTests.swift @@ -2,12 +2,30 @@ import Foundation import Testing @testable import SwiftMarkItDown +#if canImport(Vision) && canImport(CoreGraphics) && canImport(CoreText) && canImport(ImageIO) +import CoreGraphics +import CoreText +import ImageIO +import Vision +#endif + @Suite("SwiftMarkItDown core conversions") struct SwiftMarkItDownTests { @Test("infers format from extension") func infersFormatFromExtension() { #expect(DocumentFormat.infer(fileName: "report.csv", contentType: nil) == .csv) #expect(DocumentFormat.infer(fileName: "deck.pptx", contentType: nil) == .pptx) + #expect(DocumentFormat.infer(fileName: "scan.png", contentType: nil) == .png) + #expect(DocumentFormat.infer(fileName: "receipt.jpeg", contentType: nil) == .jpeg) + #expect(DocumentFormat.infer(fileName: "photo.HEIC", contentType: nil) == .heic) + } + + @Test("infers image formats from content type") + func infersImageFormatsFromContentType() { + #expect(DocumentFormat.infer(fileName: nil, contentType: "image/png") == .png) + #expect(DocumentFormat.infer(fileName: nil, contentType: "image/jpeg; charset=binary") == .jpeg) + #expect(DocumentFormat.infer(fileName: nil, contentType: "image/heic") == .heic) + #expect(DocumentFormat.infer(fileName: nil, contentType: "image/tiff") == .tiff) } @Test("converts plain text without surrounding blank lines") @@ -42,6 +60,31 @@ struct SwiftMarkItDownTests { #expect(document.markdown == "- **formats**:\n - txt\n - html\n- **title**: Roadmap") } + #if canImport(Vision) && canImport(CoreGraphics) && canImport(CoreText) && canImport(ImageIO) + @Test("uses Vision OCR to convert rendered images to Markdown") + func convertsRenderedImagesWithVisionOCR() throws { + for sample in try renderedOCRSamples() { + let request = ConversionRequest(data: sample.data, fileName: sample.fileName) + let document = try MarkItDown().convert(request) + let normalized = document.markdown.uppercased() + + #expect(document.sourceFormat == sample.format) + #expect(normalized.contains("SWIFT")) + #expect(normalized.contains("OCR")) + #expect(normalized.contains("MARKDOWN")) + #expect(document.metadata["recognizedTextLineCount"] != "0") + } + } + #else + @Test("throws unsupported for images when Vision OCR is unavailable") + func throwsForImagesWhenVisionOCRIsUnavailable() throws { + let request = ConversionRequest(data: Data(), fileName: "scan.png") + #expect(throws: ConversionError.unsupportedFormat(.png)) { + try MarkItDown().convert(request) + } + } + #endif + @Test("throws for reserved but unimplemented formats") func throwsForUnimplementedFormats() throws { let request = ConversionRequest(data: Data(), fileName: "paper.pdf") @@ -50,3 +93,70 @@ struct SwiftMarkItDownTests { } } } + +#if canImport(Vision) && canImport(CoreGraphics) && canImport(CoreText) && canImport(ImageIO) +private struct OCRSample { + let fileName: String + let format: DocumentFormat + let data: Data +} + +private func renderedOCRSamples() throws -> [OCRSample] { + [ + OCRSample(fileName: "ocr-sample.png", format: .png, data: try renderOCRImage(typeIdentifier: "public.png" as CFString)), + OCRSample(fileName: "ocr-sample.jpg", format: .jpeg, data: try renderOCRImage(typeIdentifier: "public.jpeg" as CFString)) + ] +} + +private func renderOCRImage(typeIdentifier: CFString) throws -> Data { + let width = 1_200 + let height = 520 + let colorSpace = CGColorSpaceCreateDeviceRGB() + guard let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + throw ConversionError.malformedInput("Could not create OCR test image context.") + } + + context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + context.setAllowsAntialiasing(true) + context.setShouldAntialias(true) + + let font = CTFontCreateWithName("Helvetica-Bold" as CFString, 112, nil) + let attributes: [NSAttributedString.Key: Any] = [ + NSAttributedString.Key(kCTFontAttributeName as String): font, + NSAttributedString.Key(kCTForegroundColorAttributeName as String): CGColor(red: 0, green: 0, blue: 0, alpha: 1) + ] + + for (index, line) in ["SWIFT OCR", "MARKDOWN"].enumerated() { + let attributed = NSAttributedString(string: line, attributes: attributes) + let textLine = CTLineCreateWithAttributedString(attributed) + context.textPosition = CGPoint(x: 80, y: height - 170 - (index * 150)) + CTLineDraw(textLine, context) + } + + guard let image = context.makeImage() else { + throw ConversionError.malformedInput("Could not render OCR test image.") + } + + let output = NSMutableData() + guard let destination = CGImageDestinationCreateWithData(output, typeIdentifier, 1, nil) else { + throw ConversionError.malformedInput("Could not create OCR test image destination.") + } + + let options = [kCGImageDestinationLossyCompressionQuality as String: 0.95] as CFDictionary + CGImageDestinationAddImage(destination, image, options) + guard CGImageDestinationFinalize(destination) else { + throw ConversionError.malformedInput("Could not encode OCR test image.") + } + + return output as Data +} +#endif