[WIP] Improvements to pigmentation baselining

.gitignore

@@ -3,4 +3,5 @@
 /Packages
 xcuserdata/
 /output/
+/grouped-output/
 ColonyPigmentationAnalysis.xcodeproj

Sources/ColonyPigmentationAnalysis/Additions.swift

@@ -50,14 +50,22 @@ extension CSV: StorableInDisk {
     }
 }
 
+extension Array where Element == PigmentationSample {
+    fileprivate static let csvHeader = "x, average, stddev, columns"
+
+    var csv: CSV {
+        let header = "\(Self.csvHeader)\n"
+        let contents = map({ "\($0.x),\($0.averagePigmentation),\($0.standardDeviation),\($0.includedColumnIndices.map(String.init).joined(separator: "-"))" }).joined(separator: "\n")
+
+        return CSV(contents: header + contents)
+    }
+}
+
 extension Array: StorableInDisk where Element == PigmentationSample {
     static var fileExtension: String { "csv" }
 
     func save(toPath path: String) throws {
-        let header = "x, average, stddev, columns\n"
-        let contents = map({ "\($0.x),\($0.averagePigmentation),\($0.standardDeviation),\($0.includedColumnIndices.map(String.init).joined(separator: "-"))" }).joined(separator: "\n")
-
-        try CSV(contents: header + contents).save(toPath: path)
+        try csv.save(toPath: path)
     }
 }
 
@@ -68,3 +76,61 @@ func createDirectory(_ directory: String) throws {
         throw TaskError.failedToCreateResultDirectory(path: directory, underlyingError: error)
     }
 }
+
+func readPigmentationHistogram(at path: String) throws -> [PigmentationSample] {
+    enum InvalidCSVFormatError: CustomNSError, LocalizedError {
+        case invalidHeader(contents: String)
+        case invalidNumberOfRows(Int)
+        case invalidRowFormat(rowIndex: Int, contents: String)
+        case invalidValueFormat(expectedType: Any.Type, value: String)
+
+        static let errorDomain = "InvalidCSVFormatError"
+
+        var errorDescription: String? {
+            switch self {
+            case let .invalidHeader(contents): return "Invalid header: \(contents). Expected \"\([PigmentationSample].csvHeader)\""
+            case let .invalidNumberOfRows(rows): return "Invalid number of rows: \(rows). Expected > 1"
+            case let .invalidRowFormat(rowIndex, contents): return "Invalid row format at \(rowIndex): \(contents)"
+            case let .invalidValueFormat(expectedType, value): return "Invalid value format: \(value). Expected \(expectedType)"
+            }
+        }
+    }
+
+    let contents = try String(contentsOfFile: path)
+    let rows = contents.split(separator: "\n")
+    guard rows.count > 1 else {
+        throw InvalidCSVFormatError.invalidNumberOfRows(rows.count)
+    }
+
+    guard rows[0] == [PigmentationSample].csvHeader else {
+        throw InvalidCSVFormatError.invalidHeader(contents: String(rows[0]))
+    }
+
+    let rowsWithoutHeader = rows.dropFirst()
+
+    return try rowsWithoutHeader.enumerated().map { (index, row) in
+        let columns = row.split(separator: ",")
+        guard columns.count == 4 else {
+            throw InvalidCSVFormatError.invalidRowFormat(rowIndex: index, contents: String(row))
+        }
+
+        guard let x = Double(columns[0]) else {
+            throw InvalidCSVFormatError.invalidValueFormat(expectedType: Double.self, value: String(columns[0]))
+        }
+        guard let averagePigmentation = Double(columns[1]) else {
+            throw InvalidCSVFormatError.invalidValueFormat(expectedType: Double.self, value: String(columns[1]))
+        }
+
+        guard let standardDeviation = Double(columns[2]) else {
+            throw InvalidCSVFormatError.invalidValueFormat(expectedType: Double.self, value: String(columns[2]))
+        }
+        let includedColumnIndices = try columns[3].split(separator: "-").map { (value: Substring) throws -> Int in
+            guard let intValue = Int(value) else {
+                throw InvalidCSVFormatError.invalidValueFormat(expectedType: Int.self, value: String(value))
+            }
+            return intValue
+        }
+
+        return PigmentationSample(x: x, averagePigmentation: averagePigmentation, standardDeviation: standardDeviation, includedColumnIndices: includedColumnIndices)
+    }
+}

Sources/ColonyPigmentationAnalysis/Tasks.swift

@@ -95,6 +95,7 @@ let removeBackgroundTask = Task<(ImageMap, MaskBitMap), (), ImageMap>(name: "Rem
 struct PigmentationHistogramTaskConfiguration {
     let pigmentationColor: ColonyPigmentationAnalysisKit.RGBColor
     let baselinePigmentation: Double
+    let pigmentationValuesToSubtract: [Double]?
     let pigmentationAreaOfInterestHeightPercentage: Double
     let horizontalSamples: Int?
 }
@@ -104,6 +105,7 @@ let pigmentationHistogramTask = Task<(ImageMap, MaskBitMap), PigmentationHistogr
         withColonyMask: input.1,
         keyColor: configuration.pigmentationColor,
         baselinePigmentation: configuration.baselinePigmentation,
+        pigmentationValuesToSubtract: configuration.pigmentationValuesToSubtract,
         areaOfInterestHeightPercentage: configuration.pigmentationAreaOfInterestHeightPercentage,
         horizontalSamples: configuration.horizontalSamples
     )
@@ -120,14 +122,25 @@ let pigmentationSeriesTask = Task<[PigmentationSample], Void, CSV>(name: "Pigmen
 // MARK: - Draw Pigmentation
 
 struct DrawPigmentationTaskConfiguration {
-    let pigmentationColor: ColonyPigmentationAnalysisKit.RGBColor
-    let baselinePigmentation: Double
+    var pigmentationColor: ColonyPigmentationAnalysisKit.RGBColor
+    var baselinePigmentation: Double
+    var pigmentationValuesToSubtract: [Double]? = nil
+    var areaOfInterestHeightPercentage: Double = 1
+    var cropWithinAreaOfInterest: Bool = false
 }
 
 let drawPigmentationTask = Task<(ImageMap, MaskBitMap), DrawPigmentationTaskConfiguration, ImageMap>(name: "Draw Pigmentation") { input, configuration in
-    return input.0.replacingColonyPixels(
-            withMask: input.1,
-            withPigmentationBasedOnKeyColor: configuration.pigmentationColor,
-            baselinePigmentation: configuration.baselinePigmentation
+    var result = input.0.replacingColonyPixels(
+        withMask: input.1,
+        withPigmentationBasedOnKeyColor: configuration.pigmentationColor,
+        baselinePigmentation: configuration.baselinePigmentation,
+        pigmentationValuesToSubtract: configuration.pigmentationValuesToSubtract,
+        areaOfInterestHeightPercentage: configuration.areaOfInterestHeightPercentage
     )
+
+    if configuration.cropWithinAreaOfInterest {
+        result.removePixelsOutsideAreaOfInterest(withMask: input.1, areaOfInterestHeightPercentage: configuration.areaOfInterestHeightPercentage)
+    }
+
+    return result
 }

Sources/ColonyPigmentationAnalysis/main.swift

@@ -44,6 +44,9 @@ struct Main: ParsableCommand {
 
     @Option(default: 0.436, help: "A minimum level of pigmentation that is considered 'background noise' and is subtracted from all values")
     var baselinePigmentation: Double
+
+    @Option(default: nil, help: "A file to read a pigmentation histogram for to use as baseline values. If specified, this takes precendence over --baseline-pigmentation. It must be a file with the same format as a csv output by this program")
+    var baselinePigmentationHistogramFilePath: String?
 
     @Option(default: 200, help: "Output the pigmentation histogram csv by interpolating to this many values")
     var pigmentationHistogramSampleCount: Int
@@ -56,21 +59,26 @@ struct Main: ParsableCommand {
 
     func run() throws {
         guard !images.isEmpty else {
-            Self.exit(withError: ValidationError("No images specified"))
+            throw ValidationError("No images specified")
         }
 
         logger.info("\("Analyzing \(String(images.count).onBlack()) images".blue())")
 
         try saveCurrentConfigurationToFile()
 
+        let pigmentationValuesToSubtract = try baselinePigmentationSamples()?.map { $0.averagePigmentation }
+        if let pigmentationValuesToSubtract = pigmentationValuesToSubtract {
+            precondition(pigmentationValuesToSubtract.count == pigmentationHistogramSampleCount, "The number of pigmentation samples to subtract provided via --baseline-pigmentation-histogram-file-path needs to match --pigmentation-histogram-sample-count")
+        }
+
         let queue = DispatchQueue(label: "colony-analysis-queue", qos: .userInitiated, attributes: parallelize ? .concurrent : [], autoreleaseFrequency: .inherit, target: nil)
 
         var lastCaughtError: Swift.Error?
 
         for (index, imagePath) in images.enumerated() {
             queue.async {
                 do {
-                    try self.analyzeImage(atPath: imagePath)
+                    try self.analyzeImage(atPath: imagePath, withPigmentationValuesToSubtract: pigmentationValuesToSubtract)
                 } catch {
                     lastCaughtError = error
                     logger.error("Error analyzing image \(index) (\(imagePath)): \(error)")
@@ -85,6 +93,9 @@ struct Main: ParsableCommand {
                     try averagedPigmentationSamples.save(toPath: self.outputPath.appending("average_pigmentation.\([PigmentationSample].fileExtension)"))
 
                     try pigmentationSeriesTask.run(withInput: averagedPigmentationSamples, configuration: ()).save(toPath: self.outputPath.appending("average_pigmentation_1d.csv"))
+
+//                    let minPigmentation = PigmentationSample.minAveragePigmentation(Self.pigmentationSamplesWithoutBaseline)
+//                    try CSV(contents: "\(minPigmentation)").save(toPath: self.outputPath.appending("min_pigmentation.txt"))
                 }
             } catch {
                 lastCaughtError = error
@@ -103,8 +114,9 @@ Main.main()
 
 private extension Main {
     static var pigmentationSamples: [[PigmentationSample]] = []
+    static var pigmentationSamplesWithoutBaseline: [[PigmentationSample]] = []
 
-    func analyzeImage(atPath imagePath: String) throws {
+    func analyzeImage(atPath imagePath: String, withPigmentationValuesToSubtract pigmentationValuesToSubtract: [Double]?) throws {
         let imageName = ((imagePath as NSString).lastPathComponent as NSString).deletingPathExtension
 
         try measure(name: imageName) {
@@ -138,22 +150,40 @@ private extension Main {
                 artifactDirectory: "DrawnPigmentation"
             )
 
+            try taskRunner.run(
+                drawPigmentationTask,
+                withInput: (image, colonyMask),
+                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: baselinePigmentation,
+                                     pigmentationValuesToSubtract: pigmentationValuesToSubtract,
+                                     areaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage,
+                                     cropWithinAreaOfInterest: true),
+                artifactDirectory: "DrawnPigmentationROI"
+            )
+
             try taskRunner.run(
                 pigmentationHistogramTask,
                 withInput: (image, colonyMask),
-                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: baselinePigmentation, pigmentationAreaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage, horizontalSamples: nil),
+                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: baselinePigmentation, pigmentationValuesToSubtract: nil, pigmentationAreaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage, horizontalSamples: nil),
                 artifactDirectory: "RawPigmentationHistogram"
             )
 
             let sampledPigmentation = try taskRunner.run(
                 pigmentationHistogramTask,
                 withInput: (image, colonyMask),
-                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: baselinePigmentation, pigmentationAreaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage, horizontalSamples: pigmentationHistogramSampleCount),
+                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: baselinePigmentation, pigmentationValuesToSubtract: pigmentationValuesToSubtract, pigmentationAreaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage, horizontalSamples: pigmentationHistogramSampleCount),
                 artifactDirectory: "SampledPigmentationHistogram"
             )
 
+//            let sampledPigmentationWithoutBaseline = try taskRunner.run(
+//                pigmentationHistogramTask,
+//                withInput: (image, colonyMask),
+//                configuration: .init(pigmentationColor: pigmentationColor, baselinePigmentation: 0, pigmentationValuesToSubtract: nil, pigmentationAreaOfInterestHeightPercentage: pigmentationAreaOfInterestHeightPercentage, horizontalSamples: pigmentationHistogramSampleCount),
+//                artifactDirectory: "NoBaselineSampledPigmentationHistogram"
+//            )
+
             DispatchQueue.main.sync {
                 Self.pigmentationSamples.append(sampledPigmentation)
+//                Self.pigmentationSamplesWithoutBaseline.append(sampledPigmentationWithoutBaseline)
             }
 
             try taskRunner.run(
@@ -164,6 +194,12 @@ private extension Main {
             )
         }
     }
+
+    private func baselinePigmentationSamples() throws -> [PigmentationSample]? {
+        return try baselinePigmentationHistogramFilePath.map {
+            return try readPigmentationHistogram(at: $0)
+        }
+    }
 }
 
 extension Main {
@@ -176,6 +212,16 @@ extension Main {
         guard downscaleFactor > 0 && downscaleFactor <= 1 else { throw ValidationError("downscale-factor must be a value between 0 and 1. Got \(downscaleFactor) instead") }
         guard (0...1).contains(backgroundChromaKeyThreshold) else { throw ValidationError("background-chroma-key-threshold must be a value between 0 and 1. Got \(backgroundChromaKeyThreshold) instead") }
 
+        guard (0...1).contains(baselinePigmentation) else {
+            throw ValidationError("baseline-pigmentation must be a value between 0 and 1. Got \(baselinePigmentation) instead")
+        }
+
+        if let baselinePigmentationHistogramFilePath = baselinePigmentationHistogramFilePath {
+            guard FileManager.default.fileExists(atPath: baselinePigmentationHistogramFilePath) else {
+                throw ValidationError("--baseline-pigmentation-histogram-file-path \"\(baselinePigmentationHistogramFilePath)\" doesn't exist")
+            }
+        }
+
         guard (0...1).contains(pigmentationAreaOfInterestHeightPercentage) else { throw ValidationError("pigmentation-roi-height must be a value between 0 and 1. Got \(pigmentationAreaOfInterestHeightPercentage) instead") }
     }
 
@@ -193,14 +239,15 @@ private extension Main {
     func saveCurrentConfigurationToFile() throws {
         let configuration = """
         Date: \(Date())
-        Images: \(images.count)
         Downscale Factor: \(downscaleFactor)
         Background Chroma Key Color: \(backgroundChromaKeyColor)
         Background Chroma Key Threshold: \(backgroundChromaKeyThreshold)
         Pigmentation Color: \(pigmentationColor)
-        Baseline Pigmentation: \(baselinePigmentation)
         Pigmentation Histogram Sample Count: \(pigmentationHistogramSampleCount)
         Pigmentation Area of Interest Height Percentage: \(pigmentationAreaOfInterestHeightPercentage)
+        Images (\(images.count)): \(images.joined(separator: ", "))
+        Baseline Pigmentation: \(baselinePigmentation)
+        Baseline Pigmentation Histogram Contents:\n\(try baselinePigmentationSamples()?.csv.contents ?? "N/A")
         """
 
         let path = outputPath.appending("parameters.txt")