From dceb3ca07dd0b38cd0946a4be78ab2d0339087a7 Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Wed, 17 Jul 2019 11:10:07 +0200 Subject: [PATCH] - Remove Xcode and Regnet classifier - Add MobileNet classifier - Add average color for each cap - Add option to show average colors in mosaic --- CapCollector.xcodeproj/project.pbxproj | 28 +--- CapCollector/AppDelegate.swift | 9 + CapCollector/Base.lproj/Main.storyboard | 125 +++++++------- CapCollector/Classifier.swift | 131 +++++++-------- CapCollector/Data/Cap.swift | 155 ++++++++++++++++-- CapCollector/Data/DiskManager.swift | 23 +-- CapCollector/Data/UserDefaults.swift | 26 +-- .../Extensions/UIImageExtensions.swift | 1 + .../Presentation/GridViewController.swift | 34 +++- CapCollector/Presentation/ImageSelector.swift | 50 ++---- .../Presentation/SettingsController.swift | 97 ++++++----- 11 files changed, 383 insertions(+), 296 deletions(-) diff --git a/CapCollector.xcodeproj/project.pbxproj b/CapCollector.xcodeproj/project.pbxproj index a0c39fa..0d75af1 100644 --- a/CapCollector.xcodeproj/project.pbxproj +++ b/CapCollector.xcodeproj/project.pbxproj @@ -10,13 +10,12 @@ 043EC7C35065DD26F6BB496F /* Pods_CapCollector.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 86546C4DAB5E47A540F6E8DD /* Pods_CapCollector.framework */; }; 5904C33A2199C9FA0046A573 /* SortController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5904C3392199C9FA0046A573 /* SortController.swift */; }; 5904C33C2199D0260046A573 /* AlwaysShowPopup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5904C33B2199D0260046A573 /* AlwaysShowPopup.swift */; }; - 591252EE21A837FB005B1179 /* Squeezenet.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 591252EB21A837FB005B1179 /* Squeezenet.mlmodel */; }; - 591252F021A837FB005B1179 /* Resnet.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 591252ED21A837FB005B1179 /* Resnet.mlmodel */; }; 59158B1621E37B0200D90CB0 /* GridViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 59158B1521E37B0200D90CB0 /* GridViewController.swift */; }; 59158B1821E4C9AC00D90CB0 /* NavigationController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 59158B1721E4C9AC00D90CB0 /* NavigationController.swift */; }; 591832CE21A2A97E00E5987D /* Cap.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591832CD21A2A97E00E5987D /* Cap.swift */; }; 5970380D225737F800D21B55 /* LogViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5970380C225737F800D21B55 /* LogViewController.swift */; }; - 598D60E221B6B4D200C7473E /* ImageClassifier.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 598D60E121B6B4D200C7473E /* ImageClassifier.mlmodel */; }; + 599BC9DA22CBBDA90061BCDB /* Squeezenet.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 599BC9D922CBBDA90061BCDB /* Squeezenet.mlmodel */; }; + 599BC9DC22CDE2640061BCDB /* MobileNet.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 599BC9DB22CDE2640061BCDB /* MobileNet.mlmodel */; }; 59C1BBA92174CBB800EC84BB /* SettingsController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 59C1BBA82174CBB800EC84BB /* SettingsController.swift */; }; 59C1BBAB21762D9600EC84BB /* UserDefaults.swift in Sources */ = {isa = PBXBuildFile; fileRef = 59C1BBAA21762D9600EC84BB /* UserDefaults.swift */; }; CE56CECE209D81DE00932C01 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE56CECD209D81DE00932C01 /* AppDelegate.swift */; }; @@ -48,13 +47,12 @@ 342A23CD7996DA1E7039C097 /* Pods-CapCollector.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-CapCollector.release.xcconfig"; path = "Pods/Target Support Files/Pods-CapCollector/Pods-CapCollector.release.xcconfig"; sourceTree = ""; }; 5904C3392199C9FA0046A573 /* SortController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SortController.swift; sourceTree = ""; }; 5904C33B2199D0260046A573 /* AlwaysShowPopup.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AlwaysShowPopup.swift; sourceTree = ""; }; - 591252EB21A837FB005B1179 /* Squeezenet.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = Squeezenet.mlmodel; path = ../../../../Dropbox/Models/Squeezenet.mlmodel; sourceTree = ""; }; - 591252ED21A837FB005B1179 /* Resnet.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = Resnet.mlmodel; path = ../../../../Dropbox/Models/Resnet.mlmodel; sourceTree = ""; }; 59158B1521E37B0200D90CB0 /* GridViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GridViewController.swift; sourceTree = ""; }; 59158B1721E4C9AC00D90CB0 /* NavigationController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NavigationController.swift; sourceTree = ""; }; 591832CD21A2A97E00E5987D /* Cap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Cap.swift; sourceTree = ""; }; 5970380C225737F800D21B55 /* LogViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LogViewController.swift; sourceTree = ""; }; - 598D60E121B6B4D200C7473E /* ImageClassifier.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = ImageClassifier.mlmodel; path = ../../../../Dropbox/Models/ImageClassifier.mlmodel; sourceTree = ""; }; + 599BC9D922CBBDA90061BCDB /* Squeezenet.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = Squeezenet.mlmodel; path = ../../../../Dropbox/Models/Squeezenet.mlmodel; sourceTree = ""; }; + 599BC9DB22CDE2640061BCDB /* MobileNet.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; name = MobileNet.mlmodel; path = ../../../../Dropbox/Models/MobileNet.mlmodel; sourceTree = ""; }; 59C1BBA82174CBB800EC84BB /* SettingsController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsController.swift; sourceTree = ""; }; 59C1BBAA21762D9600EC84BB /* UserDefaults.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserDefaults.swift; sourceTree = ""; }; 86546C4DAB5E47A540F6E8DD /* Pods_CapCollector.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_CapCollector.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -107,16 +105,6 @@ name = Pods; sourceTree = ""; }; - 591252E921A837B4005B1179 /* Models */ = { - isa = PBXGroup; - children = ( - 591252ED21A837FB005B1179 /* Resnet.mlmodel */, - 591252EB21A837FB005B1179 /* Squeezenet.mlmodel */, - 598D60E121B6B4D200C7473E /* ImageClassifier.mlmodel */, - ); - name = Models; - sourceTree = ""; - }; 9EAE4B3CEE704AF443897B44 /* Frameworks */ = { isa = PBXGroup; children = ( @@ -149,7 +137,8 @@ CE56CECD209D81DE00932C01 /* AppDelegate.swift */, CE56CED1209D81DE00932C01 /* Main.storyboard */, CE56CEF1209D83B500932C01 /* Classifier.swift */, - 591252E921A837B4005B1179 /* Models */, + 599BC9D922CBBDA90061BCDB /* Squeezenet.mlmodel */, + 599BC9DB22CDE2640061BCDB /* MobileNet.mlmodel */, CEF3874D209D9378001C8D3C /* Capture */, CEF3874E209D9390001C8D3C /* Sync */, CEF38750209D93D1001C8D3C /* Data */, @@ -360,7 +349,6 @@ buildActionMask = 2147483647; files = ( 59C1BBA92174CBB800EC84BB /* SettingsController.swift in Sources */, - 591252EE21A837FB005B1179 /* Squeezenet.mlmodel in Sources */, CE56CF09209D83B800932C01 /* Classifier.swift in Sources */, 5904C33A2199C9FA0046A573 /* SortController.swift in Sources */, CE56CF0B209D83B800932C01 /* Logger.swift in Sources */, @@ -376,7 +364,6 @@ 59C1BBAB21762D9600EC84BB /* UserDefaults.swift in Sources */, CE56CECE209D81DE00932C01 /* AppDelegate.swift in Sources */, CE56CF0D209D83B800932C01 /* ImageSelector.swift in Sources */, - 598D60E221B6B4D200C7473E /* ImageClassifier.mlmodel in Sources */, CE56CEFF209D83B800932C01 /* CameraController.swift in Sources */, CE56CF05209D83B800932C01 /* ViewControllerExtensions.swift in Sources */, 5970380D225737F800D21B55 /* LogViewController.swift in Sources */, @@ -387,9 +374,10 @@ CE56CF06209D83B800932C01 /* CameraView.swift in Sources */, CE56CF0A209D83B800932C01 /* CropView.swift in Sources */, 5904C33C2199D0260046A573 /* AlwaysShowPopup.swift in Sources */, + 599BC9DC22CDE2640061BCDB /* MobileNet.mlmodel in Sources */, + 599BC9DA22CBBDA90061BCDB /* Squeezenet.mlmodel in Sources */, CE56CF02209D83B800932C01 /* RoundedImageView.swift in Sources */, CE56CEF8209D83B800932C01 /* CapCell.swift in Sources */, - 591252F021A837FB005B1179 /* Resnet.mlmodel in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/CapCollector/AppDelegate.swift b/CapCollector/AppDelegate.swift index 06b3acf..300f36e 100644 --- a/CapCollector/AppDelegate.swift +++ b/CapCollector/AppDelegate.swift @@ -10,6 +10,15 @@ import UIKit import CoreData import SwiftyDropbox +/** + TODO: + - Mosaic: Prevent swap of tiles when tapping the free space at the right edge + - Show banner with number of unmatched caps when using camera comparison + - Feature: Create mosaic from image + - Feature: Delete cap + - Feature: Delete image of cap + */ + var shouldLaunchCamera = false @UIApplicationMain diff --git a/CapCollector/Base.lproj/Main.storyboard b/CapCollector/Base.lproj/Main.storyboard index 74b8aa2..fa3fb58 100644 --- a/CapCollector/Base.lproj/Main.storyboard +++ b/CapCollector/Base.lproj/Main.storyboard @@ -20,7 +20,7 @@ - + @@ -29,7 +29,29 @@ - + + + + + + + + + + + + + + + - + @@ -48,7 +70,7 @@ - + @@ -73,7 +95,7 @@ - + @@ -98,7 +120,7 @@ - + @@ -124,68 +146,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -203,7 +167,7 @@ - + @@ -220,12 +184,30 @@ + + + + + + + + + + + + - + @@ -247,7 +229,7 @@ - + @@ -523,6 +505,13 @@ + + + + + + + diff --git a/CapCollector/Classifier.swift b/CapCollector/Classifier.swift index 40c54fa..f318d09 100644 --- a/CapCollector/Classifier.swift +++ b/CapCollector/Classifier.swift @@ -33,54 +33,26 @@ class Classifier: Logger { // MARK: Stored predictions - private var predictions = [[Int : Float]]() - private var notify = false private var image: UIImage? - private func request(for model: MLModel, name: String) -> VNCoreMLRequest { - - let model = try! VNCoreMLModel(for: model) - - let request = VNCoreMLRequest(model: model, completionHandler: { [weak self] request, error in - self?.process(request: request, error: error) - self?.event("Finished \(name) prediction (\(self!.predictions.count)/\(self!.requestCount))") - }) - request.imageCropAndScaleOption = .centerCrop - return request - } - - private var requestCount = 0 - - private var requests: [VNCoreMLRequest] { - var reqs = [VNCoreMLRequest]() - if Persistence.squeezenet { - reqs.append(request(for: Squeezenet().model, name: "Squeezenet")) - } - if Persistence.resnet { - reqs.append(request(for: Resnet().model, name: "Resnet")) - } - if Persistence.xcode { - reqs.append(request(for: ImageClassifier().model, name: "Xcode")) - } - requestCount = reqs.count - return reqs - } - /** Classify an image - parameter image: The image to classify - parameter reportingImage: Set to true, if the delegate should receive the image */ func recognise(image: UIImage, reportingImage: Bool = true) { - predictions.removeAll() self.image = image notify = reportingImage - performClassifications() + if Persistence.useMobileNet { + performClassifications(model: MobileNet().model) + } else { + performClassifications(model: Squeezenet().model) + } } - private func performClassifications() { + private func performClassifications(model: MLModel) { let orientation = CGImagePropertyOrientation(image!.imageOrientation) guard let ciImage = CIImage(image: image!) else { report(error: "Unable to create CIImage") @@ -89,13 +61,18 @@ class Classifier: Logger { DispatchQueue.global(qos: .userInitiated).async { let handler = VNImageRequestHandler(ciImage: ciImage, orientation: orientation) - let requests = self.requests - guard requests.count > 0 else { - self.report(error: "No classifiers selected") - return - } + let model = try! VNCoreMLModel(for: model) + + let request = VNCoreMLRequest(model: model, completionHandler: { [weak self] request, error in + guard self != nil else { + Classifier.event("Self not captured, instance deallocated?") + return + } + self!.process(request: request, error: error) + }) + request.imageCropAndScaleOption = .centerCrop do { - try handler.perform(requests) + try handler.perform([request]) } catch { DispatchQueue.main.async { self.report(error: "Classification failed: \(error.localizedDescription)") @@ -103,64 +80,66 @@ class Classifier: Logger { } } } - + private func process(request: VNRequest, error: Error?) { - guard let result = request.results as? [VNClassificationObservation], - result.isEmpty == false else { - report(error: "Unable to classify image: \(error?.localizedDescription ?? "No error thrown")") - return + if let e = error { + report(error: "Unable to classify image: \(e.localizedDescription)") + return } - let current = dict(from: result) - predictions.append(current) + if let result = request.results as? [VNClassificationObservation] { + let classification = dict(from: result) + process(classification: classification) + return + } + if let result = (request.results as? [VNCoreMLFeatureValueObservation])?.first?.featureValue.multiArrayValue { + let classification = dict(from: result) + process(classification: classification) + return + } + report(error: "Invalid classifier result: \(String(describing: request.results))") + + + } - if predictions.count == requestCount { - updateRecognizedCapsCount() - combine() + private func process(classification: [Int : Float]) { + Cap.unsortedCaps.forEach { cap in + cap.match = classification[cap.id] ?? 0 } + Cap.hasMatches = true + + // Update the count of recognized counts + Persistence.recognizedCapCount = classification.count + + report() } /// Create a dictionary from a vision prediction private func dict(from results: [VNClassificationObservation]) -> [Int : Float] { - let array = results.map{ item -> (Int, Float) in + let array = results.map { item -> (Int, Float) in return (Int(item.identifier) ?? 0, item.confidence) } return [Int : Float](uniqueKeysWithValues: array) } - - /// Combine two predictions - private func combine() { - Cap.unsortedCaps.forEach { cap in - var result: Float = 0 - for index in 0.. [Int : Float] { + let length = results.count + let doublePtr = results.dataPointer.bindMemory(to: Double.self, capacity: length) + let doubleBuffer = UnsafeBufferPointer(start: doublePtr, count: length) + let output = Array(doubleBuffer).enumerated().map { + ($0.offset + 1, Float($0.element)) } - Cap.hasMatches = true - report() + return [Int : Float](uniqueKeysWithValues: output) } - private func updateRecognizedCapsCount() { - let recognizedCaps = predictions.map { prediction in - return prediction.count - } - Persistence.recognizedCapCount = recognizedCaps.max()! - } - // MARK: Callbacks - private func cleanup() { - predictions.removeAll() - image = nil - } - private func report(error message: String) { guard delegate != nil else { error("No delegate: " + message) return } DispatchQueue.main.async { - self.cleanup() + self.image = nil self.delegate?.classifier(error: message) } } @@ -172,7 +151,7 @@ class Classifier: Logger { } DispatchQueue.main.async { let img = self.notify ? self.image : nil - self.cleanup() + self.image = nil self.delegate?.classifier(finished: img) } } diff --git a/CapCollector/Data/Cap.swift b/CapCollector/Data/Cap.swift index a61492c..8d081e4 100644 --- a/CapCollector/Data/Cap.swift +++ b/CapCollector/Data/Cap.swift @@ -8,6 +8,8 @@ import Foundation import UIKit +import CoreImage + import SwiftyDropbox protocol CapsDelegate: class { @@ -110,6 +112,9 @@ final class Cap { } } + /// The average color of the cap + var color: UIColor? + /// The similarity of the cap to the currently processed image var match: Float? = nil @@ -130,6 +135,10 @@ final class Cap { return tiles[tile]?.thumbnail } + static func tileColor(tile: Int) -> UIColor? { + return tiles[tile]?.averageColor + } + /** Switch two tiles. */ @@ -140,7 +149,6 @@ final class Cap { r.tile = lhs tiles[rhs] = l tiles[lhs] = r - event("Switched tiles \(lhs) and \(rhs)") } // MARK: - Initialization @@ -164,7 +172,6 @@ final class Cap { Cap.shouldCreateFolderForCap(self.id) Cap.save() Cap.delegate?.capHasUpdates(self) - //Cap.updateMosaicWithNewCap(id: self.id, image) DispatchQueue.global(qos: .userInitiated).async { self.add(image: image) { _ in @@ -180,7 +187,7 @@ final class Cap { return nil } let parts = line.components(separatedBy: ";") - guard parts.count == 4 else { + guard parts.count == 4 || parts.count == 8 else { Cap.error("Cap names: Invalid line \(line)") return nil } @@ -197,6 +204,14 @@ final class Cap { Cap.error("Invalid tile in line \(line)") return nil } + if parts.count == 8 { + guard let r = Int(parts[4]), let g = Int(parts[5]), let b = Int(parts[6]), let a = Int(parts[7]) else { + Cap.error("Invalid color in line \(line)") + return nil + } + self.color = UIColor(red: CGFloat(r)/255, green: CGFloat(g)/255, blue: CGFloat(b)/255, alpha: CGFloat(a)/255) + } + self.id = nr self.name = parts[1] self.count = count @@ -211,6 +226,9 @@ final class Cap { /// The main image of the cap var image: UIImage? { guard let data = DiskManager.image(for: id) else { + self.downloadImage { _ in + Cap.delegate?.capHasUpdates(self) + } return nil } return UIImage(data: data) @@ -224,6 +242,13 @@ final class Cap { return makeThumbnail() } + var averageColor: UIColor? { + if let c = color { + return c + } + return makeAverageColor() + } + @discardableResult func makeThumbnail() -> UIImage? { guard let img = image else { @@ -249,12 +274,7 @@ final class Cap { - parameter image: The image, if the download was successful, or nil on error */ func downloadImage(_ number: Int = 0, completion: @escaping (_ image: UIImage?) -> Void) { - if number == 0, let image = self.image { - event("Main image for cap \(id) already downloaded") - completion(image) - return - } - let path = "/Images/\(id)/\(id)-\(number).jpg" + let path = imageFolderPath + "/\(id)-\(number).jpg" DropboxController.client.files.download(path: path).response { data, dbError in if let error = dbError { self.error("Failed to download image data (\(number)) for cap \(self.id): \(error)") @@ -285,7 +305,7 @@ final class Cap { func save(mainImage: UIImage) -> Bool { guard let data = mainImage.jpegData(compressionQuality: Cap.jpgQuality) else { - error("Failed to convert image to data") + error("Failed to convert main image to data for cap \(id)") return false } guard DiskManager.save(imageData: data, for: id) else { @@ -293,9 +313,7 @@ final class Cap { return false } event("Saved main image for cap \(id) to disk") - guard let _ = makeThumbnail() else { - return true - } + makeThumbnail() Cap.delegate?.capHasUpdates(self) return true @@ -313,6 +331,18 @@ final class Cap { } } + private var imageFolderPath: String { + return String(format: "/Images/%04d", id) + } + + private static func imageFolderPath(for cap: Int) -> String { + return String(format: "/Images/%04d", cap) + } + + private func imageFilePath(imageId: Int) -> String { + return imageFolderPath + "/\(id)-\(imageId).jpg" + } + // MARK: - Image upload private func folderExists(completion: @escaping (_ exists: Bool?) -> Void) { @@ -340,7 +370,7 @@ final class Cap { return } // Create folder for cap - let path = "/Images/\(cap)" + let path = imageFolderPath(for: cap) DropboxController.client.files.createFolderV2(path: path).response { _, error in if let err = error { self.event("Could not create folder for cap \(cap): \(err)") @@ -386,7 +416,7 @@ final class Cap { } private static func uploadCapImage(at url: URL, forCapWithExistingFolder cap: Int, completion: @escaping (Bool) -> Void) { - let path = "/Images/\(cap)/" + url.lastPathComponent + let path = imageFolderPath(for: cap) + "/" + url.lastPathComponent let data: Data do { @@ -439,6 +469,37 @@ final class Cap { } } + func setMainImage(to imageId: Int, image: UIImage) { + guard imageId != 0 else { + self.event("No need to switch main image with itself") + return + } + let tempFile = imageFilePath(imageId: count) + let oldFile = imageFilePath(imageId: 0) + let newFile = imageFilePath(imageId: imageId) + DropboxController.shared.move(file: oldFile, to: tempFile) { success in + guard success else { + self.error("Could not move \(oldFile) to \(tempFile)") + return + } + DropboxController.shared.move(file: newFile, to: oldFile) { success in + guard success else { + self.error("Could not move \(newFile) to \(oldFile)") + return + } + DropboxController.shared.move(file: tempFile, to: newFile) { success in + if !success { + self.error("Could not move \(tempFile) to \(newFile)") + } + guard self.save(mainImage: image) else { + return + } + self.event("Successfully set image \(imageId) to main image for cap \(self.id)") + } + } + } + } + // MARK: - Counts func updateCount(completion: @escaping (Bool) -> Void) { @@ -454,7 +515,8 @@ final class Cap { } private func getImageCount(completion: @escaping (Int?) -> Void) { - DropboxController.client.files.listFolder(path: "/Images/\(id)").response { response, error in + let path = imageFolderPath + DropboxController.client.files.listFolder(path: path).response { response, error in if let err = error { self.error("Error getting folder content of cap \(self.id): \(err)") completion(nil) @@ -578,6 +640,50 @@ final class Cap { } Persistence.folderNotCreated = oldCaps.filter { $0 != cap } } + + // MARK: - Average color + + @discardableResult + func makeAverageColor() -> UIColor? { + guard let url = DiskManager.imageUrlForCap(id) else { + event("No main image for cap \(id), no average color calculated") + return nil + } + + guard let inputImage = CIImage(contentsOf: url) else { + error("Failed to read CIImage for main image of cap \(id)") + return nil + } + let extentVector = CIVector(x: inputImage.extent.origin.x, y: inputImage.extent.origin.y, z: inputImage.extent.size.width, w: inputImage.extent.size.height) + + guard let filter = CIFilter(name: "CIAreaAverage", parameters: [kCIInputImageKey: inputImage, kCIInputExtentKey: extentVector]) else { + error("Failed to create filter to calculate average for cap \(id)") + return nil + } + guard let outputImage = filter.outputImage else { + error("Failed get filter output for image of cap \(id)") + return nil + } + + var bitmap = [UInt8](repeating: 0, count: 4) + let context = CIContext(options: [.workingColorSpace: kCFNull]) + context.render(outputImage, toBitmap: &bitmap, rowBytes: 4, bounds: CGRect(x: 0, y: 0, width: 1, height: 1), format: .RGBA8, colorSpace: nil) + + color = UIColor( + red: saturate(bitmap[0]), + green: saturate(bitmap[1]), + blue: saturate(bitmap[2]), + alpha: CGFloat(bitmap[3]) / 255) + + event("Average color updated for cap \(id)") + Cap.save() + return color + } +} + +/// Map expected range 75-200 to 0-255 +private func saturate(_ component: UInt8) -> CGFloat { + return max(min(CGFloat(component) * 2 - 150, 255), 0) / 255 } // MARK: - Protocol Hashable @@ -599,7 +705,22 @@ extension Cap: Hashable { extension Cap: CustomStringConvertible { var description: String { - return "\(id);\(name);\(count);\(tile)\n" + guard let c = color else { + return String(format: "%04d", id) + ";\(name);\(count);\(tile)\n" + } + + var fRed: CGFloat = 0 + var fGreen: CGFloat = 0 + var fBlue: CGFloat = 0 + var fAlpha: CGFloat = 0 + guard c.getRed(&fRed, green: &fGreen, blue: &fBlue, alpha: &fAlpha) else { + return String(format: "%04d", id) + ";\(name);\(count);\(tile)\n" + } + let r = Int(fRed * 255.0) + let g = Int(fGreen * 255.0) + let b = Int(fBlue * 255.0) + let a = Int(fAlpha * 255.0) + return String(format: "%04d", id) + ";\(name);\(count);\(tile);\(r);\(g);\(b);\(a)\n" } } diff --git a/CapCollector/Data/DiskManager.swift b/CapCollector/Data/DiskManager.swift index d47a7a3..b12f1f5 100644 --- a/CapCollector/Data/DiskManager.swift +++ b/CapCollector/Data/DiskManager.swift @@ -82,6 +82,14 @@ final class DiskManager { let url = localUrl(for: cap) return fm.fileExists(atPath: url.path) } + + static func imageUrlForCap(_ id: Int) -> URL? { + let url = localUrl(for: id) + guard fm.fileExists(atPath: url.path) else { + return nil + } + return url + } private static func localUrl(for cap: Int) -> URL { return URL(fileURLWithPath: "\(cap).jpg", isDirectory: true, relativeTo: LocalDirectory.images.url) @@ -104,21 +112,6 @@ final class DiskManager { return readData(from: url) } - private static let mosaicURL: URL = documentsDirectory.appendingPathComponent("mosaic.png") - - - static var mosaicExists: Bool { - return fm.fileExists(atPath: mosaicURL.path) - } - - static func saveMosaicData(_ data: Data) -> Bool { - return write(data, to: mosaicURL) - } - - static var mosaicData: Data? { - return readData(from: mosaicURL) - } - /** Get the thumbnail for a cap. If the image exists on disk, it is returned. diff --git a/CapCollector/Data/UserDefaults.swift b/CapCollector/Data/UserDefaults.swift index 5964271..d82e73d 100644 --- a/CapCollector/Data/UserDefaults.swift +++ b/CapCollector/Data/UserDefaults.swift @@ -46,33 +46,13 @@ final class Persistence { } } - static var squeezenet: Bool { + static var useMobileNet: Bool { get { - return UserDefaults.standard.bool(forKey: "squeezenet") + return UserDefaults.standard.bool(forKey: "mobileNet") } set { - UserDefaults.standard.set(newValue, forKey: "squeezenet") - } - } - - static var resnet: Bool { - get { - return UserDefaults.standard.bool(forKey: "resnet") - } - - set { - UserDefaults.standard.set(newValue, forKey: "resnet") - } - } - - static var xcode: Bool { - get { - return UserDefaults.standard.bool(forKey: "xcode") - } - - set { - UserDefaults.standard.set(newValue, forKey: "xcode") + UserDefaults.standard.set(newValue, forKey: "mobileNet") } } diff --git a/CapCollector/Extensions/UIImageExtensions.swift b/CapCollector/Extensions/UIImageExtensions.swift index c5767aa..f1bde3a 100644 --- a/CapCollector/Extensions/UIImageExtensions.swift +++ b/CapCollector/Extensions/UIImageExtensions.swift @@ -9,6 +9,7 @@ import Foundation import UIKit + extension UIImage { static func templateImage(named: String) -> UIImage { diff --git a/CapCollector/Presentation/GridViewController.swift b/CapCollector/Presentation/GridViewController.swift index 67436c1..1f86bfb 100644 --- a/CapCollector/Presentation/GridViewController.swift +++ b/CapCollector/Presentation/GridViewController.swift @@ -36,6 +36,19 @@ class GridViewController: UIViewController { return true } + private var isShowingColors = false + + @IBAction func toggleAverageColor(_ sender: Any) { + isShowingColors = !isShowingColors + for (tile, view) in installedTiles { + if isShowingColors { + view.image = nil + } else { + view.image = Cap.tileImage(tile: tile) + } + } + } + override func viewDidLoad() { super.viewDidLoad() @@ -131,10 +144,15 @@ class GridViewController: UIViewController { private func makeTile(_ tile: Int) { let view = RoundedImageView(frame: frame(for: tile)) myView.addSubview(view) - view.image = Cap.tileImage(tile: tile) + view.backgroundColor = Cap.tileColor(tile: tile) + // Only set image if images are shown + if !isShowingColors { + view.image = Cap.tileImage(tile: tile) + } + installedTiles[tile] = view } - + private func frame(for tile: Int) -> CGRect { let row = tile / columns let column = tile - row * columns @@ -144,12 +162,20 @@ class GridViewController: UIViewController { } private func switchTiles(oldTile: Int, newTile: Int) { - if oldTile != newTile { - Cap.switchTiles(oldTile, newTile) + guard oldTile != newTile else { + clearTileSelection() + return + } + Cap.switchTiles(oldTile, newTile) + // Switch cap colors + installedTiles[oldTile]?.backgroundColor = Cap.tileColor(tile: oldTile) + installedTiles[newTile]?.backgroundColor = Cap.tileColor(tile: newTile) + if !isShowingColors { installedTiles[oldTile]?.image = Cap.tileImage(tile: oldTile) installedTiles[newTile]?.image = Cap.tileImage(tile: newTile) } clearTileSelection() + } private func clearTileSelection() { diff --git a/CapCollector/Presentation/ImageSelector.swift b/CapCollector/Presentation/ImageSelector.swift index 7f78f65..60659fd 100644 --- a/CapCollector/Presentation/ImageSelector.swift +++ b/CapCollector/Presentation/ImageSelector.swift @@ -56,7 +56,19 @@ class ImageSelector: UIViewController { private func downloadImages() { images = [UIImage?](repeating: nil, count: cap.count) event("\(cap.count) images for cap \(cap.id)") - for number in 0.. 0 else { + return + } + for number in 1.. 0 || imageCount > 0 databaseUpdatesLabel.text = "\(capCount) new caps and \(imageCount) new images" } @@ -83,36 +93,34 @@ class SettingsController: UITableViewController { dropboxAccountLabel.text = DropboxController.shared.isEnabled ? "Sign out" : "Sign in" } - private func setAccessories() { - tableView.cellForRow(at: IndexPath(row: 0, section: 2))?.accessoryType = Persistence.squeezenet ? .checkmark : .none - tableView.cellForRow(at: IndexPath(row: 1, section: 2))?.accessoryType = Persistence.resnet ? .checkmark : .none - tableView.cellForRow(at: IndexPath(row: 2, section: 2))?.accessoryType = Persistence.xcode ? .checkmark : .none + private func setClassifierChoice(_ useMobileNet: Bool) { + tableView.cellForRow(at: IndexPath(row: 0, section: 0))?.accessoryType = useMobileNet ? .checkmark : .none } - private func toggleClassifier(index: Int) { - switch index { - case 0: Persistence.squeezenet = !Persistence.squeezenet - case 1: Persistence.resnet = !Persistence.resnet - case 2: Persistence.xcode = !Persistence.xcode - default: - return - } - setAccessories() + private func toggleClassifier() { + let newValue = !Persistence.useMobileNet + Persistence.useMobileNet = newValue + setClassifierChoice(newValue) } override func tableView(_ tableView: UITableView, shouldHighlightRowAt indexPath: IndexPath) -> Bool { switch indexPath.section { - case 0: // Mosaic + case 0: // Choose models return true - case 1: // Database - return indexPath.row == 2 && nameFileChanges - case 2: // Choose models + case 1: // Mosaic return true + case 2: // Database + return indexPath.row == 2 && !isUploadingNameFile case 3: // Refresh - if indexPath.row == 0 { + switch indexPath.row { + case 0: return !isUpdatingCounts - } else { + case 1: return !isUpdatingThumbnails + case 2: + return !isUpdatingColors + default: + return false } case 4: // Dropbox account return true @@ -124,17 +132,22 @@ class SettingsController: UITableViewController { override func tableView(_ tableView: UITableView, willSelectRowAt indexPath: IndexPath) -> IndexPath? { switch indexPath.section { - case 0: // Mosaic + case 0: // Choose models return indexPath - case 1: // Database - return (indexPath.row == 2 && nameFileChanges) ? indexPath : nil - case 2: // Choose models + case 1: // Mosaic return indexPath + case 2: // Database + return (indexPath.row == 2 && !isUploadingNameFile) ? indexPath : nil case 3: // Refresh count - if indexPath.row == 0 { - return isUpdatingCounts ? nil : indexPath - } else { + switch indexPath.row { + case 0: + return isUpdatingCounts ? nil : indexPath + case 1: return isUpdatingThumbnails ? nil : indexPath + case 2: + return isUpdatingColors ? nil : indexPath + default: + return nil } case 4: // Dropbox account return indexPath @@ -147,17 +160,22 @@ class SettingsController: UITableViewController { override func tableView(_ tableView: UITableView, didSelectRowAt indexPath: IndexPath) { tableView.deselectRow(at: indexPath, animated: true) switch indexPath.section { - case 1: // Upload - if indexPath.row == 2 && nameFileChanges { + case 0: // Choose models + toggleClassifier() + case 2: // Upload + if indexPath.row == 2 && !isUploadingNameFile { uploadNameFile() } - case 2: // Choose models - toggleClassifier(index: indexPath.row) case 3: // Refresh count - if indexPath.row == 0 { + switch indexPath.row { + case 0: updateCounts() - } else { + case 1: updateThumbnails() + case 2: + updateColors() + default: + break } default: break @@ -205,7 +223,10 @@ class SettingsController: UITableViewController { } private func uploadNameFile() { + event("Uploading name file") + isUploadingNameFile = true Cap.saveAndUpload() { _ in + self.isUploadingNameFile = false self.updateNameFileStats() } }