New background task modes

CapCollector.xcodeproj/project.pbxproj

@@ -14,6 +14,7 @@
 		591832CE21A2A97E00E5987D /* Cap.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591832CD21A2A97E00E5987D /* Cap.swift */; };
 		591FDD1E234E151600AA379E /* SearchAndDisplayAccessory.xib in Resources */ = {isa = PBXBuildFile; fileRef = 591FDD1D234E151600AA379E /* SearchAndDisplayAccessory.xib */; };
 		591FDD20234E162000AA379E /* SearchAndDisplayAccessory.swift in Sources */ = {isa = PBXBuildFile; fileRef = 591FDD1F234E162000AA379E /* SearchAndDisplayAccessory.swift */; };
+		88A89ECE25AF420F00323B64 /* DispatchGroup+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88A89ECD25AF420F00323B64 /* DispatchGroup+Extensions.swift */; };
 		CE0A501124752A9800A9E753 /* TileImage.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE0A501024752A9800A9E753 /* TileImage.swift */; };
 		CE0A5013247D745200A9E753 /* Colors.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE0A5012247D745200A9E753 /* Colors.swift */; };
 		CE56CECE209D81DE00932C01 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE56CECD209D81DE00932C01 /* AppDelegate.swift */; };
@@ -45,7 +46,6 @@
 		CEB269572445DB56004B74B3 /* SQLite in Frameworks */ = {isa = PBXBuildFile; productRef = CEB269562445DB56004B74B3 /* SQLite */; };
 		CEB269592445DB72004B74B3 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = CEB269582445DB72004B74B3 /* Database.swift */; };
 		CEB2695B2445E54E004B74B3 /* UIColor+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = CEB2695A2445E54E004B74B3 /* UIColor+Extensions.swift */; };
-		CEC7F815245A2B1200B896B1 /* JGProgressHUD in Frameworks */ = {isa = PBXBuildFile; productRef = CEC7F814245A2B1200B896B1 /* JGProgressHUD */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
@@ -56,6 +56,7 @@
 		591832CD21A2A97E00E5987D /* Cap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Cap.swift; sourceTree = "<group>"; };
 		591FDD1D234E151600AA379E /* SearchAndDisplayAccessory.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = SearchAndDisplayAccessory.xib; sourceTree = "<group>"; };
 		591FDD1F234E162000AA379E /* SearchAndDisplayAccessory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchAndDisplayAccessory.swift; sourceTree = "<group>"; };
+		88A89ECD25AF420F00323B64 /* DispatchGroup+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DispatchGroup+Extensions.swift"; sourceTree = "<group>"; };
 		CE0A501024752A9800A9E753 /* TileImage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TileImage.swift; sourceTree = "<group>"; };
 		CE0A5012247D745200A9E753 /* Colors.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Colors.swift; sourceTree = "<group>"; };
 		CE56CECA209D81DD00932C01 /* CapCollector.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CapCollector.app; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -96,7 +97,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				CEB269572445DB56004B74B3 /* SQLite in Frameworks */,
-				CEC7F815245A2B1200B896B1 /* JGProgressHUD in Frameworks */,
 				CE5B7D032458C921002E5C06 /* Reachability in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -168,6 +168,7 @@
 				CE56CEF7209D83B700932C01 /* UIImage+Extensions.swift */,
 				CE56CEEC209D83B400932C01 /* UIViewExtensions.swift */,
 				CE56CEED209D83B400932C01 /* ViewControllerExtensions.swift */,
+				88A89ECD25AF420F00323B64 /* DispatchGroup+Extensions.swift */,
 			);
 			path = Extensions;
 			sourceTree = "<group>";
@@ -232,7 +233,6 @@
 			packageProductDependencies = (
 				CEB269562445DB56004B74B3 /* SQLite */,
 				CE5B7D022458C921002E5C06 /* Reachability */,
-				CEC7F814245A2B1200B896B1 /* JGProgressHUD */,
 			);
 			productName = CapCollector;
 			productReference = CE56CECA209D81DD00932C01 /* CapCollector.app */;
@@ -271,7 +271,6 @@
 			packageReferences = (
 				CEB269552445DB56004B74B3 /* XCRemoteSwiftPackageReference "SQLite" */,
 				CE5B7D012458C921002E5C06 /* XCRemoteSwiftPackageReference "Reachability" */,
-				CEC7F813245A2B1200B896B1 /* XCRemoteSwiftPackageReference "JGProgressHUD" */,
 			);
 			productRefGroup = CE56CECB209D81DD00932C01 /* Products */;
 			projectDirPath = "";
@@ -330,6 +329,7 @@
 				CE85AA16246A96C3002D1074 /* UINavigationItem+Extensions.swift in Sources */,
 				CEB269592445DB72004B74B3 /* Database.swift in Sources */,
 				CE56CF02209D83B800932C01 /* RoundedImageView.swift in Sources */,
+				88A89ECE25AF420F00323B64 /* DispatchGroup+Extensions.swift in Sources */,
 				CE0A501124752A9800A9E753 /* TileImage.swift in Sources */,
 				CE56CEF8209D83B800932C01 /* CapCell.swift in Sources */,
 				CE6E4828246C304100570CB0 /* CGImagePropertyOrientation+Extensions.swift in Sources */,
@@ -560,14 +560,6 @@
 				minimumVersion = 0.12.2;
 			};
 		};
-		CEC7F813245A2B1200B896B1 /* XCRemoteSwiftPackageReference "JGProgressHUD" */ = {
-			isa = XCRemoteSwiftPackageReference;
-			repositoryURL = "https://github.com/JonasGessner/JGProgressHUD";
-			requirement = {
-				kind = upToNextMajorVersion;
-				minimumVersion = 2.1.0;
-			};
-		};
 /* End XCRemoteSwiftPackageReference section */
 
 /* Begin XCSwiftPackageProductDependency section */
@@ -581,11 +573,6 @@
 			package = CEB269552445DB56004B74B3 /* XCRemoteSwiftPackageReference "SQLite" */;
 			productName = SQLite;
 		};
-		CEC7F814245A2B1200B896B1 /* JGProgressHUD */ = {
-			isa = XCSwiftPackageProductDependency;
-			package = CEC7F813245A2B1200B896B1 /* XCRemoteSwiftPackageReference "JGProgressHUD" */;
-			productName = JGProgressHUD;
-		};
 /* End XCSwiftPackageProductDependency section */
 	};
 	rootObject = CE56CEC2209D81DD00932C01 /* Project object */;

CapCollector.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -1,15 +1,6 @@
 {
   "object": {
     "pins": [
-      {
-        "package": "JGProgressHUD",
-        "repositoryURL": "https://github.com/JonasGessner/JGProgressHUD",
-        "state": {
-          "branch": null,
-          "revision": "08d130dd614a743f813286f096804c43a6ffa3f6",
-          "version": "2.1.0"
-        }
-      },
       {
         "package": "Reachability",
         "repositoryURL": "https://github.com/ashleymills/Reachability.swift",

CapCollector.xcodeproj/xcuserdata/imac.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Bucket
+   uuid = "B1F05379-5C9B-42D8-94ED-BB89F9571BE1"
+   type = "1"
+   version = "2.0">
+</Bucket>

CapCollector/AppDelegate.swift

@@ -17,7 +17,7 @@ import Reachability
 #warning("GridController: Reorder caps by dragging")
 #warning("TableView: Fix blur background of search bar after transition")
 #warning("TableView: Add banner to jump down to unmatched caps / bottom")
-#warning("Database: Calculate thumbnails and colors in the background")
+
 var shouldLaunchCamera = false
 
 var app: AppDelegate!
@@ -34,16 +34,12 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
 
     var window: UIWindow?
     
-    var mainStoryboard: UIStoryboard {
+    var mainStoryboard: UIStoryboard { .init(name: "Main", bundle: nil) }
-        UIStoryboard(name: "Main", bundle: nil)
-    }
     
     let documentsFolder = try! FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
     
     var database: Database!
     
-    var storage: Storage!
-    
     var reachability: Reachability!
 
     var dbUrl: URL {
@@ -65,12 +61,11 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
     func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
         app = self
         
-        storage = Storage(in: documentsFolder)
         reachability = try! Reachability()
         
         //resetToFactoryState()
         
-        database = Database(url: dbUrl, server: serverUrl)
+        database = Database(url: dbUrl, server: serverUrl, storageFolder: documentsFolder)
         return true
     }
     
@@ -97,8 +92,6 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
     }
     
     func applicationDidBecomeActive(_ application: UIApplication) {
-        app.database?.uploadRemainingData()
-        
         guard shouldLaunchCamera else { return }
         shouldLaunchCamera = false
         if let c = (frontmostViewController as? UINavigationController)?.topViewController as? TableView {

CapCollector/Data/Database.swift

@@ -19,22 +19,91 @@ protocol DatabaseDelegate: class {
     
     func database(didLoadImageForCap cap: Int)
     
+    func database(completedBackgroundWorkItem title: String, subtitle: String)
+    
+    func database(needsUserConfirmation title: String, body: String, shouldProceed: @escaping (Bool) -> Void)
+    
+    func database(didFailBackgroundWork title: String, subtitle: String)
+    
+    func databaseHasNewClassifier()
+    
+    func databaseDidFinishBackgroundWork()
+    
     func databaseNeedsFullRefresh()
 }
 
+private enum BackgroundWorkTaskType: Int, CustomStringConvertible, Comparable {
+    
+    case downloadCapNames = 9
+    case downloadCounts = 8
+    case downloadClassifier = 7
+    case uploadingCaps = 6
+    case uploadingImages = 5
+    case downloadMainImages = 4
+    case creatingThumbnails = 3
+    case creatingColors = 2
+    
+    var description: String {
+        switch self {
+        case .downloadCapNames:
+            return "Downloading names"
+        case .downloadCounts:
+            return "Downloading counts"
+        case .downloadClassifier:
+            return "Downloading classifier"
+        case .uploadingCaps:
+            return "Uploading caps"
+        case .uploadingImages:
+            return "Uploading images"
+        case .downloadMainImages:
+            return "Downloading images"
+        case .creatingThumbnails:
+            return "Creating thumbnails"
+        case .creatingColors:
+            return "Creating colors"
+        }
+    }
+    
+    
+    var maximumNumberOfSimultaneousItems: Int {
+        switch self {
+        case .downloadMainImages:
+            return 50
+        case .creatingThumbnails:
+            return 10
+        case .creatingColors:
+            return 10
+        default:
+            return 1
+        }
+    }
+    
+    var nextType: BackgroundWorkTaskType? {
+        BackgroundWorkTaskType(rawValue: rawValue - 1)
+    }
+    
+    static func < (lhs: BackgroundWorkTaskType, rhs: BackgroundWorkTaskType) -> Bool {
+        lhs.rawValue < rhs.rawValue
+    }
+    
+}
+
+
 final class Database {
     
     // MARK: Variables
     
     let db: Connection
     
-    let upload: Upload
+    private let upload: Upload
     
-    let download: Download
+    private let download: Download
+    
+    let storage: Storage
     
     weak var delegate: DatabaseDelegate?
 
-    init?(url: URL, server: URL) {
+    init?(url: URL, server: URL, storageFolder: URL) {
         guard let db = try? Connection(url.path) else {
             return nil
         }
@@ -54,6 +123,7 @@ final class Database {
         self.db = db
         self.upload = upload
         self.download = download
+        self.storage = Storage(in: storageFolder)
         log("Database loaded with \(capCount) caps")
     }
     
@@ -64,6 +134,11 @@ final class Database {
         (try? db.prepare(Cap.table))?.map(Cap.init) ?? []
     }
     
+    /// The ids of all caps
+    var capIds: Set<Int> {
+        Set(caps.map { $0.id })
+    }
+    
     /// A dictionary of all caps, indexed by their ids
     var capDict: [Int : Cap] {
         caps.reduce(into: [:]) { $0[$1.id] = $1 }
@@ -90,61 +165,68 @@ final class Database {
         (try? db.prepare(Cap.table).reduce(0) { $0 + $1[Cap.columnCount] }) ?? 0
     }
     
-    /// The caps without a downloaded image
+    var nextPendingCapUpload: Cap? {
-    var capsWithoutImages: [Cap] {
-        caps.filter({ !app.storage.hasImage(for: $0.id) })
-    }
-    
-    /// The number of caps without a downloaded image
-    var capCountWithoutImages: Int {
-        capsWithoutImages.count
-    }
-    
-    /// The caps without a downloaded image
-    var capsWithoutThumbnails: [Cap] {
-        caps.filter({ !app.storage.hasThumbnail(for: $0.id) })
-    }
-    
-    /// The number of caps without a downloaded image
-    var capCountWithoutThumbnails: Int {
-        capsWithoutThumbnails.count
-    }
-
-    var pendingImageUploads: [(cap: Int, version: Int)] {
         do {
-            return try db.prepare(upload.table).map { row in
+            guard let row = try db.pluck(Cap.table.filter(Cap.columnUploaded == false).order(Cap.columnId.asc)) else {
-                (cap: row[upload.rowCapId], version: row[upload.rowCapVersion])
+                return nil
             }
+            return Cap(row: row)
         } catch {
-            log("Failed to get pending image uploads")
+            log("Failed to get next pending cap upload")
-            return []
+            return nil
         }
     }
     
-    /// Indicate if there are any unfinished uploads
+    var pendingCapUploadCount: Int {
-    var hasPendingImageUploads: Bool {
-        ((try? db.scalar(upload.table.count)) ?? 0) > 0
-    }
-    
-    var pendingCapUploads: [Cap] {
-        do {
-            return try db.prepare(Cap.table.filter(Cap.columnUploaded == false).order(Cap.columnId.asc)).map(Cap.init)
-        } catch {
-            log("Failed to get pending cap uploads")
-            return []
-        }
-    }
-    
-    var hasPendingCapUploads: Bool {
         do {
             let query = Cap.table.filter(Cap.columnUploaded == false).count
-            return try db.scalar(query) > 0
+            return try db.scalar(query)
         } catch {
             log("Failed to get pending cap upload count")
-            return false
+            return 0
         }
     }
     
+    var nextPendingImageUpload: (id: Int, version: Int)? {
+        do {
+            guard let row = try db.pluck(upload.table) else {
+                return nil
+            }
+            return (id: row[upload.rowCapId], version: row[upload.rowCapVersion])
+        } catch {
+            log("Failed to get pending image uploads")
+            return nil
+        }
+    }
+    
+    var capsWithImages: Set<Int> {
+        capIds.filter { storage.hasImage(for: $0) }
+    }
+    
+    var capsWithThumbnails: Set<Int> {
+        capIds.filter { storage.hasThumbnail(for: $0) }
+    }
+    
+    var pendingImageUploadCount: Int {
+        ((try? db.scalar(upload.table.count)) ?? 0)
+    }
+    
+    /// The number of caps without a thumbnail on disk
+    var pendingCapForThumbnailCreation: Int {
+        caps.reduce(0) { $0 + (storage.hasThumbnail(for: $1.id) ? 0 : 1) }
+    }
+    
+    var pendingCapsForColorCreation: Int {
+        do {
+            return try capCount - db.scalar(Colors.table.count)
+        } catch {
+            log("Failed to get count of caps without color: \(error)")
+            return 0
+        }
+    }
+    
+   
+    
     var classifierVersion: Int {
         set {
             UserDefaults.standard.set(newValue, forKey: Classifier.userDefaultsKey)
@@ -184,28 +266,12 @@ final class Database {
             log("Cap not inserted")
             return false
         }
-        guard app.storage.save(image: image, for: cap.id) else {
+        guard storage.save(image: image, for: cap.id) else {
             log("Cap image not saved")
             return false
         }
-        guard !isInOfflineMode else {
+        addPendingUpload(for: cap.id, version: 0)
-            log("Offline mode: Not uploading cap")
+        startBackgroundWork()
-            return true
-        }
-        upload.upload(name: name, for: cap.id) { success in
-            guard success else {
-                return
-            }
-            self.update(uploaded: true, for: cap.id)
-            self.upload.uploadImage(for: cap.id, version: 0) { actualVersion in
-                guard let actualVersion = actualVersion else {
-                    self.log("Failed to upload first image for cap \(cap.id)")
-                    return
-                }
-                self.log("Uploaded first image for cap \(cap.id)")
-                self.update(count: actualVersion + 1, for: cap.id)
-            }
-        }
         return true
     }
     
@@ -236,7 +302,7 @@ final class Database {
             log("Failed to get count for cap \(cap)")
             return false
         }
-        guard app.storage.save(image: image, for: cap, version: version) else {
+        guard storage.save(image: image, for: cap, version: version) else {
             log("Failed to save image \(version) for cap \(cap) to disk")
             return false
         }
@@ -248,22 +314,7 @@ final class Database {
             log("Failed to add cap \(cap) version \(version) to upload queue")
             return false
         }
-        guard !isInOfflineMode else {
+        startBackgroundWork()
-            log("Offline mode: Not uploading cap image")
-            return true
-        }
-        upload.uploadImage(for: cap, version: version) { actualVersion in
-            guard let actualVersion = actualVersion else {
-                self.log("Failed to upload image \(version) for cap \(cap)")
-                return
-            }
-            guard self.removePendingUpload(of: cap, version: version) else {
-                self.log("Failed to remove version \(version) for cap \(cap) from upload queue")
-                return
-            }
-            self.log("Uploaded version \(actualVersion) for cap \(cap)")
-            self.update(count: actualVersion + 1, for: cap)
-        }
         return true
     }
     
@@ -310,7 +361,7 @@ final class Database {
         guard update("name", for: cap, setter: Cap.columnName <- name, Cap.columnUploaded <- false) else {
             return false
         }
-        uploadRemainingData()
+        startBackgroundWork()
         return true
     }
     
@@ -343,8 +394,12 @@ final class Database {
     
     // MARK: Uploads
     
+    @discardableResult
     private func addPendingUpload(for cap: Int, version: Int) -> Bool {
         do {
+            guard try db.scalar(upload.existsQuery(for: cap, version: version)) == 0 else {
+                return true
+            }
             try db.run(upload.insertQuery(for: cap, version: version))
             return true
         } catch {
@@ -353,6 +408,7 @@ final class Database {
         }
     }
     
+    @discardableResult
     private func removePendingUpload(for cap: Int, version: Int) -> Bool {
         do {
             try db.run(upload.deleteQuery(for: cap, version: version))
@@ -420,32 +476,18 @@ final class Database {
     
     @discardableResult
     func downloadImage(for cap: Int, version: Int = 0, completion: @escaping (_ image: UIImage?) -> Void) -> Bool {
-        return download.image(for: cap, version: version) { image in
+        let url = storage.localImageUrl(for: cap, version: version)
-            if version == 0 && image != nil {
+        return download.image(for: cap, version: version, to: url) { success in
+            if version == 0 && success {
                 DispatchQueue.main.async {
                     self.delegate?.database(didLoadImageForCap: cap)
                 }
             }
+            let image = self.storage.image(for: cap, version: version)
             completion(image)
         }
     }
     
-    func downloadCapNames(completion: @escaping (_ success: Bool) -> Void) {
-        log("Downloading cap names")
-        download.names { names in
-            guard let names = names else {
-                DispatchQueue.main.async {
-                    completion(false)
-                }
-                return
-            }
-            self.update(names: names)
-            DispatchQueue.main.async {
-                completion(true)
-            }
-        }
-    }
-    
     private func update(names: [String]) {
         let notify = capCount > 0
         log("Downloaded cap names (initialDownload: \(!notify))")
@@ -475,46 +517,418 @@ final class Database {
         }
     }
     
-    func downloadMainCapImages(progress: @escaping (_ current: Int, _ total: Int) -> Void) {
+    var isDoingWorkInBackgound: Bool {
-        let caps = capsWithoutImages.map { $0.id }
+        backgroundTaskStatus != nil
+    }
     
-        var downloaded = 0
+    private var didUpdateBackgroundItems = false
-        let total = caps.count
+    private var backgroundTaskStatus: BackgroundWorkTaskType? = nil
+    private var expectedBackgroundWorkStatus: BackgroundWorkTaskType? = nil
     
-        func update() {
+    private var nextBackgroundWorkStatus: BackgroundWorkTaskType? {
-            DispatchQueue.main.async {
+        guard let oldType = backgroundTaskStatus else {
-                progress(downloaded, total)
+            return expectedBackgroundWorkStatus
-            }
         }
-        update()
+        guard let type = expectedBackgroundWorkStatus else {
+            return backgroundTaskStatus?.nextType
+        }
+        guard oldType > type else {
+            return type
+        }
+        return oldType.nextType
+    }
     
-        guard total > 0 else {
+    private func setNextBackgroundWorkStatus() -> BackgroundWorkTaskType? {
-            log("No images to download")
+        backgroundTaskStatus = nextBackgroundWorkStatus
+        expectedBackgroundWorkStatus = nil
+        return backgroundTaskStatus
+    }
+    
+    private let context = CIContext(options: [.workingColorSpace: kCFNull!])
+    
+    
+    func startInitialDownload() {
+        startBackgroundWork(startingWith: .downloadCapNames)
+    }
+    
+    func scheduleClassifierDownload() {
+        startBackgroundWork(startingWith: .downloadClassifier)
+    }
+    
+    func startBackgroundWork() {
+        startBackgroundWork(startingWith: .uploadingCaps)
+    }
+    
+    private func startBackgroundWork(startingWith type: BackgroundWorkTaskType) {
+        guard !isDoingWorkInBackgound else {
+            if expectedBackgroundWorkStatus?.rawValue ?? 0 < type.rawValue {
+                log("Background work scheduled: \(type)")
+                expectedBackgroundWorkStatus = type
+            }
             return
         }
-        log("Starting to download \(total) images")
+        DispatchQueue.global(qos: .utility).async {
+            self.performAllBackgroundWorkItems(allItemsStartingAt: type)
+        }
+    }
+    
+    private func performAllBackgroundWorkItems(allItemsStartingAt type: BackgroundWorkTaskType) {
+        didUpdateBackgroundItems = false
+        expectedBackgroundWorkStatus = type
+        log("Starting background task")
+        while let type = setNextBackgroundWorkStatus() {
+            log("Handling background task: \(type)")
+            guard performAllItems(for: type) else {
+                // If an error occurs, stop the background tasks
+                backgroundTaskStatus = nil
+                expectedBackgroundWorkStatus = nil
+                break
+            }
+        }
+        log("Background work completed")
+        delegate?.databaseDidFinishBackgroundWork()
+    }
+    
+    private func performAllItems(for type: BackgroundWorkTaskType) -> Bool {
+        switch type {
+        case .downloadCapNames:
+            return downloadCapNames()
+        case .downloadCounts:
+            return downloadImageCounts()
+        case .downloadClassifier:
+            return downloadClassifier()
+        case .uploadingCaps:
+            return uploadCaps()
+        case .uploadingImages:
+            return uploadImages()
+        case .downloadMainImages:
+            return downloadMainImages()
+        case .creatingThumbnails:
+            return createThumbnails()
+        case .creatingColors:
+            return createColors()
+        }
+    }
+    
+    private func downloadCapNames() -> Bool {
+        log("Downloading cap names")
+        let result = DispatchGroup.singleTask { callback in
+            download.names { names in
+                guard let names = names else {
+                    callback(false)
+                    return
+                }
+                self.update(names: names)
+                callback(true)
+            }
+        }
+        log("Completed download of cap names")
+        return result
+    }
+    
+    private func downloadImageCounts() -> Bool {
+        log("Downloading cap image counts")
+        let result = DispatchGroup.singleTask { callback in
+            download.imageCounts { counts in
+                guard let counts = counts else {
+                    self.log("Failed to download server image counts")
+                    callback(false)
+                    return
+                }
+                let newCaps = self.didDownload(imageCounts: counts)
+                
+                guard newCaps.count > 0 else {
+                    callback(true)
+                    return
+                }
+                self.log("Found \(newCaps.count) new caps on the server.")
+                self.downloadInfo(for: newCaps) { success in
+                    callback(success)
+                }
+            }
+        }
+        guard result else {
+            log("Failed download of cap image counts")
+            return false
+        }
+        log("Completed download of cap image counts")
+        return true
+    }
+    
+    private func downloadClassifier() -> Bool {
+        log("Downloading classifier (if needed)")
+        let result = DispatchGroup.singleTask { callback in
+            download.classifierVersion { version in
+                guard let version = version else {
+                    self.log("Failed to download server model version")
+                    callback(false)
+                    return
+                }
+                let ownVersion = self.classifierVersion
+                guard ownVersion < version else {
+                    self.log("Not updating classifier: Own version \(ownVersion), server version \(version)")
+                    callback(true)
+                    return
+                }
+                let title = "Download classifier"
+                let detail = ownVersion == 0 ?
+                    "A classifier to match caps is available for download (version \(version)). Would you like to download it now?" :
+                    "Version \(version) of the classifier is available for download (You have version \(ownVersion)). Would you like to download it now?"
+                self.delegate!.database(needsUserConfirmation: title, body: detail) { proceed in
+                    guard proceed else {
+                        self.log("User skipped classifier download")
+                        callback(true)
+                        return
+                    }
+                    self.download.classifier { progress, received, total in
+                        let t = ByteCountFormatter.string(fromByteCount: total, countStyle: .file)
+                        let r = ByteCountFormatter.string(fromByteCount: received, countStyle: .file)
+                        let title = String(format: "%.0f", progress * 100) + " % (\(r) / \(t))"
+                        self.delegate?.database(completedBackgroundWorkItem: "Downloading classifier", subtitle: title)
+                    } completion: { url in
+                        guard let url = url else {
+                            self.log("Failed to download classifier")
+                            callback(false)
+                            return
+                        }
+                        let compiledUrl: URL
+                        do {
+                            compiledUrl = try MLModel.compileModel(at: url)
+                        } catch {
+                            self.log("Failed to compile downloaded classifier: \(error)")
+                            callback(false)
+                            return
+                        }
+                        
+                        guard self.storage.save(recognitionModelAt: compiledUrl) else {
+                            self.log("Failed to save compiled classifier")
+                            callback(false)
+                            return
+                        }
+                        callback(true)
+                        self.classifierVersion = version
+                    }
+                }
+            }
+        }
+        log("Downloaded classifier (if new version existed)")
+        return result
+    }
+    
+    private func uploadCaps() -> Bool {
+        var completed = 0
+        while let cap = nextPendingCapUpload {
+            guard upload.upload(cap) else {
+                delegate?.database(didFailBackgroundWork: "Upload failed",
+                                   subtitle: "Cap \(cap.id) not uploaded")
+                return false
+            }
+            update(uploaded: true, for: cap.id)
+            completed += 1
+            let total = completed + pendingCapUploadCount
+            delegate?.database(completedBackgroundWorkItem: "Uploading caps", subtitle: "\(completed + 1) of \(total)")
+        }
+        return true
+    }
+    
+    private func uploadImages() -> Bool {
+        var completed = 0
+        while let (id, version) = nextPendingImageUpload {
+            guard let cap = self.cap(for: id) else {
+                log("No cap \(id) to upload image \(version)")
+                removePendingUpload(for: id, version: version)
+                continue
+            }
+            guard let url = storage.existingImageUrl(for: cap.id, version: version) else {
+                log("No image \(version) of cap \(id) to upload")
+                removePendingUpload(for: id, version: version)
+                continue
+            }
+            guard let count = upload.upload(imageAt: url, of: cap.id) else {
+                delegate?.database(didFailBackgroundWork: "Upload failed", subtitle: "Image \(version) of cap \(id)")
+                return false
+            }
+            if count > cap.count {
+                update(count: count, for: cap.id)
+            }
+            removePendingUpload(for: id, version: version)
+            
+            completed += 1
+            let total = completed + pendingImageUploadCount
+            delegate?.database(completedBackgroundWorkItem: "Uploading images", subtitle: "\(completed + 1) of \(total)")
+        }
+        return true
+    }
+    
+    private func downloadMainImages() -> Bool {
+        let missing = caps.map { $0.id }.filter { !storage.hasImage(for: $0) }
+        let count = missing.count
+        guard count > 0 else {
+            log("No images to download")
+            return true
+        }
+        log("Starting image downloads")
         
         let group = DispatchGroup()
-        let split = 50
+        group.enter()
-        DispatchQueue.global(qos: .userInitiated).async {
+        
-            for part in caps.split(intoPartsOf: split) {
+        var shouldDownload = true
-                for id in part {
+        let title = "Download images"
-                    let downloading = self.downloadImage(for: id) { _ in
+        let detail = "\(count) caps have no image. Would you like to download them now? (~ \(ByteCountFormatter.string(fromByteCount: Int64(count * 10000), countStyle: .file))). Grid view is not available until all images are downloaded."
+        delegate?.database(needsUserConfirmation: title, body: detail) { proceed in
+            shouldDownload = proceed
+            group.leave()
+        }
+        group.wait()
+        guard shouldDownload else {
+            log("User skipped image download")
+            return false
+        }
+        
+        group.enter()
+        let queue = DispatchQueue(label: "images")
+        let semaphore = DispatchSemaphore(value: 5)
+        
+        var downloadsAreSuccessful = true
+        var completed = 0
+        for cap in missing {
+            queue.async {
+                guard downloadsAreSuccessful else {
+                    return
+                }
+                semaphore.wait()
+                let url = self.storage.localImageUrl(for: cap)
+                self.download.image(for: cap, to: url, queue: queue) { success in
+                    defer { semaphore.signal() }
+                    guard success else {
+                        self.delegate?.database(didFailBackgroundWork: "Download failed", subtitle: "Image of cap \(cap)")
+                        downloadsAreSuccessful = false
+                        group.leave()
+                        return
+                    }
+                    completed += 1
+                    self.delegate?.database(completedBackgroundWorkItem: "Downloading images", subtitle: "\(completed) of \(missing.count)")
+                    if completed == missing.count {
                         group.leave()
                     }
-                    if downloading {
-                        group.enter()
-                    }
                 }
-                if group.wait(timeout: .now() + .seconds(30)) != .success {
-                    self.log("Timed out waiting for images to be downloaded")
-                }
-                downloaded += part.count
-                self.log("Finished \(downloaded) of \(total) image downloads")
-                update()
             }
-            self.log("Finished all image downloads")
         }
+        guard group.wait(timeout: .now() + TimeInterval(missing.count * 2)) == .success else {
+            log("Timed out downloading images")
+            return false
+        }
+        log("Finished all image downloads")
+        return true
+    }
+    
+    private func createThumbnails() -> Bool {
+        let missing = caps.map { $0.id }.filter { !storage.hasThumbnail(for: $0) }
+        guard missing.count > 0 else {
+            log("No thumbnails to create")
+            return true
+        }
+        log("Creating thumbnails")
+        let queue = DispatchQueue(label: "thumbnails")
+        let semaphore = DispatchSemaphore(value: 5)
+        
+        let group = DispatchGroup()
+        group.enter()
+        var thumbnailsAreSuccessful = true
+        var completed = 0
+        for cap in missing {
+            queue.async {
+                guard thumbnailsAreSuccessful else {
+                    return
+                }
+                semaphore.wait()
+                defer { semaphore.signal() }
+                guard let image = self.storage.image(for: cap) else {
+                    self.log("No image for cap \(cap) to create thumbnail")
+                    self.delegate?.database(didFailBackgroundWork: "Creation failed", subtitle: "Thumbnail of cap \(cap)")
+                    thumbnailsAreSuccessful = false
+                    group.leave()
+                    return
+                }
+                let thumb = Cap.thumbnail(for: image)
+                guard self.storage.save(thumbnail: thumb, for: cap) else {
+                    self.log("Failed to save thumbnail for cap \(cap)")
+                    self.delegate?.database(didFailBackgroundWork: "Image not saved", subtitle: "Thumbnail of cap \(cap)")
+                    thumbnailsAreSuccessful = false
+                    group.leave()
+                    return
+                }
+                completed += 1
+                self.delegate?.database(completedBackgroundWorkItem: "Creating thumbnails", subtitle: "\(completed) of \(missing.count)")
+                if completed == missing.count {
+                    group.leave()
+                }
+            }
+        }
+        guard group.wait(timeout: .now() + TimeInterval(missing.count * 2)) == .success else {
+            log("Timed out creating thumbnails")
+            return false
+        }
+        log("Finished all thumbnails")
+        return true
+    }
+    
+    private func createColors() -> Bool {
+        let missing = capIds.subtracting(capsWithColors)
+        guard missing.count > 0 else {
+            log("No colors to create")
+            return true
+        }
+        log("Creating colors")
+        let queue = DispatchQueue(label: "colors")
+        let semaphore = DispatchSemaphore(value: 5)
+        
+        let group = DispatchGroup()
+        group.enter()
+        var colorsAreSuccessful = true
+        var completed = 0
+        for cap in missing {
+            queue.async {
+                guard colorsAreSuccessful else {
+                    return
+                }
+                semaphore.wait()
+                defer { semaphore.signal() }
+                guard let image = self.storage.ciImage(for: cap) else {
+                    self.log("No image for cap \(cap) to create color")
+                    self.delegate?.database(didFailBackgroundWork: "No thumbnail found", subtitle: "Color of cap \(cap)")
+                    colorsAreSuccessful = false
+                    group.leave()
+                    return
+                }
+                defer { self.context.clearCaches() }
+                guard let color = image.averageColor(context: self.context) else {
+                    self.log("Failed to create color for cap \(cap)")
+                    self.delegate?.database(didFailBackgroundWork: "Calculation failed", subtitle: "Color of cap \(cap)")
+                    colorsAreSuccessful = false
+                    group.leave()
+                    return
+                }
+                guard self.set(color: color, for: cap) else {
+                    self.log("Failed to save color for cap \(cap)")
+                    self.delegate?.database(didFailBackgroundWork: "Color not saved", subtitle: "Color of cap \(cap)")
+                    colorsAreSuccessful = false
+                    group.leave()
+                    return
+                }
+                completed += 1
+                self.delegate?.database(completedBackgroundWorkItem: "Creating colors", subtitle: "\(completed) of \(missing.count)")
+                if completed == missing.count {
+                    group.leave()
+                }
+            }
+        }
+        guard group.wait(timeout: .now() + TimeInterval(missing.count * 2)) == .success else {
+            log("Timed out creating colors")
+            return false
+        }
+        log("Finished all colors")
+        return true
     }
     
     func hasNewClassifier(completion: @escaping (_ version: Int?, _ size: Int64?) -> Void) {
@@ -537,65 +951,6 @@ final class Database {
         }
     }
     
-    func downloadClassifier(progress: Download.Delegate.ProgressHandler? = nil, completion: @escaping (_ success: Bool) -> Void) {
-        download.classifier(progress: progress) { url in
-            guard let url = url else {
-                self.log("Failed to download classifier")
-                completion(false)
-                return
-            }
-            let compiledUrl: URL
-            do {
-                compiledUrl = try MLModel.compileModel(at: url)
-            } catch {
-                self.log("Failed to compile downloaded classifier: \(error)")
-                completion(false)
-                return
-            }
-            
-            guard app.storage.save(recognitionModelAt: compiledUrl) else {
-                self.log("Failed to save classifier")
-                completion(false)
-                return
-            }
-            completion(true)
-            self.download.classifierVersion { version in
-                guard let version = version else {
-                    self.log("Failed to download classifier version")
-                    return
-                }
-                self.classifierVersion = version
-            }
-        }
-    }
-    
-    func downloadImageCounts(completion: @escaping (_ success: Bool) -> Void) {
-        log("Refreshing server image counts")
-        download.imageCounts { counts in
-            guard let counts = counts else {
-                self.log("Failed to download server image counts")
-                DispatchQueue.main.async {
-                    completion(false)
-                }
-                return
-            }
-            let newCaps = self.didDownload(imageCounts: counts)
-            
-            guard newCaps.count > 0 else {
-                DispatchQueue.main.async {
-                    completion(true)
-                }
-                return
-            }
-            self.log("Found \(newCaps.count) new caps on the server.")
-            self.downloadInfo(for: newCaps) { success in
-                DispatchQueue.main.async {
-                    completion(success)
-                }
-            }
-        }
-    }
-    
     private func didDownload(imageCounts newCounts: [Int]) -> [Int : Int] {
         let capsCounts = capDict
         if newCounts.count != capsCounts.count {
@@ -662,119 +1017,6 @@ final class Database {
         }
     }
     
-    private func uploadNextItem() {
-        let capUploads = self.pendingCapUploads
-        if let id = capUploads.first {
-            
-            return
-        }
-        let imageUploads = pendingImageUploads
-        guard imageUploads.count > 0 else {
-            log("No pending image uploads")
-            return
-        }
-        uploadRemainingImages()
-    }
-    
-    private func upload(cap: Int) {
-        
-    }
-
-    func uploadRemainingData() {
-        guard !isInOfflineMode else {
-            log("Not uploading pending data due to offline mode")
-            return
-        }
-        let uploads = self.pendingCapUploads
-        guard uploads.count > 0 else {
-            log("No pending cap uploads")
-            uploadRemainingImages()
-            return
-        }
-        log("\(uploads.count) cap uploads pending")
-        
-        var remaining = uploads.count
-        DispatchQueue.global(qos: .background).async {
-            let group = DispatchGroup()
-            for cap in uploads {
-                group.enter()
-                self.upload.upload(name: cap.name, for: cap.id) { success in
-                    group.leave()
-                    if success {
-                        self.log("Uploaded cap \(cap.id)")
-                        self.update(uploaded: true, for: cap.id)
-                    } else {
-                        self.log("Failed to upload cap \(cap.id)")
-                        return
-                    }
-                    
-                    remaining -= 1
-                    
-                }
-                guard group.wait(timeout: .now() + .seconds(60)) == .success else {
-                    self.log("Timed out uploading cap \(cap.id)")
-                    return
-                }
-            }
-            DispatchQueue.main.async {
-                self.uploadRemainingImages()
-            }
-        }
-        
-    }
-    
-    private func uploadRemainingImages() {
-        let uploads = pendingImageUploads
-        guard uploads.count > 0 else {
-            log("No pending image uploads")
-            return
-        }
-        log("\(uploads.count) image uploads pending")
-        
-        DispatchQueue.global(qos: .background).async {
-            let group = DispatchGroup()
-            for (id, version) in uploads {
-                guard let cap = self.cap(for: id) else {
-                    self.log("No cap \(id) to upload image \(version)")
-                    self.removePendingUpload(of: id, version: version)
-                    continue
-                }
-                guard cap.uploaded else {
-                    self.log("Cap \(id) not uploaded, skipping image upload")
-                    continue
-                }
-                group.enter()
-                self.upload.uploadImage(for: id, version: version) { count in
-                    group.leave()
-                    guard let _ = count else {
-                        self.log("Failed to upload version \(version) of cap \(id)")
-                        return
-                    }
-                    self.log("Uploaded version \(version) of cap \(id)")
-                    self.removePendingUpload(of: id, version: version)
-                }
-                guard group.wait(timeout: .now() + .seconds(60)) == .success else {
-                    self.log("Timed out uploading version \(version) of cap \(id)")
-                    return
-                }
-            }
-        }
-        
-    }
-    
-    @discardableResult
-    func removePendingUpload(of cap: Int, version: Int) -> Bool {
-        do {
-            let query = upload.table.filter(upload.rowCapId == cap && upload.rowCapVersion == version).delete()
-            try db.run(query)
-            log("Deleted pending upload of cap \(cap) version \(version)")
-            return true
-        } catch {
-            log("Failed to delete pending upload of cap \(cap) version \(version)")
-            return false
-        }
-    }
-    
     func setMainImage(of cap: Int, to version: Int) {
         guard version != 0 else {
             log("No need to switch main image with itself for cap \(cap)")
@@ -785,7 +1027,7 @@ final class Database {
                 self.log("Could not make \(version) the main image for cap \(cap)")
                 return
             }
-            guard app.storage.switchMainImage(to: version, for: cap) else {
+            guard self.storage.switchMainImage(to: version, for: cap) else {
                 self.log("Could not switch \(version) to main image for cap \(cap)")
                 return
             }

CapCollector/Data/Download.swift

@@ -111,6 +111,25 @@ final class Download {
     
     // MARK: Downloading data
     
+    func image(for cap: Int, to url: URL, timeout: TimeInterval = 30) -> Bool {
+        let group = DispatchGroup()
+        group.enter()
+        var result = true
+        let success = image(for: cap, version: 0, to: url) { success in
+            result = success
+            group.leave()
+        }
+        guard success else {
+            log("Already downloading image for cap \(cap)")
+            return false
+        }
+        guard group.wait(timeout: .now() + timeout) == .success else {
+            log("Timed out downloading image for cap \(cap)")
+            return false
+        }
+        return result
+    }
+    
     /**
      Download an image for a cap.
      - Parameter cap: The id of the cap.
@@ -119,7 +138,7 @@ final class Download {
      - Returns: `true`, of the file download was started, `false`, if the image is already downloading.
      */
     @discardableResult
-    func image(for cap: Int, version: Int = 0, completion: @escaping (_ image: UIImage?) -> Void) -> Bool {
+    func image(for cap: Int, version: Int = 0, to url: URL, queue: DispatchQueue = .main, completion: @escaping (Bool) -> Void) -> Bool {
         // Check if main image, and already being downloaded
         if version == 0 {
             guard !downloadingMainImages.contains(cap) else {
@@ -127,24 +146,28 @@ final class Download {
             }
             downloadingMainImages.insert(cap)
         }
-        let url = serverImageUrl(for: cap, version: version)
+        let serverUrl = serverImageUrl(for: cap, version: version)
         let query = "Image of cap \(cap) version \(version)"
-        let task = session.downloadTask(with: url) { fileUrl, response, error in
+        let task = session.downloadTask(with: serverUrl) { fileUrl, response, error in
             if version == 0 {
-                DispatchQueue.main.async {
+                queue.async {
                     self.downloadingMainImages.remove(cap)
                 }
             }
             guard let fileUrl = self.convertResponse(to: query, fileUrl, response, error) else {
-                completion(nil)
+                completion(false)
                 return
             }
-            guard let image = app.storage.saveImage(at: fileUrl, for: cap, version: version) else {
+            do {
-                self.log("Request '\(query)' could not move downloaded file")
+                if FileManager.default.fileExists(atPath: url.path) {
-                completion(nil)
+                    try FileManager.default.removeItem(at: url)
-                return
+                }
+                try FileManager.default.moveItem(at: fileUrl, to: url)
+            } catch {
+                self.log("Failed to move downloaded image for cap \(cap): \(error)")
+                completion(false)
             }
-            completion(image)
+            completion(true)
         }
         task.resume()
         return true

CapCollector/Data/Storage.swift

@@ -11,7 +11,30 @@ import UIKit
 import CoreML
 import Vision
 
-final class Storage {
+
+
+protocol ImageProvider: class {
+    
+    func image(for cap: Int) -> UIImage?
+    
+    func image(for cap: Int, version: Int) -> UIImage?
+    
+    func ciImage(for cap: Int) -> CIImage?
+}
+
+protocol ThumbnailCreationDelegate {
+    
+    func thumbnailCreation(progress: Int, total: Int)
+    
+    func thumbnailCreationFailed()
+    
+    func thumbnailCreationIsMissingImages()
+    
+    func thumbnailCreationCompleted()
+}
+
+
+final class Storage: ImageProvider {
     
     // MARK: Paths
     
@@ -35,7 +58,7 @@ final class Storage {
         baseUrl.appendingPathComponent("model.mlmodel")
     }
     
-    private func localImageUrl(for cap: Int, version: Int) -> URL {
+    func localImageUrl(for cap: Int, version: Int = 0) -> URL {
         baseUrl.appendingPathComponent("\(cap)-\(version).jpg")
     }
     
@@ -114,7 +137,7 @@ final class Storage {
      - parameter cap: The cap id
      - returns: True, if the image was saved
      */
-    func save(thumbnailData: Data, for cap: Int) -> Bool {
+    private func save(thumbnailData: Data, for cap: Int) -> Bool {
         write(thumbnailData, to: thumbnailUrl(for: cap))
     }
     
@@ -227,7 +250,7 @@ final class Storage {
      - note: Removes invalid image data on disk, if the data is not a valid image
      - note: Must be called on the main thread
      */
-    func image(for cap: Int, version: Int = 0) -> UIImage? {
+    func image(for cap: Int, version: Int) -> UIImage? {
         guard let data = imageData(for: cap, version: version) else {
             return nil
         }
@@ -239,6 +262,10 @@ final class Storage {
         return image
     }
     
+    func image(for cap: Int) -> UIImage? {
+        image(for: cap, version: 0)
+    }
+    
     /**
      Get the thumbnail data for a cap.
      If the image exists on disk, it is returned.

CapCollector/Data/Upload.swift

@@ -57,6 +57,10 @@ struct Upload {
         }
     }
     
+    func existsQuery(for cap: Int, version: Int) -> ScalarQuery<Int> {
+        table.filter(rowCapId == cap && rowCapVersion == version).count
+    }
+    
     func insertQuery(for cap: Int, version: Int) -> Insert {
         table.insert(rowCapId <- cap, rowCapVersion <- version)
     }
@@ -97,36 +101,55 @@ struct Upload {
         task.resume()
     }
     
-    func uploadImage(for cap: Int, version: Int, completion: @escaping (_ count: Int?) -> Void) {
+    func upload(_ cap: Cap, timeout: TimeInterval = 30) -> Bool {
-        guard let url = app.storage.existingImageUrl(for: cap, version: version) else {
+        upload(name: cap.name, for: cap.id, timeout: timeout)
-            completion(nil)
+    }
-            return
+    
+    func upload(name: String, for cap: Int, timeout: TimeInterval = 30) -> Bool {
+        let group = DispatchGroup()
+        group.enter()
+        var result = true
+        upload(name: name, for: cap) { success in
+            if success {
+                self.log("Uploaded cap \(cap)")
+            } else {
+                result = false
+            }
+            group.leave()
         }
+        guard group.wait(timeout: .now() + timeout) == .success else {
+            log("Timed out uploading cap \(cap)")
+            return false
+        }
+        return result
+    }
+    
+    func upload(imageAt url: URL, for cap: Int, completion: @escaping (_ count: Int?) -> Void) {
         var request = URLRequest(url: serverImageUploadUrl(for: cap))
         request.httpMethod = "POST"
         let task = URLSession.shared.uploadTask(with: request, fromFile: url) { data, response, error in
             if let error = error {
-                self.log("Failed to upload image \(version) of cap \(cap): \(error)")
+                self.log("Failed to upload image of cap \(cap): \(error)")
                 completion(nil)
                 return
             }
             guard let response = response else {
-                self.log("Failed to upload image \(version) of cap \(cap): No response")
+                self.log("Failed to upload image of cap \(cap): No response")
                 completion(nil)
                 return
             }
             guard let urlResponse = response as? HTTPURLResponse else {
-                self.log("Failed to upload image \(version) of cap \(cap): \(response)")
+                self.log("Failed to upload image of cap \(cap): \(response)")
                 completion(nil)
                 return
             }
             guard urlResponse.statusCode == 200 else {
-                self.log("Failed to upload image \(version) of cap \(cap): Response \(urlResponse.statusCode)")
+                self.log("Failed to upload image of cap \(cap): Response \(urlResponse.statusCode)")
                 completion(nil)
                 return
             }
             guard let d = data, let string = String(data: d, encoding: .utf8), let int = Int(string) else {
-                self.log("Failed to upload image \(version) of cap \(cap): Invalid response")
+                self.log("Failed to upload image of cap \(cap): Invalid response")
                 completion(nil)
                 return
             }
@@ -135,6 +158,21 @@ struct Upload {
         task.resume()
     }
     
+    func upload(imageAt url: URL, of cap: Int, timeout: TimeInterval = 30) -> Int? {
+        let group = DispatchGroup()
+        group.enter()
+        var result: Int? = nil
+        upload(imageAt: url, for: cap) { count in
+            result = count
+            group.leave()
+        }
+        guard group.wait(timeout: .now() + timeout) == .success else {
+            log("Timed out uploading image of \(cap)")
+            return nil
+        }
+        return result
+    }
+    
     /**
      Sets the main image for a cap to a different version.
      

CapCollector/Extensions/DispatchGroup+Extensions.swift

@@ -0,0 +1,28 @@
+//
+//  DispatchGroup+Extensions.swift
+//  CapCollector
+//
+//  Created by iMac on 13.01.21.
+//  Copyright © 2021 CH. All rights reserved.
+//
+
+import Foundation
+
+extension DispatchGroup {
+    
+    typealias AsyncSuccessCallback = (Bool) -> Void
+    
+    static func singleTask(timeout: TimeInterval = 30, _ block: (@escaping AsyncSuccessCallback) -> Void) -> Bool {
+        let group = DispatchGroup()
+        group.enter()
+        var result = true
+        block { success in
+            result = success
+            group.leave()
+        }
+        guard group.wait(timeout: .now() + timeout) == .success else {
+            return false
+        }
+        return result
+    }
+}

CapCollector/Presentation/GridViewController.swift

@@ -60,7 +60,7 @@ class GridViewController: UIViewController {
                 view.backgroundColor = tileColor(tile: tile)
             } else {
                 let id = tiles[tile]
-                if let image = app.storage.thumbnail(for: id)  {
+                if let image = app.database.storage.thumbnail(for: id)  {
                     view.image = image
                     continue
                 }
@@ -216,7 +216,7 @@ class GridViewController: UIViewController {
             return
             
         }
-        if let image = app.storage.thumbnail(for: tiles[tile]) {
+        if let image = app.database.storage.thumbnail(for: tiles[tile]) {
             view.image = image
             return
         }
@@ -238,7 +238,7 @@ class GridViewController: UIViewController {
                 self.log("No installed tile for downloaded image \(id)")
                 return
             }
-            guard let image = app.storage.thumbnail(for: id) else {
+            guard let image = app.database.storage.thumbnail(for: id) else {
                 self.log("Failed to load image for cap \(id) after successful download")
                 return
             }

CapCollector/Presentation/ImageSelector.swift

@@ -37,6 +37,8 @@ class ImageSelector: UIViewController {
 
     var cap: Cap!
     
+    weak var imageProvider: ImageProvider?
+
     @IBOutlet weak var collection: UICollectionView!
     
     private var titleText: String {
@@ -97,7 +99,7 @@ class ImageSelector: UIViewController {
         images = [UIImage?](repeating: nil, count: cap.count)
         log("\(cap.count) images for cap \(cap.id)")
         for version in 0..<cap.count {
-            if let image = app.storage.image(for: cap.id, version: version) {
+            if let image = imageProvider?.image(for: cap.id, version: version) {
                 log("Image \(version) already downloaded")
                 set(image, for: version)
             } else {

CapCollector/TableView.swift

@@ -8,13 +8,27 @@
 
 import UIKit
 
-import JGProgressHUD
+enum NavigationBarDataType {
+    
+    case appInfo
+    case upload
+    case thumbnails
+}
+
+protocol NavigationBarDataSource {
+    
+    var title: String { get }
+    
+    var subtitle: String { get }
+    
+    var id: NavigationBarDataType { get }
+}
 
 class TableView: UITableViewController {
     
     @IBOutlet weak var infoButton: UIBarButtonItem!
     
-    private var classifier: Classifier?
+    private lazy var classifier: Classifier? = loadClassifier()
     
     private var accessory: SearchAndDisplayAccessory?
     
@@ -39,9 +53,9 @@ class TableView: UITableViewController {
     
     private var isUnlocked = false
     
-    private var processingScreenHud: JGProgressHUD?
+    var imageProvider: ImageProvider {
-    
+        app.database.storage
-    private var titleViewUpdateTimer: Timer?
+    }
     
     // MARK: Computed properties
     
@@ -84,11 +98,14 @@ class TableView: UITableViewController {
     // MARK: - Actions
     
     @IBAction func updateInfo(_ sender: UIBarButtonItem, forEvent event: UIEvent) {
-        guard let touch = event.allTouches?.first, touch.tapCount > 0, !app.database.isInOfflineMode else {
+        guard let touch = event.allTouches?.first, touch.tapCount > 0 else {
+            return
+        }
+        guard !app.database.isInOfflineMode else {
             showOfflineDialog()
             return
         }
-        downloadCapNames()
+        app.database.startInitialDownload()
     }
     
     @IBAction func showMosaic(_ sender: UIBarButtonItem) {
@@ -136,12 +153,10 @@ class TableView: UITableViewController {
         let count = app.database.capCount
         if count == 0 {
             log("No caps found, downloading names")
-            downloadCapNames()
+            app.database.startInitialDownload()
-            showProcessingScreen()
         } else {
             log("Loaded \(count) caps")
             reloadCapsFromDatabase()
-            loadClassifier()
         }
     }
     
@@ -151,6 +166,7 @@ class TableView: UITableViewController {
         (navigationController as? NavigationController)?.allowLandscape = false
         isUnlocked = app.isUnlocked
         log(isUnlocked ? "App is unlocked" : "App is locked")
+        app.database.startBackgroundWork()
     }
 
     override func didMove(toParent parent: UIViewController?) {
@@ -198,6 +214,13 @@ class TableView: UITableViewController {
         stackView.addGestureRecognizer(longPress)
     }
     
+    private func set(title: String, subtitle: String) {
+        DispatchQueue.main.async {
+            self.titleLabel?.text = title
+            self.subtitleLabel?.text = subtitle
+        }
+    }
+
     private func updateNavigationItemTitleView() {
         DispatchQueue.main.async {
             self.titleLabel?.text = self.titleText
@@ -208,114 +231,14 @@ class TableView: UITableViewController {
     // MARK: Starting updates
     
     private func checkThumbnailsAndColorsBeforShowingGrid() {
-        let missingImageCount = app.database.capCountWithoutImages
+        let colors = app.database.pendingCapsForColorCreation
-        guard missingImageCount == 0 else {
+        let thumbs = app.database.pendingCapForThumbnailCreation
-            askUserToDownload(capImages: missingImageCount)
+        guard colors == 0 && thumbs == 0 else {
+            app.database.startBackgroundWork()
+            showAlert("Please wait until all background work is completed. \(colors) colors and \(thumbs) thumbnails need to be created.", title: "Mosaic not ready")
             return
         }
-        createMissingThumbnailsBeforeShowingGrid()
+        showGrid()
-    }
-    
-    private func createMissingThumbnailsBeforeShowingGrid() {
-        let missing = app.database.capsWithoutThumbnails.map { $0.id }
-        guard missing.count > 0 else {
-            log("No thumbnails missing, checking colors")
-            checkColorsBeforeShowingGrid()
-            return
-        }
-        log("Generating \(missing.count) thumbnails")
-        let hud = JGProgressHUD(style: traitCollection.userInterfaceStyle == .dark ? .dark : .light)
-        hud.indicatorView = JGProgressHUDPieIndicatorView()
-        hud.detailTextLabel.text = "0 % complete (0 / \(missing.count)"
-        hud.textLabel.text = "Generating thumbnails"
-        hud.show(in: self.view)
-        
-        let group = DispatchGroup()
-        var done = 0
-        let split = 50
-        DispatchQueue.global(qos: .background).async {
-            for part in missing.split(intoPartsOf: split) {
-                for id in part {
-                    group.enter()
-                    defer {
-                        done += 1
-                        let ratio = Float(done) / Float(missing.count)
-                        let percent = Int((ratio * 100).rounded())
-                        DispatchQueue.main.async {
-                            hud.progress = ratio
-                            hud.detailTextLabel.text = "\(percent) % complete (\(done) / \(missing.count))"
-                        }
-                        group.leave()
-                    }
-                    guard let image = app.storage.image(for: id) else {
-                        return
-                    }
-                    let thumbnail = Cap.thumbnail(for: image)
-                    _ = app.storage.save(thumbnail: thumbnail, for: id)
-                }
-                if group.wait(timeout: .now() + .seconds(30)) != .success {
-                    self.log("Timed out waiting for thumbnails to be generated")
-                }
-            }
-            DispatchQueue.main.async {
-                hud.dismiss()
-                self.checkColorsBeforeShowingGrid()
-            }
-        }
-    }
-    
-    private func checkColorsBeforeShowingGrid() {
-        let missing = Array(app.database.capsWithoutColors)
-        
-        guard missing.count > 0 else {
-            log("No missing colors, showing grid")
-            showGrid()
-            return
-        }
-        log("Generating \(missing.count) colors")
-        let hud = JGProgressHUD(style: traitCollection.userInterfaceStyle == .dark ? .dark : .light)
-        hud.indicatorView = JGProgressHUDPieIndicatorView()
-        hud.detailTextLabel.text = "0 % complete (0 / \(missing.count)"
-        hud.textLabel.text = "Generating colors"
-        hud.show(in: self.view)
-        
-        let group = DispatchGroup()
-        var done = 0
-        let split = 50
-        let context = CIContext(options: [.workingColorSpace: kCFNull!])
-        
-        DispatchQueue.global(qos: .background).async {
-            for part in missing.split(intoPartsOf: split) {
-                for id in part {
-                    group.enter()
-                    defer {
-                        done += 1
-                        let ratio = Float(done) / Float(missing.count)
-                        let percent = Int((ratio * 100).rounded())
-                        DispatchQueue.main.async {
-                            hud.progress = ratio
-                            hud.detailTextLabel.text = "\(percent) % complete (\(done) / \(missing.count))"
-                        }
-                        group.leave()
-                    }
-                    guard let image = app.storage.ciImage(for: id) else {
-                        return
-                    }
-                    guard let color = image.averageColor(context: context) else {
-                        return
-                    }
-                    _ = app.database.set(color: color, for: id)
-                }
-                if group.wait(timeout: .now() + .seconds(30)) != .success {
-                    self.log("Timed out waiting for colors to be generated")
-                }
-                context.clearCaches()
-            }
-            DispatchQueue.main.async {
-                hud.dismiss()
-                self.showGrid()
-            }
-        }
     }
     
     private func showGrid() {
@@ -340,7 +263,7 @@ class TableView: UITableViewController {
         if offline {
             print("Marking as online")
             app.database.isInOfflineMode = false
-            app.database.uploadRemainingData()
+            app.database.startBackgroundWork()
             self.showAlert("Offline mode was disabled", title: "Online")
         } else {
             print("Marking as offline")
@@ -349,52 +272,6 @@ class TableView: UITableViewController {
         }
     }
     
-    private func downloadCapNames() {
-        app.database.downloadCapNames { success in
-            guard success else {
-                self.hideProcessingScreen()
-                self.showAlert("Failed to download cap names", title: "Sync failed")
-                return
-            }
-            self.downloadImageCounts()
-        }
-    }
-    
-    private func downloadImageCounts() {
-        app.database.downloadImageCounts { success in
-            guard success else {
-                self.hideProcessingScreen()
-                self.showAlert("Failed to download image counts", title: "Sync failed")
-                return
-            }
-            self.hideProcessingScreen()
-            self.checkIfCapImagesNeedDownload()
-        }
-    }
-    
-    private func checkIfCapImagesNeedDownload() {
-        let count = app.database.capCountWithoutImages
-        guard count > 0 else {
-            log("No cap images to download")
-            self.downloadNewestClassifierIfNeeded()
-            return
-        }
-        DispatchQueue.main.async {
-            self.askUserToDownload(capImages: count)
-        }
-    }
-    
-    private func downloadNewestClassifierIfNeeded() {
-        app.database.hasNewClassifier { version, size in
-            guard let version = version else {
-                return
-            }
-            DispatchQueue.main.async {
-                self.askUserToDownload(classifier: version, size: size)
-            }
-        }
-    }
-    
     private func rename(cap: Cap, at indexPath: IndexPath) {
         let detail =  "Choose a new name for the cap"
         askUserForText("Enter new name", detail: detail, existingText: cap.name, yesText: "Save") { text in
@@ -441,26 +318,6 @@ class TableView: UITableViewController {
     
     // MARK: User interaction
     
-    private func showProcessingScreen() {
-        guard processingScreenHud == nil else {
-            log("Already showing processing screen")
-            return
-        }
-        
-        let style: JGProgressHUDStyle = traitCollection.userInterfaceStyle == .dark ? .dark : .extraLight
-        let hud = JGProgressHUD(style: style)
-        hud.indicatorView = JGProgressHUDIndeterminateIndicatorView()
-        hud.detailTextLabel.text = "Please wait until the app has finished processing."
-        hud.textLabel.text = "Processing..."
-        hud.show(in: self.view)
-        self.processingScreenHud = hud
-    }
-    
-    private func hideProcessingScreen() {
-        processingScreenHud?.dismiss()
-        processingScreenHud = nil
-    }
-    
     @objc private func attemptChangeOfUserPermissions() {
         guard isUnlocked else {
             attemptAppUnlock()
@@ -497,48 +354,11 @@ class TableView: UITableViewController {
         updateNavigationItemTitleView()
     }
     
-    private func loadClassifier(reload: Bool = false) {
+    private func loadClassifier() -> Classifier? {
-        guard classifier == nil || reload else {
+        guard let model = app.database.storage.recognitionModel else {
-            return
+            return nil
-        }
-        guard let model = app.storage.recognitionModel else {
-            downloadNewestClassifierIfNeeded()
-            return
-        }
-        classifier = Classifier(model: model)
-    }
-
-    private func askUserToDownload(capImages: Int) {
-        let detail = "\(capImages) caps have no image. Would you like to download them now? (\(ByteCountFormatter.string(fromByteCount: Int64(capImages * 10000), countStyle: .file)))"
-        presentUserBinaryChoice("Download images", detail: detail, yesText: "Download", noText: "Later", dismissed: {
-            self.downloadNewestClassifierIfNeeded()
-        }) {
-            self.downloadAllCapImages()
-        }
-    }
-    
-    private func askUserToDownload(classifier version: Int, size: Int64?) {
-        let oldVersion = app.database.classifierVersion
-        let sizeText = size != nil ? " (\(ByteCountFormatter.string(fromByteCount: size!, countStyle: .file)))" : ""
-        guard oldVersion > 0 else {
-            askUserToDownloadFirst(classifier: version, sizeText: sizeText)
-            return
-        }
-        askUserToDownloadNew(classifier: version, sizeText: sizeText, oldVersion: oldVersion)
-    }
-
-    private func askUserToDownloadNew(classifier version: Int, sizeText: String, oldVersion: Int) {
-        let detail = "Version \(version) of the classifier is available for download (You have version \(oldVersion)). Would you like to download it now?"
-        presentUserBinaryChoice("New classifier", detail: detail + sizeText, yesText: "Download") {
-            self.downloadClassifier()
-        }
-    }
-    
-    private func askUserToDownloadFirst(classifier version: Int, sizeText: String) {
-        let detail = "A classifier to match caps is available for download (version \(version). Would you like to download it now?"
-        presentUserBinaryChoice("Download classifier", detail: detail + sizeText, yesText: "Download") {
-            self.downloadClassifier()
         }
+        return Classifier(model: model)
     }
     
     private func askUserForText(_ title: String, detail: String, existingText: String? = nil, placeholder: String? = "Cap name", yesText: String, noText: String = "Cancel", confirmed: @escaping (_ text: String) -> Void) {
@@ -580,55 +400,8 @@ class TableView: UITableViewController {
         }
         alert.addAction(confirm)
         alert.addAction(cancel)
-        self.present(alert, animated: true)
+        DispatchQueue.main.async {
-        
+            self.present(alert, animated: true)
-    }
-    
-    // MARK: Starting downloads
-    
-    private func downloadClassifier() {
-        let style: JGProgressHUDStyle = traitCollection.userInterfaceStyle == .dark ? .dark : .light
-        let hud = JGProgressHUD(style: style)
-        //hud.vibrancyEnabled = true
-        hud.indicatorView = JGProgressHUDPieIndicatorView()
-        hud.detailTextLabel.text = "0 % complete"
-        hud.textLabel.text = "Downloading image classifier"
-        hud.show(in: self.view)
-        
-        app.database.downloadClassifier(progress: { progress, received, total in
-            DispatchQueue.main.async {
-                hud.progress = progress
-                let t = ByteCountFormatter.string(fromByteCount: total, countStyle: .file)
-                let r = ByteCountFormatter.string(fromByteCount: received, countStyle: .file)
-                hud.detailTextLabel.text = String(format: "%.0f", progress * 100) + " % (\(r) / \(t))"
-            }
-        }) { success in
-            DispatchQueue.main.async {
-                hud.dismiss()
-                self.didDownloadClassifier(successfully: success)
-            }
-        }
-    }
-    
-    private func downloadAllCapImages() {
-        let style: JGProgressHUDStyle = traitCollection.userInterfaceStyle == .dark ? .dark : .light
-        let hud = JGProgressHUD(style: style)
-        //hud.vibrancyEnabled = true
-        hud.indicatorView = JGProgressHUDPieIndicatorView()
-        hud.detailTextLabel.text = "0 % complete"
-        hud.textLabel.text = "Downloading cap images"
-        hud.show(in: self.view)
-        
-        app.database.downloadMainCapImages { done, total in
-            let progress = Float(done) / Float(total)
-            let percent = Int((progress * 100).rounded())
-            hud.detailTextLabel.text = "\(percent) % (\(done) / \(total))"
-            hud.progress = progress
-            
-            if done >= total {
-                hud.dismiss(afterDelay: 1.0)
-                self.downloadNewestClassifierIfNeeded()
-            }
         }
     }
     
@@ -688,13 +461,12 @@ class TableView: UITableViewController {
     
     // MARK: Finishing downloads
     
-    private func didDownloadClassifier(successfully success: Bool) {
+    private func didDownloadClassifier() {
-        guard success else {
+        guard let model = app.database.storage.recognitionModel else {
-            self.log("Failed to download classifier")
+            classifier = nil
             return
         }
-        loadClassifier(reload: true)
+        classifier = Classifier(model: model)
-        self.log("Classifier was downloaded.")
         guard let image = accessory!.currentImage else {
             classifyDummyImage()
             return
@@ -853,7 +625,7 @@ extension TableView {
         cell.set(matchLabel: matchText)
         cell.set(countLabel: countText)
         
-        if let image = app.storage.image(for: cap.id) {
+        if let image = imageProvider.image(for: cap.id) {
             cell.set(image: image)
         } else {
             cell.set(image: nil)
@@ -933,6 +705,7 @@ extension TableView {
             let storyboard = UIStoryboard(name: "Main", bundle: nil)
             let controller = storyboard.instantiateViewController(withIdentifier: "ImageSelector") as! ImageSelector
             controller.cap = cap
+            controller.imageProvider = self.imageProvider
             self.navigationController?.pushViewController(controller, animated: true)
             success(true)
         }
@@ -961,7 +734,7 @@ extension TableView {
         let similar = UIContextualAction(style: .normal, title: "Similar\ncaps") { (_, _, success) in
             self.giveFeedback(.medium)
             self.accessory?.hideImageView()
-            guard let image = app.storage.image(for: cap.id) else {
+            guard let image = self.imageProvider.image(for: cap.id, version: 0) else {
                 success(false)
                 return
             }
@@ -983,6 +756,37 @@ extension TableView: Logger { }
 
 extension TableView: DatabaseDelegate {
     
+    func database(needsUserConfirmation title: String, body: String, shouldProceed: @escaping (Bool) -> Void) {
+        presentUserBinaryChoice(title, detail: body, yesText: "Download", noText: "Later", dismissed: {
+            shouldProceed(false)
+        }) {
+            shouldProceed(true)
+        }
+    }
+    
+    func databaseHasNewClassifier() {
+        didDownloadClassifier()
+    }
+    
+    func database(completedBackgroundWorkItem title: String, subtitle: String) {
+        set(title: title, subtitle: subtitle)
+    }
+    
+    func database(didFailBackgroundWork title: String, subtitle: String) {
+        set(title: title, subtitle: subtitle)
+        DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(10)) {
+            self.updateNavigationItemTitleView()
+        }
+    }
+    
+    func databaseDidFinishBackgroundWork() {
+//        set(title: "All tasks completed", subtitle: titleText)
+        self.updateNavigationItemTitleView()
+//        DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(5)) {
+//            self.updateNavigationItemTitleView()
+//        }
+    }
+    
     func database(didAddCap cap: Cap) {
         caps.append(cap)
         updateNavigationItemTitleView()
@@ -1021,14 +825,17 @@ extension TableView: DatabaseDelegate {
     }
     
     func database(didLoadImageForCap cap: Int) {
-        guard let cell = visibleCell(for: cap) else {
+        DispatchQueue.main.async {
-            return
+            guard let cell = self.visibleCell(for: cap) else {
+                return
+            }
+            guard let image = self.imageProvider.image(for: cap) else {
+                self.log("No image for cap \(cap), although it should be loaded")
+                return
+            }
+            cell.set(image: image)
         }
-        guard let image = app.storage.image(for: cap) else {
+        
-            log("No image for cap \(cap), although it should be loaded")
-            return
-        }
-        cell.set(image: image)
     }
     
     func databaseNeedsFullRefresh() {
@@ -1076,11 +883,3 @@ extension TableView: CapAccessoryDelegate {
     }
     
 }
-
-extension TableView {
-    
-    
-    private func switchBetweenNavigationTitleViewsIfNeeded() {
-        
-    }
-}