Caps-iOS/CapCollector/Data/Database.swift

738 lines
24 KiB
Swift
Raw Normal View History

2020-05-16 11:21:55 +02:00
//
// Database.swift
// CapCollector
//
// Created by Christoph on 14.04.20.
// Copyright © 2020 CH. All rights reserved.
//
import Foundation
import UIKit
import CoreML
import SQLite
protocol DatabaseDelegate: class {
func database(didAddCap cap: Cap)
func database(didChangeCap cap: Int)
2020-05-16 11:21:55 +02:00
func database(didLoadImageForCap cap: Int)
2020-05-16 11:21:55 +02:00
func databaseNeedsFullRefresh()
2020-05-16 11:21:55 +02:00
}
final class Database {
// MARK: Variables
let db: Connection
let upload: Upload
let download: Download
weak var delegate: DatabaseDelegate?
2020-05-16 11:21:55 +02:00
init?(url: URL, server: URL) {
guard let db = try? Connection(url.path) else {
return nil
}
let upload = Upload(server: server)
let download = Download(server: server)
do {
try db.run(Cap.createQuery)
try db.run(upload.createQuery)
try db.run(Database.Colors.createQuery)
try db.run(Database.TileImage.createQuery)
2020-05-16 11:21:55 +02:00
} catch {
return nil
}
self.db = db
self.upload = upload
self.download = download
log("Database loaded with \(capCount) caps")
}
// MARK: Computed properties
/// All caps currently in the database
var caps: [Cap] {
(try? db.prepare(Cap.table))?.map(Cap.init) ?? []
}
/// A dictionary of all caps, indexed by their ids
var capDict: [Int : Cap] {
caps.reduce(into: [:]) { $0[$1.id] = $1 }
}
2020-05-16 11:21:55 +02:00
/// The ids of the caps which weren't included in the last classification
var unmatchedCaps: [Int] {
let query = Cap.table.select(Cap.columnId).filter(Cap.columnMatched == false)
return (try? db.prepare(query).map { $0[Cap.columnId] }) ?? []
2020-05-16 11:21:55 +02:00
}
/// The number of caps which could be recognized during the last classification
var recognizedCapCount: Int {
(try? db.scalar(Cap.table.filter(Cap.columnMatched == true).count)) ?? 0
2020-05-16 11:21:55 +02:00
}
/// The number of caps currently in the database
var capCount: Int {
(try? db.scalar(Cap.table.count)) ?? 0
}
/// The total number of images for all caps
var imageCount: Int {
(try? db.prepare(Cap.table).reduce(0) { $0 + $1[Cap.columnCount] }) ?? 0
}
/// The caps without a downloaded image
var capsWithoutImages: [Cap] {
caps.filter({ !app.storage.hasImage(for: $0.id) })
2020-05-16 11:21:55 +02:00
}
/// The number of caps without a downloaded image
var capCountWithoutImages: Int {
capsWithoutImages.count
2020-05-16 11:21:55 +02:00
}
/// The caps without a downloaded image
var capsWithoutThumbnails: [Cap] {
caps.filter({ !app.storage.hasThumbnail(for: $0.id) })
}
/// The number of caps without a downloaded image
var capCountWithoutThumbnails: Int {
capsWithoutThumbnails.count
}
2020-05-16 11:21:55 +02:00
var pendingImageUploads: [(cap: Int, version: Int)] {
2020-05-16 11:21:55 +02:00
do {
return try db.prepare(upload.table).map { row in
(cap: row[upload.rowCapId], version: row[upload.rowCapVersion])
}
} catch {
log("Failed to get pending image uploads")
2020-05-16 11:21:55 +02:00
return []
}
}
/// Indicate if there are any unfinished uploads
var hasPendingImageUploads: Bool {
2020-05-16 11:21:55 +02:00
((try? db.scalar(upload.table.count)) ?? 0) > 0
}
var pendingCapUploads: [Cap] {
do {
return try db.prepare(Cap.table.filter(Cap.columnUploaded == false)).map(Cap.init)
} catch {
log("Failed to get pending cap uploads")
return []
}
}
2020-05-16 11:21:55 +02:00
var classifierVersion: Int {
set {
UserDefaults.standard.set(newValue, forKey: Classifier.userDefaultsKey)
log("Classifier version set to \(newValue)")
}
get {
UserDefaults.standard.integer(forKey: Classifier.userDefaultsKey)
}
}
2020-08-09 21:04:30 +02:00
var isInOfflineMode: Bool {
set {
UserDefaults.standard.set(newValue, forKey: Upload.offlineKey)
log("Offline mode set to \(newValue)")
}
get {
UserDefaults.standard.bool(forKey: Upload.offlineKey)
}
}
2020-05-16 11:21:55 +02:00
// MARK: Data updates
/**
Create a new cap with an image.
The cap is inserted into the database, and the name and image will be uploaded to the server.
- parameter image: The main image of the cap
- parameter name: The name of the cap
- note: Must be called on the main queue.
- note: The registered delegates will be informed about the added cap through `database(didAddCap:)`
- returns: `true`, if the cap was created.
*/
func createCap(image: UIImage, name: String) -> Bool {
let cap = Cap(name: name, id: capCount + 1)
2020-05-16 11:21:55 +02:00
guard insert(cap: cap) else {
log("Cap not inserted")
2020-05-16 11:21:55 +02:00
return false
}
guard app.storage.save(image: image, for: cap.id) else {
log("Cap image not saved")
2020-05-16 11:21:55 +02:00
return false
}
2020-08-09 21:04:30 +02:00
guard !isInOfflineMode else {
log("Offline mode: Not uploading cap")
return true
}
2020-05-16 11:21:55 +02:00
upload.upload(name: name, for: cap.id) { success in
guard success else {
return
}
self.update(uploaded: true, for: cap.id)
self.upload.uploadImage(for: cap.id, version: 0) { actualVersion in
guard let actualVersion = actualVersion else {
2020-05-16 11:21:55 +02:00
self.log("Failed to upload first image for cap \(cap.id)")
return
}
self.log("Uploaded first image for cap \(cap.id)")
self.update(count: actualVersion + 1, for: cap.id)
2020-05-16 11:21:55 +02:00
}
}
return true
}
/**
Insert a new cap.
Only inserts the cap into the database, and optionally notifies the delegates.
- note: When a new cap is created, use `createCap(image:name:)` instead
*/
@discardableResult
private func insert(cap: Cap, notify: Bool = true) -> Bool {
2020-05-16 11:21:55 +02:00
do {
try db.run(cap.insertQuery)
if notify {
DispatchQueue.main.async {
self.delegate?.database(didAddCap: cap)
}
2020-05-16 11:21:55 +02:00
}
return true
} catch {
log("Failed to insert cap \(cap.id): \(error)")
return false
}
}
func add(image: UIImage, for cap: Int) -> Bool {
guard let version = count(for: cap) else {
log("Failed to get count for cap \(cap)")
return false
}
guard app.storage.save(image: image, for: cap, version: version) else {
log("Failed to save image \(version) for cap \(cap) to disk")
return false
}
guard update(count: version + 1, for: cap) else {
log("Failed update count \(version) for cap \(cap)")
return false
}
guard addPendingUpload(for: cap, version: version) else {
log("Failed to add cap \(cap) version \(version) to upload queue")
return false
}
2020-08-09 21:04:30 +02:00
guard !isInOfflineMode else {
log("Offline mode: Not uploading cap image")
return true
}
upload.uploadImage(for: cap, version: version) { actualVersion in
guard let actualVersion = actualVersion else {
2020-05-16 11:21:55 +02:00
self.log("Failed to upload image \(version) for cap \(cap)")
return
}
guard self.removePendingUpload(of: cap, version: version) else {
self.log("Failed to remove version \(version) for cap \(cap) from upload queue")
return
}
self.log("Uploaded version \(actualVersion) for cap \(cap)")
self.update(count: actualVersion + 1, for: cap)
2020-05-16 11:21:55 +02:00
}
return true
}
// MARK: Updating cap properties
private func update(_ property: String, for cap: Int, notify: Bool = true, setter: Setter...) -> Bool {
2020-05-16 11:21:55 +02:00
do {
let query = updateQuery(for: cap).update(setter)
try db.run(query)
if notify {
DispatchQueue.main.async {
self.delegate?.database(didChangeCap: cap)
}
}
2020-05-16 11:21:55 +02:00
return true
} catch {
log("Failed to update \(property) for cap \(cap): \(error)")
return false
}
}
@discardableResult
private func update(uploaded: Bool, for cap: Int) -> Bool {
update("uploaded", for: cap, setter: Cap.columnUploaded <- uploaded)
2020-05-16 11:21:55 +02:00
}
2020-05-16 11:21:55 +02:00
@discardableResult
private func update(count: Int, for cap: Int) -> Bool {
update("count", for: cap, setter: Cap.columnCount <- count)
2020-05-16 11:21:55 +02:00
}
@discardableResult
private func update(matched: Bool, for cap: Int) -> Bool {
update("matched", for: cap, setter: Cap.columnMatched <- matched)
2020-05-16 11:21:55 +02:00
}
// MARK: External editing
2020-05-16 11:21:55 +02:00
/**
Update the `name` of a cap.
*/
2020-05-16 11:21:55 +02:00
@discardableResult
func update(name: String, for cap: Int) -> Bool {
guard update("name", for: cap, setter: Cap.columnName <- name, Cap.columnUploaded <- false) else {
return false
}
uploadRemainingCaps()
return true
2020-05-16 11:21:55 +02:00
}
@discardableResult
private func updateWithoutUpload(name: String, for cap: Int) -> Bool {
update("name", for: cap, notify: false, setter: Cap.columnName <- name)
2020-05-16 11:21:55 +02:00
}
func update(recognizedCaps: Set<Int>) {
let unrecognized = self.unmatchedCaps
// Update caps which haven't been recognized before
let newlyRecognized = recognizedCaps.intersection(unrecognized)
let logIndividualMessages = newlyRecognized.count < 10
if !logIndividualMessages {
log("Marking \(newlyRecognized.count) caps as matched")
}
for cap in newlyRecognized {
if logIndividualMessages {
log("Marking cap \(cap) as matched")
}
update(matched: true, for: cap)
}
// Update caps which are no longer recognized
let missing = Set(1...capCount).subtracting(recognizedCaps).subtracting(unrecognized)
for cap in missing {
log("Marking cap \(cap) as not matched")
update(matched: false, for: cap)
}
}
// MARK: Uploads
private func addPendingUpload(for cap: Int, version: Int) -> Bool {
2020-05-16 11:21:55 +02:00
do {
try db.run(upload.insertQuery(for: cap, version: version))
return true
} catch {
log("Failed to add pending upload of cap \(cap) version \(version): \(error)")
return false
}
}
private func removePendingUpload(for cap: Int, version: Int) -> Bool {
2020-05-16 11:21:55 +02:00
do {
try db.run(upload.deleteQuery(for: cap, version: version))
return true
} catch {
log("Failed to remove pending upload of cap \(cap) version \(version): \(error)")
return false
}
}
// MARK: Information retrieval
func cap(for id: Int) -> Cap? {
do {
guard let row = try db.pluck(updateQuery(for: id)) else {
log("No cap with id \(id) in database")
return nil
}
return Cap(row: row)
} catch {
log("Failed to get cap \(id): \(error)")
return nil
}
}
private func count(for cap: Int) -> Int? {
do {
let row = try db.pluck(updateQuery(for: cap).select(Cap.columnCount))
return row?[Cap.columnCount]
2020-05-16 11:21:55 +02:00
} catch {
log("Failed to get count for cap \(cap)")
return nil
}
}
func countOfCaps(withImageCountLessThan limit: Int) -> Int {
do {
return try db.scalar(Cap.table.filter(Cap.columnCount < limit).count)
2020-05-16 11:21:55 +02:00
} catch {
log("Failed to get caps with less than \(limit) images")
return 0
}
}
func lowestImageCountForCaps(startingAt start: Int) -> (count: Int, numberOfCaps: Int) {
do {
var currentCount = start - 1
var capsFound = 0
repeat {
currentCount += 1
capsFound = try db.scalar(Cap.table.filter(Cap.columnCount == currentCount).count)
2020-05-16 11:21:55 +02:00
} while capsFound == 0
return (currentCount, capsFound)
} catch {
return (0,0)
}
}
func updateQuery(for cap: Int) -> Table {
Cap.table.filter(Cap.columnId == cap)
2020-05-16 11:21:55 +02:00
}
// MARK: Downloads
@discardableResult
func downloadMainImage(for cap: Int, completion: @escaping (_ success: Bool) -> Void) -> Bool {
return download.mainImage(for: cap) { success in
guard success else {
completion(false)
2020-05-16 11:21:55 +02:00
return
}
DispatchQueue.main.async {
self.delegate?.database(didLoadImageForCap: cap)
2020-05-16 11:21:55 +02:00
}
completion(true)
2020-05-16 11:21:55 +02:00
}
}
@discardableResult
func downloadImage(for cap: Int, version: Int, completion: @escaping (_ success: Bool) -> Void) -> Bool {
2020-05-16 11:21:55 +02:00
return download.image(for: cap, version: version, completion: completion)
}
func downloadCapNames(completion: @escaping (_ success: Bool) -> Void) {
log("Downloading cap names")
download.names { names in
guard let names = names else {
DispatchQueue.main.async {
completion(false)
}
return
}
self.update(names: names)
DispatchQueue.main.async {
completion(true)
}
}
2020-05-16 11:21:55 +02:00
}
private func update(names: [String]) {
let notify = capCount > 0
log("Downloaded cap names (initialDownload: \(!notify))")
let caps = self.capDict
let changed: [Int] = names.enumerated().compactMap { id, name in
let id = id + 1
guard let existingName = caps[id]?.name else {
// Insert cap
let cap = Cap(id: id, name: name, count: 0)
guard insert(cap: cap, notify: notify) else {
return nil
}
return id
2020-05-16 11:21:55 +02:00
}
guard existingName != name else {
// Name unchanged
return nil
}
guard updateWithoutUpload(name: name, for: id) else {
return nil
}
return id
}
if !notify {
log("Added \(changed.count) new caps after initial download")
delegate?.databaseNeedsFullRefresh()
2020-05-16 11:21:55 +02:00
}
}
func downloadMainCapImages(progress: @escaping (_ current: Int, _ total: Int) -> Void) {
let caps = capsWithoutImages.map { $0.id }
2020-05-16 11:21:55 +02:00
var downloaded = 0
let total = caps.count
func update() {
DispatchQueue.main.async {
progress(downloaded, total)
}
}
update()
guard total > 0 else {
log("No images to download")
return
}
log("Starting to download \(total) images")
let group = DispatchGroup()
let split = 50
DispatchQueue.global(qos: .userInitiated).async {
for part in caps.split(intoPartsOf: split) {
for id in part {
let downloading = self.downloadMainImage(for: id) { _ in
group.leave()
}
if downloading {
group.enter()
}
}
if group.wait(timeout: .now() + .seconds(30)) != .success {
self.log("Timed out waiting for images to be downloaded")
}
downloaded += part.count
self.log("Finished \(downloaded) of \(total) image downloads")
update()
}
self.log("Finished all image downloads")
}
}
func hasNewClassifier(completion: @escaping (_ version: Int?, _ size: Int64?) -> Void) {
download.classifierVersion { version in
guard let version = version else {
self.log("Failed to download server model version")
completion(nil, nil)
return
}
let ownVersion = self.classifierVersion
guard ownVersion < version else {
self.log("Not updating classifier: Own version \(ownVersion), server version \(version)")
completion(nil, nil)
return
}
self.log("Getting size of classifier \(version)")
2020-05-16 11:21:55 +02:00
self.download.classifierSize { size in
completion(version, size)
}
}
}
func downloadClassifier(progress: Download.Delegate.ProgressHandler? = nil, completion: @escaping (_ success: Bool) -> Void) {
download.classifier(progress: progress) { url in
guard let url = url else {
self.log("Failed to download classifier")
completion(false)
return
}
let compiledUrl: URL
do {
compiledUrl = try MLModel.compileModel(at: url)
} catch {
self.log("Failed to compile downloaded classifier: \(error)")
completion(false)
return
}
guard app.storage.save(recognitionModelAt: compiledUrl) else {
self.log("Failed to save classifier")
completion(false)
return
}
completion(true)
self.download.classifierVersion { version in
guard let version = version else {
self.log("Failed to download classifier version")
return
}
self.classifierVersion = version
}
}
}
func downloadImageCounts(completion: @escaping (_ success: Bool) -> Void) {
2020-05-16 11:21:55 +02:00
log("Refreshing server image counts")
download.imageCounts { counts in
2020-05-16 11:21:55 +02:00
guard let counts = counts else {
self.log("Failed to download server image counts")
DispatchQueue.main.async {
completion(false)
}
2020-05-16 11:21:55 +02:00
return
}
let newCaps = self.didDownload(imageCounts: counts)
2020-05-16 11:21:55 +02:00
guard newCaps.count > 0 else {
DispatchQueue.main.async {
completion(true)
}
return
}
self.log("Found \(newCaps.count) new caps on the server.")
self.downloadInfo(for: newCaps) { success in
DispatchQueue.main.async {
completion(success)
}
}
2020-05-16 11:21:55 +02:00
}
}
private func didDownload(imageCounts newCounts: [Int]) -> [Int : Int] {
let capsCounts = capDict
2020-05-16 11:21:55 +02:00
if newCounts.count != capsCounts.count {
log("Downloaded \(newCounts.count) image counts, but \(capsCounts.count) caps stored locally")
2020-05-16 11:21:55 +02:00
}
var newCaps = [Int : Int]()
let changed = newCounts.enumerated().compactMap { id, newCount -> Int? in
let id = id + 1
guard let oldCount = capsCounts[id]?.count else {
2020-05-16 11:21:55 +02:00
log("Received count \(newCount) for unknown cap \(id)")
newCaps[id] = newCount
2020-05-16 11:21:55 +02:00
return nil
}
guard oldCount != newCount else {
return nil
}
self.update(count: newCount, for: id)
2020-05-16 11:21:55 +02:00
return id
}
switch changed.count {
case 0:
log("Refreshed image counts for all caps without changes")
case 1:
log("Refreshed image counts for caps, changed cap \(changed[0])")
case 2...10:
log("Refreshed image counts for caps \(changed.map(String.init).joined(separator: ", ")).")
default:
log("Refreshed image counts for all caps (\(changed.count) changed)")
}
return newCaps
}
private func downloadInfo(for newCaps: [Int : Int], completion: @escaping (_ success: Bool) -> Void) {
var success = true
let group = DispatchGroup()
for (id, count) in newCaps {
group.enter()
download.name(for: id) { name in
guard let name = name else {
self.log("Failed to get name for new cap \(id)")
success = false
group.leave()
return
}
let cap = Cap(id: id, name: name, count: count)
self.insert(cap: cap)
group.leave()
}
}
if group.wait(timeout: .now() + .seconds(30)) != .success {
self.log("Timed out waiting for images to be downloaded")
}
completion(success)
}
func downloadImageCount(for cap: Int) {
download.imageCount(for: cap) { count in
guard let count = count else {
return
}
self.update(count: count, for: cap)
}
2020-05-16 11:21:55 +02:00
}
func uploadRemainingCaps() {
let uploads = self.pendingCapUploads
guard uploads.count > 0 else {
log("No pending cap uploads")
2020-05-16 11:21:55 +02:00
return
}
log("\(uploads.count) cap uploads pending")
for cap in uploads {
upload.upload(name: cap.name, for: cap.id) { success in
guard success else {
self.log("Failed to upload cap \(cap.id)")
return
}
self.log("Uploaded cap \(cap.id)")
self.update(uploaded: true, for: cap.id)
}
2020-05-16 11:21:55 +02:00
}
}
func uploadRemainingImages() {
let uploads = pendingImageUploads
guard uploads.count > 0 else {
log("No pending image uploads")
2020-05-16 11:21:55 +02:00
return
}
log("\(uploads.count) image uploads pending")
2020-05-16 11:21:55 +02:00
for (cap, version) in uploads {
2020-05-16 11:21:55 +02:00
upload.uploadImage(for: cap, version: version) { count in
guard let _ = count else {
self.log("Failed to upload version \(version) of cap \(cap)")
return
}
self.log("Uploaded version \(version) of cap \(cap)")
self.removePendingUpload(of: cap, version: version)
}
}
}
@discardableResult
func removePendingUpload(of cap: Int, version: Int) -> Bool {
do {
let query = upload.table.filter(upload.rowCapId == cap && upload.rowCapVersion == version).delete()
try db.run(query)
log("Deleted pending upload of cap \(cap) version \(version)")
return true
} catch {
log("Failed to delete pending upload of cap \(cap) version \(version)")
return false
}
}
func setMainImage(of cap: Int, to version: Int) {
guard version != 0 else {
log("No need to switch main image with itself for cap \(cap)")
return
}
upload.setMainImage(for: cap, to: version) { success in
guard success else {
2020-05-16 11:21:55 +02:00
self.log("Could not make \(version) the main image for cap \(cap)")
return
}
guard app.storage.switchMainImage(to: version, for: cap) else {
self.log("Could not switch \(version) to main image for cap \(cap)")
return
}
DispatchQueue.main.async {
self.delegate?.database(didLoadImageForCap: cap)
}
2020-05-16 11:21:55 +02:00
}
}
}
extension Database: Logger { }