Caps-Server/Sources/App/CapServer.swift

615 lines
19 KiB
Swift
Raw Normal View History

2021-11-08 21:58:55 +01:00
import Foundation
import Vapor
2023-01-11 18:29:32 +01:00
import Clairvoyant
2021-11-08 21:58:55 +01:00
final class CapServer {
2021-11-08 21:58:55 +01:00
// MARK: Paths
private let imageFolder: URL
private let thumbnailFolder: URL
/// The file where the cap count is stored for the grid webpage
private let gridCountFile: URL
2022-05-24 14:47:50 +02:00
/// The file where the database of caps is stored
private let dbFile: URL
2022-12-16 13:32:33 +01:00
/// The file to store the HTML info of the cap count
private let htmlFile: URL
2021-11-08 21:58:55 +01:00
private let classifierVersionFile: URL
2022-06-23 22:48:58 +02:00
private let classifierFile: URL
2023-03-20 15:25:58 +01:00
private let classifierClassesFile: URL
2023-01-14 23:04:29 +01:00
private let changedImagesFile: URL
2021-11-08 21:58:55 +01:00
private let fm = FileManager.default
2023-01-15 11:21:47 +01:00
private let changedImageEntryDateFormatter: DateFormatter
/// Indicates that the data is loaded
private(set) var isOperational = false
2023-01-15 11:21:47 +01:00
2021-11-08 21:58:55 +01:00
// MARK: Caps
2022-06-23 22:48:58 +02:00
2023-01-14 23:04:29 +01:00
/// The changed images not yet written to disk
private var unwrittenImageChanges: [(cap: Int, image: Int)] = []
2023-01-11 18:28:37 +01:00
var classifierVersion: Int = 0 {
didSet {
writeClassifierVersion()
2023-02-16 23:08:58 +01:00
Task {
try? await classifierMetric.update(classifierVersion)
}
2022-06-23 22:48:58 +02:00
}
}
2022-05-27 09:25:41 +02:00
2022-05-28 21:58:51 +02:00
/**
The time to wait for changes to be written to disk.
This delay is used to prevent file writes for each small update to the caps.
*/
private let saveDelay: TimeInterval = 1
/**
The time when a save should occur.
No save is necessary if this property is `nil`.
*/
private var nextSaveTime: Date?
2022-05-24 14:47:50 +02:00
private var caps = [Int: Cap]() {
2023-01-11 18:29:32 +01:00
didSet {
scheduleSave()
2023-02-16 23:08:58 +01:00
Task {
2023-09-08 10:05:55 +02:00
_ = try? await capCountMetric.update(caps.count)
_ = try? await imageCountMetric.update(imageCount)
2023-02-16 23:08:58 +01:00
}
2023-01-11 18:29:32 +01:00
}
2022-05-24 14:47:50 +02:00
}
2023-01-11 18:28:37 +01:00
var capCount: Int {
caps.count
}
var imageCount: Int {
caps.reduce(0) { $0 + $1.value.count }
}
2023-02-16 23:08:58 +01:00
init(in folder: URL) async {
2021-11-08 21:58:55 +01:00
self.imageFolder = folder.appendingPathComponent("images")
self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
self.gridCountFile = folder.appendingPathComponent("count.js")
2022-05-24 14:47:50 +02:00
self.dbFile = folder.appendingPathComponent("caps.json")
2022-12-16 13:32:33 +01:00
self.htmlFile = folder.appendingPathComponent("count.html")
self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
2022-06-23 22:48:58 +02:00
self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
2023-03-20 15:25:58 +01:00
self.classifierClassesFile = folder.appendingPathComponent("classifier.classes")
2023-01-14 23:04:29 +01:00
self.changedImagesFile = folder.appendingPathComponent("changes.txt")
2023-01-15 11:21:47 +01:00
self.changedImageEntryDateFormatter = DateFormatter()
changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
2023-02-16 23:08:58 +01:00
// Metric initializers only fail if observer is missing or ID is duplicate
self.capCountMetric = try! await .init("caps.count",
name: "Number of caps",
description: "The total number of caps in the database")
self.imageCountMetric = try! await .init("caps.images",
name: "Total images",
description: "The total number of images for all caps")
self.classifierMetric = try! await .init("caps.classifier",
name: "Classifier Version",
description: "The current version of the image classifier")
2023-01-11 18:28:37 +01:00
}
2022-05-24 14:47:50 +02:00
2023-01-11 18:28:37 +01:00
func loadData() throws {
loadClassifierVersion(at: classifierVersionFile)
2022-05-24 14:47:50 +02:00
try loadCaps()
2022-12-16 13:32:33 +01:00
saveCapCountHTML()
2023-01-15 14:55:10 +01:00
updateGridCapCount()
try ensureExistenceOfChangedImagesFile()
organizeImages()
isOperational = true
2021-11-08 21:58:55 +01:00
}
2023-01-11 18:28:37 +01:00
private func loadClassifierVersion(at url: URL) {
2023-02-17 11:39:32 +01:00
guard exists(url) else {
2023-01-11 18:28:37 +01:00
return
}
let content: String
do {
content = try String(contentsOf: url)
.trimmingCharacters(in: .whitespacesAndNewlines)
} catch {
log("Failed to read classifier version file: \(error)")
return
}
guard let value = Int(content) else {
log("Invalid classifier version: \(content)")
return
}
self.classifierVersion = value
}
private func writeClassifierVersion() {
do {
try "\(classifierVersion)".data(using: .utf8)!
.write(to: classifierVersionFile)
} catch {
log("Failed to save classifier version: \(error)")
}
}
2022-05-24 14:47:50 +02:00
private func loadCaps() throws {
2023-02-17 12:04:32 +01:00
guard exists(dbFile) else {
log("No cap database found")
return
}
2022-10-07 21:13:21 +02:00
do {
let data = try Data(contentsOf: dbFile)
caps = try JSONDecoder().decode([Cap].self, from: data)
.reduce(into: [:]) { $0[$1.id] = $1 }
} catch {
log("Failed to load caps: \(error)")
throw error
}
2021-11-08 21:58:55 +01:00
log("\(caps.count) caps loaded")
}
2022-05-28 21:58:51 +02:00
private func scheduleSave() {
nextSaveTime = Date().addingTimeInterval(saveDelay)
2022-06-11 01:05:02 +02:00
DispatchQueue.global().asyncAfter(deadline: .now() + saveDelay) {
2022-05-28 21:58:51 +02:00
self.performScheduledSave()
}
}
private func performScheduledSave() {
guard let date = nextSaveTime else {
// No save necessary, or already saved
return
}
guard date < Date() else {
// Save pushed to future
return
}
do {
try saveCaps()
nextSaveTime = nil
} catch {
// Attempt save again
scheduleSave()
}
}
2022-05-24 14:47:50 +02:00
private func saveCaps() throws {
let data = try JSONEncoder().encode(caps.values.sorted())
try data.write(to: dbFile)
}
2022-12-16 13:32:33 +01:00
private func saveCapCountHTML() {
let count = caps.count
let content =
"""
2023-03-05 09:13:56 +01:00
<body style="margin: 0;overflow: hidden">
2022-12-16 13:32:33 +01:00
<div style="display: flex; justify-content: center;">
<div style="font-size: 60px; font-family: 'SF Pro Display',-apple-system,BlinkMacSystemFont,Helvetica,sans-serif; -webkit-font-smoothing: antialiased;">\(count)</div>
</div>
</body>
"""
try? content.data(using: .utf8)!.write(to: htmlFile)
}
private func organizeImages() {
2023-01-15 16:45:07 +01:00
caps.values.sorted().forEach(organizeImages)
}
2023-02-17 11:39:32 +01:00
private func createImageFolder(for cap: Int) throws {
let folderUrl = folder(of: cap)
do {
try fm.createDirectory(at: folderUrl, withIntermediateDirectories: true)
} catch {
log("Failed to create folder for cap \(cap): \(error)")
throw error
}
}
2023-03-12 11:49:33 +01:00
/**
Rearrange images of a cap to ensure that an image exists for each number from 0 to `image count - 1`.
This is done by using the last images to fill in possible gaps in the sequence.
E.g. If there are images `0`, `2`, `3`, then `3` will be renamed to `1`.
- Note: The main image is also changed, if the main image is renamed.
*/
2023-01-15 16:45:07 +01:00
private func organizeImages(for cap: Cap) {
var cap = cap
2023-02-17 11:39:32 +01:00
let folderUrl = folder(of: cap.id)
guard exists(folderUrl) else {
try? createImageFolder(for: cap.id)
cap.count = 0
caps[cap.id] = cap
2023-02-19 00:38:06 +01:00
log("Found cap \(cap.id) without image folder")
2023-02-17 11:39:32 +01:00
return
}
guard let images = try? images(in: folderUrl) else {
log("Failed to get image urls for cap \(cap.id)")
return
}
2023-02-18 23:47:52 +01:00
if images.count != cap.count {
log("\(images.count) instead of \(cap.count) images for cap \(cap.id)")
}
// Get list of existing images
var sorted: [(id: Int, url: URL)] = images.compactMap {
2023-01-15 02:02:28 +01:00
guard let id = Int($0.deletingPathExtension().lastPathComponent.components(separatedBy: "-").last!) else {
return nil
}
return (id, $0)
}.sorted { $0.id < $1.id }
2023-02-18 23:47:52 +01:00
// Check that all images are available
for version in 0..<images.count {
guard version != sorted[version].id else {
continue
}
let lastImage = sorted.popLast()!
2023-01-15 16:45:07 +01:00
let newUrl = imageUrl(of: cap.id, version: version)
do {
try fm.moveItem(at: lastImage.url, to: newUrl)
2023-02-18 23:47:52 +01:00
log("Moved image \(lastImage.id) to \(version) for cap \(cap.id)")
} catch {
log("Failed to move file \(lastImage.url.path) to \(newUrl.path): \(error)")
return
}
if cap.mainImage == lastImage.id {
cap.mainImage = version
}
sorted.insert((version, newUrl), at: version)
}
2023-03-12 11:49:33 +01:00
cap.count = sorted.count
2023-03-12 11:49:33 +01:00
// Fix invalid main image
if cap.mainImage >= cap.count || cap.mainImage < 0 {
cap.mainImage = 0
}
caps[cap.id] = cap
}
2022-12-16 13:32:33 +01:00
2021-11-08 21:58:55 +01:00
// MARK: Paths
func folder(of cap: Int) -> URL {
imageFolder.appendingPathComponent(String(format: "%04d", cap))
}
func thumbnail(of cap: Int) -> URL {
thumbnailFolder.appendingPathComponent(String(format: "%04d.jpg", cap))
}
2021-11-08 21:58:55 +01:00
2023-01-15 16:45:07 +01:00
func imageUrl(of cap: Int, version: Int) -> URL {
2021-11-08 21:58:55 +01:00
folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version))
}
2022-05-27 09:25:41 +02:00
2023-02-17 11:39:32 +01:00
private func exists(_ url: URL) -> Bool {
fm.fileExists(atPath: url.path)
}
2021-11-08 21:58:55 +01:00
// MARK: Counts
2022-05-24 14:47:50 +02:00
private func images(in folder: URL) throws -> [URL] {
2023-02-17 11:51:11 +01:00
do {
return try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
.filter { $0.pathExtension == "jpg" }
} catch {
log("Failed to get image urls for cap \(folder.lastPathComponent): \(error)")
throw error
}
2021-11-08 21:58:55 +01:00
}
2022-05-24 14:47:50 +02:00
/**
Get the image count of a cap.
*/
2021-11-08 21:58:55 +01:00
func count(of cap: Int) throws -> Int {
2023-02-17 11:39:32 +01:00
let capImageFolder = folder(of: cap)
guard exists(capImageFolder) else {
2021-11-08 21:58:55 +01:00
return 0
}
2023-02-17 11:39:32 +01:00
return try images(in: capImageFolder).count
2021-11-08 21:58:55 +01:00
}
2022-05-24 14:47:50 +02:00
2021-11-08 21:58:55 +01:00
// MARK: Images
/**
Save a cap image to disk.
Automatically creates the image name with the current image count.
- Parameter data: The image data
- Parameter cap: The id of the cap.
2022-05-24 14:47:50 +02:00
- Throws: `CapError.unknownId`, if the cap doesn't exist. `CapError.dataInconsistency` if an image already exists for the current count.
2021-11-08 21:58:55 +01:00
*/
func save(image data: Data, for cap: Int) throws {
guard caps[cap] != nil else {
2023-02-17 11:51:11 +01:00
log("Tried to save image for unknown cap \(cap)")
2022-05-24 14:47:50 +02:00
throw CapError.unknownId
}
var id = 0
2022-06-11 01:01:24 +02:00
let capFolder = folder(of: cap)
2023-02-17 11:39:32 +01:00
var capImageUrl = imageUrl(of: cap, version: id)
if exists(capFolder) {
while exists(capImageUrl) {
2022-06-11 01:01:24 +02:00
id += 1
2023-02-17 11:39:32 +01:00
capImageUrl = imageUrl(of: cap, version: id)
2022-06-11 01:01:24 +02:00
}
} else {
2023-02-17 11:39:32 +01:00
try createImageFolder(for: cap)
2021-11-08 21:58:55 +01:00
}
2023-02-17 11:51:11 +01:00
do {
try data.write(to: capImageUrl)
} catch {
log("Failed to write image \(id) for cap \(cap): \(error)")
throw CapError.invalidFile
}
2023-03-13 11:10:38 +01:00
let count = try count(of: cap)
caps[cap]!.count = count
2023-01-14 23:04:29 +01:00
addChangedImageToLog(cap: cap, image: id)
2023-03-13 11:10:38 +01:00
log("Added image \(id) for cap \(cap) (\(count) total)")
2021-11-08 21:58:55 +01:00
}
2023-01-14 23:04:29 +01:00
private func writeChangedImagesToDisk() throws {
guard !unwrittenImageChanges.isEmpty else {
return
}
2023-02-17 12:04:32 +01:00
try ensureExistenceOfChangedImagesFile()
2023-01-14 23:04:29 +01:00
2023-02-17 12:04:32 +01:00
let handle: FileHandle
do {
handle = try FileHandle(forWritingTo: changedImagesFile)
try handle.seekToEnd()
} catch {
log("Failed to open changed images file for writing: \(error)")
throw error
}
2023-01-14 23:04:29 +01:00
var entries = unwrittenImageChanges
defer {
unwrittenImageChanges = entries
try? handle.close()
}
2023-01-15 11:21:47 +01:00
let dateString = changedImageEntryDateFormatter.string(from: Date())
2023-01-14 23:04:29 +01:00
while let entry = entries.popLast() {
2023-01-14 23:21:35 +01:00
let content = "\(dateString):\(entry.cap):\(entry.image)\n".data(using: .utf8)!
2023-02-17 12:04:32 +01:00
do {
try handle.write(contentsOf: content)
} catch {
log("Failed to write entry to changed images file: \(error)")
throw error
}
2023-01-14 23:04:29 +01:00
}
}
private func addChangedImageToLog(cap: Int, image: Int) {
unwrittenImageChanges.append((cap, image))
do {
try writeChangedImagesToDisk()
} catch {
log("Failed to save changed image list: \(error)")
}
}
private func ensureExistenceOfChangedImagesFile() throws {
2023-02-17 11:39:32 +01:00
if exists(changedImagesFile) {
2023-01-14 23:04:29 +01:00
return
}
do {
try Data().write(to: changedImagesFile)
} catch {
log("Failed to create changed images file: \(error)")
throw error
}
}
2023-01-15 11:21:47 +01:00
func removeAllEntriesInImageChangeList(before date: Date) {
2023-02-17 12:04:32 +01:00
guard exists(changedImagesFile) else {
log("No file for changed images to update")
return
}
do {
2023-01-15 11:21:47 +01:00
try String(contentsOf: changedImagesFile)
.components(separatedBy: "\n")
.filter { $0 != "" }
.compactMap { line -> String? in
guard let entryDate = changedImageEntryDateFormatter.date(from: line.components(separatedBy: ":").first!) else {
return nil
}
guard entryDate > date else {
return nil
}
return line
}
.joined(separator: "\n")
.data(using: .utf8)!
.write(to: changedImagesFile)
2023-01-14 23:04:29 +01:00
} catch {
2023-01-15 11:21:47 +01:00
log("Failed to update changed images file: \(error)")
2023-01-14 23:04:29 +01:00
}
}
2021-11-08 21:58:55 +01:00
func switchMainImage(to version: Int, for cap: Int) throws {
2023-02-17 11:39:32 +01:00
let capImageUrl = imageUrl(of: cap, version: version)
guard exists(capImageUrl) else {
2021-11-08 21:58:55 +01:00
log("No image \(version) for cap \(cap)")
throw CapError.invalidFile
}
2022-05-24 14:47:50 +02:00
caps[cap]?.mainImage = version
2021-11-08 21:58:55 +01:00
log("Switched cap \(cap) to version \(version)")
}
func addOrUpdate(_ cap: Cap) throws {
if let existingCap = caps[cap.id] {
2022-06-11 00:38:53 +02:00
update(existingCap, with: cap)
} else {
try add(cap)
}
}
private func add(_ cap: Cap) throws {
2022-06-11 00:38:53 +02:00
guard cap.mainImage == 0 else {
2023-02-17 12:04:32 +01:00
log("Attempting to add cap \(cap.id) with main image \(cap.mainImage)")
throw CapError.invalidData
}
2022-06-11 00:38:53 +02:00
var cap = cap
cap.count = 0
caps[cap.id] = cap
2022-12-16 13:32:33 +01:00
saveCapCountHTML()
updateGridCapCount()
2022-05-28 22:09:29 +02:00
log("Added cap \(cap.id) '\(cap.name)'")
}
2022-06-11 00:38:53 +02:00
private func update(_ existingCap: Cap, with cap: Cap) {
var updatedCap = existingCap
if cap.name != "" {
updatedCap.name = cap.name
}
2023-02-17 11:39:32 +01:00
let capImageUrl = imageUrl(of: existingCap.id, version: cap.mainImage)
if exists(capImageUrl) {
updatedCap.mainImage = cap.mainImage
}
if let color = cap.color {
updatedCap.color = color
}
2022-05-28 22:09:29 +02:00
caps[existingCap.id] = updatedCap
log("Updated cap \(existingCap.id)")
}
2023-03-12 11:49:33 +01:00
func deleteImage(version: Int, for capId: Int) -> Cap? {
2023-01-15 16:45:07 +01:00
guard let cap = caps[capId] else {
2023-02-17 12:04:32 +01:00
log("Attempting to delete image \(version) of unknown cap \(capId)")
2023-03-12 11:49:33 +01:00
return nil
2023-01-15 16:45:07 +01:00
}
2023-02-17 11:39:32 +01:00
let capImageUrl = imageUrl(of: capId, version: version)
guard exists(capImageUrl) else {
2023-02-17 12:04:32 +01:00
log("Attempting to delete missing image \(version) of cap \(capId)")
2023-03-12 11:49:33 +01:00
return nil
2023-01-15 16:45:07 +01:00
}
organizeImages(for: cap)
2023-03-12 11:49:33 +01:00
return caps[capId]!
2023-01-15 16:45:07 +01:00
}
2023-03-13 10:41:24 +01:00
func delete(cap capId: Int) -> Bool {
guard caps[capId] != nil else {
log("Attempting to delete unknown cap \(capId)")
return false
}
// 1. Remove all images
do {
let imageFolderUrl = folder(of: capId)
if exists(imageFolderUrl) {
try fm.removeItem(at: imageFolderUrl)
}
} catch {
log("Failed to delete image folder of cap \(capId): \(error)")
return false
}
// 2. Remove thumbnail
do {
let url = thumbnail(of: capId)
if exists(url) {
try fm.removeItem(at: url)
}
} catch {
log("Failed to delete thumbnail of cap \(capId): \(error)")
return false
}
// 3. Remove cap
caps[capId] = nil
saveCapCountHTML()
updateGridCapCount()
return true
}
2023-01-15 16:45:07 +01:00
// MARK: Classifier
2023-03-20 15:25:58 +01:00
func saveTrainedClasses(content: String) throws {
let classes = content.components(separatedBy: ",")
// Validate input
try classes.forEach { s in
guard let id = Int(s) else {
log("Invalid id '\(s)' in uploaded id list")
throw Abort(.badRequest)
}
guard caps[id] != nil else {
log("Unknown id '\(id)' in uploaded id list")
throw Abort(.badRequest)
}
}
2023-03-20 15:25:58 +01:00
guard let data = content.data(using: .utf8) else {
log("Failed to get classes data for writing")
throw Abort(.internalServerError)
}
do {
try data.write(to: classifierClassesFile)
log("Updated \(classes.count) classifier classes")
} catch {
log("Failed to write classifier classes: \(error)")
throw Abort(.internalServerError)
}
}
2022-06-23 22:48:58 +02:00
func save(classifier: Data, version: Int) throws {
do {
2022-06-23 22:48:58 +02:00
try classifier.write(to: classifierFile)
} catch {
2023-02-17 12:04:32 +01:00
log("Failed to write classifier \(version): \(error)")
2022-06-23 22:48:58 +02:00
throw Abort(.internalServerError)
}
2022-06-23 22:48:58 +02:00
classifierVersion = version
2022-06-24 11:49:34 +02:00
log("Updated classifier to version \(version)")
}
2023-01-15 16:45:07 +01:00
// MARK: Grid
func getListOfMissingThumbnails() -> [Int] {
2023-02-17 11:39:32 +01:00
caps.keys.filter { !exists(thumbnail(of: $0)) }
}
func saveThumbnail(_ data: Data, for cap: Int) {
let url = thumbnail(of: cap)
do {
try data.write(to: url)
} catch {
log("Failed to save thumbnail \(cap): \(error)")
}
}
private func updateGridCapCount() {
do {
try "const numberOfCaps = \(capCount);"
.data(using: .utf8)!
.write(to: gridCountFile)
} catch {
log("Failed to save grid cap count: \(error)")
}
}
2023-01-11 18:29:32 +01:00
// MARK: Monitoring
2023-01-11 18:29:32 +01:00
2023-02-16 23:08:58 +01:00
private let capCountMetric: Metric<Int>
2023-01-11 18:29:32 +01:00
2023-02-16 23:08:58 +01:00
private let imageCountMetric: Metric<Int>
2023-01-11 18:29:32 +01:00
2023-02-16 23:08:58 +01:00
private let classifierMetric: Metric<Int>
2021-11-08 21:58:55 +01:00
}