Add image assets

.gitignore

@@ -8,12 +8,11 @@
 .DS_Store
 Package.resolved
 .swiftpm/
+Public/caps.json
+Public/changes.txt
+Public/classifier.*
+Public/count.*
 Public/images/
 Public/thumbnails/
-Public/classifier.version
-Public/classifier.mlmodel
-Public/caps.json
-Training/backup/
-Training/config.json
-Public/thumbnails
-Public/count.js
+Resources/config.json
+Resources/logs/

Package.swift

@@ -8,28 +8,20 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
-        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.9.0"),
-        .package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.2.0"),
+        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.11.2"),
+        .package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.4.0"),
         .package(url: "https://github.com/christophhagen/ClairvoyantBinaryCodable", from: "0.3.0"),
-        .package(url:"https://github.com/christophhagen/CBORCoding", from: "1.0.0"),
     ],
     targets: [
-        .target(name: "App",
-                dependencies: [
-                    .product(name: "Vapor", package: "vapor"),
-                    .product(name: "Clairvoyant", package: "Clairvoyant"),
-                    .product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"),
-                    .product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"),
-                    .product(name: "CBORCoding", package: "CBORCoding"),
-                ],
-                swiftSettings: [
-                    // Enable better optimizations when building in Release configuration. Despite the use of
-                    // the `.unsafeFlags` construct required by SwiftPM, this flag is recommended for Release
-                    // builds. See <https://github.com/swift-server/guides#building-for-production> for details.
-                    .unsafeFlags(["-cross-module-optimization"], .when(configuration: .release))
-                ]),
-        .executableTarget(name: "Run", dependencies: ["App"]),
-        .testTarget(name: "AppTests", dependencies: ["App"]),
+        .executableTarget(
+            name: "App",
+            dependencies: [
+                .product(name: "Vapor", package: "vapor"),
+                .product(name: "Clairvoyant", package: "Clairvoyant"),
+                .product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"),
+                .product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"),
+            ]
+        )
     ]
 )
 

Public/assets/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/assets/mstile-150x150.png?v=1"/>
+            <TileColor>#a36490</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

Public/assets/site.webmanifest

@@ -0,0 +1 @@
+{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

Public/grid.html

@@ -5,13 +5,13 @@
         <meta charset="utf-8"/>
         <meta name="robots" content="noindex"/>
         <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0"/>
-        <link rel="apple-touch-icon" sizes="180x180" href="/assets/icons/apple-touch-icon.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="32x32" href="/assets/icons/favicon-32x32.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="16x16" href="/assets/icons/favicon-16x16.png?v=1"/>
-        <link rel="manifest" href="/assets/icons/site.webmanifest?v=1"/>
-        <link rel="shortcut icon" href="/assets/icons/favicon.ico?v=1"/>
+        <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png?v=1"/>
+        <link rel="manifest" href="/assets/site.webmanifest?v=1"/>
+        <link rel="shortcut icon" href="/assets/favicon.ico?v=1"/>
         <meta name="msapplication-TileColor" content="#da532c">
-        <meta name="msapplication-config" content="/assets/icons/browserconfig.xml?v=1"/>
+        <meta name="msapplication-config" content="/assets/browserconfig.xml?v=1"/>
         <meta name="theme-color" content="#ffffff"/>
         <link href="grid.css?v=2" rel="stylesheet"/>
         <meta name="author" content="Christoph Hagen"/>

Readme.md

@@ -51,31 +51,7 @@ Note: The data for the mosaic is currently not updated automatically, since Swif
 ## Classifier training
 
 The main server is running on Linux, which doesn't provide the CreateML framework required for classifier training. 
-This has to be done on macOS using the `train.swift` script in the `Training` folder.
-It will:
-- Fetch the current image catalog by checking missing and changed images
-- Train a classifier on the image set
-- Increment the classifier version and upload the new model
-- Update the list of caps which can be recognized using the classifier
-- Create missing thumbnails for each cap for the cap grid
-
-A configuration file is required to run the training, with a valid access token for the server:
-
-```json
-{
-    "imageDirectory": "../Public/images",
-    "classifierModelPath": "../Public/classifier.mlmodel",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}
-```
-
-The configuration file `config.json` must be located in the folder from which the script is run.
-
-```bash
-swift train.swift
-```
+This has to be done on macOS using the [Caps-Train](https://christophhagen.de/git/ch/Caps-Train) repository.
 
 ## Future work
 - Create thumbnails on the server using [JPEG](https://github.com/kelvin13/jpeg)

Resources/example_config.json

@@ -3,6 +3,7 @@
     "maxBodySize" : "2mb",
     "logPath": "\/var\/log\/caps/metrics",
     "serveFiles": true,
+    "dataDirectory" : "/ch/data/caps",
     "writers" : [
         "auth_key_1"
     ]

Sources/App/CapServer.swift

@@ -4,6 +4,10 @@ import Clairvoyant
 
 final class CapServer {
     
+    private let imageSize = 360
+    
+    private let thumbnailSize = 100
+    
     // MARK: Paths
     
     private let imageFolder: URL
@@ -34,6 +38,8 @@ final class CapServer {
     /// Indicates that the data is loaded
     private(set) var isOperational = false
     
+    private(set) var canResizeImages = false
+    
     // MARK: Caps
 
     
@@ -81,7 +87,7 @@ final class CapServer {
         caps.reduce(0) { $0 + $1.value.count }
     }
 
-    init(in folder: URL) async {
+    init(in folder: URL) {
         self.imageFolder = folder.appendingPathComponent("images")
         self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
         self.gridCountFile = folder.appendingPathComponent("count.js")
@@ -95,15 +101,15 @@ final class CapServer {
         changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
 
         // Metric initializers only fail if observer is missing or ID is duplicate
-        self.capCountMetric = try! await .init("caps.count",
+        self.capCountMetric = .init("caps.count",
                                     name: "Number of caps",
                                     description: "The total number of caps in the database")
 
-        self.imageCountMetric = try! await .init("caps.images",
+        self.imageCountMetric = .init("caps.images",
                                       name: "Total images",
                                       description: "The total number of images for all caps")
 
-        self.classifierMetric = try! await .init("caps.classifier",
+        self.classifierMetric = .init("caps.classifier",
                                       name: "Classifier Version",
                                       description: "The current version of the image classifier")
     }
@@ -115,6 +121,12 @@ final class CapServer {
         updateGridCapCount()
         try ensureExistenceOfChangedImagesFile()
         organizeImages()
+        if let version = getMagickVersion() {
+            log("Using ImageMagick \(version.rawValue)")
+            canResizeImages = true
+        }
+        // shrinkImages()
+        createMissingThumbnails()
         isOperational = true
     }
 
@@ -160,7 +172,6 @@ final class CapServer {
             log("Failed to load caps: \(error)")
             throw error
         }
-        log("\(caps.count) caps loaded")
     }
 
     private func scheduleSave() {
@@ -360,6 +371,10 @@ final class CapServer {
         let count = try count(of: cap)
         caps[cap]!.count = count
         addChangedImageToLog(cap: cap, image: id)
+        if canResizeImages {
+            shrink(imageAt: capImageUrl, size: imageSize, destination: capImageUrl)
+            createThumbnail(for: cap)
+        }
         log("Added image \(id) for cap \(cap) (\(count) total)")
     }
     
@@ -450,6 +465,9 @@ final class CapServer {
             throw CapError.invalidFile
         }
         caps[cap]?.mainImage = version
+        if canResizeImages {
+            createThumbnail(for: cap)
+        }
         log("Switched cap \(cap) to version \(version)")
     }
 
@@ -603,6 +621,30 @@ final class CapServer {
             log("Failed to save grid cap count: \(error)")
         }
     }
+    
+    func createMissingThumbnails() {
+        let thumbnailsToCreate = getListOfMissingThumbnails()
+        guard !thumbnailsToCreate.isEmpty else {
+            return
+        }
+        guard canResizeImages else {
+            log("Can't create thumbnails, missing ImageMagick")
+            return
+        }
+        log("Creating \(thumbnailsToCreate.count) thumbnails")
+        for cap in thumbnailsToCreate {
+            createThumbnail(for: cap)
+        }
+    }
+    
+    func createThumbnail(for cap: Int) {
+        guard let version = caps[cap]?.mainImage else {
+            return
+        }
+        let mainImageUrl = imageUrl(of: cap, version: version)
+        let thumbnailUrl = thumbnail(of: cap)
+        shrink(imageAt: mainImageUrl, size: thumbnailSize, destination: thumbnailUrl)
+    }
 
     // MARK: Monitoring
 
@@ -611,4 +653,85 @@ final class CapServer {
     private let imageCountMetric: Metric<Int>
 
     private let classifierMetric: Metric<Int>
+    
+    // MARK: Maintenance
+    
+    private func getMagickVersion() -> SemanticVersion? {
+        do {
+            let command = "convert -version"
+            let (code, output) = try safeShell(command)
+            guard code == 0,
+                  let line = output.components(separatedBy: "\n").first,
+                  line.hasPrefix("Version: ImageMagick ") else {
+                log("Missing dependency ImageMagick: " + output)
+                return nil
+            }
+            guard let versionString = line
+                .replacingOccurrences(of: "Version: ImageMagick ", with: "")
+                .components(separatedBy: "-").first  else {
+                log("Invalid ImageMagick version: " + output)
+                return nil
+            }
+            guard let version = SemanticVersion(rawValue: versionString) else {
+                log("Invalid ImageMagick version: " + output)
+                return nil
+            }
+            return version
+        } catch {
+            log("Failed to check dependency ImageMagick: \(error)")
+            return nil
+        }
+    }
+    
+    func shrinkImages() {
+        guard canResizeImages else {
+            log("Can't resize images, missing ImageMagick")
+            return
+        }
+        let imageFolders: [URL]
+        do {
+            imageFolders = try fm.contentsOfDirectory(at: imageFolder, includingPropertiesForKeys: nil)
+        } catch {
+            log("Failed to get all image folders")
+            return
+        }
+        for folder in imageFolders {
+            guard let images = try? self.images(in: folder) else {
+                continue
+            }
+            for imageUrl in images {
+                shrink(imageAt: imageUrl, size: imageSize, destination: imageUrl)
+            }
+        }
+    }
+    
+    private func shrink(imageAt url: URL, size: Int, destination: URL) {
+        do {
+            let command = "convert \(url.path) -resize '\(size)x\(size)>' \(destination.path)"
+            let (code, output) = try safeShell(command)
+            if code != 0 {
+                log("Failed to shrink image \(url.path): " + output)
+            }
+        } catch {
+            log("Failed to shrink image \(url.path): \(error)")
+        }
+    }
+    
+    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
+        let task = Process()
+        let pipe = Pipe()
+
+        task.standardOutput = pipe
+        task.standardError = pipe
+        task.arguments = ["-cl", command]
+        task.executableURL = URL(fileURLWithPath: "/bin/bash")
+        task.standardInput = nil
+
+        try task.run()
+        task.waitUntilExit()
+        
+        let data = pipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(data: data, encoding: .utf8)!
+        return (task.terminationStatus, output)
+    }
 }

Sources/App/Config.swift

@@ -9,16 +9,39 @@ struct Config: Codable {
     let maxBodySize: String
 
     /// The path to the folder where the metric logs are stored
-    let logPath: String
+    ///
+    /// If no path is provided, then a folder `logs` in the resources directory is created
+    /// If the path is relative, then it is assumed relative to the resources directory
+    let logPath: String?
 
     /// Serve files in the Public directory using Vapor
     let serveFiles: Bool
 
     /// Authentication tokens for remotes allowed to write
     let writers: [String]
-
-    var logURL: URL {
-        .init(fileURLWithPath: logPath)
+    
+    /**
+     The folder where the data should be stored.
+     
+     If the folder is set to `nil`, then the `Resources` folder is used.
+     */
+    let dataDirectory: String?
+    
+    func customDataDirectory(or publicDirectory: String) -> URL {
+        guard let dataDirectory else {
+            return URL(fileURLWithPath: publicDirectory)
+        }
+        return URL(fileURLWithPath: dataDirectory)
+    }
+    
+    func logURL(possiblyRelativeTo resourcesDirectory: URL) -> URL {
+        guard let logPath else {
+            return resourcesDirectory.appendingPathComponent("logs")
+        }
+        guard !logPath.hasPrefix("/") else {
+            return .init(fileURLWithPath: logPath)
+        }
+        return resourcesDirectory.appendingPathComponent(logPath)
     }
 }
 

Sources/App/EventLoopScheduler.swift

@@ -0,0 +1,14 @@
+import Foundation
+import Clairvoyant
+import Vapor
+import NIOCore
+
+extension MultiThreadedEventLoopGroup: AsyncScheduler {
+    
+    func test() {
+    }
+
+    public func schedule(asyncJob: @escaping @Sendable () async throws -> Void) {
+        _ = any().makeFutureWithTask(asyncJob)
+    }
+}

Sources/App/configure.swift

@@ -5,8 +5,13 @@ import ClairvoyantVapor
 import ClairvoyantBinaryCodable
 
 private var provider: VaporMetricProvider!
+private var serverStatus: Metric<ServerStatus>!
 
-public func configure(_ app: Application) async throws {
+private let asyncScheduler = MultiThreadedEventLoopGroup(numberOfThreads: 2)
+
+private var server: CapServer!
+
+func configure(_ app: Application) async throws {
 
     let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory)
     let publicDirectory = app.directory.publicDirectory
@@ -14,20 +19,23 @@ public func configure(_ app: Application) async throws {
     let config = Config(loadFrom: resourceDirectory)
     let authenticator = Authenticator(writers: config.writers)
 
-    let monitor = MetricObserver(logFileFolder: config.logURL, logMetricId: "caps.log")
+    let logURL = config.logURL(possiblyRelativeTo: resourceDirectory)
+    let monitor = MetricObserver(logFileFolder: logURL, logMetricId: "caps.log")
     MetricObserver.standard = monitor
 
-    let status = try await Metric<ServerStatus>("caps.status",
+    serverStatus = Metric<ServerStatus>("caps.status",
                                       name: "Status",
                                       description: "The general status of the service")
-    try await status.update(.initializing)
+    try await serverStatus.update(.initializing)
 
     app.http.server.configuration.port = config.port
     app.routes.defaultMaxBodySize = .init(stringLiteral: config.maxBodySize)
 
-    let server = await CapServer(in: URL(fileURLWithPath: publicDirectory))
+    let dataDirectory = config.customDataDirectory(or: publicDirectory)
+    server = CapServer(in: dataDirectory)
 
     provider = .init(observer: monitor, accessManager: config.writers)
+    provider.asyncScheduler = asyncScheduler
     provider.registerRoutes(app)
 
     if config.serveFiles {
@@ -42,9 +50,27 @@ public func configure(_ app: Application) async throws {
     do {
         try server.loadData()
     } catch {
-        try await status.update(.initializationFailure)
+        try await serverStatus.update(.initializationFailure)
+        print("[\(df.string(from: Date()))] Server failed to start: \(error)")
+        return
+    }
+    if server.canResizeImages {
+        try await serverStatus.update(.nominal)
+    } else {
+        try await serverStatus.update(.reducedFunctionality)
+    }
+    print("[\(df.string(from: Date()))] Server started (\(app.environment.name), \(server.capCount) caps)")
+}
+
+func shutdown() {
+    Task {
+        print("[\(df.string(from: Date()))] Server shutdown")
+        do {
+            try await asyncScheduler.shutdownGracefully()
+        } catch {
+            print("Failed to shut down MultiThreadedEventLoopGroup: \(error)")
+        }
     }
-    try await status.update(.nominal)
 }
 
 func log(_ message: String) {
@@ -52,114 +78,14 @@ func log(_ message: String) {
         print(message)
         return
     }
-    observer.log(message)
-}
-
-import CBORCoding
-
-public func migrate(folder: URL) throws {
-    try migrateMetric("caps.log", containing: String.self, in: folder)
-    try migrateMetric("caps.status", containing: ServerStatus.self, in: folder)
-    try migrateMetric("caps.count", containing: Int.self, in: folder)
-    try migrateMetric("caps.images", containing: Int.self, in: folder)
-    try migrateMetric("caps.classifier", containing: Int.self, in: folder)
-}
-
-private func migrateMetric<T>(_ id: String, containing type: T.Type, in folder: URL) throws where T: MetricValue {
-    print("Processing metric \(id)")
-    let file = id.hashed()
-    let url = folder.appendingPathComponent(file)
-
-    let files = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil)
-        .filter { Int($0.lastPathComponent) != nil }
-    print("Found \(files.count) files for \(id)")
-
-    let all: [Timestamped<T>] = try files.map(readElements(from:))
-        .reduce([], +)
-        .sorted { $0.timestamp < $1.timestamp }
-
-    print("Found \(all.count) items for \(id)")
-
-    try FileManager.default.removeItem(at: url)
-
-    print("Removed log folder")
-
-    // TODO: Write values back to disk
-    let observer = MetricObserver(logFileFolder: folder, logMetricId: "sesame.migration")
-    let metric: Metric<T> = observer.addMetric(id: id)
-    let semaphore = DispatchSemaphore(value: 0)
-    Task {
-        try await metric.update(all)
-        print("Saved all values for metric \(id)")
-        semaphore.signal()
-    }
-    semaphore.wait()
-    print("Finished metric \(id)")
-}
-
-private func readElements<T>(from url: URL) throws -> [Timestamped<T>] where T: MetricValue {
-    let data = try Data(contentsOf: url)
-    let file = url.lastPathComponent
-    print("File \(file): Loaded \(data.count) bytes")
-
-
-
-    let decoder = CBORDecoder()
-    let timestampLength = 9
-    let byteCountLength = 2
-
-    var result: [Timestamped<T>] = []
-    var currentIndex = data.startIndex
-    var skippedValues = 0
-    while currentIndex < data.endIndex {
-        let startIndexOfTimestamp = currentIndex + byteCountLength
-        guard startIndexOfTimestamp <= data.endIndex else {
-            print("File \(file): Only \(data.endIndex - currentIndex) bytes, needed \(byteCountLength) for byte count")
-            throw MetricError.logFileCorrupted
-        }
-        guard let byteCount = UInt16(fromData: data[currentIndex..<startIndexOfTimestamp]) else {
-            print("File \(file): Invalid byte count")
-            throw MetricError.logFileCorrupted
-        }
-        let nextIndex = startIndexOfTimestamp + Int(byteCount)
-        guard nextIndex <= data.endIndex else {
-            print("File \(file): Needed \(byteCountLength + Int(byteCount)) for timestamped value, has \(data.endIndex - startIndexOfTimestamp)")
-            throw MetricError.logFileCorrupted
-        }
-        guard byteCount >= timestampLength else {
-            print("File \(file): Only \(byteCount) bytes, needed \(timestampLength) for timestamp")
-            throw MetricError.logFileCorrupted
-        }
-        let timestampData = data[startIndexOfTimestamp..<startIndexOfTimestamp+timestampLength]
-        let timestamp = try decoder.decode(Double.self, from: timestampData)
-        let date = Date(timeIntervalSince1970: timestamp)
-        let elementData = data[startIndexOfTimestamp+timestampLength..<nextIndex]
-        do {
-            let element: T = try decoder.decode(from: elementData)
-            result.append(.init(value: element, timestamp: date))
-        } catch {
-            skippedValues += 1
-        }
-        currentIndex = nextIndex
-        if result.count % 100 == 1 {
-            print("File \(file): \(result.count) entries loaded (\(currentIndex)/\(data.endIndex) bytes)")
-        }
-    }
-    print("Loaded \(result.count) data points (\(skippedValues) skipped)")
-    return result
-}
-
-extension UInt16 {
-
-    func toData() -> Data {
-        Data([UInt8(self >> 8 & 0xFF), UInt8(self & 0xFF)])
-    }
-
-    init?<T: DataProtocol>(fromData data: T) {
-        guard data.count == 2 else {
-            return nil
-        }
-        let bytes = Array(data)
-        self = UInt16(UInt32(bytes[0]) << 8 | UInt32(bytes[1]))
+    asyncScheduler.schedule {
+        await observer.log(message)
     }
 }
+
+private let df: DateFormatter = {
+    let df = DateFormatter()
+    df.dateStyle = .short
+    df.timeStyle = .short
+    return df
+}()

Sources/App/entrypoint.swift

@@ -0,0 +1,43 @@
+import Vapor
+import Dispatch
+import Logging
+
+/// This extension is temporary and can be removed once Vapor gets this support.
+private extension Vapor.Application {
+    static let baseExecutionQueue = DispatchQueue(label: "vapor.codes.entrypoint")
+    
+    func runFromAsyncMainEntrypoint() async throws {
+        try await withCheckedThrowingContinuation { continuation in
+            Vapor.Application.baseExecutionQueue.async { [self] in
+                do {
+                    try self.run()
+                    continuation.resume()
+                } catch {
+                    continuation.resume(throwing: error)
+                }
+            }
+        }
+    }
+}
+
+@main
+enum Entrypoint {
+    static func main() async throws {
+        var env = try Environment.detect()
+        try LoggingSystem.bootstrap(from: &env)
+        
+        let app = Application(env)
+        defer {
+            shutdown()
+            app.shutdown()
+        }
+        
+        do {
+            try await configure(app)
+        } catch {
+            app.logger.report(error: error)
+            throw error
+        }
+        try await app.runFromAsyncMainEntrypoint()
+    }
+}

Sources/Run/main.swift

@@ -1,23 +0,0 @@
-import App
-import Vapor
-
-var env = Environment.production
-try LoggingSystem.bootstrap(from: &env)
-let app = Application(env)
-defer { app.shutdown() }
-
-let storageFolder = URL(fileURLWithPath: app.directory.resourcesDirectory)
-let logFolder = storageFolder.appendingPathComponent("logs")
-print("Starting migration")
-try migrate(folder: logFolder)
-print("Finished migration")
-/*
- 
-private let semaphore = DispatchSemaphore(value: 0)
-Task {
-    try await configure(app)
-    semaphore.signal()
-}
-semaphore.wait()
-try app.run()
-*/

Tests/AppTests/AppTests.swift

@@ -1,13 +0,0 @@
-import App
-import Dispatch
-import XCTest
-
-final class AppTests : XCTestCase {
-    func testNothing() throws {
-        XCTAssert(true)
-    }
-
-    static let allTests = [
-        ("testNothing", testNothing),
-    ]
-}

Training/config_example.json

@@ -1,6 +0,0 @@
-{
-    "contentDirectory": "../Public",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}

Training/train.swift

@@ -1,521 +0,0 @@
-import Foundation
-import Cocoa
-import CreateML
-
-struct Configuration: Codable {
-    
-    let contentFolder: String
-
-    let trainingIterations: Int
-    
-    let serverPath: String
-    
-    let authenticationToken: String
-    
-    init?(at url: URL) {
-        do {
-            let configData = try Data(contentsOf: url)
-            self = try JSONDecoder().decode(Configuration.self, from: configData)
-        } catch {
-            print("[ERROR] Failed to load configuration at \(url.absoluteURL.path): \(error)")
-            return nil
-        }
-    }
-}
-
-struct Cap: Codable {
-
-    let id: Int
-
-    let count: Int
-
-    enum CodingKeys: String, CodingKey {
-        case id = "i"
-        case count = "c"
-    }
-}
-
-final class ClassifierCreator {
-    
-    static let configurationFileUrl = URL(fileURLWithPath: "config.json")
-    
-    let server: URL
-    
-    let configuration: Configuration
-    
-    let imageDirectory: URL
-    
-    let thumbnailDirectory: URL
-    
-    let classifierUrl: URL
-    
-    let df = DateFormatter()
-    
-    
-    // MARK: Step 1: Load configuration
-    
-    init?() {
-        guard let configuration = Configuration(at: ClassifierCreator.configurationFileUrl) else {
-            return nil
-        }
-        self.configuration = configuration
-        guard let serverUrl = URL(string: configuration.serverPath) else {
-            print("[ERROR] Configuration: Invalid server path \(configuration.serverPath)")
-            return nil
-        }
-        self.server = serverUrl
-        let contentDirectory = URL(fileURLWithPath: configuration.contentFolder)
-        self.imageDirectory = contentDirectory.appendingPathComponent("images")
-        self.classifierUrl = contentDirectory.appendingPathComponent("classifier.mlmodel")
-        self.thumbnailDirectory = contentDirectory.appendingPathComponent("thumbnails")
-        df.dateFormat = "yy-MM-dd-HH-mm-ss"
-    }
-    
-    // MARK: Main function
-    
-    func run() async {
-        let imagesSnapshotDate = Date()
-        guard let (classes, changedImageCount, changedMainImages) = await loadImages() else {
-            return
-        }
-
-        guard !classes.isEmpty else {
-            return
-        }
-        
-        guard changedImageCount > 0 else {
-            print("[INFO] No changed images, so no new classifier trained")
-            await createThumbnails(changed: changedMainImages)
-            print("[INFO] Done")
-            return
-        }
-
-        guard let classifierVersion = await getClassifierVersion() else {
-            return
-        }
-        let newVersion = classifierVersion + 1
-
-        print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
-        print("[INFO] Model path: \(classifierUrl.path)")
-        print("[INFO] Version: \(newVersion)")
-        print("[INFO] Classes: \(classes.count)")
-        print("[INFO] Iterations: \(configuration.trainingIterations)")
-
-        guard trainModel() else {
-            return
-        }
-
-        guard await upload(version: newVersion) else {
-            return
-        }
-
-        guard await upload(classes: classes, lastUpdate: imagesSnapshotDate) else {
-            return
-        }
-        await createThumbnails(changed: changedMainImages)
-        print("[INFO] Done")
-    }
-    
-    // MARK: Step 2: Load changed images
-    
-    func loadImages() async -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int])? {
-        guard createImageFolderIfMissing() else {
-            return nil
-        }
-        let imageCounts = await getImageCounts()
-        let missingImageList: [(cap: Int, image: Int)] = imageCounts
-            .sorted { $0.key < $1.key }
-            .reduce(into: []) { list, pair in
-                let missingImagesForCap: [(cap: Int, image: Int)] = (0..<pair.value).compactMap { image in
-                    let url = imageUrl(base: imageDirectory, cap: pair.key, image: image)
-                    guard !FileManager.default.fileExists(atPath: url.path) else {
-                        return nil
-                    }
-                    return (cap: pair.key, image: image)
-                }
-                list.append(contentsOf: missingImagesForCap)
-            }
-        if missingImageList.isEmpty {
-            print("[INFO] No missing images to load")
-        } else {
-            print("[INFO] Loading \(missingImageList.count) missing images...")
-        }
-        guard await loadImages(missingImageList) else {
-            return nil
-        }
-        let changedImageList = await getChangedImageList()
-            .filter { $0.image < imageCounts[$0.cap] ?? 0 } // Filter non-existent images
-        
-        if changedImageList.isEmpty {
-            print("[INFO] No changed images to load")
-        } else {
-            print("[INFO] Loading \(changedImageList.count) changed images...")
-        }
-        guard await loadImages(changedImageList) else {
-            return nil
-        }
-        let changedMainImages = changedImageList.filter { $0.image == 0 }.map { $0.cap }
-        let classes = imageCounts.keys.sorted()
-        
-        // Delete any image folders not present as caps
-        guard deleteUnnecessaryImageFolders(caps: classes) else {
-            return nil
-        }
-        
-        return (classes, missingImageList.count + changedImageList.count, changedMainImages)
-    }
-    
-    private func createImageFolderIfMissing() -> Bool {
-        createFolderIfMissing(imageDirectory)
-    }
-    
-    private func getImageCounts() async -> [Int : Int] {
-        guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
-            return [:]
-        }
-        do {
-            return try JSONDecoder().decode([Cap].self, from: data)
-                .reduce(into: [:]) { $0[$1.id] = $1.count }
-        } catch {
-            print("[ERROR] Failed to decode cap database: \(error)")
-            return [:]
-        }
-    }
-    
-    private func deleteUnnecessaryImageFolders(caps: [Int]) -> Bool {
-        let validNames = caps.map { String(format: "%04d", $0) }
-        let folders: [String]
-        do {
-            folders = try FileManager.default.contentsOfDirectory(atPath: imageDirectory.path)
-        } catch {
-            print("[ERROR] Failed to get list of image folders: \(error)")
-            return false
-        }
-        for folder in folders {
-            if validNames.contains(folder) {
-                continue
-            }
-            
-            // Not a valid cap folder
-            let url = imageDirectory.appendingPathComponent(folder)
-            do {
-                try FileManager.default.removeItem(at: url)
-                print("[ERROR] Removed unused image folder '\(folder)'")
-            } catch {
-                print("[ERROR] Failed to delete unused image folder \(folder): \(error)")
-                return false
-            }
-        }
-        return true
-    }
-    
-    private func imageUrl(base: URL, cap: Int, image: Int) -> URL {
-        base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", cap, cap, image))
-    }
-    
-    private func loadImage(cap: Int, image: Int) async -> Bool {
-        guard createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", cap))) else {
-            return false
-        }
-        let url = imageUrl(base: server.appendingPathComponent("images"), cap: cap, image: image)
-        let tempFile: URL, response: URLResponse
-        do {
-            (tempFile, response) = try await URLSession.shared.download(from: url)
-        } catch {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): \(error)")
-            return false
-        }
-        let responseCode = (response as! HTTPURLResponse).statusCode
-        guard responseCode == 200 else {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): Response \(responseCode)")
-            return false
-        }
-        do {
-            let localUrl = imageUrl(base: imageDirectory, cap: cap, image: image)
-            if FileManager.default.fileExists(atPath: localUrl.path) {
-                try FileManager.default.removeItem(at: localUrl)
-            }
-            try FileManager.default.moveItem(at: tempFile, to: localUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save image \(image) of cap \(cap): \(error)")
-            return false
-        }
-    }
-    
-    private func loadImages(_ list: [(cap: Int, image: Int)]) async -> Bool {
-        guard !list.isEmpty else {
-            return true
-        }
-        var loadedImages = 0
-        await withTaskGroup(of: Bool.self) { group in
-            for (cap, image) in list {
-                group.addTask {
-                    await self.loadImage(cap: cap, image: image)
-                }
-            }
-            for await loaded in group {
-                if loaded {
-                    loadedImages += 1
-                }
-            }
-        }
-        if loadedImages != list.count {
-            print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
-            return false
-        }
-        return true
-    }
-    
-    func getChangedImageList() async -> [(cap: Int, image: Int)] {
-        guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
-            print("[ERROR] Failed to get list of changed images")
-            return []
-        }
-        
-        return string
-            .components(separatedBy: "\n")
-            .filter { $0 != "" }
-            .compactMap {
-                let parts = $0.components(separatedBy: ":")
-                guard parts.count == 3 else {
-                    return nil
-                }
-                /*
-                guard let date = df.date(from: parts[0]) else {
-                    print("[WARN] Invalid date \(parts[0]) in change list")
-                    return nil
-                }
-                 */
-                guard let cap = Int(parts[1]) else {
-                    print("[WARN] Invalid cap id \(parts[1]) in change list")
-                    return nil
-                }
-                guard let image = Int(parts[2]) else {
-                    print("[WARN] Invalid image id \(parts[2]) in change list")
-                    return nil
-                }
-                return (cap, image)
-            }
-    }
-    
-    // MARK: Step 3: Compute version
-    
-    func getClassifierVersion() async -> Int? {
-        guard let string: String = await get(server.appendingPathComponent("version")) else {
-            print("[ERROR] Failed to get classifier version")
-            return nil
-        }
-        guard let version = Int(string) else {
-            print("[ERROR] Invalid classifier version \(string)")
-            return nil
-        }
-        return version
-    }
-    
-    // MARK: Step 4: Train classifier
-    
-    func trainModel() -> Bool {
-        var params = MLImageClassifier.ModelParameters(augmentation: [])
-        params.maxIterations = configuration.trainingIterations
-        let model: MLImageClassifier
-        do {
-            model = try MLImageClassifier(
-                trainingData: .labeledDirectories(at: imageDirectory),
-                parameters: params)
-        } catch {
-            print("[ERROR] Failed to create classifier: \(error)")
-            return false
-        }
-
-        print("[INFO] Saving classifier...")
-        do {
-            try model.write(to: classifierUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save model to file: \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Step 5: Upload classifier
-    
-    func upload(version: Int) async -> Bool {
-        print("[INFO] Uploading classifier...")
-        let modelData: Data
-        do {
-            modelData = try Data(contentsOf: classifierUrl)
-        } catch {
-            print("[ERROR] Failed to read classifier data: \(error)")
-            return false
-        }
-        
-        return await post(
-            url: server.appendingPathComponent("classifier/\(version)"),
-            body: modelData)
-    }
-    
-    // MARK: Step 6: Update classes
-    
-    func upload(classes: [Int], lastUpdate: Date) async -> Bool {
-        print("[INFO] Uploading trained classes...")
-        let dateString = df.string(from: lastUpdate)
-        return await post(
-            url: server.appendingPathComponent("classes/\(dateString)"),
-            body: classes.map(String.init).joined(separator: ",").data(using: .utf8)!)
-    }
-    
-    // MARK: Step 7: Create thumbnails
-    
-    func createThumbnails(changed: [Int]) async {
-        guard checkMagickAvailability() else {
-            return
-        }
-        guard createFolderIfMissing(thumbnailDirectory) else {
-            print("[ERROR] Failed to create folder for thumbnails")
-            return
-        }
-        let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
-        let all = Set(capIdsOfMissingThumbnails).union(changed)
-        print("[INFO] Creating \(all.count) thumbnails...")
-        for cap in all {
-            await createThumbnail(for: cap)
-        }
-    }
-    
-    func checkMagickAvailability() -> Bool {
-        do {
-            let (code, output) = try safeShell("magick --version")
-            guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
-                .components(separatedBy: " ").first else {
-                print("[ERROR] Magick not found, install using 'brew install imagemagick'")
-                return false
-            }
-            print("[INFO] Using magick \(version)")
-        } catch {
-            print("[ERROR] Failed to get version of magick: (\(error))")
-            return false
-        }
-        return true
-    }
-    
-    private func getMissingThumbnailIds() async -> [Int] {
-        guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
-            print("[ERROR] Failed to get missing thumbnails")
-            return []
-        }
-        return string.components(separatedBy: ",").compactMap(Int.init)
-    }
-    
-    private func createThumbnail(for cap: Int) async {
-        let inputUrl = imageUrl(base: imageDirectory, cap: cap, image: 0)
-        guard FileManager.default.fileExists(atPath: inputUrl.path) else {
-            print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
-            return
-        }
-        
-        let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
-        do {
-            let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
-            let (code, output) = try safeShell(command)
-            if code != 0 {
-                print("Failed to create thumbnail for cap \(cap): \(output)")
-                return
-            }
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        
-        let data: Data
-        do {
-            data = try Data(contentsOf: output)
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
-            print("Failed to upload thumbnail for cap \(cap)")
-            return
-        }
-    }
-    
-    // MARK: Helper
-    
-    @discardableResult
-    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
-        let task = Process()
-        let pipe = Pipe()
-
-        task.standardOutput = pipe
-        task.standardError = pipe
-        task.arguments = ["-cl", command]
-        task.executableURL = URL(fileURLWithPath: "/bin/zsh")
-        task.standardInput = nil
-
-        try task.run()
-        task.waitUntilExit()
-        
-        let data = pipe.fileHandleForReading.readDataToEndOfFile()
-        let output = String(data: data, encoding: .utf8)!
-        return (task.terminationStatus, output)
-    }
-    
-    
-    private func createFolderIfMissing(_ folder: URL) -> Bool {
-        guard !FileManager.default.fileExists(atPath: folder.path) else {
-            return true
-        }
-        do {
-            try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
-            return true
-        } catch {
-            print("[ERROR] Failed to create directory \(folder.path): \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Requests
-    
-    private func post(url: URL, body: Data) async -> Bool {
-        var request = URLRequest(url: url)
-        request.httpMethod = "POST"
-        request.httpBody = body
-        request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
-        return await perform(request) != nil
-    }
-    
-    private func perform(_ request: URLRequest) async -> Data? {
-        let data: Data
-        let response: URLResponse
-        do {
-            (data, response) = try await URLSession.shared.data(for: request)
-        } catch {
-            print("[ERROR] Request to \(request.url!.absoluteString) failed: \(error)")
-            return nil
-        }
-        let code = (response as! HTTPURLResponse).statusCode
-        guard code == 200 else {
-            print("[ERROR] Request to \(request.url!.absoluteString): Invalid response \(code)")
-            return nil
-        }
-        return data
-    }
-    
-    private func get(_ url: URL) async -> Data? {
-        await perform(URLRequest(url: url))
-    }
-    
-    private func get(_ url: URL) async -> String? {
-        guard let data: Data = await get(url) else {
-            return nil
-        }
-        guard let string = String(data: data, encoding: .utf8) else {
-            print("[ERROR] Invalid string response \(data)")
-            return nil
-        }
-        return string
-    }
-}
-
-await ClassifierCreator()?.run()

update.sh

@@ -1,15 +0,0 @@
-echo "[1/8] Stopping server..."
-sudo supervisorctl stop caps
-echo "[2/8] Changing permissions..."
-sudo chown -R pi:pi .
-echo "[3/8] Pulling changes..."
-git pull
-echo "[4/8] Updating dependencies..."
-swift package update
-echo "[5/8] Building project..."
-swift build -c release
-echo "[6/8] Restoring permissions..."
-sudo chown -R www-data:www-data .
-echo "[7/8] Starting server..."
-sudo supervisorctl start caps
-echo "[8/8] Done"