Add image assets

.gitignore

@@ -8,12 +8,11 @@
 .DS_Store
 Package.resolved
 .swiftpm/
+Public/caps.json
+Public/changes.txt
+Public/classifier.*
+Public/count.*
 Public/images/
 Public/thumbnails/
-Public/classifier.version
+Resources/config.json
-Public/classifier.mlmodel
+Resources/logs/
-Public/caps.json
-Training/backup/
-Training/config.json
-Public/thumbnails
-Public/count.js

Package.swift

@@ -8,22 +8,20 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
-        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.5.0"),
+        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.11.2"),
+        .package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.4.0"),
+        .package(url: "https://github.com/christophhagen/ClairvoyantBinaryCodable", from: "0.3.0"),
     ],
     targets: [
-        .target(name: "App",
+        .executableTarget(
+            name: "App",
             dependencies: [
                 .product(name: "Vapor", package: "vapor"),
                 .product(name: "Clairvoyant", package: "Clairvoyant"),
-                ],
+                .product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"),
-                swiftSettings: [
+                .product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"),
-                    // Enable better optimizations when building in Release configuration. Despite the use of
+            ]
-                    // the `.unsafeFlags` construct required by SwiftPM, this flag is recommended for Release
+        )
-                    // builds. See <https://github.com/swift-server/guides#building-for-production> for details.
-                    .unsafeFlags(["-cross-module-optimization"], .when(configuration: .release))
-                ]),
-        .executableTarget(name: "Run", dependencies: ["App"]),
-        .testTarget(name: "AppTests", dependencies: ["App"]),
     ]
 )
 

Public/assets/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/assets/mstile-150x150.png?v=1"/>
+            <TileColor>#a36490</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

Public/assets/site.webmanifest

@@ -0,0 +1 @@
+{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

Public/grid.html

@@ -5,13 +5,13 @@
         <meta charset="utf-8"/>
         <meta name="robots" content="noindex"/>
         <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0"/>
-        <link rel="apple-touch-icon" sizes="180x180" href="/assets/icons/apple-touch-icon.png?v=1"/>
+        <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="32x32" href="/assets/icons/favicon-32x32.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="16x16" href="/assets/icons/favicon-16x16.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png?v=1"/>
-        <link rel="manifest" href="/assets/icons/site.webmanifest?v=1"/>
+        <link rel="manifest" href="/assets/site.webmanifest?v=1"/>
-        <link rel="shortcut icon" href="/assets/icons/favicon.ico?v=1"/>
+        <link rel="shortcut icon" href="/assets/favicon.ico?v=1"/>
         <meta name="msapplication-TileColor" content="#da532c">
-        <meta name="msapplication-config" content="/assets/icons/browserconfig.xml?v=1"/>
+        <meta name="msapplication-config" content="/assets/browserconfig.xml?v=1"/>
         <meta name="theme-color" content="#ffffff"/>
         <link href="grid.css?v=2" rel="stylesheet"/>
         <meta name="author" content="Christoph Hagen"/>

Readme.md

@@ -51,31 +51,7 @@ Note: The data for the mosaic is currently not updated automatically, since Swif
 ## Classifier training
 
 The main server is running on Linux, which doesn't provide the CreateML framework required for classifier training. 
-This has to be done on macOS using the `train.swift` script in the `Training` folder.
+This has to be done on macOS using the [Caps-Train](https://christophhagen.de/git/ch/Caps-Train) repository.
-It will:
-- Fetch the current image catalog by checking missing and changed images
-- Train a classifier on the image set
-- Increment the classifier version and upload the new model
-- Update the list of caps which can be recognized using the classifier
-- Create missing thumbnails for each cap for the cap grid
-
-A configuration file is required to run the training, with a valid access token for the server:
-
-```json
-{
-    "imageDirectory": "../Public/images",
-    "classifierModelPath": "../Public/classifier.mlmodel",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}
-```
-
-The configuration file `config.json` must be located in the folder from which the script is run.
-
-```bash
-swift train.swift
-```
 
 ## Future work
 - Create thumbnails on the server using [JPEG](https://github.com/kelvin13/jpeg)

Resources/example_config.json

@@ -3,6 +3,7 @@
     "maxBodySize" : "2mb",
     "logPath": "\/var\/log\/caps/metrics",
     "serveFiles": true,
+    "dataDirectory" : "/ch/data/caps",
     "writers" : [
         "auth_key_1"
     ]

Sources/App/Authenticator.swift

@@ -1,8 +1,7 @@
 import Foundation
-import Clairvoyant
 import Vapor
 
-final class Authenticator: MetricAccessManager {
+final class Authenticator {
 
     private var writers: Set<String>
 
@@ -10,7 +9,6 @@ final class Authenticator: MetricAccessManager {
         self.writers = Set(writers)
     }
 
-
     func hasAuthorization(for key: String) -> Bool {
         // Note: This is not a constant-time compare, so there may be an opportunity
         // for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
@@ -20,20 +18,6 @@ final class Authenticator: MetricAccessManager {
         writers.contains(key)
     }
 
-    func metricListAccess(isAllowedForToken accessToken: AccessToken) throws {
-        guard let key = String(data: accessToken, encoding: .utf8) else {
-            return
-        }
-        guard hasAuthorization(for: key) else {
-            throw MetricError.accessDenied
-        }
-    }
-
-    func metricAccess(to metric: MetricId, isAllowedForToken accessToken: AccessToken) throws {
-        try metricListAccess(isAllowedForToken: accessToken)
-    }
-
-
     func authorize(_ request: Request) throws {
         guard let key = request.headers.first(name: "key") else {
             throw Abort(.badRequest) // 400

Sources/App/Cap.swift

@@ -10,9 +10,6 @@ struct Cap: Codable {
 
     var mainImage: Int
 
-    /// The version of the first classifier trained on this cap
-    var classifierVersion: Int?
-
     var color: Color?
 
     enum CodingKeys: String, CodingKey {
@@ -20,7 +17,6 @@ struct Cap: Codable {
         case name = "n"
         case count = "c"
         case mainImage = "m"
-        case classifierVersion = "v"
         case color = "f"
     }
 

Sources/App/CapServer+Routes.swift

@@ -79,7 +79,7 @@ extension CapServer {
             try authenticator.authorize(request)
             let body = try request.getStringBody(request: "/classes/:date")
 
-            self.updateTrainedClasses(content: body)
+            try self.saveTrainedClasses(content: body)
             self.removeAllEntriesInImageChangeList(before: date)
         }
 

Sources/App/CapServer.swift

@@ -4,6 +4,10 @@ import Clairvoyant
 
 final class CapServer {
     
+    private let imageSize = 360
+    
+    private let thumbnailSize = 100
+    
     // MARK: Paths
     
     private let imageFolder: URL
@@ -23,6 +27,8 @@ final class CapServer {
     
     private let classifierFile: URL
     
+    private let classifierClassesFile: URL
+    
     private let changedImagesFile: URL
     
     private let fm = FileManager.default
@@ -32,6 +38,8 @@ final class CapServer {
     /// Indicates that the data is loaded
     private(set) var isOperational = false
     
+    private(set) var canResizeImages = false
+    
     // MARK: Caps
 
     
@@ -65,16 +73,12 @@ final class CapServer {
         didSet {
             scheduleSave()
             Task {
-                try? await capCountMetric.update(caps.count)
+                _ = try? await capCountMetric.update(caps.count)
-                try? await imageCountMetric.update(imageCount)
+                _ = try? await imageCountMetric.update(imageCount)
             }
         }
     }
 
-    var nextClassifierVersion: Int {
-        caps.values.compactMap { $0.classifierVersion }.max() ?? 1
-    }
-
     var capCount: Int {
         caps.count
     }
@@ -83,7 +87,7 @@ final class CapServer {
         caps.reduce(0) { $0 + $1.value.count }
     }
 
-    init(in folder: URL) async {
+    init(in folder: URL) {
         self.imageFolder = folder.appendingPathComponent("images")
         self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
         self.gridCountFile = folder.appendingPathComponent("count.js")
@@ -91,20 +95,21 @@ final class CapServer {
         self.htmlFile = folder.appendingPathComponent("count.html")
         self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
         self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
+        self.classifierClassesFile = folder.appendingPathComponent("classifier.classes")
         self.changedImagesFile = folder.appendingPathComponent("changes.txt")
         self.changedImageEntryDateFormatter = DateFormatter()
         changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
 
         // Metric initializers only fail if observer is missing or ID is duplicate
-        self.capCountMetric = try! await .init("caps.count",
+        self.capCountMetric = .init("caps.count",
                                     name: "Number of caps",
                                     description: "The total number of caps in the database")
 
-        self.imageCountMetric = try! await .init("caps.images",
+        self.imageCountMetric = .init("caps.images",
                                       name: "Total images",
                                       description: "The total number of images for all caps")
 
-        self.classifierMetric = try! await .init("caps.classifier",
+        self.classifierMetric = .init("caps.classifier",
                                       name: "Classifier Version",
                                       description: "The current version of the image classifier")
     }
@@ -116,6 +121,12 @@ final class CapServer {
         updateGridCapCount()
         try ensureExistenceOfChangedImagesFile()
         organizeImages()
+        if let version = getMagickVersion() {
+            log("Using ImageMagick \(version.rawValue)")
+            canResizeImages = true
+        }
+        // shrinkImages()
+        createMissingThumbnails()
         isOperational = true
     }
 
@@ -161,7 +172,6 @@ final class CapServer {
             log("Failed to load caps: \(error)")
             throw error
         }
-        log("\(caps.count) caps loaded")
     }
 
     private func scheduleSave() {
@@ -361,6 +371,10 @@ final class CapServer {
         let count = try count(of: cap)
         caps[cap]!.count = count
         addChangedImageToLog(cap: cap, image: id)
+        if canResizeImages {
+            shrink(imageAt: capImageUrl, size: imageSize, destination: capImageUrl)
+            createThumbnail(for: cap)
+        }
         log("Added image \(id) for cap \(cap) (\(count) total)")
     }
     
@@ -451,6 +465,9 @@ final class CapServer {
             throw CapError.invalidFile
         }
         caps[cap]?.mainImage = version
+        if canResizeImages {
+            createThumbnail(for: cap)
+        }
         log("Switched cap \(cap) to version \(version)")
     }
 
@@ -469,7 +486,6 @@ final class CapServer {
         }
         var cap = cap
         cap.count = 0
-        cap.classifierVersion = nextClassifierVersion
         caps[cap.id] = cap
         saveCapCountHTML()
         updateGridCapCount()
@@ -541,17 +557,33 @@ final class CapServer {
     
     // MARK: Classifier
     
-    func updateTrainedClasses(content: String) {
+    func saveTrainedClasses(content: String) throws {
-        let trainedCaps = content
+        let classes = content.components(separatedBy: ",")
-            .components(separatedBy: "\n")
+        
-            .compactMap(Int.init)
+        // Validate input
-        let version = classifierVersion
+        try classes.forEach { s in
-        for cap in trainedCaps {
+            guard let id = Int(s) else {
-            if caps[cap]?.classifierVersion == nil {
+                log("Invalid id '\(s)' in uploaded id list")
-                caps[cap]?.classifierVersion = version
+                throw Abort(.badRequest)
+            }
+            guard caps[id] != nil else {
+                log("Unknown id '\(id)' in uploaded id list")
+                throw Abort(.badRequest)
             }
         }
-        log("Updated \(trainedCaps.count) classifier classes")
+        
+        guard let data = content.data(using: .utf8) else {
+            log("Failed to get classes data for writing")
+            throw Abort(.internalServerError)
+        }
+        
+        do {
+            try data.write(to: classifierClassesFile)
+            log("Updated \(classes.count) classifier classes")
+        } catch {
+            log("Failed to write classifier classes: \(error)")
+            throw Abort(.internalServerError)
+        }
     }
     
     func save(classifier: Data, version: Int) throws {
@@ -590,6 +622,30 @@ final class CapServer {
         }
     }
     
+    func createMissingThumbnails() {
+        let thumbnailsToCreate = getListOfMissingThumbnails()
+        guard !thumbnailsToCreate.isEmpty else {
+            return
+        }
+        guard canResizeImages else {
+            log("Can't create thumbnails, missing ImageMagick")
+            return
+        }
+        log("Creating \(thumbnailsToCreate.count) thumbnails")
+        for cap in thumbnailsToCreate {
+            createThumbnail(for: cap)
+        }
+    }
+    
+    func createThumbnail(for cap: Int) {
+        guard let version = caps[cap]?.mainImage else {
+            return
+        }
+        let mainImageUrl = imageUrl(of: cap, version: version)
+        let thumbnailUrl = thumbnail(of: cap)
+        shrink(imageAt: mainImageUrl, size: thumbnailSize, destination: thumbnailUrl)
+    }
+
     // MARK: Monitoring
 
     private let capCountMetric: Metric<Int>
@@ -597,4 +653,85 @@ final class CapServer {
     private let imageCountMetric: Metric<Int>
 
     private let classifierMetric: Metric<Int>
+    
+    // MARK: Maintenance
+    
+    private func getMagickVersion() -> SemanticVersion? {
+        do {
+            let command = "convert -version"
+            let (code, output) = try safeShell(command)
+            guard code == 0,
+                  let line = output.components(separatedBy: "\n").first,
+                  line.hasPrefix("Version: ImageMagick ") else {
+                log("Missing dependency ImageMagick: " + output)
+                return nil
+            }
+            guard let versionString = line
+                .replacingOccurrences(of: "Version: ImageMagick ", with: "")
+                .components(separatedBy: "-").first  else {
+                log("Invalid ImageMagick version: " + output)
+                return nil
+            }
+            guard let version = SemanticVersion(rawValue: versionString) else {
+                log("Invalid ImageMagick version: " + output)
+                return nil
+            }
+            return version
+        } catch {
+            log("Failed to check dependency ImageMagick: \(error)")
+            return nil
+        }
+    }
+    
+    func shrinkImages() {
+        guard canResizeImages else {
+            log("Can't resize images, missing ImageMagick")
+            return
+        }
+        let imageFolders: [URL]
+        do {
+            imageFolders = try fm.contentsOfDirectory(at: imageFolder, includingPropertiesForKeys: nil)
+        } catch {
+            log("Failed to get all image folders")
+            return
+        }
+        for folder in imageFolders {
+            guard let images = try? self.images(in: folder) else {
+                continue
+            }
+            for imageUrl in images {
+                shrink(imageAt: imageUrl, size: imageSize, destination: imageUrl)
+            }
+        }
+    }
+    
+    private func shrink(imageAt url: URL, size: Int, destination: URL) {
+        do {
+            let command = "convert \(url.path) -resize '\(size)x\(size)>' \(destination.path)"
+            let (code, output) = try safeShell(command)
+            if code != 0 {
+                log("Failed to shrink image \(url.path): " + output)
+            }
+        } catch {
+            log("Failed to shrink image \(url.path): \(error)")
+        }
+    }
+    
+    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
+        let task = Process()
+        let pipe = Pipe()
+
+        task.standardOutput = pipe
+        task.standardError = pipe
+        task.arguments = ["-cl", command]
+        task.executableURL = URL(fileURLWithPath: "/bin/bash")
+        task.standardInput = nil
+
+        try task.run()
+        task.waitUntilExit()
+        
+        let data = pipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(data: data, encoding: .utf8)!
+        return (task.terminationStatus, output)
+    }
 }

Sources/App/Config.swift

@@ -9,7 +9,10 @@ struct Config: Codable {
     let maxBodySize: String
 
     /// The path to the folder where the metric logs are stored
-    let logPath: String
+    ///
+    /// If no path is provided, then a folder `logs` in the resources directory is created
+    /// If the path is relative, then it is assumed relative to the resources directory
+    let logPath: String?
 
     /// Serve files in the Public directory using Vapor
     let serveFiles: Bool
@@ -17,8 +20,28 @@ struct Config: Codable {
     /// Authentication tokens for remotes allowed to write
     let writers: [String]
     
-    var logURL: URL {
+    /**
-        .init(fileURLWithPath: logPath)
+     The folder where the data should be stored.
+     
+     If the folder is set to `nil`, then the `Resources` folder is used.
+     */
+    let dataDirectory: String?
+    
+    func customDataDirectory(or publicDirectory: String) -> URL {
+        guard let dataDirectory else {
+            return URL(fileURLWithPath: publicDirectory)
+        }
+        return URL(fileURLWithPath: dataDirectory)
+    }
+    
+    func logURL(possiblyRelativeTo resourcesDirectory: URL) -> URL {
+        guard let logPath else {
+            return resourcesDirectory.appendingPathComponent("logs")
+        }
+        guard !logPath.hasPrefix("/") else {
+            return .init(fileURLWithPath: logPath)
+        }
+        return resourcesDirectory.appendingPathComponent(logPath)
     }
 }
 
@@ -31,12 +54,11 @@ extension Config {
     init(loadFrom directory: URL) {
         let configFileUrl = Config.file(in: directory)
 
-        if FileManager.default.fileExists(atPath: configFileUrl.path) {
+        guard FileManager.default.fileExists(atPath: configFileUrl.path) else {
-            self.init(loadAt: configFileUrl)
+            print("No configuration found at \(configFileUrl.path)")
-        } else {
+            exit(-1)
-            self.init(standardIn: directory)
-            write(to: configFileUrl)
         }
+        self.init(loadAt: configFileUrl)
     }
 
     private init(loadAt url: URL) {
@@ -45,23 +67,7 @@ extension Config {
             self = try JSONDecoder().decode(Config.self, from: configData)
         } catch {
             print("Failed to load configuration from \(url.path): \(error)")
-            print("Using default configuration")
+            exit(-1)
-            self.init(standardIn: url.deletingLastPathComponent())
-        }
-    }
-
-    private init(standardIn directory: URL) {
-        let defaultLogPath = directory.appendingPathComponent("logs").path
-        self.init(port: 8000, maxBodySize: "2mb", logPath: defaultLogPath, serveFiles: true, writers: [])
-    }
-
-    private func write(to url: URL) {
-        do {
-            let configData = try JSONEncoder().encode(self)
-            try configData.write(to: url)
-            print("Configuration written at \(url.path)")
-        } catch {
-            print("Failed to write default configuration to \(url.path)")
         }
     }
 }

Sources/App/EventLoopScheduler.swift

@@ -0,0 +1,14 @@
+import Foundation
+import Clairvoyant
+import Vapor
+import NIOCore
+
+extension MultiThreadedEventLoopGroup: AsyncScheduler {
+    
+    func test() {
+    }
+
+    public func schedule(asyncJob: @escaping @Sendable () async throws -> Void) {
+        _ = any().makeFutureWithTask(asyncJob)
+    }
+}

Sources/App/configure.swift

@@ -1,8 +1,17 @@
 import Vapor
 import Foundation
 import Clairvoyant
+import ClairvoyantVapor
+import ClairvoyantBinaryCodable
 
-public func configure(_ app: Application) async throws {
+private var provider: VaporMetricProvider!
+private var serverStatus: Metric<ServerStatus>!
+
+private let asyncScheduler = MultiThreadedEventLoopGroup(numberOfThreads: 2)
+
+private var server: CapServer!
+
+func configure(_ app: Application) async throws {
 
     let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory)
     let publicDirectory = app.directory.publicDirectory
@@ -10,25 +19,24 @@ public func configure(_ app: Application) async throws {
     let config = Config(loadFrom: resourceDirectory)
     let authenticator = Authenticator(writers: config.writers)
 
-    let monitor = await MetricObserver(
+    let logURL = config.logURL(possiblyRelativeTo: resourceDirectory)
-        logFolder: config.logURL,
+    let monitor = MetricObserver(logFileFolder: logURL, logMetricId: "caps.log")
-        accessManager: authenticator,
-        logMetricId: "caps.log")
-
-    // All new metrics are automatically registered with the standard observer
     MetricObserver.standard = monitor
 
-    let status = try await Metric<ServerStatus>("caps.status",
+    serverStatus = Metric<ServerStatus>("caps.status",
                                       name: "Status",
                                       description: "The general status of the service")
-    try await status.update(.initializing)
+    try await serverStatus.update(.initializing)
 
     app.http.server.configuration.port = config.port
     app.routes.defaultMaxBodySize = .init(stringLiteral: config.maxBodySize)
 
-    let server = await CapServer(in: URL(fileURLWithPath: publicDirectory))
+    let dataDirectory = config.customDataDirectory(or: publicDirectory)
+    server = CapServer(in: dataDirectory)
 
-    await monitor.registerRoutes(app)
+    provider = .init(observer: monitor, accessManager: config.writers)
+    provider.asyncScheduler = asyncScheduler
+    provider.registerRoutes(app)
 
     if config.serveFiles {
         let middleware = FileMiddleware(publicDirectory: publicDirectory)
@@ -42,9 +50,27 @@ public func configure(_ app: Application) async throws {
     do {
         try server.loadData()
     } catch {
-        try await status.update(.initializationFailure)
+        try await serverStatus.update(.initializationFailure)
+        print("[\(df.string(from: Date()))] Server failed to start: \(error)")
+        return
+    }
+    if server.canResizeImages {
+        try await serverStatus.update(.nominal)
+    } else {
+        try await serverStatus.update(.reducedFunctionality)
+    }
+    print("[\(df.string(from: Date()))] Server started (\(app.environment.name), \(server.capCount) caps)")
+}
+
+func shutdown() {
+    Task {
+        print("[\(df.string(from: Date()))] Server shutdown")
+        do {
+            try await asyncScheduler.shutdownGracefully()
+        } catch {
+            print("Failed to shut down MultiThreadedEventLoopGroup: \(error)")
+        }
     }
-    try await status.update(.nominal)
 }
 
 func log(_ message: String) {
@@ -52,7 +78,14 @@ func log(_ message: String) {
         print(message)
         return
     }
-    Task {
+    asyncScheduler.schedule {
         await observer.log(message)
     }
 }
+
+private let df: DateFormatter = {
+    let df = DateFormatter()
+    df.dateStyle = .short
+    df.timeStyle = .short
+    return df
+}()

Sources/App/entrypoint.swift

@@ -0,0 +1,43 @@
+import Vapor
+import Dispatch
+import Logging
+
+/// This extension is temporary and can be removed once Vapor gets this support.
+private extension Vapor.Application {
+    static let baseExecutionQueue = DispatchQueue(label: "vapor.codes.entrypoint")
+    
+    func runFromAsyncMainEntrypoint() async throws {
+        try await withCheckedThrowingContinuation { continuation in
+            Vapor.Application.baseExecutionQueue.async { [self] in
+                do {
+                    try self.run()
+                    continuation.resume()
+                } catch {
+                    continuation.resume(throwing: error)
+                }
+            }
+        }
+    }
+}
+
+@main
+enum Entrypoint {
+    static func main() async throws {
+        var env = try Environment.detect()
+        try LoggingSystem.bootstrap(from: &env)
+        
+        let app = Application(env)
+        defer {
+            shutdown()
+            app.shutdown()
+        }
+        
+        do {
+            try await configure(app)
+        } catch {
+            app.logger.report(error: error)
+            throw error
+        }
+        try await app.runFromAsyncMainEntrypoint()
+    }
+}

Sources/Run/main.swift

@@ -1,15 +0,0 @@
-import App
-import Vapor
-
-var env = Environment.production
-try LoggingSystem.bootstrap(from: &env)
-let app = Application(env)
-defer { app.shutdown() }
-
-private let semaphore = DispatchSemaphore(value: 0)
-Task {
-    try await configure(app)
-    semaphore.signal()
-}
-semaphore.wait()
-try app.run()

Tests/AppTests/AppTests.swift

@@ -1,13 +0,0 @@
-import App
-import Dispatch
-import XCTest
-
-final class AppTests : XCTestCase {
-    func testNothing() throws {
-        XCTAssert(true)
-    }
-
-    static let allTests = [
-        ("testNothing", testNothing),
-    ]
-}

Training/config_example.json

@@ -1,6 +0,0 @@
-{
-    "contentDirectory": "../Public",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}

Training/train.swift

@@ -1,488 +0,0 @@
-import Foundation
-import Cocoa
-import CreateML
-
-struct Configuration: Codable {
-    
-    let contentFolder: String
-
-    let trainingIterations: Int
-    
-    let serverPath: String
-    
-    let authenticationToken: String
-    
-    init?(at url: URL) {
-        do {
-            let configData = try Data(contentsOf: url)
-            self = try JSONDecoder().decode(Configuration.self, from: configData)
-        } catch {
-            print("[ERROR] Failed to load configuration at \(url.absoluteURL.path): \(error)")
-            return nil
-        }
-    }
-}
-
-struct Cap: Codable {
-
-    let id: Int
-
-    let count: Int
-
-    enum CodingKeys: String, CodingKey {
-        case id = "i"
-        case count = "c"
-    }
-}
-
-final class ClassifierCreator {
-    
-    static let configurationFileUrl = URL(fileURLWithPath: "config.json")
-    
-    let server: URL
-    
-    let configuration: Configuration
-    
-    let imageDirectory: URL
-    
-    let thumbnailDirectory: URL
-    
-    let classifierUrl: URL
-    
-    let df = DateFormatter()
-    
-    
-    // MARK: Step 1: Load configuration
-    
-    init?() {
-        guard let configuration = Configuration(at: ClassifierCreator.configurationFileUrl) else {
-            return nil
-        }
-        self.configuration = configuration
-        guard let serverUrl = URL(string: configuration.serverPath) else {
-            print("[ERROR] Configuration: Invalid server path \(configuration.serverPath)")
-            return nil
-        }
-        self.server = serverUrl
-        let contentDirectory = URL(fileURLWithPath: configuration.contentFolder)
-        self.imageDirectory = contentDirectory.appendingPathComponent("images")
-        self.classifierUrl = contentDirectory.appendingPathComponent("classifier.mlmodel")
-        self.thumbnailDirectory = contentDirectory.appendingPathComponent("thumbnails")
-        df.dateFormat = "yy-MM-dd-HH-mm-ss"
-    }
-    
-    // MARK: Main function
-    
-    func run() async {
-        let imagesSnapshotDate = Date()
-        guard let (classes, changedImageCount, changedMainImages) = await loadImages() else {
-            return
-        }
-
-        guard !classes.isEmpty else {
-            return
-        }
-        
-        guard changedImageCount > 0 else {
-            print("[INFO] No changed images, so no new classifier trained")
-            await createThumbnails(changed: changedMainImages)
-            print("[INFO] Done")
-            return
-        }
-
-        guard let classifierVersion = await getClassifierVersion() else {
-            return
-        }
-        let newVersion = classifierVersion + 1
-
-        print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
-        print("[INFO] Model path: \(classifierUrl.path)")
-        print("[INFO] Version: \(newVersion)")
-        print("[INFO] Classes: \(classes.count)")
-        print("[INFO] Iterations: \(configuration.trainingIterations)")
-
-        guard trainModel() else {
-            return
-        }
-
-        guard await upload(version: newVersion) else {
-            return
-        }
-
-        guard await upload(classes: classes, lastUpdate: imagesSnapshotDate) else {
-            return
-        }
-        await createThumbnails(changed: changedMainImages)
-        print("[INFO] Done")
-    }
-    
-    // MARK: Step 2: Load changed images
-    
-    func loadImages() async -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int])? {
-        guard createImageFolderIfMissing() else {
-            return nil
-        }
-        let imageCounts = await getImageCounts()
-        let missingImageList: [(cap: Int, image: Int)] = imageCounts
-            .sorted { $0.key < $1.key }
-            .reduce(into: []) { list, pair in
-                let missingImagesForCap: [(cap: Int, image: Int)] = (0..<pair.value).compactMap { image in
-                    let url = imageUrl(base: imageDirectory, cap: pair.key, image: image)
-                    guard !FileManager.default.fileExists(atPath: url.path) else {
-                        return nil
-                    }
-                    return (cap: pair.key, image: image)
-                }
-                list.append(contentsOf: missingImagesForCap)
-            }
-        if missingImageList.isEmpty {
-            print("[INFO] No missing images to load")
-        } else {
-            print("[INFO] Loading \(missingImageList.count) missing images...")
-        }
-        guard await loadImages(missingImageList) else {
-            return nil
-        }
-        let changedImageList = await getChangedImageList()
-            .filter { $0.image < imageCounts[$0.cap] ?? 0 } // Filter non-existent images
-        
-        if changedImageList.isEmpty {
-            print("[INFO] No changed images to load")
-        } else {
-            print("[INFO] Loading \(changedImageList.count) changed images...")
-        }
-        guard await loadImages(changedImageList) else {
-            return nil
-        }
-        let changedMainImages = changedImageList.filter { $0.image == 0 }.map { $0.cap }
-        let classes = imageCounts.keys.sorted()
-        return (classes, missingImageList.count + changedImageList.count, changedMainImages)
-    }
-    
-    private func createImageFolderIfMissing() -> Bool {
-        createFolderIfMissing(imageDirectory)
-    }
-    
-    private func getImageCounts() async -> [Int : Int] {
-        guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
-            return [:]
-        }
-        do {
-            return try JSONDecoder().decode([Cap].self, from: data)
-                .reduce(into: [:]) { $0[$1.id] = $1.count }
-        } catch {
-            print("[ERROR] Failed to decode cap database: \(error)")
-            return [:]
-        }
-    }
-    
-    private func imageUrl(base: URL, cap: Int, image: Int) -> URL {
-        base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", cap, cap, image))
-    }
-    
-    private func loadImage(cap: Int, image: Int) async -> Bool {
-        guard createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", cap))) else {
-            return false
-        }
-        let url = imageUrl(base: server.appendingPathComponent("images"), cap: cap, image: image)
-        let tempFile: URL, response: URLResponse
-        do {
-            (tempFile, response) = try await URLSession.shared.download(from: url)
-        } catch {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): \(error)")
-            return false
-        }
-        let responseCode = (response as! HTTPURLResponse).statusCode
-        guard responseCode == 200 else {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): Response \(responseCode)")
-            return false
-        }
-        do {
-            let localUrl = imageUrl(base: imageDirectory, cap: cap, image: image)
-            if FileManager.default.fileExists(atPath: localUrl.path) {
-                try FileManager.default.removeItem(at: localUrl)
-            }
-            try FileManager.default.moveItem(at: tempFile, to: localUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save image \(image) of cap \(cap): \(error)")
-            return false
-        }
-    }
-    
-    private func loadImages(_ list: [(cap: Int, image: Int)]) async -> Bool {
-        guard !list.isEmpty else {
-            return true
-        }
-        var loadedImages = 0
-        await withTaskGroup(of: Bool.self) { group in
-            for (cap, image) in list {
-                group.addTask {
-                    await self.loadImage(cap: cap, image: image)
-                }
-            }
-            for await loaded in group {
-                if loaded {
-                    loadedImages += 1
-                }
-            }
-        }
-        if loadedImages != list.count {
-            print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
-            return false
-        }
-        return true
-    }
-    
-    func getChangedImageList() async -> [(cap: Int, image: Int)] {
-        guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
-            print("[ERROR] Failed to get list of changed images")
-            return []
-        }
-        
-        return string
-            .components(separatedBy: "\n")
-            .filter { $0 != "" }
-            .compactMap {
-                let parts = $0.components(separatedBy: ":")
-                guard parts.count == 3 else {
-                    return nil
-                }
-                /*
-                guard let date = df.date(from: parts[0]) else {
-                    print("[WARN] Invalid date \(parts[0]) in change list")
-                    return nil
-                }
-                 */
-                guard let cap = Int(parts[1]) else {
-                    print("[WARN] Invalid cap id \(parts[1]) in change list")
-                    return nil
-                }
-                guard let image = Int(parts[2]) else {
-                    print("[WARN] Invalid image id \(parts[2]) in change list")
-                    return nil
-                }
-                return (cap, image)
-            }
-    }
-    
-    // MARK: Step 3: Compute version
-    
-    func getClassifierVersion() async -> Int? {
-        guard let string: String = await get(server.appendingPathComponent("version")) else {
-            print("[ERROR] Failed to get classifier version")
-            return nil
-        }
-        guard let version = Int(string) else {
-            print("[ERROR] Invalid classifier version \(string)")
-            return nil
-        }
-        return version
-    }
-    
-    // MARK: Step 4: Train classifier
-    
-    func trainModel() -> Bool {
-        var params = MLImageClassifier.ModelParameters(augmentation: [])
-        params.maxIterations = configuration.trainingIterations
-        let model: MLImageClassifier
-        do {
-            model = try MLImageClassifier(
-                trainingData: .labeledDirectories(at: imageDirectory),
-                parameters: params)
-        } catch {
-            print("[ERROR] Failed to create classifier: \(error)")
-            return false
-        }
-
-        print("[INFO] Saving classifier...")
-        do {
-            try model.write(to: classifierUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save model to file: \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Step 5: Upload classifier
-    
-    func upload(version: Int) async -> Bool {
-        print("[INFO] Uploading classifier...")
-        let modelData: Data
-        do {
-            modelData = try Data(contentsOf: classifierUrl)
-        } catch {
-            print("[ERROR] Failed to read classifier data: \(error)")
-            return false
-        }
-        
-        return await post(
-            url: server.appendingPathComponent("classifier/\(version)"),
-            body: modelData)
-    }
-    
-    // MARK: Step 6: Update classes
-    
-    func upload(classes: [Int], lastUpdate: Date) async -> Bool {
-        print("[INFO] Uploading trained classes...")
-        let dateString = df.string(from: lastUpdate)
-        return await post(
-            url: server.appendingPathComponent("classes/\(dateString)"),
-            body: classes.map(String.init).joined(separator: "\n").data(using: .utf8)!)
-    }
-    
-    // MARK: Step 7: Create thumbnails
-    
-    func createThumbnails(changed: [Int]) async {
-        guard checkMagickAvailability() else {
-            return
-        }
-        guard createFolderIfMissing(thumbnailDirectory) else {
-            print("[ERROR] Failed to create folder for thumbnails")
-            return
-        }
-        let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
-        let all = Set(capIdsOfMissingThumbnails).union(changed)
-        print("[INFO] Creating \(all.count) thumbnails...")
-        for cap in all {
-            await createThumbnail(for: cap)
-        }
-    }
-    
-    func checkMagickAvailability() -> Bool {
-        do {
-            let (code, output) = try safeShell("magick --version")
-            guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
-                .components(separatedBy: " ").first else {
-                print("[ERROR] Magick not found, install using 'brew install imagemagick'")
-                return false
-            }
-            print("[INFO] Using magick \(version)")
-        } catch {
-            print("[ERROR] Failed to get version of magick: (\(error))")
-            return false
-        }
-        return true
-    }
-    
-    private func getMissingThumbnailIds() async -> [Int] {
-        guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
-            print("[ERROR] Failed to get missing thumbnails")
-            return []
-        }
-        return string.components(separatedBy: ",").compactMap(Int.init)
-    }
-    
-    private func createThumbnail(for cap: Int) async {
-        let inputUrl = imageUrl(base: imageDirectory, cap: cap, image: 0)
-        guard FileManager.default.fileExists(atPath: inputUrl.path) else {
-            print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
-            return
-        }
-        
-        let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
-        do {
-            let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
-            let (code, output) = try safeShell(command)
-            if code != 0 {
-                print("Failed to create thumbnail for cap \(cap): \(output)")
-                return
-            }
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        
-        let data: Data
-        do {
-            data = try Data(contentsOf: output)
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
-            print("Failed to upload thumbnail for cap \(cap)")
-            return
-        }
-    }
-    
-    // MARK: Helper
-    
-    @discardableResult
-    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
-        let task = Process()
-        let pipe = Pipe()
-
-        task.standardOutput = pipe
-        task.standardError = pipe
-        task.arguments = ["-cl", command]
-        task.executableURL = URL(fileURLWithPath: "/bin/zsh")
-        task.standardInput = nil
-
-        try task.run()
-        task.waitUntilExit()
-        
-        let data = pipe.fileHandleForReading.readDataToEndOfFile()
-        let output = String(data: data, encoding: .utf8)!
-        return (task.terminationStatus, output)
-    }
-    
-    
-    private func createFolderIfMissing(_ folder: URL) -> Bool {
-        guard !FileManager.default.fileExists(atPath: folder.path) else {
-            return true
-        }
-        do {
-            try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
-            return true
-        } catch {
-            print("[ERROR] Failed to create directory \(folder.path): \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Requests
-    
-    private func post(url: URL, body: Data) async -> Bool {
-        var request = URLRequest(url: url)
-        request.httpMethod = "POST"
-        request.httpBody = body
-        request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
-        return await perform(request) != nil
-    }
-    
-    private func perform(_ request: URLRequest) async -> Data? {
-        let data: Data
-        let response: URLResponse
-        do {
-            (data, response) = try await URLSession.shared.data(for: request)
-        } catch {
-            print("[ERROR] Request to \(request.url!.absoluteString) failed: \(error)")
-            return nil
-        }
-        let code = (response as! HTTPURLResponse).statusCode
-        guard code == 200 else {
-            print("[ERROR] Request to \(request.url!.absoluteString): Invalid response \(code)")
-            return nil
-        }
-        return data
-    }
-    
-    private func get(_ url: URL) async -> Data? {
-        await perform(URLRequest(url: url))
-    }
-    
-    private func get(_ url: URL) async -> String? {
-        guard let data: Data = await get(url) else {
-            return nil
-        }
-        guard let string = String(data: data, encoding: .utf8) else {
-            print("[ERROR] Invalid string response \(data)")
-            return nil
-        }
-        return string
-    }
-}
-
-await ClassifierCreator()?.run()

build.sh

@@ -1,13 +0,0 @@
-echo "[1/7] Stopping server..."
-sudo supervisorctl stop caps
-echo "[2/7] Changing permissions..."
-sudo chown -R pi:pi .
-echo "[3/7] Pulling changes..."
-git pull
-echo "[4/7] Building project..."
-swift build -c release
-echo "[5/7] Restoring permissions..."
-sudo chown -R www-data:www-data .
-echo "[6/7] Starting server..."
-sudo supervisorctl start caps
-echo "[7/7] Done"