Add image assets

.gitignore

@@ -8,12 +8,11 @@
 .DS_Store
 Package.resolved
 .swiftpm/
+Public/caps.json
+Public/changes.txt
+Public/classifier.*
+Public/count.*
 Public/images/
 Public/thumbnails/
-Public/classifier.version
+Resources/config.json
-Public/classifier.mlmodel
+Resources/logs/
-Public/caps.json
-Training/backup/
-Training/config.json
-Public/thumbnails
-Public/count.js

Package.swift

@@ -8,22 +8,20 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
-        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.2.0"),
+        .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.11.2"),
+        .package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.4.0"),
+        .package(url: "https://github.com/christophhagen/ClairvoyantBinaryCodable", from: "0.3.0"),
     ],
     targets: [
-        .target(name: "App",
+        .executableTarget(
-                dependencies: [
+            name: "App",
-                    .product(name: "Vapor", package: "vapor"),
+            dependencies: [
-                    .product(name: "Clairvoyant", package: "Clairvoyant"),
+                .product(name: "Vapor", package: "vapor"),
-                ],
+                .product(name: "Clairvoyant", package: "Clairvoyant"),
-                swiftSettings: [
+                .product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"),
-                    // Enable better optimizations when building in Release configuration. Despite the use of
+                .product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"),
-                    // the `.unsafeFlags` construct required by SwiftPM, this flag is recommended for Release
+            ]
-                    // builds. See <https://github.com/swift-server/guides#building-for-production> for details.
+        )
-                    .unsafeFlags(["-cross-module-optimization"], .when(configuration: .release))
-                ]),
-        .executableTarget(name: "Run", dependencies: ["App"]),
-        .testTarget(name: "AppTests", dependencies: ["App"]),
     ]
 )
 

Public/assets/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/assets/mstile-150x150.png?v=1"/>
+            <TileColor>#a36490</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

Public/assets/site.webmanifest

@@ -0,0 +1 @@
+{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

Public/grid.html

@@ -5,13 +5,13 @@
         <meta charset="utf-8"/>
         <meta name="robots" content="noindex"/>
         <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0"/>
-        <link rel="apple-touch-icon" sizes="180x180" href="/assets/icons/apple-touch-icon.png?v=1"/>
+        <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="32x32" href="/assets/icons/favicon-32x32.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png?v=1"/>
-        <link rel="icon" type="image/png" sizes="16x16" href="/assets/icons/favicon-16x16.png?v=1"/>
+        <link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png?v=1"/>
-        <link rel="manifest" href="/assets/icons/site.webmanifest?v=1"/>
+        <link rel="manifest" href="/assets/site.webmanifest?v=1"/>
-        <link rel="shortcut icon" href="/assets/icons/favicon.ico?v=1"/>
+        <link rel="shortcut icon" href="/assets/favicon.ico?v=1"/>
         <meta name="msapplication-TileColor" content="#da532c">
-        <meta name="msapplication-config" content="/assets/icons/browserconfig.xml?v=1"/>
+        <meta name="msapplication-config" content="/assets/browserconfig.xml?v=1"/>
         <meta name="theme-color" content="#ffffff"/>
         <link href="grid.css?v=2" rel="stylesheet"/>
         <meta name="author" content="Christoph Hagen"/>

Readme.md

@@ -51,31 +51,7 @@ Note: The data for the mosaic is currently not updated automatically, since Swif
 ## Classifier training
 
 The main server is running on Linux, which doesn't provide the CreateML framework required for classifier training. 
-This has to be done on macOS using the `train.swift` script in the `Training` folder.
+This has to be done on macOS using the [Caps-Train](https://christophhagen.de/git/ch/Caps-Train) repository.
-It will:
-- Fetch the current image catalog by checking missing and changed images
-- Train a classifier on the image set
-- Increment the classifier version and upload the new model
-- Update the list of caps which can be recognized using the classifier
-- Create missing thumbnails for each cap for the cap grid
-
-A configuration file is required to run the training, with a valid access token for the server:
-
-```json
-{
-    "imageDirectory": "../Public/images",
-    "classifierModelPath": "../Public/classifier.mlmodel",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}
-```
-
-The configuration file `config.json` must be located in the folder from which the script is run.
-
-```bash
-swift train.swift
-```
 
 ## Future work
 - Create thumbnails on the server using [JPEG](https://github.com/kelvin13/jpeg)

Resources/example_config.json

@@ -1,8 +1,9 @@
 {
     "port" : 6001,
     "maxBodySize" : "2mb",
-    "logPath": "\/var\/log\/caps.log",
+    "logPath": "\/var\/log\/caps/metrics",
     "serveFiles": true,
+    "dataDirectory" : "/ch/data/caps",
     "writers" : [
         "auth_key_1"
     ]

Sources/App/Authenticator.swift

@@ -0,0 +1,29 @@
+import Foundation
+import Vapor
+
+final class Authenticator {
+
+    private var writers: Set<String>
+
+    init(writers: [String]) {
+        self.writers = Set(writers)
+    }
+
+    func hasAuthorization(for key: String) -> Bool {
+        // Note: This is not a constant-time compare, so there may be an opportunity
+        // for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
+        // and we're not doing anything critical in this application.
+        // Worst case, an unauthorized person with a lot of free time and energy to hack this system
+        // is able to change contents of the database, which are backed up in any case.
+        writers.contains(key)
+    }
+
+    func authorize(_ request: Request) throws {
+        guard let key = request.headers.first(name: "key") else {
+            throw Abort(.badRequest) // 400
+        }
+        guard hasAuthorization(for: key) else {
+            throw Abort(.forbidden) // 403
+        }
+    }
+}

Sources/App/Cap.swift

@@ -10,9 +10,6 @@ struct Cap: Codable {
 
     var mainImage: Int
 
-    /// The version of the first classifier trained on this cap
-    var classifierVersion: Int?
-
     var color: Color?
 
     enum CodingKeys: String, CodingKey {
@@ -20,7 +17,6 @@ struct Cap: Codable {
         case name = "n"
         case count = "c"
         case mainImage = "m"
-        case classifierVersion = "v"
         case color = "f"
     }
 

Sources/App/CapServer+Routes.swift

@@ -0,0 +1,157 @@
+import Vapor
+import Foundation
+
+/// The decoder to extract caps from JSON payloads given to the `cap` route.
+private let decoder = JSONDecoder()
+
+private let encoder = JSONEncoder()
+
+/// The date formatter to decode dates in requests
+private let dateFormatter: DateFormatter = {
+    let df = DateFormatter()
+    df.dateFormat = "yy-MM-dd-HH-mm-ss"
+    return df
+}()
+
+extension CapServer {
+
+    private func ensureOperability() throws {
+        guard isOperational else {
+            throw Abort(.noContent)
+        }
+    }
+
+    func registerRoutes(with app: Application, authenticator: Authenticator) {
+        app.get("version") { _ in
+            try self.ensureOperability()
+            return "\(self.classifierVersion)"
+        }
+
+        // Add or change a cap
+        app.postCatching("cap") { request in
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            let data = try request.getBodyData(request: "/cap")
+            let cap = try decoder.decode(Cap.self, from: data)
+            try self.addOrUpdate(cap)
+        }
+
+        // Upload an image
+        app.postCatching("image", .parameter("cap")) { request in
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            guard let cap = request.parameters.get("cap", as: Int.self) else {
+                log("/images/:cap: Invalid 'cap' parameter")
+                throw Abort(.badRequest)
+            }
+            let data = try request.getBodyData(request: "/images/:cap")
+            try self.save(image: data, for: cap)
+        }
+        // Update the classifier
+        app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in
+            guard let version = request.parameters.get("version", as: Int.self) else {
+                log("/classifier/:version: Invalid parameter for version")
+                throw Abort(.badRequest)
+            }
+            guard version > self.classifierVersion else {
+                throw Abort(.alreadyReported) // 208
+            }
+
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            let data = try request.getBodyData(request: "/classifier/:version")
+            try self.save(classifier: data, version: version)
+            return .ok
+        }
+
+        // Update the trained classes
+        app.postCatching("classes", ":date") { request in
+            guard let dateString = request.parameters.get("date") else {
+                log("/classes/:date: Invalid 'date' parameter")
+                throw Abort(.badRequest)
+            }
+            guard let date = dateFormatter.date(from: dateString) else {
+                log("/classes/:date: Invalid 'date' specification")
+                throw Abort(.badRequest)
+            }
+
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            let body = try request.getStringBody(request: "/classes/:date")
+
+            try self.saveTrainedClasses(content: body)
+            self.removeAllEntriesInImageChangeList(before: date)
+        }
+
+        // Get the list of missing thumbnails
+        app.get("thumbnails", "missing") { request in
+            try self.ensureOperability()
+            let missingThumbnails = self.getListOfMissingThumbnails()
+            return missingThumbnails.map(String.init).joined(separator: ",")
+        }
+
+        // Upload the thumbnail of a cap
+        app.postCatching("thumbnails", ":cap") { request in
+            guard let cap = request.parameters.get("cap", as: Int.self) else {
+                log("/thumbnails/:cap: Invalid cap parameter for thumbnail upload")
+                throw Abort(.badRequest)
+            }
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            let data = try request.getBodyData(request: "/thumbnails/:cap")
+            self.saveThumbnail(data, for: cap)
+        }
+
+        // Delete the image of a cap
+        app.postCatching("delete", ":cap", ":version") { request in
+            guard let cap = request.parameters.get("cap", as: Int.self) else {
+                log("/delete/:cap/:version: Invalid 'cap' parameter for image deletion")
+                throw Abort(.badRequest)
+            }
+            guard let version = request.parameters.get("version", as: Int.self) else {
+                log("/delete/:cap/:version: Invalid 'version' parameter for image deletion")
+                throw Abort(.badRequest)
+            }
+
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            guard let cap = self.deleteImage(version: version, for: cap) else {
+                throw Abort(.gone)
+            }
+            return try encoder.encode(cap)
+        }
+
+        // Delete a cap completely, with all images
+        app.postCatching("delete", ":cap") { request in
+            guard let cap = request.parameters.get("cap", as: Int.self) else {
+                log("/delete/:cap/: Invalid 'cap' parameter for image deletion")
+                throw Abort(.badRequest)
+            }
+            try self.ensureOperability()
+            try authenticator.authorize(request)
+            guard self.delete(cap: cap) else {
+                throw Abort(.gone)
+            }
+        }
+    }
+}
+
+private extension Request {
+
+    func getBodyData(request: String) throws -> Data {
+        guard let buffer = body.data else {
+            log("\(request): Missing body data")
+            throw CapError.invalidBody
+        }
+        return Data(buffer: buffer)
+    }
+
+    func getStringBody(request: String) throws -> String {
+        let data = try getBodyData(request: request)
+        guard let content = String(data: data, encoding: .utf8) else {
+            log("\(request): Invalid string body")
+            throw CapError.invalidBody
+        }
+        return content
+    }
+}

Sources/App/CapServer.swift

@@ -2,7 +2,11 @@ import Foundation
 import Vapor
 import Clairvoyant
 
-final class CapServer: ServerOwner {
+final class CapServer {
+    
+    private let imageSize = 360
+    
+    private let thumbnailSize = 100
     
     // MARK: Paths
     
@@ -23,15 +27,21 @@ final class CapServer: ServerOwner {
     
     private let classifierFile: URL
     
+    private let classifierClassesFile: URL
+    
     private let changedImagesFile: URL
     
     private let fm = FileManager.default
     
     private let changedImageEntryDateFormatter: DateFormatter
+
+    /// Indicates that the data is loaded
+    private(set) var isOperational = false
+    
+    private(set) var canResizeImages = false
     
     // MARK: Caps
 
-    private var writers: Set<String>
     
     /// The changed images not yet written to disk
     private var unwrittenImageChanges: [(cap: Int, image: Int)] = []
@@ -39,7 +49,9 @@ final class CapServer: ServerOwner {
     var classifierVersion: Int = 0 {
         didSet {
             writeClassifierVersion()
-            updateMonitoredClassifierVersionProperty()
+            Task {
+                try? await classifierMetric.update(classifierVersion)
+            }
         }
     }
 
@@ -60,14 +72,13 @@ final class CapServer: ServerOwner {
     private var caps = [Int: Cap]() {
         didSet {
             scheduleSave()
-            updateMonitoredPropertiesOnCapChange()
+            Task {
+                _ = try? await capCountMetric.update(caps.count)
+                _ = try? await imageCountMetric.update(imageCount)
+            }
         }
     }
 
-    var nextClassifierVersion: Int {
-        caps.values.compactMap { $0.classifierVersion }.max() ?? 1
-    }
-
     var capCount: Int {
         caps.count
     }
@@ -76,7 +87,7 @@ final class CapServer: ServerOwner {
         caps.reduce(0) { $0 + $1.value.count }
     }
 
-    init(in folder: URL, writers: [String]) {
+    init(in folder: URL) {
         self.imageFolder = folder.appendingPathComponent("images")
         self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
         self.gridCountFile = folder.appendingPathComponent("count.js")
@@ -84,10 +95,23 @@ final class CapServer: ServerOwner {
         self.htmlFile = folder.appendingPathComponent("count.html")
         self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
         self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
+        self.classifierClassesFile = folder.appendingPathComponent("classifier.classes")
         self.changedImagesFile = folder.appendingPathComponent("changes.txt")
-        self.writers = Set(writers)
         self.changedImageEntryDateFormatter = DateFormatter()
         changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
+
+        // Metric initializers only fail if observer is missing or ID is duplicate
+        self.capCountMetric = .init("caps.count",
+                                    name: "Number of caps",
+                                    description: "The total number of caps in the database")
+
+        self.imageCountMetric = .init("caps.images",
+                                      name: "Total images",
+                                      description: "The total number of images for all caps")
+
+        self.classifierMetric = .init("caps.classifier",
+                                      name: "Classifier Version",
+                                      description: "The current version of the image classifier")
     }
 
     func loadData() throws {
@@ -97,10 +121,17 @@ final class CapServer: ServerOwner {
         updateGridCapCount()
         try ensureExistenceOfChangedImagesFile()
         organizeImages()
+        if let version = getMagickVersion() {
+            log("Using ImageMagick \(version.rawValue)")
+            canResizeImages = true
+        }
+        // shrinkImages()
+        createMissingThumbnails()
+        isOperational = true
     }
 
     private func loadClassifierVersion(at url: URL) {
-        guard fm.fileExists(atPath: url.path) else {
+        guard exists(url) else {
             return
         }
         let content: String
@@ -129,6 +160,10 @@ final class CapServer: ServerOwner {
     }
 
     private func loadCaps() throws {
+        guard exists(dbFile) else {
+            log("No cap database found")
+            return
+        }
         do {
             let data = try Data(contentsOf: dbFile)
             caps = try JSONDecoder().decode([Cap].self, from: data)
@@ -137,7 +172,6 @@ final class CapServer: ServerOwner {
             log("Failed to load caps: \(error)")
             throw error
         }
-        log("\(caps.count) caps loaded")
     }
 
     private func scheduleSave() {
@@ -174,7 +208,7 @@ final class CapServer: ServerOwner {
         let count = caps.count
         let content =
         """
-        <body style="margin: 0;">
+        <body style="margin: 0;overflow: hidden">
             <div style="display: flex; justify-content: center;">
                 <div style="font-size: 60px; font-family: 'SF Pro Display',-apple-system,BlinkMacSystemFont,Helvetica,sans-serif; -webkit-font-smoothing: antialiased;">\(count)</div>
             </div>
@@ -186,19 +220,52 @@ final class CapServer: ServerOwner {
     private func organizeImages() {
         caps.values.sorted().forEach(organizeImages)
     }
+
+    private func createImageFolder(for cap: Int) throws {
+        let folderUrl = folder(of: cap)
+        do {
+            try fm.createDirectory(at: folderUrl, withIntermediateDirectories: true)
+        } catch {
+            log("Failed to create folder for cap \(cap): \(error)")
+            throw error
+        }
+    }
     
+    /**
+     Rearrange images of a cap to ensure that an image exists for each number from 0 to `image count - 1`.
+     This is done by using the last images to fill in possible gaps in the sequence.
+     E.g. If there are images `0`, `2`, `3`, then `3` will be renamed to `1`.
+     - Note: The main image is also changed, if the main image is renamed.
+     */
     private func organizeImages(for cap: Cap) {
         var cap = cap
-        guard let images = try? images(in: folder(of: cap.id)) else {
+        let folderUrl = folder(of: cap.id)
+        guard exists(folderUrl) else {
+            try? createImageFolder(for: cap.id)
+            cap.count = 0
+            caps[cap.id] = cap
+            log("Found cap \(cap.id) without image folder")
+            return
+        }
+
+        guard let images = try? images(in: folderUrl) else {
             log("Failed to get image urls for cap \(cap.id)")
             return
         }
+        
+        if images.count != cap.count {
+            log("\(images.count) instead of \(cap.count) images for cap \(cap.id)")
+        }
+        
+        // Get list of existing images
         var sorted: [(id: Int, url: URL)] = images.compactMap {
             guard let id = Int($0.deletingPathExtension().lastPathComponent.components(separatedBy: "-").last!) else {
                 return nil
             }
             return (id, $0)
         }.sorted { $0.id < $1.id }
+        
+        // Check that all images are available
         for version in 0..<images.count {
             guard version != sorted[version].id else {
                 continue
@@ -207,6 +274,7 @@ final class CapServer: ServerOwner {
             let newUrl = imageUrl(of: cap.id, version: version)
             do {
                 try fm.moveItem(at: lastImage.url, to: newUrl)
+                log("Moved image \(lastImage.id) to \(version) for cap \(cap.id)")
             } catch {
                 log("Failed to move file \(lastImage.url.path) to \(newUrl.path): \(error)")
                 return
@@ -216,7 +284,14 @@ final class CapServer: ServerOwner {
             }
             sorted.insert((version, newUrl), at: version)
         }
+
         cap.count = sorted.count
+
+        // Fix invalid main image
+        if cap.mainImage >= cap.count || cap.mainImage < 0 {
+            cap.mainImage = 0
+        }
+
         caps[cap.id] = cap
     }
 
@@ -234,33 +309,31 @@ final class CapServer: ServerOwner {
         folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version))
     }
 
-    // MARK: Authentication
+    private func exists(_ url: URL) -> Bool {
-
+        fm.fileExists(atPath: url.path)
-    func hasAuthorization(for key: String) -> Bool {
-        // Note: This is not a constant-time compare, so there may be an opportunity
-        // for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
-        // and we're not doing anything critical in this application.
-        // Worst case, an unauthorized person with a lot of free time and energy to hack this system
-        // is able to change contents of the database, which are backed up in any case.
-        writers.contains(key)
     }
-    
+
     // MARK: Counts
 
     private func images(in folder: URL) throws -> [URL] {
-        try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
+        do {
-            .filter { $0.pathExtension == "jpg" }
+            return try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
+                .filter { $0.pathExtension == "jpg" }
+        } catch {
+            log("Failed to get image urls for cap \(folder.lastPathComponent): \(error)")
+            throw error
+        }
     }
 
     /**
      Get the image count of a cap.
      */
     func count(of cap: Int) throws -> Int {
-        let f = folder(of: cap)
+        let capImageFolder = folder(of: cap)
-        guard fm.fileExists(atPath: f.path) else {
+        guard exists(capImageFolder) else {
             return 0
         }
-        return try images(in: f).count
+        return try images(in: capImageFolder).count
     }
 
     // MARK: Images
@@ -275,32 +348,52 @@ final class CapServer: ServerOwner {
      */
     func save(image data: Data, for cap: Int) throws {
         guard caps[cap] != nil else {
+            log("Tried to save image for unknown cap \(cap)")
             throw CapError.unknownId
         }
         var id = 0
         let capFolder = folder(of: cap)
-        var f = imageUrl(of: cap, version: id)
+        var capImageUrl = imageUrl(of: cap, version: id)
-        if fm.fileExists(atPath: capFolder.path) {
+        if exists(capFolder) {
-            while fm.fileExists(atPath: f.path) {
+            while exists(capImageUrl) {
                 id += 1
-                f = imageUrl(of: cap, version: id)
+                capImageUrl = imageUrl(of: cap, version: id)
             }
         } else {
-            try fm.createDirectory(at: capFolder, withIntermediateDirectories: true)
+            try createImageFolder(for: cap)
         }
-        try data.write(to: f)
+        do {
-        caps[cap]!.count = try count(of: cap)
+            try data.write(to: capImageUrl)
+        } catch {
+            log("Failed to write image \(id) for cap \(cap): \(error)")
+            throw CapError.invalidFile
+        }
+        let count = try count(of: cap)
+        caps[cap]!.count = count
         addChangedImageToLog(cap: cap, image: id)
-        log("Added image \(id) for cap \(cap)")
+        if canResizeImages {
+            shrink(imageAt: capImageUrl, size: imageSize, destination: capImageUrl)
+            createThumbnail(for: cap)
+        }
+        log("Added image \(id) for cap \(cap) (\(count) total)")
     }
     
     private func writeChangedImagesToDisk() throws {
         guard !unwrittenImageChanges.isEmpty else {
             return
         }
+
+        try ensureExistenceOfChangedImagesFile()
         
-        let handle = try FileHandle(forWritingTo: changedImagesFile)
+        let handle: FileHandle
-        try handle.seekToEnd()
+        do {
+            handle = try FileHandle(forWritingTo: changedImagesFile)
+            try handle.seekToEnd()
+        } catch {
+            log("Failed to open changed images file for writing: \(error)")
+            throw error
+        }
+
         var entries = unwrittenImageChanges
         defer {
             unwrittenImageChanges = entries
@@ -309,7 +402,12 @@ final class CapServer: ServerOwner {
         let dateString = changedImageEntryDateFormatter.string(from: Date())
         while let entry = entries.popLast() {
             let content = "\(dateString):\(entry.cap):\(entry.image)\n".data(using: .utf8)!
-            try handle.write(contentsOf: content)
+            do {
+                try handle.write(contentsOf: content)
+            } catch {
+                log("Failed to write entry to changed images file: \(error)")
+                throw error
+            }
         }
     }
     
@@ -323,7 +421,7 @@ final class CapServer: ServerOwner {
     }
     
     private func ensureExistenceOfChangedImagesFile() throws {
-        guard !fm.fileExists(atPath: changedImagesFile.path) else {
+        if exists(changedImagesFile) {
             return
         }
         do {
@@ -335,6 +433,10 @@ final class CapServer: ServerOwner {
     }
     
     func removeAllEntriesInImageChangeList(before date: Date) {
+        guard exists(changedImagesFile) else {
+            log("No file for changed images to update")
+            return
+        }
         do {
             try String(contentsOf: changedImagesFile)
                 .components(separatedBy: "\n")
@@ -357,12 +459,15 @@ final class CapServer: ServerOwner {
     }
     
     func switchMainImage(to version: Int, for cap: Int) throws {
-        let file2 = imageUrl(of: cap, version: version)
+        let capImageUrl = imageUrl(of: cap, version: version)
-        guard fm.fileExists(atPath: file2.path) else {
+        guard exists(capImageUrl) else {
             log("No image \(version) for cap \(cap)")
             throw CapError.invalidFile
         }
         caps[cap]?.mainImage = version
+        if canResizeImages {
+            createThumbnail(for: cap)
+        }
         log("Switched cap \(cap) to version \(version)")
     }
 
@@ -376,11 +481,11 @@ final class CapServer: ServerOwner {
 
     private func add(_ cap: Cap) throws {
         guard cap.mainImage == 0 else {
+            log("Attempting to add cap \(cap.id) with main image \(cap.mainImage)")
             throw CapError.invalidData
         }
         var cap = cap
         cap.count = 0
-        cap.classifierVersion = nextClassifierVersion
         caps[cap.id] = cap
         saveCapCountHTML()
         updateGridCapCount()
@@ -392,8 +497,8 @@ final class CapServer: ServerOwner {
         if cap.name != "" {
             updatedCap.name = cap.name
         }
-        let url = imageUrl(of: existingCap.id, version: cap.mainImage)
+        let capImageUrl = imageUrl(of: existingCap.id, version: cap.mainImage)
-        if fm.fileExists(atPath: url.path) {
+        if exists(capImageUrl) {
             updatedCap.mainImage = cap.mainImage
         }
         if let color = cap.color {
@@ -403,38 +508,89 @@ final class CapServer: ServerOwner {
         log("Updated cap \(existingCap.id)")
     }
     
-    func deleteImage(version: Int, for capId: Int) -> Bool {
+    func deleteImage(version: Int, for capId: Int) -> Cap? {
         guard let cap = caps[capId] else {
-            return false
+            log("Attempting to delete image \(version) of unknown cap \(capId)")
+            return nil
         }
-        let url = imageUrl(of: capId, version: version)
+        let capImageUrl = imageUrl(of: capId, version: version)
-        guard fm.fileExists(atPath: url.path) else {
+        guard exists(capImageUrl) else {
-            return false
+            log("Attempting to delete missing image \(version) of cap \(capId)")
+            return nil
         }
         organizeImages(for: cap)
+        return caps[capId]!
+    }
+
+    func delete(cap capId: Int) -> Bool {
+        guard caps[capId] != nil else {
+            log("Attempting to delete unknown cap \(capId)")
+            return false
+        }
+        // 1. Remove all images
+        do {
+            let imageFolderUrl = folder(of: capId)
+            if exists(imageFolderUrl) {
+                try fm.removeItem(at: imageFolderUrl)
+            }
+        } catch {
+            log("Failed to delete image folder of cap \(capId): \(error)")
+            return false
+        }
+        // 2. Remove thumbnail
+        do {
+            let url = thumbnail(of: capId)
+            if exists(url) {
+                try fm.removeItem(at: url)
+            }
+        } catch {
+            log("Failed to delete thumbnail of cap \(capId): \(error)")
+            return false
+        }
+
+        // 3. Remove cap
+        caps[capId] = nil
+        saveCapCountHTML()
+        updateGridCapCount()
         return true
     }
     
     // MARK: Classifier
     
-    func updateTrainedClasses(content: String) {
+    func saveTrainedClasses(content: String) throws {
-        let trainedCaps = content
+        let classes = content.components(separatedBy: ",")
-            .components(separatedBy: "\n")
+        
-            .compactMap(Int.init)
+        // Validate input
-        let version = classifierVersion
+        try classes.forEach { s in
-        for cap in trainedCaps {
+            guard let id = Int(s) else {
-            if caps[cap]?.classifierVersion == nil {
+                log("Invalid id '\(s)' in uploaded id list")
-                caps[cap]?.classifierVersion = version
+                throw Abort(.badRequest)
+            }
+            guard caps[id] != nil else {
+                log("Unknown id '\(id)' in uploaded id list")
+                throw Abort(.badRequest)
             }
         }
-        log("Updated \(trainedCaps.count) classifier classes")
+        
+        guard let data = content.data(using: .utf8) else {
+            log("Failed to get classes data for writing")
+            throw Abort(.internalServerError)
+        }
+        
+        do {
+            try data.write(to: classifierClassesFile)
+            log("Updated \(classes.count) classifier classes")
+        } catch {
+            log("Failed to write classifier classes: \(error)")
+            throw Abort(.internalServerError)
+        }
     }
     
     func save(classifier: Data, version: Int) throws {
         do {
             try classifier.write(to: classifierFile)
         } catch {
-            log("Failed to write classifier: \(error)")
+            log("Failed to write classifier \(version): \(error)")
             throw Abort(.internalServerError)
         }
         classifierVersion = version
@@ -444,7 +600,7 @@ final class CapServer: ServerOwner {
     // MARK: Grid
     
     func getListOfMissingThumbnails() -> [Int] {
-        caps.keys.filter { !fm.fileExists(atPath: thumbnail(of: $0).path) }
+        caps.keys.filter { !exists(thumbnail(of: $0)) }
     }
     
     func saveThumbnail(_ data: Data, for cap: Int) {
@@ -465,83 +621,117 @@ final class CapServer: ServerOwner {
             log("Failed to save grid cap count: \(error)")
         }
     }
-
+    
-    // MARK: ServerOwner
+    func createMissingThumbnails() {
-
+        let thumbnailsToCreate = getListOfMissingThumbnails()
-    let authenticationMethod: PropertyAuthenticationMethod = .accessToken
+        guard !thumbnailsToCreate.isEmpty else {
-
+            return
-    func hasReadPermission(for property: UInt32, accessData: Data) -> Bool {
+        }
-        guard let key = String(data: accessData, encoding: .utf8) else {
+        guard canResizeImages else {
-            return false
+            log("Can't create thumbnails, missing ImageMagick")
+            return
+        }
+        log("Creating \(thumbnailsToCreate.count) thumbnails")
+        for cap in thumbnailsToCreate {
+            createThumbnail(for: cap)
         }
-        return writers.contains(key)
     }
-
+    
-    func hasWritePermission(for property: UInt32, accessData: Data) -> Bool {
+    func createThumbnail(for cap: Int) {
-        guard let key = String(data: accessData, encoding: .utf8) else {
+        guard let version = caps[cap]?.mainImage else {
-            return false
+            return
         }
-        return writers.contains(key)
+        let mainImageUrl = imageUrl(of: cap, version: version)
-    }
+        let thumbnailUrl = thumbnail(of: cap)
-
+        shrink(imageAt: mainImageUrl, size: thumbnailSize, destination: thumbnailUrl)
-    func hasListAccessPermission(_ accessData: Data) -> Bool {
-        guard let key = String(data: accessData, encoding: .utf8) else {
-            return false
-        }
-        return writers.contains(key)
     }
 
     // MARK: Monitoring
 
-    public let name = "caps"
+    private let capCountMetric: Metric<Int>
 
-    private let capCountPropertyId = PropertyId(owner: "caps", uniqueId: 2)
+    private let imageCountMetric: Metric<Int>
 
-    private let imageCountPropertyId = PropertyId(owner: "caps", uniqueId: 3)
+    private let classifierMetric: Metric<Int>
-
+    
-    private let classifierVersionPropertyId = PropertyId(owner: "caps", uniqueId: 4)
+    // MARK: Maintenance
-
+    
-    func registerProperties(with monitor: PropertyManager) {
+    private func getMagickVersion() -> SemanticVersion? {
-        let capCountProperty = PropertyRegistration(
+        do {
-            uniqueId: capCountPropertyId.uniqueId,
+            let command = "convert -version"
-            name: "caps",
+            let (code, output) = try safeShell(command)
-            updates: .continuous,
+            guard code == 0,
-            isLogged: true,
+                  let line = output.components(separatedBy: "\n").first,
-            allowsManualUpdate: false,
+                  line.hasPrefix("Version: ImageMagick ") else {
-            read: { [weak self] in
+                log("Missing dependency ImageMagick: " + output)
-                return (self?.capCount ?? 0).timestamped()
+                return nil
-            })
+            }
-        monitor.register(capCountProperty, for: self)
+            guard let versionString = line
-
+                .replacingOccurrences(of: "Version: ImageMagick ", with: "")
-        let imageCountProperty = PropertyRegistration(
+                .components(separatedBy: "-").first  else {
-            uniqueId: imageCountPropertyId.uniqueId,
+                log("Invalid ImageMagick version: " + output)
-            name: "images",
+                return nil
-            updates: .continuous,
+            }
-            isLogged: true,
+            guard let version = SemanticVersion(rawValue: versionString) else {
-            allowsManualUpdate: false,
+                log("Invalid ImageMagick version: " + output)
-            read: { [weak self] in
+                return nil
-                return (self?.imageCount ?? 0).timestamped()
+            }
-            })
+            return version
-        monitor.register(imageCountProperty, for: self)
+        } catch {
-
+            log("Failed to check dependency ImageMagick: \(error)")
-        let classifierVersionProperty = PropertyRegistration(
+            return nil
-            uniqueId: classifierVersionPropertyId.uniqueId,
+        }
-            name: "classifier",
-            updates: .continuous,
-            isLogged: true,
-            allowsManualUpdate: false,
-            read: { [weak self] in
-                return (self?.classifierVersion ?? 0).timestamped()
-            })
-        monitor.register(classifierVersionProperty, for: self)
     }
-
+    
-    private func updateMonitoredPropertiesOnCapChange() {
+    func shrinkImages() {
-        try? monitor.logChanged(property: capCountPropertyId, value: capCount.timestamped())
+        guard canResizeImages else {
-        try? monitor.logChanged(property: imageCountPropertyId, value: imageCount.timestamped())
+            log("Can't resize images, missing ImageMagick")
+            return
+        }
+        let imageFolders: [URL]
+        do {
+            imageFolders = try fm.contentsOfDirectory(at: imageFolder, includingPropertiesForKeys: nil)
+        } catch {
+            log("Failed to get all image folders")
+            return
+        }
+        for folder in imageFolders {
+            guard let images = try? self.images(in: folder) else {
+                continue
+            }
+            for imageUrl in images {
+                shrink(imageAt: imageUrl, size: imageSize, destination: imageUrl)
+            }
+        }
     }
+    
+    private func shrink(imageAt url: URL, size: Int, destination: URL) {
+        do {
+            let command = "convert \(url.path) -resize '\(size)x\(size)>' \(destination.path)"
+            let (code, output) = try safeShell(command)
+            if code != 0 {
+                log("Failed to shrink image \(url.path): " + output)
+            }
+        } catch {
+            log("Failed to shrink image \(url.path): \(error)")
+        }
+    }
+    
+    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
+        let task = Process()
+        let pipe = Pipe()
 
-    private func updateMonitoredClassifierVersionProperty() {
+        task.standardOutput = pipe
-        try? monitor.logChanged(property: classifierVersionPropertyId, value: classifierVersion.timestamped())
+        task.standardError = pipe
+        task.arguments = ["-cl", command]
+        task.executableURL = URL(fileURLWithPath: "/bin/bash")
+        task.standardInput = nil
+
+        try task.run()
+        task.waitUntilExit()
+        
+        let data = pipe.fileHandleForReading.readDataToEndOfFile()
+        let output = String(data: data, encoding: .utf8)!
+        return (task.terminationStatus, output)
     }
 }

Sources/App/Config.swift

@@ -8,17 +8,40 @@ struct Config: Codable {
     /// The maximum size of the request body
     let maxBodySize: String
 
-    /// The path to the log file
+    /// The path to the folder where the metric logs are stored
-    let logPath: String
+    ///
+    /// If no path is provided, then a folder `logs` in the resources directory is created
+    /// If the path is relative, then it is assumed relative to the resources directory
+    let logPath: String?
 
     /// Serve files in the Public directory using Vapor
     let serveFiles: Bool
 
     /// Authentication tokens for remotes allowed to write
     let writers: [String]
-
+    
-    var logURL: URL {
+    /**
-        .init(fileURLWithPath: logPath)
+     The folder where the data should be stored.
+     
+     If the folder is set to `nil`, then the `Resources` folder is used.
+     */
+    let dataDirectory: String?
+    
+    func customDataDirectory(or publicDirectory: String) -> URL {
+        guard let dataDirectory else {
+            return URL(fileURLWithPath: publicDirectory)
+        }
+        return URL(fileURLWithPath: dataDirectory)
+    }
+    
+    func logURL(possiblyRelativeTo resourcesDirectory: URL) -> URL {
+        guard let logPath else {
+            return resourcesDirectory.appendingPathComponent("logs")
+        }
+        guard !logPath.hasPrefix("/") else {
+            return .init(fileURLWithPath: logPath)
+        }
+        return resourcesDirectory.appendingPathComponent(logPath)
     }
 }
 
@@ -31,12 +54,11 @@ extension Config {
     init(loadFrom directory: URL) {
         let configFileUrl = Config.file(in: directory)
 
-        if FileManager.default.fileExists(atPath: configFileUrl.path) {
+        guard FileManager.default.fileExists(atPath: configFileUrl.path) else {
-            self.init(loadAt: configFileUrl)
+            print("No configuration found at \(configFileUrl.path)")
-        } else {
+            exit(-1)
-            self.init(standardIn: directory)
-            write(to: configFileUrl)
         }
+        self.init(loadAt: configFileUrl)
     }
 
     private init(loadAt url: URL) {
@@ -45,23 +67,7 @@ extension Config {
             self = try JSONDecoder().decode(Config.self, from: configData)
         } catch {
             print("Failed to load configuration from \(url.path): \(error)")
-            print("Using default configuration")
+            exit(-1)
-            self.init(standardIn: url.deletingLastPathComponent())
-        }
-    }
-
-    private init(standardIn directory: URL) {
-        let defaultLogPath = directory.appendingPathComponent("logs").path
-        self.init(port: 8000, maxBodySize: "2mb", logPath: defaultLogPath, serveFiles: true, writers: [])
-    }
-
-    private func write(to url: URL) {
-        do {
-            let configData = try JSONEncoder().encode(self)
-            try configData.write(to: url)
-            print("Configuration written at \(url.path)")
-        } catch {
-            print("Failed to write default configuration to \(url.path)")
         }
     }
 }

Sources/App/Error.swift

@@ -12,7 +12,7 @@ enum CapError: Error {
     
     /**
      
-     HTTP Code: 404
+     HTTP Code: 410
      */
     case unknownId
     
@@ -45,11 +45,18 @@ enum CapError: Error {
      HTTP Code: 406
      */
     case invalidData
+
+    /**
+     The server failed to initialize the data and is not operational
+
+     HTTP Code: 204
+     */
+    case serviceUnavailable
     
     var response: HTTPResponseStatus {
         switch self {
-        /// 404
+        /// 410
-        case .unknownId: return .notFound
+        case .unknownId: return .gone
         /// 400
         case .invalidBody: return .badRequest
         /// 406
@@ -60,6 +67,7 @@ enum CapError: Error {
         case .invalidFile: return .preconditionFailed
         /// 500
         case .invalidConfiguration: return .internalServerError
+        case .serviceUnavailable: return .noContent
         }
     }
 }

Sources/App/EventLoopScheduler.swift

@@ -0,0 +1,14 @@
+import Foundation
+import Clairvoyant
+import Vapor
+import NIOCore
+
+extension MultiThreadedEventLoopGroup: AsyncScheduler {
+    
+    func test() {
+    }
+
+    public func schedule(asyncJob: @escaping @Sendable () async throws -> Void) {
+        _ = any().makeFutureWithTask(asyncJob)
+    }
+}

Sources/App/Router+Extensions.swift

@@ -1,10 +1,3 @@
-//
-//  Router+Extensions.swift
-//  App
-//
-//  Created by Christoph on 05.05.20.
-//
-
 import Vapor
 
 extension Application {

Sources/App/configure.swift

@@ -1,49 +1,91 @@
 import Vapor
 import Foundation
 import Clairvoyant
+import ClairvoyantVapor
+import ClairvoyantBinaryCodable
 
+private var provider: VaporMetricProvider!
+private var serverStatus: Metric<ServerStatus>!
 
-private(set) var server: CapServer!
+private let asyncScheduler = MultiThreadedEventLoopGroup(numberOfThreads: 2)
-private(set) var monitor: PropertyManager!
 
-public func configure(_ app: Application) throws {
+private var server: CapServer!
+
+func configure(_ app: Application) async throws {
 
     let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory)
     let publicDirectory = app.directory.publicDirectory
 
     let config = Config(loadFrom: resourceDirectory)
+    let authenticator = Authenticator(writers: config.writers)
 
-    server = CapServer(in: URL(fileURLWithPath: publicDirectory),
+    let logURL = config.logURL(possiblyRelativeTo: resourceDirectory)
-                       writers: config.writers)
+    let monitor = MetricObserver(logFileFolder: logURL, logMetricId: "caps.log")
+    MetricObserver.standard = monitor
 
-    monitor = .init(logFolder: config.logURL, serverOwner: server)
+    serverStatus = Metric<ServerStatus>("caps.status",
-    monitor.update(status: .initializing)
+                                      name: "Status",
-
+                                      description: "The general status of the service")
-    server.registerProperties(with: monitor)
+    try await serverStatus.update(.initializing)
-    monitor.registerRoutes(app)
 
     app.http.server.configuration.port = config.port
     app.routes.defaultMaxBodySize = .init(stringLiteral: config.maxBodySize)
 
+    let dataDirectory = config.customDataDirectory(or: publicDirectory)
+    server = CapServer(in: dataDirectory)
+
+    provider = .init(observer: monitor, accessManager: config.writers)
+    provider.asyncScheduler = asyncScheduler
+    provider.registerRoutes(app)
+
     if config.serveFiles {
         let middleware = FileMiddleware(publicDirectory: publicDirectory)
         app.middleware.use(middleware)
     }
 
+    // Register routes to the router
+    server.registerRoutes(with: app, authenticator: authenticator)
+
+    // Initialize the server data
     do {
         try server.loadData()
     } catch {
-        monitor.update(status: .initializationFailure)
+        try await serverStatus.update(.initializationFailure)
+        print("[\(df.string(from: Date()))] Server failed to start: \(error)")
         return
     }
+    if server.canResizeImages {
+        try await serverStatus.update(.nominal)
+    } else {
+        try await serverStatus.update(.reducedFunctionality)
+    }
+    print("[\(df.string(from: Date()))] Server started (\(app.environment.name), \(server.capCount) caps)")
+}
 
-    // Register routes to the router
+func shutdown() {
-    routes(app)
+    Task {
-
+        print("[\(df.string(from: Date()))] Server shutdown")
-    monitor.update(status: .nominal)
+        do {
+            try await asyncScheduler.shutdownGracefully()
+        } catch {
+            print("Failed to shut down MultiThreadedEventLoopGroup: \(error)")
+        }
+    }
 }
 
 func log(_ message: String) {
-    monitor.log(message)
+    guard let observer = MetricObserver.standard else {
-    print(message)
+        print(message)
+        return
+    }
+    asyncScheduler.schedule {
+        await observer.log(message)
+    }
 }
+
+private let df: DateFormatter = {
+    let df = DateFormatter()
+    df.dateStyle = .short
+    df.timeStyle = .short
+    return df
+}()

Sources/App/entrypoint.swift

@@ -0,0 +1,43 @@
+import Vapor
+import Dispatch
+import Logging
+
+/// This extension is temporary and can be removed once Vapor gets this support.
+private extension Vapor.Application {
+    static let baseExecutionQueue = DispatchQueue(label: "vapor.codes.entrypoint")
+    
+    func runFromAsyncMainEntrypoint() async throws {
+        try await withCheckedThrowingContinuation { continuation in
+            Vapor.Application.baseExecutionQueue.async { [self] in
+                do {
+                    try self.run()
+                    continuation.resume()
+                } catch {
+                    continuation.resume(throwing: error)
+                }
+            }
+        }
+    }
+}
+
+@main
+enum Entrypoint {
+    static func main() async throws {
+        var env = try Environment.detect()
+        try LoggingSystem.bootstrap(from: &env)
+        
+        let app = Application(env)
+        defer {
+            shutdown()
+            app.shutdown()
+        }
+        
+        do {
+            try await configure(app)
+        } catch {
+            app.logger.report(error: error)
+            throw error
+        }
+        try await app.runFromAsyncMainEntrypoint()
+    }
+}

Sources/App/routes.swift

@@ -1,133 +0,0 @@
-import Vapor
-import Foundation
-
-/// The decoder to extract caps from JSON payloads given to the `cap` route.
-private let decoder = JSONDecoder()
-
-/// The date formatter to decode dates in requests
-private let dateFormatter: DateFormatter = {
-    let df = DateFormatter()
-    df.dateFormat = "yy-MM-dd-HH-mm-ss"
-    return df
-}()
-
-private func authorize(_ request: Request) throws {
-    guard let key = request.headers.first(name: "key") else {
-        throw Abort(.badRequest) // 400
-    }
-    guard server.hasAuthorization(for: key) else {
-        throw Abort(.forbidden) // 403
-    }
-}
-
-func routes(_ app: Application) {
-    
-    app.get("version") { _ in
-        "\(server.classifierVersion)"
-    }
-
-    // Add or change a cap
-    app.postCatching("cap") { request in
-        try authorize(request)
-        let data = try request.getBodyData()
-        let cap = try decoder.decode(Cap.self, from: data)
-        try server.addOrUpdate(cap)
-    }
-    
-    // Upload an image
-    app.postCatching("images", ":n") { request in
-        try authorize(request)
-        guard let cap = request.parameters.get("n", as: Int.self) else {
-            log("Invalid parameter for cap")
-            throw Abort(.badRequest)
-        }
-        let data = try request.getBodyData()
-        try server.save(image: data, for: cap)
-    }
-    
-    // Update the classifier
-    app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in
-        try authorize(request)
-        guard let version = request.parameters.get("version", as: Int.self) else {
-            log("Invalid parameter for version")
-            throw Abort(.badRequest)
-        }
-        guard version > server.classifierVersion else {
-            throw Abort(.alreadyReported) // 208
-        }
-        let data = try request.getBodyData()
-        try server.save(classifier: data, version: version)
-        return .ok
-    }
-    
-    // Update the trained classes
-    app.postCatching("classes", ":date") { request in
-        guard let dateString = request.parameters.get("date") else {
-            log("Invalid parameter for date")
-            throw Abort(.badRequest)
-        }
-        guard let date = dateFormatter.date(from: dateString) else {
-            log("Invalid date specification")
-            throw Abort(.badRequest)
-        }
-        
-        try authorize(request)
-        let body = try request.getStringBody()
-        
-        server.updateTrainedClasses(content: body)
-        server.removeAllEntriesInImageChangeList(before: date)
-    }
-    
-    // Get the list of missing thumbnails
-    app.get("thumbnails", "missing") { request in
-        let missingThumbnails = server.getListOfMissingThumbnails()
-        return missingThumbnails.map(String.init).joined(separator: ",")
-    }
-    
-    // Upload the thumbnail of a cap
-    app.postCatching("thumbnails", ":cap") { request in
-        guard let cap = request.parameters.get("cap", as: Int.self) else {
-            log("Invalid cap parameter for thumbnail upload")
-            throw Abort(.badRequest)
-        }
-        try authorize(request)
-        let data = try request.getBodyData()
-        server.saveThumbnail(data, for: cap)
-    }
-    
-    // Delete the image of a cap
-    app.postCatching("delete", ":cap", ":version") { request in
-        guard let cap = request.parameters.get("cap", as: Int.self) else {
-            log("Invalid cap parameter for image deletion")
-            throw Abort(.badRequest)
-        }
-        try authorize(request)
-        guard let version = request.parameters.get("version", as: Int.self) else {
-            log("Invalid version parameter for image deletion")
-            throw Abort(.badRequest)
-        }
-        guard server.deleteImage(version: version, for: cap) else {
-            throw Abort(.gone)
-        }
-    }
-}
-
-private extension Request {
-    
-    func getBodyData() throws -> Data {
-        guard let buffer = body.data else {
-            log("Missing body data")
-            throw CapError.invalidBody
-        }
-        return Data(buffer: buffer)
-    }
-    
-    func getStringBody() throws -> String {
-        let data = try getBodyData()
-        guard let content = String(data: data, encoding: .utf8) else {
-            log("Invalid string body")
-            throw CapError.invalidBody
-        }
-        return content
-    }
-}

Sources/Run/main.swift

@@ -1,9 +0,0 @@
-import App
-import Vapor
-
-var env = try Environment.detect()
-try LoggingSystem.bootstrap(from: &env)
-let app = Application(env)
-defer { app.shutdown() }
-try configure(app)
-try app.run()

Tests/AppTests/AppTests.swift

@@ -1,13 +0,0 @@
-import App
-import Dispatch
-import XCTest
-
-final class AppTests : XCTestCase {
-    func testNothing() throws {
-        XCTAssert(true)
-    }
-
-    static let allTests = [
-        ("testNothing", testNothing),
-    ]
-}

Training/config_example.json

@@ -1,6 +0,0 @@
-{
-    "contentDirectory": "../Public",
-    "trainingIterations": 20,
-    "serverPath": "https://mydomain.com/caps",
-    "authenticationToken": "mysecretkey",
-}

Training/train.swift

@@ -1,484 +0,0 @@
-import Foundation
-import Cocoa
-import CreateML
-
-struct Configuration: Codable {
-    
-    let contentFolder: String
-
-    let trainingIterations: Int
-    
-    let serverPath: String
-    
-    let authenticationToken: String
-    
-    init?(at url: URL) {
-        do {
-            let configData = try Data(contentsOf: url)
-            self = try JSONDecoder().decode(Configuration.self, from: configData)
-        } catch {
-            print("[ERROR] Failed to load configuration at \(url.absoluteURL.path): \(error)")
-            return nil
-        }
-    }
-}
-
-struct Cap: Codable {
-
-    let id: Int
-
-    let count: Int
-
-    enum CodingKeys: String, CodingKey {
-        case id = "i"
-        case count = "c"
-    }
-}
-
-final class ClassifierCreator {
-    
-    static let configurationFileUrl = URL(fileURLWithPath: "config.json")
-    
-    let server: URL
-    
-    let configuration: Configuration
-    
-    let imageDirectory: URL
-    
-    let thumbnailDirectory: URL
-    
-    let classifierUrl: URL
-    
-    let df = DateFormatter()
-    
-    
-    // MARK: Step 1: Load configuration
-    
-    init?() {
-        guard let configuration = Configuration(at: ClassifierCreator.configurationFileUrl) else {
-            return nil
-        }
-        self.configuration = configuration
-        guard let serverUrl = URL(string: configuration.serverPath) else {
-            print("[ERROR] Configuration: Invalid server path \(configuration.serverPath)")
-            return nil
-        }
-        self.server = serverUrl
-        let contentDirectory = URL(fileURLWithPath: configuration.contentFolder)
-        self.imageDirectory = contentDirectory.appendingPathComponent("images")
-        self.classifierUrl = contentDirectory.appendingPathComponent("classifier.mlmodel")
-        self.thumbnailDirectory = contentDirectory.appendingPathComponent("thumbnails")
-        df.dateFormat = "yy-MM-dd-HH-mm-ss"
-    }
-    
-    // MARK: Main function
-    
-    func run() async {
-        let imagesSnapshotDate = Date()
-        guard let (classes, changedImageCount, changedMainImages) = await loadImages() else {
-            return
-        }
-
-        guard !classes.isEmpty else {
-            return
-        }
-        
-        guard changedImageCount > 0 else {
-            print("[INFO] No changed images, so no new classifier trained")
-            await createThumbnails(changed: changedMainImages)
-            print("[INFO] Done")
-            return
-        }
-
-        guard let classifierVersion = await getClassifierVersion() else {
-            return
-        }
-        let newVersion = classifierVersion + 1
-
-        print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
-        print("[INFO] Model path: \(classifierUrl.path)")
-        print("[INFO] Version: \(newVersion)")
-        print("[INFO] Classes: \(classes.count)")
-        print("[INFO] Iterations: \(configuration.trainingIterations)")
-
-        guard trainModel() else {
-            return
-        }
-
-        guard await upload(version: newVersion) else {
-            return
-        }
-
-        guard await upload(classes: classes, lastUpdate: imagesSnapshotDate) else {
-            return
-        }
-        await createThumbnails(changed: changedMainImages)
-        print("[INFO] Done")
-    }
-    
-    // MARK: Step 2: Load changed images
-    
-    func loadImages() async -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int])? {
-        guard createImageFolderIfMissing() else {
-            return nil
-        }
-        let imageCounts = await getImageCounts()
-        let missingImageList: [(cap: Int, image: Int)] = imageCounts
-            .sorted { $0.key < $1.key }
-            .reduce(into: []) { list, pair in
-                let missingImagesForCap: [(cap: Int, image: Int)] = (0..<pair.value).compactMap { image in
-                    let url = imageUrl(base: imageDirectory, cap: pair.key, image: image)
-                    guard !FileManager.default.fileExists(atPath: url.path) else {
-                        return nil
-                    }
-                    return (cap: pair.key, image: image)
-                }
-                list.append(contentsOf: missingImagesForCap)
-            }
-        if missingImageList.isEmpty {
-            print("[INFO] No missing images to load")
-        } else {
-            print("[INFO] Loading \(missingImageList.count) missing images...")
-        }
-        guard await loadImages(missingImageList) else {
-            return nil
-        }
-        let changedImageList = await getChangedImageList()
-        
-        if changedImageList.isEmpty {
-            print("[INFO] No changed images to load")
-        } else {
-            print("[INFO] Loading \(changedImageList.count) changed images...")
-        }
-        guard await loadImages(changedImageList) else {
-            return nil
-        }
-        let changedMainImages = changedImageList.filter { $0.image == 0 }.map { $0.cap }
-        let classes = imageCounts.keys.sorted()
-        return (classes, missingImageList.count + changedImageList.count, changedMainImages)
-    }
-    
-    private func createImageFolderIfMissing() -> Bool {
-        createFolderIfMissing(imageDirectory)
-    }
-    
-    private func getImageCounts() async -> [Int : Int] {
-        guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
-            return [:]
-        }
-        do {
-            return try JSONDecoder().decode([Cap].self, from: data)
-                .reduce(into: [:]) { $0[$1.id] = $1.count }
-        } catch {
-            print("[ERROR] Failed to decode cap database: \(error)")
-            return [:]
-        }
-    }
-    
-    private func imageUrl(base: URL, cap: Int, image: Int) -> URL {
-        base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", cap, cap, image))
-    }
-    
-    private func loadImage(cap: Int, image: Int) async -> Bool {
-        let url = imageUrl(base: server.appendingPathComponent("images"), cap: cap, image: image)
-        let tempFile: URL, response: URLResponse
-        do {
-            (tempFile, response) = try await URLSession.shared.download(from: url)
-        } catch {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): \(error)")
-            return false
-        }
-        let responseCode = (response as! HTTPURLResponse).statusCode
-        guard responseCode == 200 else {
-            print("[ERROR] Failed to load image \(image) of cap \(cap): Response \(responseCode)")
-            return false
-        }
-        do {
-            let localUrl = imageUrl(base: imageDirectory, cap: cap, image: image)
-            if FileManager.default.fileExists(atPath: localUrl.path) {
-                try FileManager.default.removeItem(at: localUrl)
-            }
-            try FileManager.default.moveItem(at: tempFile, to: localUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save image \(image) of cap \(cap): \(error)")
-            return false
-        }
-    }
-    
-    private func loadImages(_ list: [(cap: Int, image: Int)]) async -> Bool {
-        guard !list.isEmpty else {
-            return true
-        }
-        var loadedImages = 0
-        await withTaskGroup(of: Bool.self) { group in
-            for (cap, image) in list {
-                group.addTask {
-                    await self.loadImage(cap: cap, image: image)
-                }
-            }
-            for await loaded in group {
-                if loaded {
-                    loadedImages += 1
-                }
-            }
-        }
-        if loadedImages != list.count {
-            print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
-            return false
-        }
-        return true
-    }
-    
-    func getChangedImageList() async -> [(cap: Int, image: Int)] {
-        guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
-            print("[ERROR] Failed to get list of changed images")
-            return []
-        }
-        
-        return string
-            .components(separatedBy: "\n")
-            .filter { $0 != "" }
-            .compactMap {
-                let parts = $0.components(separatedBy: ":")
-                guard parts.count == 3 else {
-                    return nil
-                }
-                /*
-                guard let date = df.date(from: parts[0]) else {
-                    print("[WARN] Invalid date \(parts[0]) in change list")
-                    return nil
-                }
-                 */
-                guard let cap = Int(parts[1]) else {
-                    print("[WARN] Invalid cap id \(parts[1]) in change list")
-                    return nil
-                }
-                guard let image = Int(parts[2]) else {
-                    print("[WARN] Invalid image id \(parts[2]) in change list")
-                    return nil
-                }
-                return (cap, image)
-            }
-    }
-    
-    // MARK: Step 3: Compute version
-    
-    func getClassifierVersion() async -> Int? {
-        guard let string: String = await get(server.appendingPathComponent("version")) else {
-            print("[ERROR] Failed to get classifier version")
-            return nil
-        }
-        guard let version = Int(string) else {
-            print("[ERROR] Invalid classifier version \(string)")
-            return nil
-        }
-        return version
-    }
-    
-    // MARK: Step 4: Train classifier
-    
-    func trainModel() -> Bool {
-        var params = MLImageClassifier.ModelParameters(augmentation: [])
-        params.maxIterations = configuration.trainingIterations
-        let model: MLImageClassifier
-        do {
-            model = try MLImageClassifier(
-                trainingData: .labeledDirectories(at: imageDirectory),
-                parameters: params)
-        } catch {
-            print("[ERROR] Failed to create classifier: \(error)")
-            return false
-        }
-
-        print("[INFO] Saving classifier...")
-        do {
-            try model.write(to: classifierUrl)
-            return true
-        } catch {
-            print("[ERROR] Failed to save model to file: \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Step 5: Upload classifier
-    
-    func upload(version: Int) async -> Bool {
-        print("[INFO] Uploading classifier...")
-        let modelData: Data
-        do {
-            modelData = try Data(contentsOf: classifierUrl)
-        } catch {
-            print("[ERROR] Failed to read classifier data: \(error)")
-            return false
-        }
-        
-        return await post(
-            url: server.appendingPathComponent("classifier/\(version)"),
-            body: modelData)
-    }
-    
-    // MARK: Step 6: Update classes
-    
-    func upload(classes: [Int], lastUpdate: Date) async -> Bool {
-        print("[INFO] Uploading trained classes...")
-        let dateString = df.string(from: lastUpdate)
-        return await post(
-            url: server.appendingPathComponent("classes/\(dateString)"),
-            body: classes.map(String.init).joined(separator: "\n").data(using: .utf8)!)
-    }
-    
-    // MARK: Step 7: Create thumbnails
-    
-    func createThumbnails(changed: [Int]) async {
-        guard checkMagickAvailability() else {
-            return
-        }
-        guard createFolderIfMissing(thumbnailDirectory) else {
-            print("[ERROR] Failed to create folder for thumbnails")
-            return
-        }
-        let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
-        let all = Set(capIdsOfMissingThumbnails).union(changed)
-        print("[INFO] Creating \(all.count) thumbnails...")
-        for cap in all {
-            await createThumbnail(for: cap)
-        }
-    }
-    
-    func checkMagickAvailability() -> Bool {
-        do {
-            let (code, output) = try safeShell("magick --version")
-            guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
-                .components(separatedBy: " ").first else {
-                print("[ERROR] Magick not found, install using 'brew install imagemagick'")
-                return false
-            }
-            print("[INFO] Using magick \(version)")
-        } catch {
-            print("[ERROR] Failed to get version of magick: (\(error))")
-            return false
-        }
-        return true
-    }
-    
-    private func getMissingThumbnailIds() async -> [Int] {
-        guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
-            print("[ERROR] Failed to get missing thumbnails")
-            return []
-        }
-        return string.components(separatedBy: ",").compactMap(Int.init)
-    }
-    
-    private func createThumbnail(for cap: Int) async {
-        let inputUrl = imageUrl(base: imageDirectory, cap: cap, image: 0)
-        guard FileManager.default.fileExists(atPath: inputUrl.path) else {
-            print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
-            return
-        }
-        
-        let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
-        do {
-            let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
-            let (code, output) = try safeShell(command)
-            if code != 0 {
-                print("Failed to create thumbnail for cap \(cap): \(output)")
-                return
-            }
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        
-        let data: Data
-        do {
-            data = try Data(contentsOf: output)
-        } catch {
-            print("Failed to read created thumbnail for cap \(cap): \(error)")
-            return
-        }
-        guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
-            print("Failed to upload thumbnail for cap \(cap)")
-            return
-        }
-    }
-    
-    // MARK: Helper
-    
-    @discardableResult
-    private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
-        let task = Process()
-        let pipe = Pipe()
-
-        task.standardOutput = pipe
-        task.standardError = pipe
-        task.arguments = ["-cl", command]
-        task.executableURL = URL(fileURLWithPath: "/bin/zsh")
-        task.standardInput = nil
-
-        try task.run()
-        task.waitUntilExit()
-        
-        let data = pipe.fileHandleForReading.readDataToEndOfFile()
-        let output = String(data: data, encoding: .utf8)!
-        return (task.terminationStatus, output)
-    }
-    
-    
-    private func createFolderIfMissing(_ folder: URL) -> Bool {
-        guard !FileManager.default.fileExists(atPath: folder.path) else {
-            return true
-        }
-        do {
-            try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
-            return true
-        } catch {
-            print("[ERROR] Failed to create directory \(folder.path): \(error)")
-            return false
-        }
-    }
-    
-    // MARK: Requests
-    
-    private func post(url: URL, body: Data) async -> Bool {
-        var request = URLRequest(url: url)
-        request.httpMethod = "POST"
-        request.httpBody = body
-        request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
-        return await perform(request) != nil
-    }
-    
-    private func perform(_ request: URLRequest) async -> Data? {
-        let data: Data
-        let response: URLResponse
-        do {
-            (data, response) = try await URLSession.shared.data(for: request)
-        } catch {
-            print("[ERROR] Request failed: \(error)")
-            return nil
-        }
-        let code = (response as! HTTPURLResponse).statusCode
-        guard code == 200 else {
-            print("[ERROR] Invalid response \(code)")
-            return nil
-        }
-        return data
-    }
-    
-    private func get(_ url: URL) async -> Data? {
-        await perform(URLRequest(url: url))
-    }
-    
-    private func get(_ url: URL) async -> String? {
-        guard let data: Data = await get(url) else {
-            return nil
-        }
-        guard let string = String(data: data, encoding: .utf8) else {
-            print("[ERROR] Invalid string response \(data)")
-            return nil
-        }
-        return string
-    }
-}
-
-await ClassifierCreator()?.run()

build.sh

@@ -1,13 +0,0 @@
-echo "[1/7] Stopping server..."
-sudo supervisorctl stop caps
-echo "[2/7] Changing permissions..."
-sudo chown -R pi:pi .
-echo "[3/7] Pulling changes..."
-git pull
-echo "[4/7] Building project..."
-swift build -c release
-echo "[5/7] Restoring permissions..."
-sudo chown -R www-data:www-data .
-echo "[6/7] Starting server..."
-sudo supervisorctl start caps
-echo "[7/7] Done"