Compare commits

..

15 Commits

26 changed files with 289 additions and 760 deletions

13
.gitignore vendored
View File

@ -8,12 +8,11 @@
.DS_Store
Package.resolved
.swiftpm/
Public/caps.json
Public/changes.txt
Public/classifier.*
Public/count.*
Public/images/
Public/thumbnails/
Public/classifier.version
Public/classifier.mlmodel
Public/caps.json
Training/backup/
Training/config.json
Public/thumbnails
Public/count.js
Resources/config.json
Resources/logs/

View File

@ -8,28 +8,20 @@ let package = Package(
],
dependencies: [
.package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
.package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.9.0"),
.package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.2.0"),
.package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.11.2"),
.package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.4.0"),
.package(url: "https://github.com/christophhagen/ClairvoyantBinaryCodable", from: "0.3.0"),
.package(url:"https://github.com/christophhagen/CBORCoding", from: "1.0.0"),
],
targets: [
.target(name: "App",
.executableTarget(
name: "App",
dependencies: [
.product(name: "Vapor", package: "vapor"),
.product(name: "Clairvoyant", package: "Clairvoyant"),
.product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"),
.product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"),
.product(name: "CBORCoding", package: "CBORCoding"),
],
swiftSettings: [
// Enable better optimizations when building in Release configuration. Despite the use of
// the `.unsafeFlags` construct required by SwiftPM, this flag is recommended for Release
// builds. See <https://github.com/swift-server/guides#building-for-production> for details.
.unsafeFlags(["-cross-module-optimization"], .when(configuration: .release))
]),
.executableTarget(name: "Run", dependencies: ["App"]),
.testTarget(name: "AppTests", dependencies: ["App"]),
]
)
]
)

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 171 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<browserconfig>
<msapplication>
<tile>
<square150x150logo src="/assets/mstile-150x150.png?v=1"/>
<TileColor>#a36490</TileColor>
</tile>
</msapplication>
</browserconfig>

Binary file not shown.

After

Width:  |  Height:  |  Size: 953 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

BIN
Public/assets/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

View File

@ -0,0 +1 @@
{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"}

View File

@ -5,13 +5,13 @@
<meta charset="utf-8"/>
<meta name="robots" content="noindex"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0"/>
<link rel="apple-touch-icon" sizes="180x180" href="/assets/icons/apple-touch-icon.png?v=1"/>
<link rel="icon" type="image/png" sizes="32x32" href="/assets/icons/favicon-32x32.png?v=1"/>
<link rel="icon" type="image/png" sizes="16x16" href="/assets/icons/favicon-16x16.png?v=1"/>
<link rel="manifest" href="/assets/icons/site.webmanifest?v=1"/>
<link rel="shortcut icon" href="/assets/icons/favicon.ico?v=1"/>
<link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png?v=1"/>
<link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png?v=1"/>
<link rel="icon" type="image/png" sizes="16x16" href="/assets/favicon-16x16.png?v=1"/>
<link rel="manifest" href="/assets/site.webmanifest?v=1"/>
<link rel="shortcut icon" href="/assets/favicon.ico?v=1"/>
<meta name="msapplication-TileColor" content="#da532c">
<meta name="msapplication-config" content="/assets/icons/browserconfig.xml?v=1"/>
<meta name="msapplication-config" content="/assets/browserconfig.xml?v=1"/>
<meta name="theme-color" content="#ffffff"/>
<link href="grid.css?v=2" rel="stylesheet"/>
<meta name="author" content="Christoph Hagen"/>

View File

@ -51,31 +51,7 @@ Note: The data for the mosaic is currently not updated automatically, since Swif
## Classifier training
The main server is running on Linux, which doesn't provide the CreateML framework required for classifier training.
This has to be done on macOS using the `train.swift` script in the `Training` folder.
It will:
- Fetch the current image catalog by checking missing and changed images
- Train a classifier on the image set
- Increment the classifier version and upload the new model
- Update the list of caps which can be recognized using the classifier
- Create missing thumbnails for each cap for the cap grid
A configuration file is required to run the training, with a valid access token for the server:
```json
{
"imageDirectory": "../Public/images",
"classifierModelPath": "../Public/classifier.mlmodel",
"trainingIterations": 20,
"serverPath": "https://mydomain.com/caps",
"authenticationToken": "mysecretkey",
}
```
The configuration file `config.json` must be located in the folder from which the script is run.
```bash
swift train.swift
```
This has to be done on macOS using the [Caps-Train](https://christophhagen.de/git/ch/Caps-Train) repository.
## Future work
- Create thumbnails on the server using [JPEG](https://github.com/kelvin13/jpeg)

View File

@ -3,6 +3,7 @@
"maxBodySize" : "2mb",
"logPath": "\/var\/log\/caps/metrics",
"serveFiles": true,
"dataDirectory" : "/ch/data/caps",
"writers" : [
"auth_key_1"
]

View File

@ -4,6 +4,10 @@ import Clairvoyant
final class CapServer {
private let imageSize = 360
private let thumbnailSize = 100
// MARK: Paths
private let imageFolder: URL
@ -34,6 +38,8 @@ final class CapServer {
/// Indicates that the data is loaded
private(set) var isOperational = false
private(set) var canResizeImages = false
// MARK: Caps
@ -81,7 +87,7 @@ final class CapServer {
caps.reduce(0) { $0 + $1.value.count }
}
init(in folder: URL) async {
init(in folder: URL) {
self.imageFolder = folder.appendingPathComponent("images")
self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
self.gridCountFile = folder.appendingPathComponent("count.js")
@ -95,15 +101,15 @@ final class CapServer {
changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
// Metric initializers only fail if observer is missing or ID is duplicate
self.capCountMetric = try! await .init("caps.count",
self.capCountMetric = .init("caps.count",
name: "Number of caps",
description: "The total number of caps in the database")
self.imageCountMetric = try! await .init("caps.images",
self.imageCountMetric = .init("caps.images",
name: "Total images",
description: "The total number of images for all caps")
self.classifierMetric = try! await .init("caps.classifier",
self.classifierMetric = .init("caps.classifier",
name: "Classifier Version",
description: "The current version of the image classifier")
}
@ -115,6 +121,12 @@ final class CapServer {
updateGridCapCount()
try ensureExistenceOfChangedImagesFile()
organizeImages()
if let version = getMagickVersion() {
log("Using ImageMagick \(version.rawValue)")
canResizeImages = true
}
// shrinkImages()
createMissingThumbnails()
isOperational = true
}
@ -160,7 +172,6 @@ final class CapServer {
log("Failed to load caps: \(error)")
throw error
}
log("\(caps.count) caps loaded")
}
private func scheduleSave() {
@ -360,6 +371,10 @@ final class CapServer {
let count = try count(of: cap)
caps[cap]!.count = count
addChangedImageToLog(cap: cap, image: id)
if canResizeImages {
shrink(imageAt: capImageUrl, size: imageSize, destination: capImageUrl)
createThumbnail(for: cap)
}
log("Added image \(id) for cap \(cap) (\(count) total)")
}
@ -450,6 +465,9 @@ final class CapServer {
throw CapError.invalidFile
}
caps[cap]?.mainImage = version
if canResizeImages {
createThumbnail(for: cap)
}
log("Switched cap \(cap) to version \(version)")
}
@ -604,6 +622,30 @@ final class CapServer {
}
}
func createMissingThumbnails() {
let thumbnailsToCreate = getListOfMissingThumbnails()
guard !thumbnailsToCreate.isEmpty else {
return
}
guard canResizeImages else {
log("Can't create thumbnails, missing ImageMagick")
return
}
log("Creating \(thumbnailsToCreate.count) thumbnails")
for cap in thumbnailsToCreate {
createThumbnail(for: cap)
}
}
func createThumbnail(for cap: Int) {
guard let version = caps[cap]?.mainImage else {
return
}
let mainImageUrl = imageUrl(of: cap, version: version)
let thumbnailUrl = thumbnail(of: cap)
shrink(imageAt: mainImageUrl, size: thumbnailSize, destination: thumbnailUrl)
}
// MARK: Monitoring
private let capCountMetric: Metric<Int>
@ -611,4 +653,85 @@ final class CapServer {
private let imageCountMetric: Metric<Int>
private let classifierMetric: Metric<Int>
// MARK: Maintenance
private func getMagickVersion() -> SemanticVersion? {
do {
let command = "convert -version"
let (code, output) = try safeShell(command)
guard code == 0,
let line = output.components(separatedBy: "\n").first,
line.hasPrefix("Version: ImageMagick ") else {
log("Missing dependency ImageMagick: " + output)
return nil
}
guard let versionString = line
.replacingOccurrences(of: "Version: ImageMagick ", with: "")
.components(separatedBy: "-").first else {
log("Invalid ImageMagick version: " + output)
return nil
}
guard let version = SemanticVersion(rawValue: versionString) else {
log("Invalid ImageMagick version: " + output)
return nil
}
return version
} catch {
log("Failed to check dependency ImageMagick: \(error)")
return nil
}
}
func shrinkImages() {
guard canResizeImages else {
log("Can't resize images, missing ImageMagick")
return
}
let imageFolders: [URL]
do {
imageFolders = try fm.contentsOfDirectory(at: imageFolder, includingPropertiesForKeys: nil)
} catch {
log("Failed to get all image folders")
return
}
for folder in imageFolders {
guard let images = try? self.images(in: folder) else {
continue
}
for imageUrl in images {
shrink(imageAt: imageUrl, size: imageSize, destination: imageUrl)
}
}
}
private func shrink(imageAt url: URL, size: Int, destination: URL) {
do {
let command = "convert \(url.path) -resize '\(size)x\(size)>' \(destination.path)"
let (code, output) = try safeShell(command)
if code != 0 {
log("Failed to shrink image \(url.path): " + output)
}
} catch {
log("Failed to shrink image \(url.path): \(error)")
}
}
private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/bash")
task.standardInput = nil
try task.run()
task.waitUntilExit()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return (task.terminationStatus, output)
}
}

View File

@ -9,7 +9,10 @@ struct Config: Codable {
let maxBodySize: String
/// The path to the folder where the metric logs are stored
let logPath: String
///
/// If no path is provided, then a folder `logs` in the resources directory is created
/// If the path is relative, then it is assumed relative to the resources directory
let logPath: String?
/// Serve files in the Public directory using Vapor
let serveFiles: Bool
@ -17,8 +20,28 @@ struct Config: Codable {
/// Authentication tokens for remotes allowed to write
let writers: [String]
var logURL: URL {
.init(fileURLWithPath: logPath)
/**
The folder where the data should be stored.
If the folder is set to `nil`, then the `Resources` folder is used.
*/
let dataDirectory: String?
func customDataDirectory(or publicDirectory: String) -> URL {
guard let dataDirectory else {
return URL(fileURLWithPath: publicDirectory)
}
return URL(fileURLWithPath: dataDirectory)
}
func logURL(possiblyRelativeTo resourcesDirectory: URL) -> URL {
guard let logPath else {
return resourcesDirectory.appendingPathComponent("logs")
}
guard !logPath.hasPrefix("/") else {
return .init(fileURLWithPath: logPath)
}
return resourcesDirectory.appendingPathComponent(logPath)
}
}

View File

@ -0,0 +1,14 @@
import Foundation
import Clairvoyant
import Vapor
import NIOCore
extension MultiThreadedEventLoopGroup: AsyncScheduler {
func test() {
}
public func schedule(asyncJob: @escaping @Sendable () async throws -> Void) {
_ = any().makeFutureWithTask(asyncJob)
}
}

View File

@ -5,8 +5,13 @@ import ClairvoyantVapor
import ClairvoyantBinaryCodable
private var provider: VaporMetricProvider!
private var serverStatus: Metric<ServerStatus>!
public func configure(_ app: Application) async throws {
private let asyncScheduler = MultiThreadedEventLoopGroup(numberOfThreads: 2)
private var server: CapServer!
func configure(_ app: Application) async throws {
let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory)
let publicDirectory = app.directory.publicDirectory
@ -14,20 +19,23 @@ public func configure(_ app: Application) async throws {
let config = Config(loadFrom: resourceDirectory)
let authenticator = Authenticator(writers: config.writers)
let monitor = MetricObserver(logFileFolder: config.logURL, logMetricId: "caps.log")
let logURL = config.logURL(possiblyRelativeTo: resourceDirectory)
let monitor = MetricObserver(logFileFolder: logURL, logMetricId: "caps.log")
MetricObserver.standard = monitor
let status = try await Metric<ServerStatus>("caps.status",
serverStatus = Metric<ServerStatus>("caps.status",
name: "Status",
description: "The general status of the service")
try await status.update(.initializing)
try await serverStatus.update(.initializing)
app.http.server.configuration.port = config.port
app.routes.defaultMaxBodySize = .init(stringLiteral: config.maxBodySize)
let server = await CapServer(in: URL(fileURLWithPath: publicDirectory))
let dataDirectory = config.customDataDirectory(or: publicDirectory)
server = CapServer(in: dataDirectory)
provider = .init(observer: monitor, accessManager: config.writers)
provider.asyncScheduler = asyncScheduler
provider.registerRoutes(app)
if config.serveFiles {
@ -42,9 +50,27 @@ public func configure(_ app: Application) async throws {
do {
try server.loadData()
} catch {
try await status.update(.initializationFailure)
try await serverStatus.update(.initializationFailure)
print("[\(df.string(from: Date()))] Server failed to start: \(error)")
return
}
if server.canResizeImages {
try await serverStatus.update(.nominal)
} else {
try await serverStatus.update(.reducedFunctionality)
}
print("[\(df.string(from: Date()))] Server started (\(app.environment.name), \(server.capCount) caps)")
}
func shutdown() {
Task {
print("[\(df.string(from: Date()))] Server shutdown")
do {
try await asyncScheduler.shutdownGracefully()
} catch {
print("Failed to shut down MultiThreadedEventLoopGroup: \(error)")
}
}
try await status.update(.nominal)
}
func log(_ message: String) {
@ -52,114 +78,14 @@ func log(_ message: String) {
print(message)
return
}
observer.log(message)
}
import CBORCoding
public func migrate(folder: URL) throws {
try migrateMetric("caps.log", containing: String.self, in: folder)
try migrateMetric("caps.status", containing: ServerStatus.self, in: folder)
try migrateMetric("caps.count", containing: Int.self, in: folder)
try migrateMetric("caps.images", containing: Int.self, in: folder)
try migrateMetric("caps.classifier", containing: Int.self, in: folder)
}
private func migrateMetric<T>(_ id: String, containing type: T.Type, in folder: URL) throws where T: MetricValue {
print("Processing metric \(id)")
let file = id.hashed()
let url = folder.appendingPathComponent(file)
let files = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil)
.filter { Int($0.lastPathComponent) != nil }
print("Found \(files.count) files for \(id)")
let all: [Timestamped<T>] = try files.map(readElements(from:))
.reduce([], +)
.sorted { $0.timestamp < $1.timestamp }
print("Found \(all.count) items for \(id)")
try FileManager.default.removeItem(at: url)
print("Removed log folder")
// TODO: Write values back to disk
let observer = MetricObserver(logFileFolder: folder, logMetricId: "sesame.migration")
let metric: Metric<T> = observer.addMetric(id: id)
let semaphore = DispatchSemaphore(value: 0)
Task {
try await metric.update(all)
print("Saved all values for metric \(id)")
semaphore.signal()
}
semaphore.wait()
print("Finished metric \(id)")
}
private func readElements<T>(from url: URL) throws -> [Timestamped<T>] where T: MetricValue {
let data = try Data(contentsOf: url)
let file = url.lastPathComponent
print("File \(file): Loaded \(data.count) bytes")
let decoder = CBORDecoder()
let timestampLength = 9
let byteCountLength = 2
var result: [Timestamped<T>] = []
var currentIndex = data.startIndex
var skippedValues = 0
while currentIndex < data.endIndex {
let startIndexOfTimestamp = currentIndex + byteCountLength
guard startIndexOfTimestamp <= data.endIndex else {
print("File \(file): Only \(data.endIndex - currentIndex) bytes, needed \(byteCountLength) for byte count")
throw MetricError.logFileCorrupted
}
guard let byteCount = UInt16(fromData: data[currentIndex..<startIndexOfTimestamp]) else {
print("File \(file): Invalid byte count")
throw MetricError.logFileCorrupted
}
let nextIndex = startIndexOfTimestamp + Int(byteCount)
guard nextIndex <= data.endIndex else {
print("File \(file): Needed \(byteCountLength + Int(byteCount)) for timestamped value, has \(data.endIndex - startIndexOfTimestamp)")
throw MetricError.logFileCorrupted
}
guard byteCount >= timestampLength else {
print("File \(file): Only \(byteCount) bytes, needed \(timestampLength) for timestamp")
throw MetricError.logFileCorrupted
}
let timestampData = data[startIndexOfTimestamp..<startIndexOfTimestamp+timestampLength]
let timestamp = try decoder.decode(Double.self, from: timestampData)
let date = Date(timeIntervalSince1970: timestamp)
let elementData = data[startIndexOfTimestamp+timestampLength..<nextIndex]
do {
let element: T = try decoder.decode(from: elementData)
result.append(.init(value: element, timestamp: date))
} catch {
skippedValues += 1
}
currentIndex = nextIndex
if result.count % 100 == 1 {
print("File \(file): \(result.count) entries loaded (\(currentIndex)/\(data.endIndex) bytes)")
asyncScheduler.schedule {
await observer.log(message)
}
}
print("Loaded \(result.count) data points (\(skippedValues) skipped)")
return result
}
extension UInt16 {
func toData() -> Data {
Data([UInt8(self >> 8 & 0xFF), UInt8(self & 0xFF)])
}
init?<T: DataProtocol>(fromData data: T) {
guard data.count == 2 else {
return nil
}
let bytes = Array(data)
self = UInt16(UInt32(bytes[0]) << 8 | UInt32(bytes[1]))
}
}
private let df: DateFormatter = {
let df = DateFormatter()
df.dateStyle = .short
df.timeStyle = .short
return df
}()

View File

@ -0,0 +1,43 @@
import Vapor
import Dispatch
import Logging
/// This extension is temporary and can be removed once Vapor gets this support.
private extension Vapor.Application {
static let baseExecutionQueue = DispatchQueue(label: "vapor.codes.entrypoint")
func runFromAsyncMainEntrypoint() async throws {
try await withCheckedThrowingContinuation { continuation in
Vapor.Application.baseExecutionQueue.async { [self] in
do {
try self.run()
continuation.resume()
} catch {
continuation.resume(throwing: error)
}
}
}
}
}
@main
enum Entrypoint {
static func main() async throws {
var env = try Environment.detect()
try LoggingSystem.bootstrap(from: &env)
let app = Application(env)
defer {
shutdown()
app.shutdown()
}
do {
try await configure(app)
} catch {
app.logger.report(error: error)
throw error
}
try await app.runFromAsyncMainEntrypoint()
}
}

View File

@ -1,23 +0,0 @@
import App
import Vapor
var env = Environment.production
try LoggingSystem.bootstrap(from: &env)
let app = Application(env)
defer { app.shutdown() }
let storageFolder = URL(fileURLWithPath: app.directory.resourcesDirectory)
let logFolder = storageFolder.appendingPathComponent("logs")
print("Starting migration")
try migrate(folder: logFolder)
print("Finished migration")
/*
private let semaphore = DispatchSemaphore(value: 0)
Task {
try await configure(app)
semaphore.signal()
}
semaphore.wait()
try app.run()
*/

View File

View File

@ -1,13 +0,0 @@
import App
import Dispatch
import XCTest
final class AppTests : XCTestCase {
func testNothing() throws {
XCTAssert(true)
}
static let allTests = [
("testNothing", testNothing),
]
}

View File

View File

@ -1,6 +0,0 @@
{
"contentDirectory": "../Public",
"trainingIterations": 20,
"serverPath": "https://mydomain.com/caps",
"authenticationToken": "mysecretkey",
}

View File

@ -1,521 +0,0 @@
import Foundation
import Cocoa
import CreateML
struct Configuration: Codable {
let contentFolder: String
let trainingIterations: Int
let serverPath: String
let authenticationToken: String
init?(at url: URL) {
do {
let configData = try Data(contentsOf: url)
self = try JSONDecoder().decode(Configuration.self, from: configData)
} catch {
print("[ERROR] Failed to load configuration at \(url.absoluteURL.path): \(error)")
return nil
}
}
}
struct Cap: Codable {
let id: Int
let count: Int
enum CodingKeys: String, CodingKey {
case id = "i"
case count = "c"
}
}
final class ClassifierCreator {
static let configurationFileUrl = URL(fileURLWithPath: "config.json")
let server: URL
let configuration: Configuration
let imageDirectory: URL
let thumbnailDirectory: URL
let classifierUrl: URL
let df = DateFormatter()
// MARK: Step 1: Load configuration
init?() {
guard let configuration = Configuration(at: ClassifierCreator.configurationFileUrl) else {
return nil
}
self.configuration = configuration
guard let serverUrl = URL(string: configuration.serverPath) else {
print("[ERROR] Configuration: Invalid server path \(configuration.serverPath)")
return nil
}
self.server = serverUrl
let contentDirectory = URL(fileURLWithPath: configuration.contentFolder)
self.imageDirectory = contentDirectory.appendingPathComponent("images")
self.classifierUrl = contentDirectory.appendingPathComponent("classifier.mlmodel")
self.thumbnailDirectory = contentDirectory.appendingPathComponent("thumbnails")
df.dateFormat = "yy-MM-dd-HH-mm-ss"
}
// MARK: Main function
func run() async {
let imagesSnapshotDate = Date()
guard let (classes, changedImageCount, changedMainImages) = await loadImages() else {
return
}
guard !classes.isEmpty else {
return
}
guard changedImageCount > 0 else {
print("[INFO] No changed images, so no new classifier trained")
await createThumbnails(changed: changedMainImages)
print("[INFO] Done")
return
}
guard let classifierVersion = await getClassifierVersion() else {
return
}
let newVersion = classifierVersion + 1
print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
print("[INFO] Model path: \(classifierUrl.path)")
print("[INFO] Version: \(newVersion)")
print("[INFO] Classes: \(classes.count)")
print("[INFO] Iterations: \(configuration.trainingIterations)")
guard trainModel() else {
return
}
guard await upload(version: newVersion) else {
return
}
guard await upload(classes: classes, lastUpdate: imagesSnapshotDate) else {
return
}
await createThumbnails(changed: changedMainImages)
print("[INFO] Done")
}
// MARK: Step 2: Load changed images
func loadImages() async -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int])? {
guard createImageFolderIfMissing() else {
return nil
}
let imageCounts = await getImageCounts()
let missingImageList: [(cap: Int, image: Int)] = imageCounts
.sorted { $0.key < $1.key }
.reduce(into: []) { list, pair in
let missingImagesForCap: [(cap: Int, image: Int)] = (0..<pair.value).compactMap { image in
let url = imageUrl(base: imageDirectory, cap: pair.key, image: image)
guard !FileManager.default.fileExists(atPath: url.path) else {
return nil
}
return (cap: pair.key, image: image)
}
list.append(contentsOf: missingImagesForCap)
}
if missingImageList.isEmpty {
print("[INFO] No missing images to load")
} else {
print("[INFO] Loading \(missingImageList.count) missing images...")
}
guard await loadImages(missingImageList) else {
return nil
}
let changedImageList = await getChangedImageList()
.filter { $0.image < imageCounts[$0.cap] ?? 0 } // Filter non-existent images
if changedImageList.isEmpty {
print("[INFO] No changed images to load")
} else {
print("[INFO] Loading \(changedImageList.count) changed images...")
}
guard await loadImages(changedImageList) else {
return nil
}
let changedMainImages = changedImageList.filter { $0.image == 0 }.map { $0.cap }
let classes = imageCounts.keys.sorted()
// Delete any image folders not present as caps
guard deleteUnnecessaryImageFolders(caps: classes) else {
return nil
}
return (classes, missingImageList.count + changedImageList.count, changedMainImages)
}
private func createImageFolderIfMissing() -> Bool {
createFolderIfMissing(imageDirectory)
}
private func getImageCounts() async -> [Int : Int] {
guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
return [:]
}
do {
return try JSONDecoder().decode([Cap].self, from: data)
.reduce(into: [:]) { $0[$1.id] = $1.count }
} catch {
print("[ERROR] Failed to decode cap database: \(error)")
return [:]
}
}
private func deleteUnnecessaryImageFolders(caps: [Int]) -> Bool {
let validNames = caps.map { String(format: "%04d", $0) }
let folders: [String]
do {
folders = try FileManager.default.contentsOfDirectory(atPath: imageDirectory.path)
} catch {
print("[ERROR] Failed to get list of image folders: \(error)")
return false
}
for folder in folders {
if validNames.contains(folder) {
continue
}
// Not a valid cap folder
let url = imageDirectory.appendingPathComponent(folder)
do {
try FileManager.default.removeItem(at: url)
print("[ERROR] Removed unused image folder '\(folder)'")
} catch {
print("[ERROR] Failed to delete unused image folder \(folder): \(error)")
return false
}
}
return true
}
private func imageUrl(base: URL, cap: Int, image: Int) -> URL {
base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", cap, cap, image))
}
private func loadImage(cap: Int, image: Int) async -> Bool {
guard createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", cap))) else {
return false
}
let url = imageUrl(base: server.appendingPathComponent("images"), cap: cap, image: image)
let tempFile: URL, response: URLResponse
do {
(tempFile, response) = try await URLSession.shared.download(from: url)
} catch {
print("[ERROR] Failed to load image \(image) of cap \(cap): \(error)")
return false
}
let responseCode = (response as! HTTPURLResponse).statusCode
guard responseCode == 200 else {
print("[ERROR] Failed to load image \(image) of cap \(cap): Response \(responseCode)")
return false
}
do {
let localUrl = imageUrl(base: imageDirectory, cap: cap, image: image)
if FileManager.default.fileExists(atPath: localUrl.path) {
try FileManager.default.removeItem(at: localUrl)
}
try FileManager.default.moveItem(at: tempFile, to: localUrl)
return true
} catch {
print("[ERROR] Failed to save image \(image) of cap \(cap): \(error)")
return false
}
}
private func loadImages(_ list: [(cap: Int, image: Int)]) async -> Bool {
guard !list.isEmpty else {
return true
}
var loadedImages = 0
await withTaskGroup(of: Bool.self) { group in
for (cap, image) in list {
group.addTask {
await self.loadImage(cap: cap, image: image)
}
}
for await loaded in group {
if loaded {
loadedImages += 1
}
}
}
if loadedImages != list.count {
print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
return false
}
return true
}
func getChangedImageList() async -> [(cap: Int, image: Int)] {
guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
print("[ERROR] Failed to get list of changed images")
return []
}
return string
.components(separatedBy: "\n")
.filter { $0 != "" }
.compactMap {
let parts = $0.components(separatedBy: ":")
guard parts.count == 3 else {
return nil
}
/*
guard let date = df.date(from: parts[0]) else {
print("[WARN] Invalid date \(parts[0]) in change list")
return nil
}
*/
guard let cap = Int(parts[1]) else {
print("[WARN] Invalid cap id \(parts[1]) in change list")
return nil
}
guard let image = Int(parts[2]) else {
print("[WARN] Invalid image id \(parts[2]) in change list")
return nil
}
return (cap, image)
}
}
// MARK: Step 3: Compute version
func getClassifierVersion() async -> Int? {
guard let string: String = await get(server.appendingPathComponent("version")) else {
print("[ERROR] Failed to get classifier version")
return nil
}
guard let version = Int(string) else {
print("[ERROR] Invalid classifier version \(string)")
return nil
}
return version
}
// MARK: Step 4: Train classifier
func trainModel() -> Bool {
var params = MLImageClassifier.ModelParameters(augmentation: [])
params.maxIterations = configuration.trainingIterations
let model: MLImageClassifier
do {
model = try MLImageClassifier(
trainingData: .labeledDirectories(at: imageDirectory),
parameters: params)
} catch {
print("[ERROR] Failed to create classifier: \(error)")
return false
}
print("[INFO] Saving classifier...")
do {
try model.write(to: classifierUrl)
return true
} catch {
print("[ERROR] Failed to save model to file: \(error)")
return false
}
}
// MARK: Step 5: Upload classifier
func upload(version: Int) async -> Bool {
print("[INFO] Uploading classifier...")
let modelData: Data
do {
modelData = try Data(contentsOf: classifierUrl)
} catch {
print("[ERROR] Failed to read classifier data: \(error)")
return false
}
return await post(
url: server.appendingPathComponent("classifier/\(version)"),
body: modelData)
}
// MARK: Step 6: Update classes
func upload(classes: [Int], lastUpdate: Date) async -> Bool {
print("[INFO] Uploading trained classes...")
let dateString = df.string(from: lastUpdate)
return await post(
url: server.appendingPathComponent("classes/\(dateString)"),
body: classes.map(String.init).joined(separator: ",").data(using: .utf8)!)
}
// MARK: Step 7: Create thumbnails
func createThumbnails(changed: [Int]) async {
guard checkMagickAvailability() else {
return
}
guard createFolderIfMissing(thumbnailDirectory) else {
print("[ERROR] Failed to create folder for thumbnails")
return
}
let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
let all = Set(capIdsOfMissingThumbnails).union(changed)
print("[INFO] Creating \(all.count) thumbnails...")
for cap in all {
await createThumbnail(for: cap)
}
}
func checkMagickAvailability() -> Bool {
do {
let (code, output) = try safeShell("magick --version")
guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
.components(separatedBy: " ").first else {
print("[ERROR] Magick not found, install using 'brew install imagemagick'")
return false
}
print("[INFO] Using magick \(version)")
} catch {
print("[ERROR] Failed to get version of magick: (\(error))")
return false
}
return true
}
private func getMissingThumbnailIds() async -> [Int] {
guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
print("[ERROR] Failed to get missing thumbnails")
return []
}
return string.components(separatedBy: ",").compactMap(Int.init)
}
private func createThumbnail(for cap: Int) async {
let inputUrl = imageUrl(base: imageDirectory, cap: cap, image: 0)
guard FileManager.default.fileExists(atPath: inputUrl.path) else {
print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
return
}
let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
do {
let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
let (code, output) = try safeShell(command)
if code != 0 {
print("Failed to create thumbnail for cap \(cap): \(output)")
return
}
} catch {
print("Failed to read created thumbnail for cap \(cap): \(error)")
return
}
let data: Data
do {
data = try Data(contentsOf: output)
} catch {
print("Failed to read created thumbnail for cap \(cap): \(error)")
return
}
guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
print("Failed to upload thumbnail for cap \(cap)")
return
}
}
// MARK: Helper
@discardableResult
private func safeShell(_ command: String) throws -> (code: Int32, output: String) {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/zsh")
task.standardInput = nil
try task.run()
task.waitUntilExit()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return (task.terminationStatus, output)
}
private func createFolderIfMissing(_ folder: URL) -> Bool {
guard !FileManager.default.fileExists(atPath: folder.path) else {
return true
}
do {
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
return true
} catch {
print("[ERROR] Failed to create directory \(folder.path): \(error)")
return false
}
}
// MARK: Requests
private func post(url: URL, body: Data) async -> Bool {
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.httpBody = body
request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
return await perform(request) != nil
}
private func perform(_ request: URLRequest) async -> Data? {
let data: Data
let response: URLResponse
do {
(data, response) = try await URLSession.shared.data(for: request)
} catch {
print("[ERROR] Request to \(request.url!.absoluteString) failed: \(error)")
return nil
}
let code = (response as! HTTPURLResponse).statusCode
guard code == 200 else {
print("[ERROR] Request to \(request.url!.absoluteString): Invalid response \(code)")
return nil
}
return data
}
private func get(_ url: URL) async -> Data? {
await perform(URLRequest(url: url))
}
private func get(_ url: URL) async -> String? {
guard let data: Data = await get(url) else {
return nil
}
guard let string = String(data: data, encoding: .utf8) else {
print("[ERROR] Invalid string response \(data)")
return nil
}
return string
}
}
await ClassifierCreator()?.run()

View File

@ -1,15 +0,0 @@
echo "[1/8] Stopping server..."
sudo supervisorctl stop caps
echo "[2/8] Changing permissions..."
sudo chown -R pi:pi .
echo "[3/8] Pulling changes..."
git pull
echo "[4/8] Updating dependencies..."
swift package update
echo "[5/8] Building project..."
swift build -c release
echo "[6/8] Restoring permissions..."
sudo chown -R www-data:www-data .
echo "[7/8] Starting server..."
sudo supervisorctl start caps
echo "[8/8] Done"