Caps-Server/Sources/App/CapServer.swift

545 lines
17 KiB
Swift
Raw Normal View History

2021-11-08 21:58:55 +01:00
import Foundation
import Vapor
2023-01-11 18:29:32 +01:00
import Clairvoyant
2021-11-08 21:58:55 +01:00
2023-01-11 18:29:32 +01:00
final class CapServer: ServerOwner {
2021-11-08 21:58:55 +01:00
// MARK: Paths
private let imageFolder: URL
private let thumbnailFolder: URL
/// The file where the cap count is stored for the grid webpage
private let gridCountFile: URL
2022-05-24 14:47:50 +02:00
/// The file where the database of caps is stored
private let dbFile: URL
2022-12-16 13:32:33 +01:00
/// The file to store the HTML info of the cap count
private let htmlFile: URL
2021-11-08 21:58:55 +01:00
private let classifierVersionFile: URL
2022-06-23 22:48:58 +02:00
private let classifierFile: URL
2023-01-14 23:04:29 +01:00
private let changedImagesFile: URL
2021-11-08 21:58:55 +01:00
private let fm = FileManager.default
2023-01-15 11:21:47 +01:00
private let changedImageEntryDateFormatter: DateFormatter
2021-11-08 21:58:55 +01:00
// MARK: Caps
2022-05-27 09:25:41 +02:00
private var writers: Set<String>
2022-06-23 22:48:58 +02:00
2023-01-14 23:04:29 +01:00
/// The changed images not yet written to disk
private var unwrittenImageChanges: [(cap: Int, image: Int)] = []
2023-01-11 18:28:37 +01:00
var classifierVersion: Int = 0 {
didSet {
writeClassifierVersion()
2023-01-11 18:29:32 +01:00
updateMonitoredClassifierVersionProperty()
2022-06-23 22:48:58 +02:00
}
}
2022-05-27 09:25:41 +02:00
2022-05-28 21:58:51 +02:00
/**
The time to wait for changes to be written to disk.
This delay is used to prevent file writes for each small update to the caps.
*/
private let saveDelay: TimeInterval = 1
/**
The time when a save should occur.
No save is necessary if this property is `nil`.
*/
private var nextSaveTime: Date?
2022-05-24 14:47:50 +02:00
private var caps = [Int: Cap]() {
2023-01-11 18:29:32 +01:00
didSet {
scheduleSave()
updateMonitoredPropertiesOnCapChange()
}
2022-05-24 14:47:50 +02:00
}
var nextClassifierVersion: Int {
2022-05-27 09:25:41 +02:00
caps.values.compactMap { $0.classifierVersion }.max() ?? 1
2022-05-24 14:47:50 +02:00
}
2023-01-11 18:28:37 +01:00
var capCount: Int {
caps.count
}
var imageCount: Int {
caps.reduce(0) { $0 + $1.value.count }
}
init(in folder: URL, writers: [String]) {
2021-11-08 21:58:55 +01:00
self.imageFolder = folder.appendingPathComponent("images")
self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
self.gridCountFile = folder.appendingPathComponent("count.js")
2022-05-24 14:47:50 +02:00
self.dbFile = folder.appendingPathComponent("caps.json")
2022-12-16 13:32:33 +01:00
self.htmlFile = folder.appendingPathComponent("count.html")
self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
2022-06-23 22:48:58 +02:00
self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
2023-01-14 23:04:29 +01:00
self.changedImagesFile = folder.appendingPathComponent("changes.txt")
2022-05-27 09:25:41 +02:00
self.writers = Set(writers)
2023-01-15 11:21:47 +01:00
self.changedImageEntryDateFormatter = DateFormatter()
changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
2023-01-11 18:28:37 +01:00
}
2022-05-24 14:47:50 +02:00
2023-01-11 18:28:37 +01:00
func loadData() throws {
loadClassifierVersion(at: classifierVersionFile)
2022-05-24 14:47:50 +02:00
try loadCaps()
2022-12-16 13:32:33 +01:00
saveCapCountHTML()
2023-01-15 14:55:10 +01:00
updateGridCapCount()
try ensureExistenceOfChangedImagesFile()
organizeImages()
2021-11-08 21:58:55 +01:00
}
2023-01-11 18:28:37 +01:00
private func loadClassifierVersion(at url: URL) {
guard fm.fileExists(atPath: url.path) else {
return
}
let content: String
do {
content = try String(contentsOf: url)
.trimmingCharacters(in: .whitespacesAndNewlines)
} catch {
log("Failed to read classifier version file: \(error)")
return
}
guard let value = Int(content) else {
log("Invalid classifier version: \(content)")
return
}
self.classifierVersion = value
}
private func writeClassifierVersion() {
do {
try "\(classifierVersion)".data(using: .utf8)!
.write(to: classifierVersionFile)
} catch {
log("Failed to save classifier version: \(error)")
}
}
2022-05-24 14:47:50 +02:00
private func loadCaps() throws {
2022-10-07 21:13:21 +02:00
do {
let data = try Data(contentsOf: dbFile)
caps = try JSONDecoder().decode([Cap].self, from: data)
.reduce(into: [:]) { $0[$1.id] = $1 }
} catch {
log("Failed to load caps: \(error)")
throw error
}
2021-11-08 21:58:55 +01:00
log("\(caps.count) caps loaded")
}
2022-05-28 21:58:51 +02:00
private func scheduleSave() {
nextSaveTime = Date().addingTimeInterval(saveDelay)
2022-06-11 01:05:02 +02:00
DispatchQueue.global().asyncAfter(deadline: .now() + saveDelay) {
2022-05-28 21:58:51 +02:00
self.performScheduledSave()
}
}
private func performScheduledSave() {
guard let date = nextSaveTime else {
// No save necessary, or already saved
return
}
guard date < Date() else {
// Save pushed to future
return
}
do {
try saveCaps()
nextSaveTime = nil
} catch {
// Attempt save again
scheduleSave()
}
}
2022-05-24 14:47:50 +02:00
private func saveCaps() throws {
let data = try JSONEncoder().encode(caps.values.sorted())
try data.write(to: dbFile)
}
2022-12-16 13:32:33 +01:00
private func saveCapCountHTML() {
let count = caps.count
let content =
"""
<body style="margin: 0;">
<div style="display: flex; justify-content: center;">
<div style="font-size: 60px; font-family: 'SF Pro Display',-apple-system,BlinkMacSystemFont,Helvetica,sans-serif; -webkit-font-smoothing: antialiased;">\(count)</div>
</div>
</body>
"""
try? content.data(using: .utf8)!.write(to: htmlFile)
}
private func organizeImages() {
2023-01-15 02:17:47 +01:00
caps.values.sorted().forEach(organizeImage)
}
private func organizeImage(for cap: Cap) {
var cap = cap
guard let images = try? images(in: folder(of: cap.id)) else {
log("Failed to get image urls for cap \(cap.id)")
return
}
var sorted: [(id: Int, url: URL)] = images.compactMap {
2023-01-15 02:02:28 +01:00
guard let id = Int($0.deletingPathExtension().lastPathComponent.components(separatedBy: "-").last!) else {
return nil
}
return (id, $0)
}.sorted { $0.id < $1.id }
for version in 0..<images.count {
guard version != sorted[version].id else {
continue
}
let lastImage = sorted.popLast()!
let newUrl = file(of: cap.id, version: version)
do {
try fm.moveItem(at: lastImage.url, to: newUrl)
} catch {
log("Failed to move file \(lastImage.url.path) to \(newUrl.path): \(error)")
return
}
if cap.mainImage == lastImage.id {
cap.mainImage = version
}
sorted.insert((version, newUrl), at: version)
}
cap.count = sorted.count
caps[cap.id] = cap
}
2022-12-16 13:32:33 +01:00
2021-11-08 21:58:55 +01:00
// MARK: Paths
func folder(of cap: Int) -> URL {
imageFolder.appendingPathComponent(String(format: "%04d", cap))
}
func thumbnail(of cap: Int) -> URL {
thumbnailFolder.appendingPathComponent(String(format: "%04d.jpg", cap))
}
2021-11-08 21:58:55 +01:00
func file(of cap: Int, version: Int) -> URL {
folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version))
}
2022-05-27 09:25:41 +02:00
// MARK: Authentication
func hasAuthorization(for key: String) -> Bool {
// Note: This is not a constant-time compare, so there may be an opportunity
// for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
// and we're not doing anything critical in this application.
// Worst case, an unauthorized person with a lot of free time and energy to hack this system
// is able to change contents of the database, which are backed up in any case.
writers.contains(key)
}
2021-11-08 21:58:55 +01:00
// MARK: Counts
2022-05-24 14:47:50 +02:00
private func updateCounts() throws {
2022-10-07 21:13:21 +02:00
do {
caps = try caps.mapValues {
var cap = $0
cap.count = try count(of: $0.id)
return cap
}
} catch {
log("Failed to update counts: \(error)")
throw error
2022-05-24 14:47:50 +02:00
}
}
private func images(in folder: URL) throws -> [URL] {
2022-05-24 14:47:50 +02:00
try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
.filter { $0.pathExtension == "jpg" }
2021-11-08 21:58:55 +01:00
}
2022-05-24 14:47:50 +02:00
/**
Get the image count of a cap.
*/
2021-11-08 21:58:55 +01:00
func count(of cap: Int) throws -> Int {
let f = folder(of: cap)
guard fm.fileExists(atPath: f.path) else {
return 0
}
return try images(in: f).count
2021-11-08 21:58:55 +01:00
}
2022-05-24 14:47:50 +02:00
2021-11-08 21:58:55 +01:00
// MARK: Images
/**
Save a cap image to disk.
Automatically creates the image name with the current image count.
- Parameter data: The image data
- Parameter cap: The id of the cap.
2022-05-24 14:47:50 +02:00
- Throws: `CapError.unknownId`, if the cap doesn't exist. `CapError.dataInconsistency` if an image already exists for the current count.
2021-11-08 21:58:55 +01:00
*/
func save(image data: Data, for cap: Int) throws {
guard caps[cap] != nil else {
2022-05-24 14:47:50 +02:00
throw CapError.unknownId
}
var id = 0
2022-06-11 01:01:24 +02:00
let capFolder = folder(of: cap)
2022-05-24 14:47:50 +02:00
var f = file(of: cap, version: id)
2022-06-11 01:01:24 +02:00
if fm.fileExists(atPath: capFolder.path) {
while fm.fileExists(atPath: f.path) {
id += 1
f = file(of: cap, version: id)
}
} else {
try fm.createDirectory(at: capFolder, withIntermediateDirectories: true)
2021-11-08 21:58:55 +01:00
}
try data.write(to: f)
2022-06-11 00:38:53 +02:00
caps[cap]!.count = try count(of: cap)
2023-01-14 23:04:29 +01:00
addChangedImageToLog(cap: cap, image: id)
2022-05-24 14:47:50 +02:00
log("Added image \(id) for cap \(cap)")
2021-11-08 21:58:55 +01:00
}
2023-01-14 23:04:29 +01:00
private func writeChangedImagesToDisk() throws {
guard !unwrittenImageChanges.isEmpty else {
return
}
let handle = try FileHandle(forWritingTo: changedImagesFile)
try handle.seekToEnd()
var entries = unwrittenImageChanges
defer {
unwrittenImageChanges = entries
try? handle.close()
}
2023-01-15 11:21:47 +01:00
let dateString = changedImageEntryDateFormatter.string(from: Date())
2023-01-14 23:04:29 +01:00
while let entry = entries.popLast() {
2023-01-14 23:21:35 +01:00
let content = "\(dateString):\(entry.cap):\(entry.image)\n".data(using: .utf8)!
2023-01-14 23:04:29 +01:00
try handle.write(contentsOf: content)
}
}
private func addChangedImageToLog(cap: Int, image: Int) {
unwrittenImageChanges.append((cap, image))
do {
try writeChangedImagesToDisk()
} catch {
log("Failed to save changed image list: \(error)")
}
}
private func ensureExistenceOfChangedImagesFile() throws {
guard !fm.fileExists(atPath: changedImagesFile.path) else {
2023-01-14 23:04:29 +01:00
return
}
do {
try Data().write(to: changedImagesFile)
} catch {
log("Failed to create changed images file: \(error)")
throw error
}
}
2023-01-15 11:21:47 +01:00
func removeAllEntriesInImageChangeList(before date: Date) {
do {
2023-01-15 11:21:47 +01:00
try String(contentsOf: changedImagesFile)
.components(separatedBy: "\n")
.filter { $0 != "" }
.compactMap { line -> String? in
guard let entryDate = changedImageEntryDateFormatter.date(from: line.components(separatedBy: ":").first!) else {
return nil
}
guard entryDate > date else {
return nil
}
return line
}
.joined(separator: "\n")
.data(using: .utf8)!
.write(to: changedImagesFile)
2023-01-14 23:04:29 +01:00
} catch {
2023-01-15 11:21:47 +01:00
log("Failed to update changed images file: \(error)")
2023-01-14 23:04:29 +01:00
}
}
2021-11-08 21:58:55 +01:00
func switchMainImage(to version: Int, for cap: Int) throws {
let file2 = file(of: cap, version: version)
guard fm.fileExists(atPath: file2.path) else {
log("No image \(version) for cap \(cap)")
throw CapError.invalidFile
}
2022-05-24 14:47:50 +02:00
caps[cap]?.mainImage = version
2021-11-08 21:58:55 +01:00
log("Switched cap \(cap) to version \(version)")
}
func addOrUpdate(_ cap: Cap) throws {
if let existingCap = caps[cap.id] {
2022-06-11 00:38:53 +02:00
update(existingCap, with: cap)
} else {
try add(cap)
}
}
private func add(_ cap: Cap) throws {
2022-06-11 00:38:53 +02:00
guard cap.mainImage == 0 else {
throw CapError.invalidData
}
2022-06-11 00:38:53 +02:00
var cap = cap
cap.count = 0
cap.classifierVersion = nextClassifierVersion
caps[cap.id] = cap
2022-12-16 13:32:33 +01:00
saveCapCountHTML()
updateGridCapCount()
2022-05-28 22:09:29 +02:00
log("Added cap \(cap.id) '\(cap.name)'")
}
2022-06-11 00:38:53 +02:00
private func update(_ existingCap: Cap, with cap: Cap) {
var updatedCap = existingCap
if cap.name != "" {
updatedCap.name = cap.name
}
2022-05-28 22:09:29 +02:00
let url = file(of: existingCap.id, version: cap.mainImage)
if fm.fileExists(atPath: url.path) {
updatedCap.mainImage = cap.mainImage
}
if let color = cap.color {
updatedCap.color = color
}
2022-05-28 22:09:29 +02:00
caps[existingCap.id] = updatedCap
log("Updated cap \(existingCap.id)")
}
2022-06-23 22:48:58 +02:00
func updateTrainedClasses(content: String) {
let trainedCaps = content
.components(separatedBy: "\n")
.compactMap(Int.init)
2022-06-23 22:48:58 +02:00
let version = classifierVersion
for cap in trainedCaps {
if caps[cap]?.classifierVersion == nil {
caps[cap]?.classifierVersion = version
}
}
2022-06-24 11:49:34 +02:00
log("Updated \(trainedCaps.count) classifier classes")
}
2022-06-23 22:48:58 +02:00
func save(classifier: Data, version: Int) throws {
do {
2022-06-23 22:48:58 +02:00
try classifier.write(to: classifierFile)
} catch {
2022-06-23 22:48:58 +02:00
log("Failed to write classifier: \(error)")
throw Abort(.internalServerError)
}
2022-06-23 22:48:58 +02:00
classifierVersion = version
2022-06-24 11:49:34 +02:00
log("Updated classifier to version \(version)")
}
func getListOfMissingThumbnails() -> [Int] {
caps.keys.filter { !fm.fileExists(atPath: thumbnail(of: $0).path) }
}
func saveThumbnail(_ data: Data, for cap: Int) {
let url = thumbnail(of: cap)
do {
try data.write(to: url)
} catch {
log("Failed to save thumbnail \(cap): \(error)")
}
}
private func updateGridCapCount() {
do {
try "const numberOfCaps = \(capCount);"
.data(using: .utf8)!
.write(to: gridCountFile)
} catch {
log("Failed to save grid cap count: \(error)")
}
}
2023-01-11 18:29:32 +01:00
// MARK: ServerOwner
let authenticationMethod: PropertyAuthenticationMethod = .accessToken
func hasReadPermission(for property: UInt32, accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
func hasWritePermission(for property: UInt32, accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
func hasListAccessPermission(_ accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
// MARK: Monitoring
2023-01-11 19:56:39 +01:00
public let name = "caps"
2023-01-11 18:29:32 +01:00
2023-01-11 19:56:39 +01:00
private let capCountPropertyId = PropertyId(owner: "caps", uniqueId: 2)
2023-01-11 18:29:32 +01:00
2023-01-11 19:56:39 +01:00
private let imageCountPropertyId = PropertyId(owner: "caps", uniqueId: 3)
private let classifierVersionPropertyId = PropertyId(owner: "caps", uniqueId: 4)
2023-01-11 18:29:32 +01:00
func registerProperties(with monitor: PropertyManager) {
let capCountProperty = PropertyRegistration(
uniqueId: capCountPropertyId.uniqueId,
2023-01-11 19:56:39 +01:00
name: "caps",
2023-01-11 18:29:32 +01:00
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.capCount ?? 0).timestamped()
})
monitor.register(capCountProperty, for: self)
let imageCountProperty = PropertyRegistration(
uniqueId: imageCountPropertyId.uniqueId,
2023-01-11 19:56:39 +01:00
name: "images",
2023-01-11 18:29:32 +01:00
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.imageCount ?? 0).timestamped()
})
monitor.register(imageCountProperty, for: self)
let classifierVersionProperty = PropertyRegistration(
uniqueId: classifierVersionPropertyId.uniqueId,
2023-01-11 19:56:39 +01:00
name: "classifier",
2023-01-11 18:29:32 +01:00
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.classifierVersion ?? 0).timestamped()
})
monitor.register(classifierVersionProperty, for: self)
}
private func updateMonitoredPropertiesOnCapChange() {
try? monitor.logChanged(property: capCountPropertyId, value: capCount.timestamped())
try? monitor.logChanged(property: imageCountPropertyId, value: imageCount.timestamped())
}
private func updateMonitoredClassifierVersionProperty() {
try? monitor.logChanged(property: classifierVersionPropertyId, value: classifierVersion.timestamped())
}
2021-11-08 21:58:55 +01:00
}