Caps-Server/Sources/App/CapServer.swift

import Foundation
import Vapor
import Clairvoyant

final class CapServer: ServerOwner {
    
    // MARK: Paths
    
    private let imageFolder: URL

    private let thumbnailFolder: URL
    
    /// The file where the cap count is stored for the grid webpage
    private let gridCountFile: URL
    
    /// The file where the database of caps is stored
    private let dbFile: URL

    /// The file to store the HTML info of the cap count
    private let htmlFile: URL
    
    private let classifierVersionFile: URL
    
    private let classifierFile: URL
    
    private let changedImagesFile: URL
    
    private let fm = FileManager.default
    
    private let changedImageEntryDateFormatter: DateFormatter
    
    // MARK: Caps

    private var writers: Set<String>
    
    /// The changed images not yet written to disk
    private var unwrittenImageChanges: [(cap: Int, image: Int)] = []
    
    var classifierVersion: Int = 0 {
        didSet {
            writeClassifierVersion()
            updateMonitoredClassifierVersionProperty()
        }
    }

    /**
     The time to wait for changes to be written to disk.

     This delay is used to prevent file writes for each small update to the caps.
     */
    private let saveDelay: TimeInterval = 1

    /**
     The time when a save should occur.

     No save is necessary if this property is `nil`.
     */
    private var nextSaveTime: Date?

    private var caps = [Int: Cap]() {
        didSet {
            scheduleSave()
            updateMonitoredPropertiesOnCapChange()
        }
    }

    var nextClassifierVersion: Int {
        caps.values.compactMap { $0.classifierVersion }.max() ?? 1
    }

    var capCount: Int {
        caps.count
    }

    var imageCount: Int {
        caps.reduce(0) { $0 + $1.value.count }
    }

    init(in folder: URL, writers: [String]) {
        self.imageFolder = folder.appendingPathComponent("images")
        self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
        self.gridCountFile = folder.appendingPathComponent("count.js")
        self.dbFile = folder.appendingPathComponent("caps.json")
        self.htmlFile = folder.appendingPathComponent("count.html")
        self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
        self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
        self.changedImagesFile = folder.appendingPathComponent("changes.txt")
        self.writers = Set(writers)
        self.changedImageEntryDateFormatter = DateFormatter()
        changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
    }

    func loadData() throws {
        loadClassifierVersion(at: classifierVersionFile)
        try loadCaps()
        saveCapCountHTML()
        updateGridCapCount()
        try ensureExistenceOfChangedImagesFile()
        organizeImages()
    }

    private func loadClassifierVersion(at url: URL) {
        guard fm.fileExists(atPath: url.path) else {
            return
        }
        let content: String
        do {
            content = try String(contentsOf: url)
                .trimmingCharacters(in: .whitespacesAndNewlines)
        } catch {
            log("Failed to read classifier version file: \(error)")
            return
        }

        guard let value = Int(content) else {
            log("Invalid classifier version: \(content)")
            return
        }
        self.classifierVersion = value
    }

    private func writeClassifierVersion() {
        do {
            try "\(classifierVersion)".data(using: .utf8)!
                .write(to: classifierVersionFile)
        } catch {
            log("Failed to save classifier version: \(error)")
        }
    }

    private func loadCaps() throws {
        do {
            let data = try Data(contentsOf: dbFile)
            caps = try JSONDecoder().decode([Cap].self, from: data)
                .reduce(into: [:]) { $0[$1.id] = $1 }
        } catch {
            log("Failed to load caps: \(error)")
            throw error
        }
        log("\(caps.count) caps loaded")
    }

    private func scheduleSave() {
        nextSaveTime = Date().addingTimeInterval(saveDelay)
        DispatchQueue.global().asyncAfter(deadline: .now() + saveDelay) {
            self.performScheduledSave()
        }
    }

    private func performScheduledSave() {
        guard let date = nextSaveTime else {
            // No save necessary, or already saved
            return
        }
        guard date < Date() else {
            // Save pushed to future
            return
        }
        do {
            try saveCaps()
            nextSaveTime = nil
        } catch {
            // Attempt save again
            scheduleSave()
        }
    }

    private func saveCaps() throws {
        let data = try JSONEncoder().encode(caps.values.sorted())
        try data.write(to: dbFile)
    }

    private func saveCapCountHTML() {
        let count = caps.count
        let content =
        """
        <body style="margin: 0;">
            <div style="display: flex; justify-content: center;">
                <div style="font-size: 60px; font-family: 'SF Pro Display',-apple-system,BlinkMacSystemFont,Helvetica,sans-serif; -webkit-font-smoothing: antialiased;">\(count)</div>
            </div>
        </body>
        """
        try? content.data(using: .utf8)!.write(to: htmlFile)
    }
    
    private func organizeImages() {
        caps.values.sorted().forEach(organizeImage)
    }
    
    private func organizeImage(for cap: Cap) {
        var cap = cap
        guard let images = try? images(in: folder(of: cap.id)) else {
            log("Failed to get image urls for cap \(cap.id)")
            return
        }
        var sorted: [(id: Int, url: URL)] = images.compactMap {
            guard let id = Int($0.deletingPathExtension().lastPathComponent.components(separatedBy: "-").last!) else {
                return nil
            }
            return (id, $0)
        }.sorted { $0.id < $1.id }
        for version in 0..<images.count {
            guard version != sorted[version].id else {
                continue
            }
            let lastImage = sorted.popLast()!
            let newUrl = file(of: cap.id, version: version)
            do {
                try fm.moveItem(at: lastImage.url, to: newUrl)
            } catch {
                log("Failed to move file \(lastImage.url.path) to \(newUrl.path): \(error)")
                return
            }
            if cap.mainImage == lastImage.id {
                cap.mainImage = version
            }
            sorted.insert((version, newUrl), at: version)
        }
        cap.count = sorted.count
        caps[cap.id] = cap
    }

    // MARK: Paths

    func folder(of cap: Int) -> URL {
        imageFolder.appendingPathComponent(String(format: "%04d", cap))
    }
    
    func thumbnail(of cap: Int) -> URL {
        thumbnailFolder.appendingPathComponent(String(format: "%04d.jpg", cap))
    }

    func file(of cap: Int, version: Int) -> URL {
        folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version))
    }

    // MARK: Authentication

    func hasAuthorization(for key: String) -> Bool {
        // Note: This is not a constant-time compare, so there may be an opportunity
        // for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
        // and we're not doing anything critical in this application.
        // Worst case, an unauthorized person with a lot of free time and energy to hack this system
        // is able to change contents of the database, which are backed up in any case.
        writers.contains(key)
    }
    
    // MARK: Counts
    
    private func updateCounts() throws {
        do {
            caps = try caps.mapValues {
                var cap = $0
                cap.count = try count(of: $0.id)
                return cap
            }
        } catch {
            log("Failed to update counts: \(error)")
            throw error
        }
    }

    private func images(in folder: URL) throws -> [URL] {
        try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
            .filter { $0.pathExtension == "jpg" }
    }

    /**
     Get the image count of a cap.
     */
    func count(of cap: Int) throws -> Int {
        let f = folder(of: cap)
        guard fm.fileExists(atPath: f.path) else {
            return 0
        }
        return try images(in: f).count
    }

    // MARK: Images
    
    /**
     Save a cap image to disk.
     
     Automatically creates the image name with the current image count.
     - Parameter data: The image data
     - Parameter cap: The id of the cap.
     - Throws: `CapError.unknownId`, if the cap doesn't exist. `CapError.dataInconsistency` if an image already exists for the current count.
     */
    func save(image data: Data, for cap: Int) throws {
        guard caps[cap] != nil else {
            throw CapError.unknownId
        }
        var id = 0
        let capFolder = folder(of: cap)
        var f = file(of: cap, version: id)
        if fm.fileExists(atPath: capFolder.path) {
            while fm.fileExists(atPath: f.path) {
                id += 1
                f = file(of: cap, version: id)
            }
        } else {
            try fm.createDirectory(at: capFolder, withIntermediateDirectories: true)
        }
        try data.write(to: f)
        caps[cap]!.count = try count(of: cap)
        addChangedImageToLog(cap: cap, image: id)
        log("Added image \(id) for cap \(cap)")
    }
    
    private func writeChangedImagesToDisk() throws {
        guard !unwrittenImageChanges.isEmpty else {
            return
        }
        
        let handle = try FileHandle(forWritingTo: changedImagesFile)
        try handle.seekToEnd()
        var entries = unwrittenImageChanges
        defer {
            unwrittenImageChanges = entries
            try? handle.close()
        }
        let dateString = changedImageEntryDateFormatter.string(from: Date())
        while let entry = entries.popLast() {
            let content = "\(dateString):\(entry.cap):\(entry.image)\n".data(using: .utf8)!
            try handle.write(contentsOf: content)
        }
    }
    
    private func addChangedImageToLog(cap: Int, image: Int) {
        unwrittenImageChanges.append((cap, image))
        do {
            try writeChangedImagesToDisk()
        } catch {
            log("Failed to save changed image list: \(error)")
        }
    }
    
    private func ensureExistenceOfChangedImagesFile() throws {
        guard !fm.fileExists(atPath: changedImagesFile.path) else {
            return
        }
        do {
            try Data().write(to: changedImagesFile)
        } catch {
            log("Failed to create changed images file: \(error)")
            throw error
        }
    }
    
    func removeAllEntriesInImageChangeList(before date: Date) {
        do {
            try String(contentsOf: changedImagesFile)
                .components(separatedBy: "\n")
                .filter { $0 != "" }
                .compactMap { line -> String? in
                    guard let entryDate = changedImageEntryDateFormatter.date(from: line.components(separatedBy: ":").first!) else {
                        return nil
                    }
                    guard entryDate > date else {
                        return nil
                    }
                    return line
                }
                .joined(separator: "\n")
                .data(using: .utf8)!
                .write(to: changedImagesFile)
        } catch {
            log("Failed to update changed images file: \(error)")
        }
    }
    
    func switchMainImage(to version: Int, for cap: Int) throws {
        let file2 = file(of: cap, version: version)
        guard fm.fileExists(atPath: file2.path) else {
            log("No image \(version) for cap \(cap)")
            throw CapError.invalidFile
        }
        caps[cap]?.mainImage = version
        log("Switched cap \(cap) to version \(version)")
    }

    func addOrUpdate(_ cap: Cap) throws {
        if let existingCap = caps[cap.id] {
            update(existingCap, with: cap)
        } else {
            try add(cap)
        }
    }

    private func add(_ cap: Cap) throws {
        guard cap.mainImage == 0 else {
            throw CapError.invalidData
        }
        var cap = cap
        cap.count = 0
        cap.classifierVersion = nextClassifierVersion
        caps[cap.id] = cap
        saveCapCountHTML()
        updateGridCapCount()
        log("Added cap \(cap.id) '\(cap.name)'")
    }

    private func update(_ existingCap: Cap, with cap: Cap) {
        var updatedCap = existingCap
        if cap.name != "" {
            updatedCap.name = cap.name
        }
        let url = file(of: existingCap.id, version: cap.mainImage)
        if fm.fileExists(atPath: url.path) {
            updatedCap.mainImage = cap.mainImage
        }
        if let color = cap.color {
            updatedCap.color = color
        }
        caps[existingCap.id] = updatedCap
        log("Updated cap \(existingCap.id)")
    }
    
    func updateTrainedClasses(content: String) {
        let trainedCaps = content
            .components(separatedBy: "\n")
            .compactMap(Int.init)
        let version = classifierVersion
        for cap in trainedCaps {
            if caps[cap]?.classifierVersion == nil {
                caps[cap]?.classifierVersion = version
            }
        }
        log("Updated \(trainedCaps.count) classifier classes")
    }
    
    func save(classifier: Data, version: Int) throws {
        do {
            try classifier.write(to: classifierFile)
        } catch {
            log("Failed to write classifier: \(error)")
            throw Abort(.internalServerError)
        }
        classifierVersion = version
        log("Updated classifier to version \(version)")
    }
    
    func getListOfMissingThumbnails() -> [Int] {
        caps.keys.filter { !fm.fileExists(atPath: thumbnail(of: $0).path) }
    }
    
    func saveThumbnail(_ data: Data, for cap: Int) {
        let url = thumbnail(of: cap)
        do {
            try data.write(to: url)
        } catch {
            log("Failed to save thumbnail \(cap): \(error)")
        }
    }
    
    private func updateGridCapCount() {
        do {
            try "const numberOfCaps = \(capCount);"
                .data(using: .utf8)!
                .write(to: gridCountFile)
        } catch {
            log("Failed to save grid cap count: \(error)")
        }
    }

    // MARK: ServerOwner

    let authenticationMethod: PropertyAuthenticationMethod = .accessToken

    func hasReadPermission(for property: UInt32, accessData: Data) -> Bool {
        guard let key = String(data: accessData, encoding: .utf8) else {
            return false
        }
        return writers.contains(key)
    }

    func hasWritePermission(for property: UInt32, accessData: Data) -> Bool {
        guard let key = String(data: accessData, encoding: .utf8) else {
            return false
        }
        return writers.contains(key)
    }

    func hasListAccessPermission(_ accessData: Data) -> Bool {
        guard let key = String(data: accessData, encoding: .utf8) else {
            return false
        }
        return writers.contains(key)
    }

    // MARK: Monitoring

    public let name = "caps"

    private let capCountPropertyId = PropertyId(owner: "caps", uniqueId: 2)

    private let imageCountPropertyId = PropertyId(owner: "caps", uniqueId: 3)

    private let classifierVersionPropertyId = PropertyId(owner: "caps", uniqueId: 4)

    func registerProperties(with monitor: PropertyManager) {
        let capCountProperty = PropertyRegistration(
            uniqueId: capCountPropertyId.uniqueId,
            name: "caps",
            updates: .continuous,
            isLogged: true,
            allowsManualUpdate: false,
            read: { [weak self] in
                return (self?.capCount ?? 0).timestamped()
            })
        monitor.register(capCountProperty, for: self)

        let imageCountProperty = PropertyRegistration(
            uniqueId: imageCountPropertyId.uniqueId,
            name: "images",
            updates: .continuous,
            isLogged: true,
            allowsManualUpdate: false,
            read: { [weak self] in
                return (self?.imageCount ?? 0).timestamped()
            })
        monitor.register(imageCountProperty, for: self)

        let classifierVersionProperty = PropertyRegistration(
            uniqueId: classifierVersionPropertyId.uniqueId,
            name: "classifier",
            updates: .continuous,
            isLogged: true,
            allowsManualUpdate: false,
            read: { [weak self] in
                return (self?.classifierVersion ?? 0).timestamped()
            })
        monitor.register(classifierVersionProperty, for: self)
    }

    private func updateMonitoredPropertiesOnCapChange() {
        try? monitor.logChanged(property: capCountPropertyId, value: capCount.timestamped())
        try? monitor.logChanged(property: imageCountPropertyId, value: imageCount.timestamped())
    }

    private func updateMonitoredClassifierVersionProperty() {
        try? monitor.logChanged(property: classifierVersionPropertyId, value: classifierVersion.timestamped())
    }
}