From c5ce5414a9385a43f765e2b0d45502ed90cd7373 Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Tue, 17 Jan 2023 22:02:27 +0100 Subject: [PATCH] Update authentication and metrics logging --- Package.swift | 2 +- Sources/App/Authenticator.swift | 39 ++++++++ Sources/App/CapServer+Routes.swift | 141 +++++++++++++++++++++++++++++ Sources/App/CapServer.swift | 103 +++------------------ Sources/App/Error.swift | 8 ++ Sources/App/configure.swift | 36 ++++---- Sources/App/routes.swift | 133 --------------------------- 7 files changed, 220 insertions(+), 242 deletions(-) create mode 100644 Sources/App/Authenticator.swift create mode 100755 Sources/App/CapServer+Routes.swift delete mode 100755 Sources/App/routes.swift diff --git a/Package.swift b/Package.swift index 499dd9b..42cb4b5 100755 --- a/Package.swift +++ b/Package.swift @@ -8,7 +8,7 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/vapor/vapor", from: "4.0.0"), - .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.2.0"), + .package(url: "https://github.com/christophhagen/Clairvoyant", branch: "main"), ], targets: [ .target(name: "App", diff --git a/Sources/App/Authenticator.swift b/Sources/App/Authenticator.swift new file mode 100644 index 0000000..b7eb71d --- /dev/null +++ b/Sources/App/Authenticator.swift @@ -0,0 +1,39 @@ +import Foundation +import Clairvoyant +import Vapor + +final class Authenticator: MetricAccessAuthenticator { + + private var writers: Set + + init(writers: [String]) { + self.writers = Set(writers) + } + + + func hasAuthorization(for key: String) -> Bool { + // Note: This is not a constant-time compare, so there may be an opportunity + // for timing attack here. Sets perform hashed lookups, so this may be less of an issue, + // and we're not doing anything critical in this application. + // Worst case, an unauthorized person with a lot of free time and energy to hack this system + // is able to change contents of the database, which are backed up in any case. + writers.contains(key) + } + + func metricAccess(isAllowedForToken accessToken: Data) -> Bool { + guard let key = String(data: accessToken, encoding: .utf8) else { + return false + } + return hasAuthorization(for: key) + } + + + func authorize(_ request: Request) throws { + guard let key = request.headers.first(name: "key") else { + throw Abort(.badRequest) // 400 + } + guard hasAuthorization(for: key) else { + throw Abort(.forbidden) // 403 + } + } +} diff --git a/Sources/App/CapServer+Routes.swift b/Sources/App/CapServer+Routes.swift new file mode 100755 index 0000000..05aca13 --- /dev/null +++ b/Sources/App/CapServer+Routes.swift @@ -0,0 +1,141 @@ +import Vapor +import Foundation + +/// The decoder to extract caps from JSON payloads given to the `cap` route. +private let decoder = JSONDecoder() + +/// The date formatter to decode dates in requests +private let dateFormatter: DateFormatter = { + let df = DateFormatter() + df.dateFormat = "yy-MM-dd-HH-mm-ss" + return df +}() + +extension CapServer { + + private func ensureOperability() throws { + guard isOperational else { + throw Abort(.noContent) + } + } + + func registerRoutes(with app: Application, authenticator: Authenticator) { + app.get("version") { _ in + try self.ensureOperability() + return "\(self.classifierVersion)" + } + + // Add or change a cap + app.postCatching("cap") { request in + try self.ensureOperability() + try authenticator.authorize(request) + let data = try request.getBodyData() + let cap = try decoder.decode(Cap.self, from: data) + try self.addOrUpdate(cap) + } + + // Upload an image + app.postCatching("images", ":n") { request in + try self.ensureOperability() + try authenticator.authorize(request) + guard let cap = request.parameters.get("n", as: Int.self) else { + log("Invalid parameter for cap") + throw Abort(.badRequest) + } + let data = try request.getBodyData() + try self.save(image: data, for: cap) + } + // Update the classifier + app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in + guard let version = request.parameters.get("version", as: Int.self) else { + log("Invalid parameter for version") + throw Abort(.badRequest) + } + guard version > self.classifierVersion else { + throw Abort(.alreadyReported) // 208 + } + + try self.ensureOperability() + try authenticator.authorize(request) + let data = try request.getBodyData() + try self.save(classifier: data, version: version) + return .ok + } + + // Update the trained classes + app.postCatching("classes", ":date") { request in + guard let dateString = request.parameters.get("date") else { + log("Invalid parameter for date") + throw Abort(.badRequest) + } + guard let date = dateFormatter.date(from: dateString) else { + log("Invalid date specification") + throw Abort(.badRequest) + } + + try self.ensureOperability() + try authenticator.authorize(request) + let body = try request.getStringBody() + + self.updateTrainedClasses(content: body) + self.removeAllEntriesInImageChangeList(before: date) + } + + // Get the list of missing thumbnails + app.get("thumbnails", "missing") { request in + try self.ensureOperability() + let missingThumbnails = self.getListOfMissingThumbnails() + return missingThumbnails.map(String.init).joined(separator: ",") + } + + // Upload the thumbnail of a cap + app.postCatching("thumbnails", ":cap") { request in + guard let cap = request.parameters.get("cap", as: Int.self) else { + log("Invalid cap parameter for thumbnail upload") + throw Abort(.badRequest) + } + try self.ensureOperability() + try authenticator.authorize(request) + let data = try request.getBodyData() + self.saveThumbnail(data, for: cap) + } + + // Delete the image of a cap + app.postCatching("delete", ":cap", ":version") { request in + guard let cap = request.parameters.get("cap", as: Int.self) else { + log("Invalid cap parameter for image deletion") + throw Abort(.badRequest) + } + guard let version = request.parameters.get("version", as: Int.self) else { + log("Invalid version parameter for image deletion") + throw Abort(.badRequest) + } + + try self.ensureOperability() + try authenticator.authorize(request) + guard self.deleteImage(version: version, for: cap) else { + throw Abort(.gone) + } + } + } +} + +private extension Request { + + func getBodyData() throws -> Data { + guard let buffer = body.data else { + log("Missing body data") + throw CapError.invalidBody + } + return Data(buffer: buffer) + } + + func getStringBody() throws -> String { + let data = try getBodyData() + guard let content = String(data: data, encoding: .utf8) else { + log("Invalid string body") + throw CapError.invalidBody + } + return content + } +} diff --git a/Sources/App/CapServer.swift b/Sources/App/CapServer.swift index c02170f..c1c63e7 100644 --- a/Sources/App/CapServer.swift +++ b/Sources/App/CapServer.swift @@ -2,7 +2,7 @@ import Foundation import Vapor import Clairvoyant -final class CapServer: ServerOwner { +final class CapServer { // MARK: Paths @@ -28,10 +28,12 @@ final class CapServer: ServerOwner { private let fm = FileManager.default private let changedImageEntryDateFormatter: DateFormatter + + /// Indicates that the data is loaded + private(set) var isOperational = false // MARK: Caps - private var writers: Set /// The changed images not yet written to disk private var unwrittenImageChanges: [(cap: Int, image: Int)] = [] @@ -39,7 +41,7 @@ final class CapServer: ServerOwner { var classifierVersion: Int = 0 { didSet { writeClassifierVersion() - updateMonitoredClassifierVersionProperty() + classifierMetric.update(classifierVersion) } } @@ -60,7 +62,8 @@ final class CapServer: ServerOwner { private var caps = [Int: Cap]() { didSet { scheduleSave() - updateMonitoredPropertiesOnCapChange() + capCountMetric.update(caps.count) + imageCountMetric.update(imageCount) } } @@ -76,7 +79,7 @@ final class CapServer: ServerOwner { caps.reduce(0) { $0 + $1.value.count } } - init(in folder: URL, writers: [String]) { + init(in folder: URL) { self.imageFolder = folder.appendingPathComponent("images") self.thumbnailFolder = folder.appendingPathComponent("thumbnails") self.gridCountFile = folder.appendingPathComponent("count.js") @@ -85,7 +88,6 @@ final class CapServer: ServerOwner { self.classifierVersionFile = folder.appendingPathComponent("classifier.version") self.classifierFile = folder.appendingPathComponent("classifier.mlmodel") self.changedImagesFile = folder.appendingPathComponent("changes.txt") - self.writers = Set(writers) self.changedImageEntryDateFormatter = DateFormatter() changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss" } @@ -97,6 +99,7 @@ final class CapServer: ServerOwner { updateGridCapCount() try ensureExistenceOfChangedImagesFile() organizeImages() + isOperational = true } private func loadClassifierVersion(at url: URL) { @@ -234,17 +237,6 @@ final class CapServer: ServerOwner { folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version)) } - // MARK: Authentication - - func hasAuthorization(for key: String) -> Bool { - // Note: This is not a constant-time compare, so there may be an opportunity - // for timing attack here. Sets perform hashed lookups, so this may be less of an issue, - // and we're not doing anything critical in this application. - // Worst case, an unauthorized person with a lot of free time and energy to hack this system - // is able to change contents of the database, which are backed up in any case. - writers.contains(key) - } - // MARK: Counts private func images(in folder: URL) throws -> [URL] { @@ -466,82 +458,11 @@ final class CapServer: ServerOwner { } } - // MARK: ServerOwner - - let authenticationMethod: PropertyAuthenticationMethod = .accessToken - - func hasReadPermission(for property: UInt32, accessData: Data) -> Bool { - guard let key = String(data: accessData, encoding: .utf8) else { - return false - } - return writers.contains(key) - } - - func hasWritePermission(for property: UInt32, accessData: Data) -> Bool { - guard let key = String(data: accessData, encoding: .utf8) else { - return false - } - return writers.contains(key) - } - - func hasListAccessPermission(_ accessData: Data) -> Bool { - guard let key = String(data: accessData, encoding: .utf8) else { - return false - } - return writers.contains(key) - } - // MARK: Monitoring - public let name = "caps" + private let capCountMetric = Metric("caps.count") - private let capCountPropertyId = PropertyId(owner: "caps", uniqueId: 2) + private let imageCountMetric = Metric("caps.images") - private let imageCountPropertyId = PropertyId(owner: "caps", uniqueId: 3) - - private let classifierVersionPropertyId = PropertyId(owner: "caps", uniqueId: 4) - - func registerProperties(with monitor: PropertyManager) { - let capCountProperty = PropertyRegistration( - uniqueId: capCountPropertyId.uniqueId, - name: "caps", - updates: .continuous, - isLogged: true, - allowsManualUpdate: false, - read: { [weak self] in - return (self?.capCount ?? 0).timestamped() - }) - monitor.register(capCountProperty, for: self) - - let imageCountProperty = PropertyRegistration( - uniqueId: imageCountPropertyId.uniqueId, - name: "images", - updates: .continuous, - isLogged: true, - allowsManualUpdate: false, - read: { [weak self] in - return (self?.imageCount ?? 0).timestamped() - }) - monitor.register(imageCountProperty, for: self) - - let classifierVersionProperty = PropertyRegistration( - uniqueId: classifierVersionPropertyId.uniqueId, - name: "classifier", - updates: .continuous, - isLogged: true, - allowsManualUpdate: false, - read: { [weak self] in - return (self?.classifierVersion ?? 0).timestamped() - }) - monitor.register(classifierVersionProperty, for: self) - } - - private func updateMonitoredPropertiesOnCapChange() { - try? monitor.logChanged(property: capCountPropertyId, value: capCount.timestamped()) - try? monitor.logChanged(property: imageCountPropertyId, value: imageCount.timestamped()) - } - - private func updateMonitoredClassifierVersionProperty() { - try? monitor.logChanged(property: classifierVersionPropertyId, value: classifierVersion.timestamped()) - } + private let classifierMetric = Metric("caps.classifier") } diff --git a/Sources/App/Error.swift b/Sources/App/Error.swift index 55ad49c..f3de109 100644 --- a/Sources/App/Error.swift +++ b/Sources/App/Error.swift @@ -45,6 +45,13 @@ enum CapError: Error { HTTP Code: 406 */ case invalidData + + /** + The server failed to initialize the data and is not operational + + HTTP Code: 204 + */ + case serviceUnavailable var response: HTTPResponseStatus { switch self { @@ -60,6 +67,7 @@ enum CapError: Error { case .invalidFile: return .preconditionFailed /// 500 case .invalidConfiguration: return .internalServerError + case .serviceUnavailable: return .noContent } } } diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 89957cf..78f2dd9 100755 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -2,24 +2,27 @@ import Vapor import Foundation import Clairvoyant - -private(set) var server: CapServer! -private(set) var monitor: PropertyManager! - public func configure(_ app: Application) throws { let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory) let publicDirectory = app.directory.publicDirectory let config = Config(loadFrom: resourceDirectory) + let authenticator = Authenticator(writers: config.writers) - server = CapServer(in: URL(fileURLWithPath: publicDirectory), - writers: config.writers) + let monitor = MetricObserver( + logFolder: config.logURL, + authenticator: authenticator, + logMetricId: "caps.log") - monitor = .init(logFolder: config.logURL, serverOwner: server) - monitor.update(status: .initializing) + // All new metrics are automatically registered with the standard observer + MetricObserver.standard = monitor + + let status = Metric("caps.status") + status.update(.initializing) + + let server = CapServer(in: URL(fileURLWithPath: publicDirectory)) - server.registerProperties(with: monitor) monitor.registerRoutes(app) app.http.server.configuration.port = config.port @@ -30,20 +33,19 @@ public func configure(_ app: Application) throws { app.middleware.use(middleware) } + // Register routes to the router + server.registerRoutes(with: app, authenticator: authenticator) + + // Initialize the server data do { try server.loadData() + status.update(.nominal) } catch { - monitor.update(status: .initializationFailure) - return + status.update(.initializationFailure) } - - // Register routes to the router - routes(app) - - monitor.update(status: .nominal) } func log(_ message: String) { - monitor.log(message) + MetricObserver.standard?.log(message) print(message) } diff --git a/Sources/App/routes.swift b/Sources/App/routes.swift deleted file mode 100755 index 00a52a5..0000000 --- a/Sources/App/routes.swift +++ /dev/null @@ -1,133 +0,0 @@ -import Vapor -import Foundation - -/// The decoder to extract caps from JSON payloads given to the `cap` route. -private let decoder = JSONDecoder() - -/// The date formatter to decode dates in requests -private let dateFormatter: DateFormatter = { - let df = DateFormatter() - df.dateFormat = "yy-MM-dd-HH-mm-ss" - return df -}() - -private func authorize(_ request: Request) throws { - guard let key = request.headers.first(name: "key") else { - throw Abort(.badRequest) // 400 - } - guard server.hasAuthorization(for: key) else { - throw Abort(.forbidden) // 403 - } -} - -func routes(_ app: Application) { - - app.get("version") { _ in - "\(server.classifierVersion)" - } - - // Add or change a cap - app.postCatching("cap") { request in - try authorize(request) - let data = try request.getBodyData() - let cap = try decoder.decode(Cap.self, from: data) - try server.addOrUpdate(cap) - } - - // Upload an image - app.postCatching("images", ":n") { request in - try authorize(request) - guard let cap = request.parameters.get("n", as: Int.self) else { - log("Invalid parameter for cap") - throw Abort(.badRequest) - } - let data = try request.getBodyData() - try server.save(image: data, for: cap) - } - - // Update the classifier - app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in - try authorize(request) - guard let version = request.parameters.get("version", as: Int.self) else { - log("Invalid parameter for version") - throw Abort(.badRequest) - } - guard version > server.classifierVersion else { - throw Abort(.alreadyReported) // 208 - } - let data = try request.getBodyData() - try server.save(classifier: data, version: version) - return .ok - } - - // Update the trained classes - app.postCatching("classes", ":date") { request in - guard let dateString = request.parameters.get("date") else { - log("Invalid parameter for date") - throw Abort(.badRequest) - } - guard let date = dateFormatter.date(from: dateString) else { - log("Invalid date specification") - throw Abort(.badRequest) - } - - try authorize(request) - let body = try request.getStringBody() - - server.updateTrainedClasses(content: body) - server.removeAllEntriesInImageChangeList(before: date) - } - - // Get the list of missing thumbnails - app.get("thumbnails", "missing") { request in - let missingThumbnails = server.getListOfMissingThumbnails() - return missingThumbnails.map(String.init).joined(separator: ",") - } - - // Upload the thumbnail of a cap - app.postCatching("thumbnails", ":cap") { request in - guard let cap = request.parameters.get("cap", as: Int.self) else { - log("Invalid cap parameter for thumbnail upload") - throw Abort(.badRequest) - } - try authorize(request) - let data = try request.getBodyData() - server.saveThumbnail(data, for: cap) - } - - // Delete the image of a cap - app.postCatching("delete", ":cap", ":version") { request in - guard let cap = request.parameters.get("cap", as: Int.self) else { - log("Invalid cap parameter for image deletion") - throw Abort(.badRequest) - } - try authorize(request) - guard let version = request.parameters.get("version", as: Int.self) else { - log("Invalid version parameter for image deletion") - throw Abort(.badRequest) - } - guard server.deleteImage(version: version, for: cap) else { - throw Abort(.gone) - } - } -} - -private extension Request { - - func getBodyData() throws -> Data { - guard let buffer = body.data else { - log("Missing body data") - throw CapError.invalidBody - } - return Data(buffer: buffer) - } - - func getStringBody() throws -> String { - let data = try getBodyData() - guard let content = String(data: data, encoding: .utf8) else { - log("Invalid string body") - throw CapError.invalidBody - } - return content - } -}