ch/Caps-Server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update authentication and metrics logging

Browse Source
This commit is contained in:
Christoph Hagen 2023-01-17 22:02:27 +01:00
parent f3ee7a4fb4
commit c5ce5414a9
7 changed files with 220 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Package.swift
View File

@ -8,7 +8,7 @@ let package = Package(
],
dependencies: [
.package(url: "https://github.com/vapor/vapor", from: "4.0.0"),
.package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.2.0"),
.package(url: "https://github.com/christophhagen/Clairvoyant", branch: "main"),
],
targets: [
.target(name: "App",

39
Sources/App/Authenticator.swift Normal file
View File

@ -0,0 +1,39 @@
import Foundation
import Clairvoyant
import Vapor
final class Authenticator: MetricAccessAuthenticator {
private var writers: Set<String>
init(writers: [String]) {
self.writers = Set(writers)
}
func hasAuthorization(for key: String) -> Bool {
// Note: This is not a constant-time compare, so there may be an opportunity
// for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
// and we're not doing anything critical in this application.
// Worst case, an unauthorized person with a lot of free time and energy to hack this system
// is able to change contents of the database, which are backed up in any case.
writers.contains(key)
}
func metricAccess(isAllowedForToken accessToken: Data) -> Bool {
guard let key = String(data: accessToken, encoding: .utf8) else {
return false
}
return hasAuthorization(for: key)
}
func authorize(_ request: Request) throws {
guard let key = request.headers.first(name: "key") else {
throw Abort(.badRequest) // 400
}
guard hasAuthorization(for: key) else {
throw Abort(.forbidden) // 403
}
}
}

141
Sources/App/CapServer+Routes.swift Executable file
View File

@ -0,0 +1,141 @@
import Vapor
import Foundation
/// The decoder to extract caps from JSON payloads given to the `cap` route.
private let decoder = JSONDecoder()
/// The date formatter to decode dates in requests
private let dateFormatter: DateFormatter = {
let df = DateFormatter()
df.dateFormat = "yy-MM-dd-HH-mm-ss"
return df
}()
extension CapServer {
private func ensureOperability() throws {
guard isOperational else {
throw Abort(.noContent)
}
}
func registerRoutes(with app: Application, authenticator: Authenticator) {
app.get("version") { _ in
try self.ensureOperability()
return "\(self.classifierVersion)"
}
// Add or change a cap
app.postCatching("cap") { request in
try self.ensureOperability()
try authenticator.authorize(request)
let data = try request.getBodyData()
let cap = try decoder.decode(Cap.self, from: data)
try self.addOrUpdate(cap)
}
// Upload an image
app.postCatching("images", ":n") { request in
try self.ensureOperability()
try authenticator.authorize(request)
guard let cap = request.parameters.get("n", as: Int.self) else {
log("Invalid parameter for cap")
throw Abort(.badRequest)
}
let data = try request.getBodyData()
try self.save(image: data, for: cap)
}
// Update the classifier
app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in
guard let version = request.parameters.get("version", as: Int.self) else {
log("Invalid parameter for version")
throw Abort(.badRequest)
}
guard version > self.classifierVersion else {
throw Abort(.alreadyReported) // 208
}
try self.ensureOperability()
try authenticator.authorize(request)
let data = try request.getBodyData()
try self.save(classifier: data, version: version)
return .ok
}
// Update the trained classes
app.postCatching("classes", ":date") { request in
guard let dateString = request.parameters.get("date") else {
log("Invalid parameter for date")
throw Abort(.badRequest)
}
guard let date = dateFormatter.date(from: dateString) else {
log("Invalid date specification")
throw Abort(.badRequest)
}
try self.ensureOperability()
try authenticator.authorize(request)
let body = try request.getStringBody()
self.updateTrainedClasses(content: body)
self.removeAllEntriesInImageChangeList(before: date)
}
// Get the list of missing thumbnails
app.get("thumbnails", "missing") { request in
try self.ensureOperability()
let missingThumbnails = self.getListOfMissingThumbnails()
return missingThumbnails.map(String.init).joined(separator: ",")
}
// Upload the thumbnail of a cap
app.postCatching("thumbnails", ":cap") { request in
guard let cap = request.parameters.get("cap", as: Int.self) else {
log("Invalid cap parameter for thumbnail upload")
throw Abort(.badRequest)
}
try self.ensureOperability()
try authenticator.authorize(request)
let data = try request.getBodyData()
self.saveThumbnail(data, for: cap)
}
// Delete the image of a cap
app.postCatching("delete", ":cap", ":version") { request in
guard let cap = request.parameters.get("cap", as: Int.self) else {
log("Invalid cap parameter for image deletion")
throw Abort(.badRequest)
}
guard let version = request.parameters.get("version", as: Int.self) else {
log("Invalid version parameter for image deletion")
throw Abort(.badRequest)
}
try self.ensureOperability()
try authenticator.authorize(request)
guard self.deleteImage(version: version, for: cap) else {
throw Abort(.gone)
}
}
}
}
private extension Request {
func getBodyData() throws -> Data {
guard let buffer = body.data else {
log("Missing body data")
throw CapError.invalidBody
}
return Data(buffer: buffer)
}
func getStringBody() throws -> String {
let data = try getBodyData()
guard let content = String(data: data, encoding: .utf8) else {
log("Invalid string body")
throw CapError.invalidBody
}
return content
}
}

103
Sources/App/CapServer.swift
View File

@ -2,7 +2,7 @@ import Foundation
import Vapor
import Clairvoyant
final class CapServer: ServerOwner {
final class CapServer {
// MARK: Paths
@ -29,9 +29,11 @@ final class CapServer: ServerOwner {
private let changedImageEntryDateFormatter: DateFormatter
/// Indicates that the data is loaded
private(set) var isOperational = false
// MARK: Caps
private var writers: Set<String>
/// The changed images not yet written to disk
private var unwrittenImageChanges: [(cap: Int, image: Int)] = []
@ -39,7 +41,7 @@ final class CapServer: ServerOwner {
var classifierVersion: Int = 0 {
didSet {
writeClassifierVersion()
updateMonitoredClassifierVersionProperty()
classifierMetric.update(classifierVersion)
}
}
@ -60,7 +62,8 @@ final class CapServer: ServerOwner {
private var caps = [Int: Cap]() {
didSet {
scheduleSave()
updateMonitoredPropertiesOnCapChange()
capCountMetric.update(caps.count)
imageCountMetric.update(imageCount)
}
}
@ -76,7 +79,7 @@ final class CapServer: ServerOwner {
caps.reduce(0) { $0 + $1.value.count }
}
init(in folder: URL, writers: [String]) {
init(in folder: URL) {
self.imageFolder = folder.appendingPathComponent("images")
self.thumbnailFolder = folder.appendingPathComponent("thumbnails")
self.gridCountFile = folder.appendingPathComponent("count.js")
@ -85,7 +88,6 @@ final class CapServer: ServerOwner {
self.classifierVersionFile = folder.appendingPathComponent("classifier.version")
self.classifierFile = folder.appendingPathComponent("classifier.mlmodel")
self.changedImagesFile = folder.appendingPathComponent("changes.txt")
self.writers = Set(writers)
self.changedImageEntryDateFormatter = DateFormatter()
changedImageEntryDateFormatter.dateFormat = "yy-MM-dd-HH-mm-ss"
}
@ -97,6 +99,7 @@ final class CapServer: ServerOwner {
updateGridCapCount()
try ensureExistenceOfChangedImagesFile()
organizeImages()
isOperational = true
}
private func loadClassifierVersion(at url: URL) {
@ -234,17 +237,6 @@ final class CapServer: ServerOwner {
folder(of: cap).appendingPathComponent(String(format: "%04d-%02d.jpg", cap, version))
}
// MARK: Authentication
func hasAuthorization(for key: String) -> Bool {
// Note: This is not a constant-time compare, so there may be an opportunity
// for timing attack here. Sets perform hashed lookups, so this may be less of an issue,
// and we're not doing anything critical in this application.
// Worst case, an unauthorized person with a lot of free time and energy to hack this system
// is able to change contents of the database, which are backed up in any case.
writers.contains(key)
}
// MARK: Counts
private func images(in folder: URL) throws -> [URL] {
@ -466,82 +458,11 @@ final class CapServer: ServerOwner {
}
}
// MARK: ServerOwner
let authenticationMethod: PropertyAuthenticationMethod = .accessToken
func hasReadPermission(for property: UInt32, accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
func hasWritePermission(for property: UInt32, accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
func hasListAccessPermission(_ accessData: Data) -> Bool {
guard let key = String(data: accessData, encoding: .utf8) else {
return false
}
return writers.contains(key)
}
// MARK: Monitoring
public let name = "caps"
private let capCountMetric = Metric<Int>("caps.count")
private let capCountPropertyId = PropertyId(owner: "caps", uniqueId: 2)
private let imageCountMetric = Metric<Int>("caps.images")
private let imageCountPropertyId = PropertyId(owner: "caps", uniqueId: 3)
private let classifierVersionPropertyId = PropertyId(owner: "caps", uniqueId: 4)
func registerProperties(with monitor: PropertyManager) {
let capCountProperty = PropertyRegistration(
uniqueId: capCountPropertyId.uniqueId,
name: "caps",
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.capCount ?? 0).timestamped()
})
monitor.register(capCountProperty, for: self)
let imageCountProperty = PropertyRegistration(
uniqueId: imageCountPropertyId.uniqueId,
name: "images",
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.imageCount ?? 0).timestamped()
})
monitor.register(imageCountProperty, for: self)
let classifierVersionProperty = PropertyRegistration(
uniqueId: classifierVersionPropertyId.uniqueId,
name: "classifier",
updates: .continuous,
isLogged: true,
allowsManualUpdate: false,
read: { [weak self] in
return (self?.classifierVersion ?? 0).timestamped()
})
monitor.register(classifierVersionProperty, for: self)
}
private func updateMonitoredPropertiesOnCapChange() {
try? monitor.logChanged(property: capCountPropertyId, value: capCount.timestamped())
try? monitor.logChanged(property: imageCountPropertyId, value: imageCount.timestamped())
}
private func updateMonitoredClassifierVersionProperty() {
try? monitor.logChanged(property: classifierVersionPropertyId, value: classifierVersion.timestamped())
}
private let classifierMetric = Metric<Int>("caps.classifier")
}

8
Sources/App/Error.swift
View File

@ -46,6 +46,13 @@ enum CapError: Error {
*/
case invalidData
/**
The server failed to initialize the data and is not operational
HTTP Code: 204
*/
case serviceUnavailable
var response: HTTPResponseStatus {
switch self {
/// 404
@ -60,6 +67,7 @@ enum CapError: Error {
case .invalidFile: return .preconditionFailed
/// 500
case .invalidConfiguration: return .internalServerError
case .serviceUnavailable: return .noContent
}
}
}

36
Sources/App/configure.swift
View File

@ -2,24 +2,27 @@ import Vapor
import Foundation
import Clairvoyant
private(set) var server: CapServer!
private(set) var monitor: PropertyManager!
public func configure(_ app: Application) throws {
let resourceDirectory = URL(fileURLWithPath: app.directory.resourcesDirectory)
let publicDirectory = app.directory.publicDirectory
let config = Config(loadFrom: resourceDirectory)
let authenticator = Authenticator(writers: config.writers)
server = CapServer(in: URL(fileURLWithPath: publicDirectory),
writers: config.writers)
let monitor = MetricObserver(
logFolder: config.logURL,
authenticator: authenticator,
logMetricId: "caps.log")
monitor = .init(logFolder: config.logURL, serverOwner: server)
monitor.update(status: .initializing)
// All new metrics are automatically registered with the standard observer
MetricObserver.standard = monitor
let status = Metric<ServerStatus>("caps.status")
status.update(.initializing)
let server = CapServer(in: URL(fileURLWithPath: publicDirectory))
server.registerProperties(with: monitor)
monitor.registerRoutes(app)
app.http.server.configuration.port = config.port
@ -30,20 +33,19 @@ public func configure(_ app: Application) throws {
app.middleware.use(middleware)
}
// Register routes to the router
server.registerRoutes(with: app, authenticator: authenticator)
// Initialize the server data
do {
try server.loadData()
status.update(.nominal)
} catch {
monitor.update(status: .initializationFailure)
return
status.update(.initializationFailure)
}
// Register routes to the router
routes(app)
monitor.update(status: .nominal)
}
func log(_ message: String) {
monitor.log(message)
MetricObserver.standard?.log(message)
print(message)
}

133
Sources/App/routes.swift
View File

@ -1,133 +0,0 @@
import Vapor
import Foundation
/// The decoder to extract caps from JSON payloads given to the `cap` route.
private let decoder = JSONDecoder()
/// The date formatter to decode dates in requests
private let dateFormatter: DateFormatter = {
let df = DateFormatter()
df.dateFormat = "yy-MM-dd-HH-mm-ss"
return df
}()
private func authorize(_ request: Request) throws {
guard let key = request.headers.first(name: "key") else {
throw Abort(.badRequest) // 400
}
guard server.hasAuthorization(for: key) else {
throw Abort(.forbidden) // 403
}
}
func routes(_ app: Application) {
app.get("version") { _ in
"\(server.classifierVersion)"
}
// Add or change a cap
app.postCatching("cap") { request in
try authorize(request)
let data = try request.getBodyData()
let cap = try decoder.decode(Cap.self, from: data)
try server.addOrUpdate(cap)
}
// Upload an image
app.postCatching("images", ":n") { request in
try authorize(request)
guard let cap = request.parameters.get("n", as: Int.self) else {
log("Invalid parameter for cap")
throw Abort(.badRequest)
}
let data = try request.getBodyData()
try server.save(image: data, for: cap)
}
// Update the classifier
app.on(.POST, "classifier", ":version", body: .collect(maxSize: "50mb")) { request -> HTTPStatus in
try authorize(request)
guard let version = request.parameters.get("version", as: Int.self) else {
log("Invalid parameter for version")
throw Abort(.badRequest)
}
guard version > server.classifierVersion else {
throw Abort(.alreadyReported) // 208
}
let data = try request.getBodyData()
try server.save(classifier: data, version: version)
return .ok
}
// Update the trained classes
app.postCatching("classes", ":date") { request in
guard let dateString = request.parameters.get("date") else {
log("Invalid parameter for date")
throw Abort(.badRequest)
}
guard let date = dateFormatter.date(from: dateString) else {
log("Invalid date specification")
throw Abort(.badRequest)
}
try authorize(request)
let body = try request.getStringBody()
server.updateTrainedClasses(content: body)
server.removeAllEntriesInImageChangeList(before: date)
}
// Get the list of missing thumbnails
app.get("thumbnails", "missing") { request in
let missingThumbnails = server.getListOfMissingThumbnails()
return missingThumbnails.map(String.init).joined(separator: ",")
}
// Upload the thumbnail of a cap
app.postCatching("thumbnails", ":cap") { request in
guard let cap = request.parameters.get("cap", as: Int.self) else {
log("Invalid cap parameter for thumbnail upload")
throw Abort(.badRequest)
}
try authorize(request)
let data = try request.getBodyData()
server.saveThumbnail(data, for: cap)
}
// Delete the image of a cap
app.postCatching("delete", ":cap", ":version") { request in
guard let cap = request.parameters.get("cap", as: Int.self) else {
log("Invalid cap parameter for image deletion")
throw Abort(.badRequest)
}
try authorize(request)
guard let version = request.parameters.get("version", as: Int.self) else {
log("Invalid version parameter for image deletion")
throw Abort(.badRequest)
}
guard server.deleteImage(version: version, for: cap) else {
throw Abort(.gone)
}
}
}
private extension Request {
func getBodyData() throws -> Data {
guard let buffer = body.data else {
log("Missing body data")
throw CapError.invalidBody
}
return Data(buffer: buffer)
}
func getStringBody() throws -> String {
let data = try getBodyData()
guard let content = String(data: data, encoding: .utf8) else {
log("Invalid string body")
throw CapError.invalidBody
}
return content
}
}