diff --git a/Package.swift b/Package.swift index 79fd2cb..ae59f79 100755 --- a/Package.swift +++ b/Package.swift @@ -11,6 +11,7 @@ let package = Package( .package(url: "https://github.com/christophhagen/Clairvoyant", from: "0.9.0"), .package(url: "https://github.com/christophhagen/ClairvoyantVapor", from: "0.2.0"), .package(url: "https://github.com/christophhagen/ClairvoyantBinaryCodable", from: "0.3.0"), + .package(url:"https://github.com/christophhagen/CBORCoding", from: "1.0.0"), ], targets: [ .target(name: "App", @@ -19,6 +20,7 @@ let package = Package( .product(name: "Clairvoyant", package: "Clairvoyant"), .product(name: "ClairvoyantVapor", package: "ClairvoyantVapor"), .product(name: "ClairvoyantBinaryCodable", package: "ClairvoyantBinaryCodable"), + .product(name: "CBORCoding", package: "CBORCoding"), ], swiftSettings: [ // Enable better optimizations when building in Release configuration. Despite the use of diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 93bb7cb..512a783 100755 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -54,3 +54,112 @@ func log(_ message: String) { } observer.log(message) } + +import CBORCoding + +public func migrate(folder: URL) throws { + try migrateMetric("caps.log", containing: String.self, in: folder) + try migrateMetric("caps.status", containing: ServerStatus.self, in: folder) + try migrateMetric("caps.count", containing: Int.self, in: folder) + try migrateMetric("caps.images", containing: Int.self, in: folder) + try migrateMetric("caps.classifier", containing: Int.self, in: folder) +} + +private func migrateMetric(_ id: String, containing type: T.Type, in folder: URL) throws where T: MetricValue { + print("Processing metric \(id)") + let file = id.hashed() + let url = folder.appendingPathComponent(file) + + let files = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil) + .filter { Int($0.lastPathComponent) != nil } + print("Found \(files.count) files for \(id)") + + let all: [Timestamped] = try files.map(readElements(from:)) + .reduce([], +) + .sorted { $0.timestamp < $1.timestamp } + + print("Found \(all.count) items for \(id)") + + try FileManager.default.removeItem(at: url) + + print("Removed log folder") + + // TODO: Write values back to disk + let observer = MetricObserver(logFileFolder: folder, logMetricId: "sesame.migration") + let metric: Metric = observer.addMetric(id: id) + let semaphore = DispatchSemaphore(value: 0) + Task { + try await metric.update(all) + print("Saved all values for metric \(id)") + semaphore.signal() + } + semaphore.wait() + print("Finished metric \(id)") +} + +private func readElements(from url: URL) throws -> [Timestamped] where T: MetricValue { + let data = try Data(contentsOf: url) + let file = url.lastPathComponent + print("File \(file): Loaded \(data.count) bytes") + + + + let decoder = CBORDecoder() + let timestampLength = 9 + let byteCountLength = 2 + + var result: [Timestamped] = [] + var currentIndex = data.startIndex + var skippedValues = 0 + while currentIndex < data.endIndex { + let startIndexOfTimestamp = currentIndex + byteCountLength + guard startIndexOfTimestamp <= data.endIndex else { + print("File \(file): Only \(data.endIndex - currentIndex) bytes, needed \(byteCountLength) for byte count") + throw MetricError.logFileCorrupted + } + guard let byteCount = UInt16(fromData: data[currentIndex..= timestampLength else { + print("File \(file): Only \(byteCount) bytes, needed \(timestampLength) for timestamp") + throw MetricError.logFileCorrupted + } + let timestampData = data[startIndexOfTimestamp.. Data { + Data([UInt8(self >> 8 & 0xFF), UInt8(self & 0xFF)]) + } + + init?(fromData data: T) { + guard data.count == 2 else { + return nil + } + let bytes = Array(data) + self = UInt16(UInt32(bytes[0]) << 8 | UInt32(bytes[1])) + } +} diff --git a/Sources/Run/main.swift b/Sources/Run/main.swift index 05937c6..400ecf7 100755 --- a/Sources/Run/main.swift +++ b/Sources/Run/main.swift @@ -6,6 +6,13 @@ try LoggingSystem.bootstrap(from: &env) let app = Application(env) defer { app.shutdown() } +let storageFolder = URL(fileURLWithPath: app.directory.resourcesDirectory) +let logFolder = storageFolder.appendingPathComponent("logs") +print("Starting migration") +try migrate(folder: logFolder) +print("Finished migration") +/* + private let semaphore = DispatchSemaphore(value: 0) Task { try await configure(app) @@ -13,3 +20,4 @@ Task { } semaphore.wait() try app.run() +*/