Improve error handling and printing

This commit is contained in:
Christoph Hagen 2023-10-23 16:49:56 +02:00
parent d654699b52
commit 874688b43f
4 changed files with 262 additions and 177 deletions

View File

@ -18,6 +18,9 @@ final class ClassifierCreator {
let df = DateFormatter()
private func print(info: String) {
Swift.print("[INFO] " + info)
}
// MARK: Step 1: Load configuration
@ -39,40 +42,42 @@ final class ClassifierCreator {
let (classes, changedImageCount, changedMainImages) = try await loadImages()
guard !classes.isEmpty else {
print("[INFO] No image classes found, exiting...")
print(info: "No image classes found, exiting...")
return
}
guard changedImageCount > 0 else {
print("[INFO] No changed images, so no new classifier trained")
await createThumbnails(changed: changedMainImages)
print("[INFO] Done")
print(info: "No changed images, so no new classifier trained")
try await createThumbnails(changed: changedMainImages)
print(info: "Done")
return
}
let classifierVersion = try await getClassifierVersion()
let newVersion = classifierVersion + 1
print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
print("[INFO] Model path: \(classifierUrl.path)")
print("[INFO] Version: \(newVersion)")
print("[INFO] Classes: \(classes.count)")
print("[INFO] Iterations: \(configuration.trainingIterations)")
print(info: "Image directory: \(imageDirectory.absoluteURL.path)")
print(info: "Model path: \(classifierUrl.path)")
print(info: "Version: \(newVersion)")
print(info: "Classes: \(classes.count)")
print(info: "Iterations: \(configuration.trainingIterations)")
try await trainAndSaveModel()
try await uploadModel(version: newVersion)
try await upload(classes: classes, lastUpdate: imagesSnapshotDate)
await createThumbnails(changed: changedMainImages)
print("[INFO] Done")
try await createThumbnails(changed: changedMainImages)
print(info: "Done")
}
// MARK: Step 2: Load changed images
func loadImages() async throws -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int]) {
guard createFolderIfMissing(imageDirectory) else {
throw TrainingError.mainImageFolderNotCreated
do {
try createFolderIfMissing(imageDirectory)
} catch {
throw TrainingError.mainImageFolderNotCreated(error)
}
let imageCounts = await getImageCounts()
let imageCounts = try await getImageCounts()
let missingImageList: [CapImage] = imageCounts
.sorted { $0.key < $1.key }
.reduce(into: []) { list, pair in
@ -87,9 +92,9 @@ final class ClassifierCreator {
list.append(contentsOf: missingImagesForCap)
}
if missingImageList.isEmpty {
print("[INFO] No missing images to load")
print(info: "No missing images to load")
} else {
print("[INFO] Loading \(missingImageList.count) missing images...")
print(info: "Loading \(missingImageList.count) missing images...")
try await loadImages(missingImageList)
}
@ -102,9 +107,9 @@ final class ClassifierCreator {
let imagesAlreadyLoad = changedImageList.count - filteredChangeList.count
let suffix = imagesAlreadyLoad > 0 ? " (\(imagesAlreadyLoad) already loaded)" : ""
if filteredChangeList.isEmpty {
print("[INFO] No changed images to load" + suffix)
print(info: "No changed images to load" + suffix)
} else {
print("[INFO] Loading \(filteredChangeList.count) changed images" + suffix)
print(info: "Loading \(filteredChangeList.count) changed images" + suffix)
try await loadImages(filteredChangeList)
}
@ -117,16 +122,19 @@ final class ClassifierCreator {
return (classes, missingImageList.count + changedImageList.count, changedMainImages)
}
private func getImageCounts() async -> [Int : Int] {
guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
return [:]
private func getImageCounts() async throws -> [Int : Int] {
let data: Data
do {
data = try await get(server.appendingPathComponent("caps.json"))
} catch {
throw TrainingError.failedToGetCapDatabase(error)
}
do {
return try JSONDecoder().decode([Cap].self, from: data)
return try JSONDecoder()
.decode([Cap].self, from: data)
.reduce(into: [:]) { $0[$1.id] = $1.count }
} catch {
print("[ERROR] Failed to decode cap database: \(error)")
return [:]
throw TrainingError.failedToDecodeCapDatabase(error)
}
}
@ -136,8 +144,7 @@ final class ClassifierCreator {
do {
folders = try FileManager.default.contentsOfDirectory(atPath: imageDirectory.path)
} catch {
print("[ERROR] Failed to get list of image folders: \(error)")
throw TrainingError.failedToGetListOfImageFolders
throw TrainingError.failedToGetListOfImageFolders(error)
}
for folder in folders {
if validNames.contains(folder) {
@ -148,10 +155,9 @@ final class ClassifierCreator {
let url = imageDirectory.appendingPathComponent(folder)
do {
try FileManager.default.removeItem(at: url)
print("[INFO] Removed unused image folder '\(folder)'")
print(info: "Removed unused image folder '\(folder)'")
} catch {
print("[ERROR] Failed to delete unused image folder \(folder): \(error)")
throw TrainingError.failedToRemoveImageFolder
throw TrainingError.failedToRemoveImageFolder(folder, error)
}
}
}
@ -160,22 +166,22 @@ final class ClassifierCreator {
base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", image.cap, image.cap, image.image))
}
private func load(image: CapImage) async -> Bool {
guard createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", image.cap))) else {
return false
private func load(image: CapImage) async throws {
do {
try createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", image.cap)))
} catch {
throw TrainingError.failedToCreateImageFolder(image.cap, error)
}
let url = imageUrl(base: server.appendingPathComponent("images"), image: image)
let tempFile: URL, response: URLResponse
do {
(tempFile, response) = try await URLSession.shared.download(from: url)
} catch {
print("[ERROR] Failed to load \(image): \(error)")
return false
throw TrainingError.failedToLoadImage(image, error)
}
let responseCode = (response as! HTTPURLResponse).statusCode
guard responseCode == 200 else {
print("[ERROR] Failed to load \(image): Response \(responseCode)")
return false
throw TrainingError.invalidImageRequestResponse(image, responseCode)
}
do {
let localUrl = imageUrl(base: imageDirectory, image: image)
@ -183,60 +189,62 @@ final class ClassifierCreator {
try FileManager.default.removeItem(at: localUrl)
}
try FileManager.default.moveItem(at: tempFile, to: localUrl)
return true
} catch {
print("[ERROR] Failed to save \(image): \(error)")
return false
throw TrainingError.failedToSaveImage(image, error)
}
}
private func loadImages(_ list: [CapImage]) async throws {
var loadedImages = 0
await withTaskGroup(of: Bool.self) { group in
var loadedImageCount = 0
var errors = [Error]()
await withTaskGroup(of: Error?.self) { group in
for image in list {
group.addTask {
await self.load(image: image)
do {
try await self.load(image: image)
return nil
} catch {
return error
}
}
}
for await loaded in group {
if loaded {
loadedImages += 1
for await error in group {
if let error {
errors.append(error)
} else {
loadedImageCount += 1
}
}
}
if loadedImages != list.count {
print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
throw TrainingError.failedToLoadImages
for error in errors {
Swift.print(error.localizedDescription)
}
let expectedCount = list.count
if loadedImageCount != expectedCount {
throw TrainingError.failedToLoadImages(expected: list.count, loaded: loadedImageCount)
}
}
func getChangedImageList() async throws -> [CapImage] {
guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
print("[ERROR] Failed to get list of changed images")
throw TrainingError.failedToGetChangedImagesList
let string: String
do {
string = try await get(server.appendingPathComponent("changes.txt"))
} catch {
throw TrainingError.failedToGetChangedImagesList(error)
}
return string
return try string
.components(separatedBy: "\n")
.map { $0.trimmingCharacters(in: .whitespaces) }
.filter { $0 != "" }
.compactMap {
let parts = $0.components(separatedBy: ":")
guard parts.count == 3 else {
return nil
}
/*
guard let date = df.date(from: parts[0]) else {
print("[WARN] Invalid date \(parts[0]) in change list")
return nil
}
*/
guard let cap = Int(parts[1]) else {
print("[WARN] Invalid cap id \(parts[1]) in change list")
return nil
}
guard let image = Int(parts[2]) else {
print("[WARN] Invalid image id \(parts[2]) in change list")
return nil
guard parts.count == 3,
let _ = df.date(from: parts[0]),
let cap = Int(parts[1]),
let image = Int(parts[2]) else {
throw TrainingError.invalidEntryInChangeList($0)
}
return CapImage(cap: cap, image: image)
}
@ -245,12 +253,13 @@ final class ClassifierCreator {
// MARK: Step 3: Compute version
func getClassifierVersion() async throws -> Int {
guard let string: String = await get(server.appendingPathComponent("version")) else {
print("[ERROR] Failed to get classifier version")
throw TrainingError.failedToGetClassifierVersion
let string: String
do {
string = try await get(server.appendingPathComponent("version"))
} catch {
throw TrainingError.failedToGetClassifierVersion(error)
}
guard let version = Int(string) else {
print("[ERROR] Invalid classifier version \(string)")
throw TrainingError.invalidClassifierVersion(string)
}
return version
@ -284,13 +293,13 @@ final class ClassifierCreator {
parameters: params,
sessionParameters: sessionParameters)
} catch {
throw TrainingError.failedToCreateClassifier
throw TrainingError.failedToCreateClassifier("\(error)")
}
job.progress
.publisher(for: \.fractionCompleted)
.sink { completed in
print(String(format: " %.1f %% completed", completed * 100), terminator: "\r")
Swift.print(String(format: " %.1f %% completed", completed * 100), terminator: "\r")
fflush(stdout)
//guard let progress = MLProgress(progress: job.progress) else {
@ -305,11 +314,10 @@ final class ClassifierCreator {
return try await withCheckedThrowingContinuation { continuation in
// Register a sink to receive the resulting model.
job.result.sink { result in
print("[ERROR] \(result)")
continuation.resume(throwing: TrainingError.failedToCreateClassifier)
} receiveValue: { model in
continuation.resume(throwing: TrainingError.failedToCreateClassifier("\(result)"))
} receiveValue: { [weak self] model in
// Use model
print("[INFO] Created model")
self?.print(info: "Created model")
continuation.resume(returning: model)
}
.store(in: &subscriptions)
@ -326,18 +334,16 @@ final class ClassifierCreator {
trainingData: .labeledDirectories(at: imageDirectory),
parameters: params)
} catch {
print("[ERROR] Failed to create classifier: \(error)")
throw TrainingError.failedToCreateClassifier
throw TrainingError.failedToCreateClassifier("\(error)")
}
}
private func save(model: MLImageClassifier) throws {
print("[INFO] Saving classifier...")
print(info: "Saving classifier...")
do {
try model.write(to: classifierUrl)
} catch {
print("[ERROR] Failed to save model to file: \(error)")
throw TrainingError.failedToWriteClassifier(error)
}
}
@ -345,83 +351,81 @@ final class ClassifierCreator {
// MARK: Step 5: Upload classifier
func uploadModel(version: Int) async throws {
print("[INFO] Uploading classifier...")
print(info: "Uploading classifier...")
let modelData: Data
do {
modelData = try Data(contentsOf: classifierUrl)
} catch {
print("[ERROR] Failed to read classifier data: \(error)")
throw TrainingError.failedToReadClassifierData(error)
}
let success = await post(
url: server.appendingPathComponent("classifier/\(version)"),
body: modelData)
guard success else {
throw TrainingError.failedToUploadClassifier
let url = server.appendingPathComponent("classifier/\(version)")
do {
try await post(url: url, body: modelData)
} catch {
throw TrainingError.failedToUploadClassifier(error)
}
}
// MARK: Step 6: Update classes
func upload(classes: [Int], lastUpdate: Date) async throws {
print("[INFO] Uploading trained classes...")
print(info: "Uploading trained classes...")
let dateString = df.string(from: lastUpdate)
let url = server.appendingPathComponent("classes/\(dateString)")
let body = classes.map(String.init).joined(separator: ",").data(using: .utf8)!
guard await post(url: url, body: body) else {
throw TrainingError.failedToUploadClassifierClasses
do {
try await post(url: url, body: body)
} catch {
throw TrainingError.failedToUploadClassifierClasses(error)
}
}
// MARK: Step 7: Create thumbnails
func createThumbnails(changed: [Int]) async {
guard checkMagickAvailability() else {
return
func createThumbnails(changed: [Int]) async throws {
try ensureMagickAvailability()
do {
try createFolderIfMissing(thumbnailDirectory)
} catch {
throw TrainingError.failedToCreateThumbnailFolder(error)
}
guard createFolderIfMissing(thumbnailDirectory) else {
print("[ERROR] Failed to create folder for thumbnails")
return
}
let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
let capIdsOfMissingThumbnails = try await getMissingThumbnailIds()
let all = Set(capIdsOfMissingThumbnails).union(changed)
print("[INFO] Creating \(all.count) thumbnails...")
print(info: "Creating \(all.count) thumbnails...")
for cap in all {
await createThumbnail(for: cap)
try await createThumbnail(for: cap)
}
}
func checkMagickAvailability() -> Bool {
func ensureMagickAvailability() throws {
do {
let (code, output) = try safeShell("magick --version")
guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
.components(separatedBy: " ").first else {
print("[ERROR] Magick not found, install using 'brew install imagemagick'")
return false
throw TrainingError.magickDependencyNotFound
}
print("[INFO] Using magick \(version)")
print(info: "Using magick \(version)")
} catch {
print("[ERROR] Failed to get version of magick: (\(error))")
return false
throw TrainingError.magickDependencyCheckFailed(error)
}
return true
}
private func getMissingThumbnailIds() async -> [Int] {
guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
print("[ERROR] Failed to get missing thumbnails")
return []
private func getMissingThumbnailIds() async throws -> [Int] {
let string: String
do {
string = try await get(server.appendingPathComponent("thumbnails/missing"))
} catch {
throw TrainingError.failedToGetMissingThumbnailIds(error)
}
return string.components(separatedBy: ",").compactMap(Int.init)
}
private func createThumbnail(for cap: Int) async {
private func createThumbnail(for cap: Int) async throws {
let mainImage = CapImage(cap: cap, image: 0)
let inputUrl = imageUrl(base: imageDirectory, image: mainImage)
guard FileManager.default.fileExists(atPath: inputUrl.path) else {
print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
return
throw TrainingError.missingMainImage(cap)
}
let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
@ -429,24 +433,22 @@ final class ClassifierCreator {
let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
let (code, output) = try safeShell(command)
if code != 0 {
print("Failed to create thumbnail for cap \(cap): \(output)")
return
throw TrainingError.failedToCreateThumbnail(cap, output)
}
} catch {
print("Failed to read created thumbnail for cap \(cap): \(error)")
return
throw TrainingError.failedToCreateThumbnail(cap, "\(error)")
}
let data: Data
do {
data = try Data(contentsOf: output)
} catch {
print("Failed to read created thumbnail for cap \(cap): \(error)")
return
throw TrainingError.failedToReadCreatedThumbnail(cap, error)
}
guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
print("Failed to upload thumbnail for cap \(cap)")
return
do {
try await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data)
} catch {
throw TrainingError.failedToUploadCreatedThumbnail(cap, error)
}
}
@ -472,57 +474,41 @@ final class ClassifierCreator {
}
private func createFolderIfMissing(_ folder: URL) -> Bool {
private func createFolderIfMissing(_ folder: URL) throws {
guard !FileManager.default.fileExists(atPath: folder.path) else {
return true
}
do {
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
return true
} catch {
print("[ERROR] Failed to create directory \(folder.path): \(error)")
return false
return
}
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
}
// MARK: Requests
private func post(url: URL, body: Data) async -> Bool {
private func post(url: URL, body: Data) async throws {
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.httpBody = body
request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
return await perform(request) != nil
_ = try await perform(request)
}
private func perform(_ request: URLRequest) async -> Data? {
let data: Data
let response: URLResponse
do {
(data, response) = try await URLSession.shared.data(for: request)
} catch {
print("[ERROR] Request to \(request.url!.absoluteString) failed: \(error)")
return nil
}
private func perform(_ request: URLRequest) async throws -> Data {
let (data, response) = try await URLSession.shared.data(for: request)
let code = (response as! HTTPURLResponse).statusCode
guard code == 200 else {
print("[ERROR] Request to \(request.url!.absoluteString): Invalid response \(code)")
return nil
throw TrainingError.invalidResponse(request.url!, code)
}
return data
}
private func get(_ url: URL) async -> Data? {
await perform(URLRequest(url: url))
private func get(_ url: URL) async throws -> Data {
try await perform(URLRequest(url: url))
}
private func get(_ url: URL) async -> String? {
guard let data: Data = await get(url) else {
return nil
}
private func get(_ url: URL) async throws -> String {
let data: Data = try await get(url)
guard let string = String(data: data, encoding: .utf8) else {
print("[ERROR] Invalid string response \(data)")
return nil
throw TrainingError.invalidGetResponseData(data.count)
}
return string
}

View File

@ -15,8 +15,7 @@ extension Configuration {
func serverUrl() throws -> URL {
guard let serverUrl = URL(string: serverPath) else {
print("[ERROR] Configuration: Invalid server path \(serverPath)")
throw TrainingError.invalidServerPath
throw TrainingError.invalidServerPath(serverPath)
}
return serverUrl
}

View File

@ -19,21 +19,18 @@ extension ConfigurationFile {
init(at url: URL) throws {
guard FileManager.default.fileExists(atPath: url.path) else {
print("[ERROR] No configuration at \(url.absoluteURL.path)")
throw TrainingError.configurationFileMissing
throw TrainingError.configurationFileMissing(url)
}
let data: Data
do {
data = try Data(contentsOf: url)
} catch {
print("[ERROR] Failed to load configuration data at \(url.absoluteURL.path): \(error)")
throw TrainingError.configurationFileUnreadable
throw TrainingError.configurationFileUnreadable(url, error)
}
do {
self = try JSONDecoder().decode(ConfigurationFile.self, from: data)
} catch {
print("[ERROR] Failed to decode configuration at \(url.absoluteURL.path): \(error)")
throw TrainingError.configurationFileDecodingFailed
throw TrainingError.configurationFileDecodingFailed(url, error)
}
}
}

View File

@ -1,29 +1,132 @@
import Foundation
enum TrainingError: Error {
case invalidGetResponseData(Int)
case invalidResponse(URL, Int)
case missingArguments
case configurationFileMissing
case configurationFileUnreadable
case configurationFileDecodingFailed
case invalidServerPath
case mainImageFolderNotCreated
case failedToLoadImages
case configurationFileMissing(URL)
case configurationFileUnreadable(URL, Error)
case configurationFileDecodingFailed(URL, Error)
case invalidServerPath(String)
case mainImageFolderNotCreated(Error)
case failedToGetListOfImageFolders
case failedToRemoveImageFolder
case failedToGetCapDatabase(Error)
case failedToDecodeCapDatabase(Error)
case failedToGetClassifierVersion
case failedToLoadImage(CapImage, Error)
case failedToCreateImageFolder(Int, Error)
case invalidImageRequestResponse(CapImage, Int)
case failedToSaveImage(CapImage, Error)
case failedToLoadImages(expected: Int, loaded: Int)
case failedToGetChangedImagesList(Error)
case invalidEntryInChangeList(String)
case failedToGetListOfImageFolders(Error)
case failedToRemoveImageFolder(String, Error)
case failedToGetClassifierVersion(Error)
case invalidClassifierVersion(String)
case failedToCreateClassifier
case failedToCreateClassifier(String)
case failedToWriteClassifier(Error)
case failedToGetChangedImagesList
case failedToReadClassifierData(Error)
case failedToUploadClassifier
case failedToUploadClassifier(Error)
case failedToUploadClassifierClasses
case failedToUploadClassifierClasses(Error)
case magickDependencyCheckFailed(Error)
case magickDependencyNotFound
case failedToCreateThumbnailFolder(Error)
case failedToGetMissingThumbnailIds(Error)
case missingMainImage(Int)
case failedToCreateThumbnail(Int, String)
case failedToReadCreatedThumbnail(Int, Error)
case failedToUploadCreatedThumbnail(Int, Error)
}
extension TrainingError: CustomStringConvertible {
var description: String {
switch self {
case .missingArguments:
return "Missing arguments"
case .configurationFileMissing(let url):
return "No configuration at \(url.absoluteURL.path)"
case .configurationFileUnreadable(let url, let error):
return "Failed to load configuration data at \(url.absoluteURL.path): \(error)"
case .configurationFileDecodingFailed(let url, let error):
return "Failed to decode configuration at \(url.absoluteURL.path): \(error)"
case .invalidServerPath(let path):
return "Configuration: Invalid server path \(path)"
case .mainImageFolderNotCreated(let error):
return "Failed to create main image folder: \(error)"
case .failedToGetCapDatabase(let error):
return "Failed to get cap database from server: \(error)"
case .failedToDecodeCapDatabase(let error):
return "Failed to decode cap database: \(error)"
case .failedToGetListOfImageFolders(let error):
return "Failed to get list of image folders: \(error)"
case .failedToRemoveImageFolder(let folder, let error):
return "Failed to delete unused image folder \(folder): \(error)"
case .failedToCreateImageFolder(let cap, let error):
return "Failed to create image folder for cap \(cap): \(error)"
case .failedToLoadImage(let image, let error):
return "Failed to load \(image): \(error)"
case .invalidImageRequestResponse(let image, let responseCode):
return "Failed to load \(image): Response \(responseCode)"
case .failedToSaveImage(let image, let error):
return "Failed to save \(image): \(error)"
case .failedToLoadImages(let expected, let loaded):
return "Only \(expected) of \(loaded) images loaded"
case .failedToGetChangedImagesList(let error):
return "Failed to get list of changed images: \(error)"
case .invalidEntryInChangeList(let entry):
return "Invalid change list entry: '\(entry)'"
case .failedToGetClassifierVersion(let error):
return "Failed to get classifier version: \(error)"
case .invalidClassifierVersion(let string):
return "Invalid classifier version \(string)"
case .failedToCreateClassifier(let result):
return "Failed to create classifier: \(result)"
case .failedToWriteClassifier(let error):
return "Failed to save model to file: \(error)"
case .failedToReadClassifierData(let error):
return "Failed to read classifier data: \(error)"
case .failedToUploadClassifier(let error):
return "Failed to upload classifier: \(error)"
case .failedToUploadClassifierClasses(let error):
return "Failed to upload classifier classes: \(error)"
case .magickDependencyCheckFailed(let error):
return "Failed to get version of magick: (\(error))"
case .magickDependencyNotFound:
return "Magick not found, install using 'brew install imagemagick'"
case .failedToCreateThumbnailFolder(let error):
return "Failed to create folder for thumbnails: \(error)"
case .failedToGetMissingThumbnailIds(let error):
return "Failed to get missing thumbnails ids: \(error)"
case .missingMainImage(let cap):
return "Local main image not found for cap \(cap)"
case .failedToCreateThumbnail(let cap, let message):
return "Failed to read created thumbnail for cap \(cap): \(message)"
case .failedToReadCreatedThumbnail(let cap, let error):
return "Failed to read created thumbnail for cap \(cap): \(error)"
case .failedToUploadCreatedThumbnail(let cap, let error):
return "Failed to upload thumbnail for cap \(cap): \(error)"
case .invalidGetResponseData(let count):
return "Response (\(count) bytes) is not a valid string"
case .invalidResponse(let url, let code):
return "Invalid response \(code) to \(url.absoluteString)"
}
}
}