Improve error handling and printing
This commit is contained in:
parent
d654699b52
commit
874688b43f
@ -18,6 +18,9 @@ final class ClassifierCreator {
|
|||||||
|
|
||||||
let df = DateFormatter()
|
let df = DateFormatter()
|
||||||
|
|
||||||
|
private func print(info: String) {
|
||||||
|
Swift.print("[INFO] " + info)
|
||||||
|
}
|
||||||
|
|
||||||
// MARK: Step 1: Load configuration
|
// MARK: Step 1: Load configuration
|
||||||
|
|
||||||
@ -39,40 +42,42 @@ final class ClassifierCreator {
|
|||||||
let (classes, changedImageCount, changedMainImages) = try await loadImages()
|
let (classes, changedImageCount, changedMainImages) = try await loadImages()
|
||||||
|
|
||||||
guard !classes.isEmpty else {
|
guard !classes.isEmpty else {
|
||||||
print("[INFO] No image classes found, exiting...")
|
print(info: "No image classes found, exiting...")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
guard changedImageCount > 0 else {
|
guard changedImageCount > 0 else {
|
||||||
print("[INFO] No changed images, so no new classifier trained")
|
print(info: "No changed images, so no new classifier trained")
|
||||||
await createThumbnails(changed: changedMainImages)
|
try await createThumbnails(changed: changedMainImages)
|
||||||
print("[INFO] Done")
|
print(info: "Done")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
let classifierVersion = try await getClassifierVersion()
|
let classifierVersion = try await getClassifierVersion()
|
||||||
let newVersion = classifierVersion + 1
|
let newVersion = classifierVersion + 1
|
||||||
|
|
||||||
print("[INFO] Image directory: \(imageDirectory.absoluteURL.path)")
|
print(info: "Image directory: \(imageDirectory.absoluteURL.path)")
|
||||||
print("[INFO] Model path: \(classifierUrl.path)")
|
print(info: "Model path: \(classifierUrl.path)")
|
||||||
print("[INFO] Version: \(newVersion)")
|
print(info: "Version: \(newVersion)")
|
||||||
print("[INFO] Classes: \(classes.count)")
|
print(info: "Classes: \(classes.count)")
|
||||||
print("[INFO] Iterations: \(configuration.trainingIterations)")
|
print(info: "Iterations: \(configuration.trainingIterations)")
|
||||||
|
|
||||||
try await trainAndSaveModel()
|
try await trainAndSaveModel()
|
||||||
try await uploadModel(version: newVersion)
|
try await uploadModel(version: newVersion)
|
||||||
try await upload(classes: classes, lastUpdate: imagesSnapshotDate)
|
try await upload(classes: classes, lastUpdate: imagesSnapshotDate)
|
||||||
await createThumbnails(changed: changedMainImages)
|
try await createThumbnails(changed: changedMainImages)
|
||||||
print("[INFO] Done")
|
print(info: "Done")
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: Step 2: Load changed images
|
// MARK: Step 2: Load changed images
|
||||||
|
|
||||||
func loadImages() async throws -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int]) {
|
func loadImages() async throws -> (classes: [Int], changedImageCount: Int, changedMainImages: [Int]) {
|
||||||
guard createFolderIfMissing(imageDirectory) else {
|
do {
|
||||||
throw TrainingError.mainImageFolderNotCreated
|
try createFolderIfMissing(imageDirectory)
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.mainImageFolderNotCreated(error)
|
||||||
}
|
}
|
||||||
let imageCounts = await getImageCounts()
|
let imageCounts = try await getImageCounts()
|
||||||
let missingImageList: [CapImage] = imageCounts
|
let missingImageList: [CapImage] = imageCounts
|
||||||
.sorted { $0.key < $1.key }
|
.sorted { $0.key < $1.key }
|
||||||
.reduce(into: []) { list, pair in
|
.reduce(into: []) { list, pair in
|
||||||
@ -87,9 +92,9 @@ final class ClassifierCreator {
|
|||||||
list.append(contentsOf: missingImagesForCap)
|
list.append(contentsOf: missingImagesForCap)
|
||||||
}
|
}
|
||||||
if missingImageList.isEmpty {
|
if missingImageList.isEmpty {
|
||||||
print("[INFO] No missing images to load")
|
print(info: "No missing images to load")
|
||||||
} else {
|
} else {
|
||||||
print("[INFO] Loading \(missingImageList.count) missing images...")
|
print(info: "Loading \(missingImageList.count) missing images...")
|
||||||
try await loadImages(missingImageList)
|
try await loadImages(missingImageList)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -102,9 +107,9 @@ final class ClassifierCreator {
|
|||||||
let imagesAlreadyLoad = changedImageList.count - filteredChangeList.count
|
let imagesAlreadyLoad = changedImageList.count - filteredChangeList.count
|
||||||
let suffix = imagesAlreadyLoad > 0 ? " (\(imagesAlreadyLoad) already loaded)" : ""
|
let suffix = imagesAlreadyLoad > 0 ? " (\(imagesAlreadyLoad) already loaded)" : ""
|
||||||
if filteredChangeList.isEmpty {
|
if filteredChangeList.isEmpty {
|
||||||
print("[INFO] No changed images to load" + suffix)
|
print(info: "No changed images to load" + suffix)
|
||||||
} else {
|
} else {
|
||||||
print("[INFO] Loading \(filteredChangeList.count) changed images" + suffix)
|
print(info: "Loading \(filteredChangeList.count) changed images" + suffix)
|
||||||
try await loadImages(filteredChangeList)
|
try await loadImages(filteredChangeList)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -117,16 +122,19 @@ final class ClassifierCreator {
|
|||||||
return (classes, missingImageList.count + changedImageList.count, changedMainImages)
|
return (classes, missingImageList.count + changedImageList.count, changedMainImages)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func getImageCounts() async -> [Int : Int] {
|
private func getImageCounts() async throws -> [Int : Int] {
|
||||||
guard let data: Data = await get(server.appendingPathComponent("caps.json")) else {
|
let data: Data
|
||||||
return [:]
|
do {
|
||||||
|
data = try await get(server.appendingPathComponent("caps.json"))
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToGetCapDatabase(error)
|
||||||
}
|
}
|
||||||
do {
|
do {
|
||||||
return try JSONDecoder().decode([Cap].self, from: data)
|
return try JSONDecoder()
|
||||||
|
.decode([Cap].self, from: data)
|
||||||
.reduce(into: [:]) { $0[$1.id] = $1.count }
|
.reduce(into: [:]) { $0[$1.id] = $1.count }
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to decode cap database: \(error)")
|
throw TrainingError.failedToDecodeCapDatabase(error)
|
||||||
return [:]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -136,8 +144,7 @@ final class ClassifierCreator {
|
|||||||
do {
|
do {
|
||||||
folders = try FileManager.default.contentsOfDirectory(atPath: imageDirectory.path)
|
folders = try FileManager.default.contentsOfDirectory(atPath: imageDirectory.path)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to get list of image folders: \(error)")
|
throw TrainingError.failedToGetListOfImageFolders(error)
|
||||||
throw TrainingError.failedToGetListOfImageFolders
|
|
||||||
}
|
}
|
||||||
for folder in folders {
|
for folder in folders {
|
||||||
if validNames.contains(folder) {
|
if validNames.contains(folder) {
|
||||||
@ -148,10 +155,9 @@ final class ClassifierCreator {
|
|||||||
let url = imageDirectory.appendingPathComponent(folder)
|
let url = imageDirectory.appendingPathComponent(folder)
|
||||||
do {
|
do {
|
||||||
try FileManager.default.removeItem(at: url)
|
try FileManager.default.removeItem(at: url)
|
||||||
print("[INFO] Removed unused image folder '\(folder)'")
|
print(info: "Removed unused image folder '\(folder)'")
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to delete unused image folder \(folder): \(error)")
|
throw TrainingError.failedToRemoveImageFolder(folder, error)
|
||||||
throw TrainingError.failedToRemoveImageFolder
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -160,22 +166,22 @@ final class ClassifierCreator {
|
|||||||
base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", image.cap, image.cap, image.image))
|
base.appendingPathComponent(String(format: "%04d/%04d-%02d.jpg", image.cap, image.cap, image.image))
|
||||||
}
|
}
|
||||||
|
|
||||||
private func load(image: CapImage) async -> Bool {
|
private func load(image: CapImage) async throws {
|
||||||
guard createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", image.cap))) else {
|
do {
|
||||||
return false
|
try createFolderIfMissing(imageDirectory.appendingPathComponent(String(format: "%04d", image.cap)))
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToCreateImageFolder(image.cap, error)
|
||||||
}
|
}
|
||||||
let url = imageUrl(base: server.appendingPathComponent("images"), image: image)
|
let url = imageUrl(base: server.appendingPathComponent("images"), image: image)
|
||||||
let tempFile: URL, response: URLResponse
|
let tempFile: URL, response: URLResponse
|
||||||
do {
|
do {
|
||||||
(tempFile, response) = try await URLSession.shared.download(from: url)
|
(tempFile, response) = try await URLSession.shared.download(from: url)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to load \(image): \(error)")
|
throw TrainingError.failedToLoadImage(image, error)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
let responseCode = (response as! HTTPURLResponse).statusCode
|
let responseCode = (response as! HTTPURLResponse).statusCode
|
||||||
guard responseCode == 200 else {
|
guard responseCode == 200 else {
|
||||||
print("[ERROR] Failed to load \(image): Response \(responseCode)")
|
throw TrainingError.invalidImageRequestResponse(image, responseCode)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
do {
|
do {
|
||||||
let localUrl = imageUrl(base: imageDirectory, image: image)
|
let localUrl = imageUrl(base: imageDirectory, image: image)
|
||||||
@ -183,60 +189,62 @@ final class ClassifierCreator {
|
|||||||
try FileManager.default.removeItem(at: localUrl)
|
try FileManager.default.removeItem(at: localUrl)
|
||||||
}
|
}
|
||||||
try FileManager.default.moveItem(at: tempFile, to: localUrl)
|
try FileManager.default.moveItem(at: tempFile, to: localUrl)
|
||||||
return true
|
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to save \(image): \(error)")
|
throw TrainingError.failedToSaveImage(image, error)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private func loadImages(_ list: [CapImage]) async throws {
|
private func loadImages(_ list: [CapImage]) async throws {
|
||||||
var loadedImages = 0
|
var loadedImageCount = 0
|
||||||
await withTaskGroup(of: Bool.self) { group in
|
var errors = [Error]()
|
||||||
|
await withTaskGroup(of: Error?.self) { group in
|
||||||
for image in list {
|
for image in list {
|
||||||
group.addTask {
|
group.addTask {
|
||||||
await self.load(image: image)
|
do {
|
||||||
}
|
try await self.load(image: image)
|
||||||
}
|
return nil
|
||||||
for await loaded in group {
|
} catch {
|
||||||
if loaded {
|
return error
|
||||||
loadedImages += 1
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if loadedImages != list.count {
|
for await error in group {
|
||||||
print("[ERROR] Only \(loadedImages) of \(list.count) images loaded")
|
if let error {
|
||||||
throw TrainingError.failedToLoadImages
|
errors.append(error)
|
||||||
|
} else {
|
||||||
|
loadedImageCount += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for error in errors {
|
||||||
|
Swift.print(error.localizedDescription)
|
||||||
|
}
|
||||||
|
let expectedCount = list.count
|
||||||
|
|
||||||
|
if loadedImageCount != expectedCount {
|
||||||
|
throw TrainingError.failedToLoadImages(expected: list.count, loaded: loadedImageCount)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getChangedImageList() async throws -> [CapImage] {
|
func getChangedImageList() async throws -> [CapImage] {
|
||||||
guard let string: String = await get(server.appendingPathComponent("changes.txt")) else {
|
let string: String
|
||||||
print("[ERROR] Failed to get list of changed images")
|
do {
|
||||||
throw TrainingError.failedToGetChangedImagesList
|
string = try await get(server.appendingPathComponent("changes.txt"))
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToGetChangedImagesList(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
return string
|
return try string
|
||||||
.components(separatedBy: "\n")
|
.components(separatedBy: "\n")
|
||||||
|
.map { $0.trimmingCharacters(in: .whitespaces) }
|
||||||
.filter { $0 != "" }
|
.filter { $0 != "" }
|
||||||
.compactMap {
|
.compactMap {
|
||||||
let parts = $0.components(separatedBy: ":")
|
let parts = $0.components(separatedBy: ":")
|
||||||
guard parts.count == 3 else {
|
guard parts.count == 3,
|
||||||
return nil
|
let _ = df.date(from: parts[0]),
|
||||||
}
|
let cap = Int(parts[1]),
|
||||||
/*
|
let image = Int(parts[2]) else {
|
||||||
guard let date = df.date(from: parts[0]) else {
|
throw TrainingError.invalidEntryInChangeList($0)
|
||||||
print("[WARN] Invalid date \(parts[0]) in change list")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
guard let cap = Int(parts[1]) else {
|
|
||||||
print("[WARN] Invalid cap id \(parts[1]) in change list")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
guard let image = Int(parts[2]) else {
|
|
||||||
print("[WARN] Invalid image id \(parts[2]) in change list")
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return CapImage(cap: cap, image: image)
|
return CapImage(cap: cap, image: image)
|
||||||
}
|
}
|
||||||
@ -245,12 +253,13 @@ final class ClassifierCreator {
|
|||||||
// MARK: Step 3: Compute version
|
// MARK: Step 3: Compute version
|
||||||
|
|
||||||
func getClassifierVersion() async throws -> Int {
|
func getClassifierVersion() async throws -> Int {
|
||||||
guard let string: String = await get(server.appendingPathComponent("version")) else {
|
let string: String
|
||||||
print("[ERROR] Failed to get classifier version")
|
do {
|
||||||
throw TrainingError.failedToGetClassifierVersion
|
string = try await get(server.appendingPathComponent("version"))
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToGetClassifierVersion(error)
|
||||||
}
|
}
|
||||||
guard let version = Int(string) else {
|
guard let version = Int(string) else {
|
||||||
print("[ERROR] Invalid classifier version \(string)")
|
|
||||||
throw TrainingError.invalidClassifierVersion(string)
|
throw TrainingError.invalidClassifierVersion(string)
|
||||||
}
|
}
|
||||||
return version
|
return version
|
||||||
@ -284,13 +293,13 @@ final class ClassifierCreator {
|
|||||||
parameters: params,
|
parameters: params,
|
||||||
sessionParameters: sessionParameters)
|
sessionParameters: sessionParameters)
|
||||||
} catch {
|
} catch {
|
||||||
throw TrainingError.failedToCreateClassifier
|
throw TrainingError.failedToCreateClassifier("\(error)")
|
||||||
}
|
}
|
||||||
|
|
||||||
job.progress
|
job.progress
|
||||||
.publisher(for: \.fractionCompleted)
|
.publisher(for: \.fractionCompleted)
|
||||||
.sink { completed in
|
.sink { completed in
|
||||||
print(String(format: " %.1f %% completed", completed * 100), terminator: "\r")
|
Swift.print(String(format: " %.1f %% completed", completed * 100), terminator: "\r")
|
||||||
fflush(stdout)
|
fflush(stdout)
|
||||||
|
|
||||||
//guard let progress = MLProgress(progress: job.progress) else {
|
//guard let progress = MLProgress(progress: job.progress) else {
|
||||||
@ -305,11 +314,10 @@ final class ClassifierCreator {
|
|||||||
return try await withCheckedThrowingContinuation { continuation in
|
return try await withCheckedThrowingContinuation { continuation in
|
||||||
// Register a sink to receive the resulting model.
|
// Register a sink to receive the resulting model.
|
||||||
job.result.sink { result in
|
job.result.sink { result in
|
||||||
print("[ERROR] \(result)")
|
continuation.resume(throwing: TrainingError.failedToCreateClassifier("\(result)"))
|
||||||
continuation.resume(throwing: TrainingError.failedToCreateClassifier)
|
} receiveValue: { [weak self] model in
|
||||||
} receiveValue: { model in
|
|
||||||
// Use model
|
// Use model
|
||||||
print("[INFO] Created model")
|
self?.print(info: "Created model")
|
||||||
continuation.resume(returning: model)
|
continuation.resume(returning: model)
|
||||||
}
|
}
|
||||||
.store(in: &subscriptions)
|
.store(in: &subscriptions)
|
||||||
@ -326,18 +334,16 @@ final class ClassifierCreator {
|
|||||||
trainingData: .labeledDirectories(at: imageDirectory),
|
trainingData: .labeledDirectories(at: imageDirectory),
|
||||||
parameters: params)
|
parameters: params)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to create classifier: \(error)")
|
throw TrainingError.failedToCreateClassifier("\(error)")
|
||||||
throw TrainingError.failedToCreateClassifier
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private func save(model: MLImageClassifier) throws {
|
private func save(model: MLImageClassifier) throws {
|
||||||
|
|
||||||
print("[INFO] Saving classifier...")
|
print(info: "Saving classifier...")
|
||||||
do {
|
do {
|
||||||
try model.write(to: classifierUrl)
|
try model.write(to: classifierUrl)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to save model to file: \(error)")
|
|
||||||
throw TrainingError.failedToWriteClassifier(error)
|
throw TrainingError.failedToWriteClassifier(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -345,83 +351,81 @@ final class ClassifierCreator {
|
|||||||
// MARK: Step 5: Upload classifier
|
// MARK: Step 5: Upload classifier
|
||||||
|
|
||||||
func uploadModel(version: Int) async throws {
|
func uploadModel(version: Int) async throws {
|
||||||
print("[INFO] Uploading classifier...")
|
print(info: "Uploading classifier...")
|
||||||
let modelData: Data
|
let modelData: Data
|
||||||
do {
|
do {
|
||||||
modelData = try Data(contentsOf: classifierUrl)
|
modelData = try Data(contentsOf: classifierUrl)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to read classifier data: \(error)")
|
|
||||||
throw TrainingError.failedToReadClassifierData(error)
|
throw TrainingError.failedToReadClassifierData(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
let success = await post(
|
let url = server.appendingPathComponent("classifier/\(version)")
|
||||||
url: server.appendingPathComponent("classifier/\(version)"),
|
do {
|
||||||
body: modelData)
|
try await post(url: url, body: modelData)
|
||||||
guard success else {
|
} catch {
|
||||||
throw TrainingError.failedToUploadClassifier
|
throw TrainingError.failedToUploadClassifier(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: Step 6: Update classes
|
// MARK: Step 6: Update classes
|
||||||
|
|
||||||
func upload(classes: [Int], lastUpdate: Date) async throws {
|
func upload(classes: [Int], lastUpdate: Date) async throws {
|
||||||
print("[INFO] Uploading trained classes...")
|
print(info: "Uploading trained classes...")
|
||||||
let dateString = df.string(from: lastUpdate)
|
let dateString = df.string(from: lastUpdate)
|
||||||
let url = server.appendingPathComponent("classes/\(dateString)")
|
let url = server.appendingPathComponent("classes/\(dateString)")
|
||||||
let body = classes.map(String.init).joined(separator: ",").data(using: .utf8)!
|
let body = classes.map(String.init).joined(separator: ",").data(using: .utf8)!
|
||||||
guard await post(url: url, body: body) else {
|
do {
|
||||||
throw TrainingError.failedToUploadClassifierClasses
|
try await post(url: url, body: body)
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToUploadClassifierClasses(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: Step 7: Create thumbnails
|
// MARK: Step 7: Create thumbnails
|
||||||
|
|
||||||
func createThumbnails(changed: [Int]) async {
|
func createThumbnails(changed: [Int]) async throws {
|
||||||
guard checkMagickAvailability() else {
|
try ensureMagickAvailability()
|
||||||
return
|
do {
|
||||||
|
try createFolderIfMissing(thumbnailDirectory)
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToCreateThumbnailFolder(error)
|
||||||
}
|
}
|
||||||
guard createFolderIfMissing(thumbnailDirectory) else {
|
let capIdsOfMissingThumbnails = try await getMissingThumbnailIds()
|
||||||
print("[ERROR] Failed to create folder for thumbnails")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
let capIdsOfMissingThumbnails = await getMissingThumbnailIds()
|
|
||||||
let all = Set(capIdsOfMissingThumbnails).union(changed)
|
let all = Set(capIdsOfMissingThumbnails).union(changed)
|
||||||
print("[INFO] Creating \(all.count) thumbnails...")
|
print(info: "Creating \(all.count) thumbnails...")
|
||||||
for cap in all {
|
for cap in all {
|
||||||
await createThumbnail(for: cap)
|
try await createThumbnail(for: cap)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkMagickAvailability() -> Bool {
|
func ensureMagickAvailability() throws {
|
||||||
do {
|
do {
|
||||||
let (code, output) = try safeShell("magick --version")
|
let (code, output) = try safeShell("magick --version")
|
||||||
guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
|
guard code == 0, let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
|
||||||
.components(separatedBy: " ").first else {
|
.components(separatedBy: " ").first else {
|
||||||
print("[ERROR] Magick not found, install using 'brew install imagemagick'")
|
throw TrainingError.magickDependencyNotFound
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
print("[INFO] Using magick \(version)")
|
print(info: "Using magick \(version)")
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to get version of magick: (\(error))")
|
throw TrainingError.magickDependencyCheckFailed(error)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private func getMissingThumbnailIds() async -> [Int] {
|
private func getMissingThumbnailIds() async throws -> [Int] {
|
||||||
guard let string: String = await get(server.appendingPathComponent("thumbnails/missing")) else {
|
let string: String
|
||||||
print("[ERROR] Failed to get missing thumbnails")
|
do {
|
||||||
return []
|
string = try await get(server.appendingPathComponent("thumbnails/missing"))
|
||||||
|
} catch {
|
||||||
|
throw TrainingError.failedToGetMissingThumbnailIds(error)
|
||||||
}
|
}
|
||||||
return string.components(separatedBy: ",").compactMap(Int.init)
|
return string.components(separatedBy: ",").compactMap(Int.init)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func createThumbnail(for cap: Int) async {
|
private func createThumbnail(for cap: Int) async throws {
|
||||||
let mainImage = CapImage(cap: cap, image: 0)
|
let mainImage = CapImage(cap: cap, image: 0)
|
||||||
let inputUrl = imageUrl(base: imageDirectory, image: mainImage)
|
let inputUrl = imageUrl(base: imageDirectory, image: mainImage)
|
||||||
guard FileManager.default.fileExists(atPath: inputUrl.path) else {
|
guard FileManager.default.fileExists(atPath: inputUrl.path) else {
|
||||||
print("[ERROR] Local main image not found for cap \(cap): \(inputUrl.path)")
|
throw TrainingError.missingMainImage(cap)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
|
let output = thumbnailDirectory.appendingPathComponent(String(format: "%04d.jpg", cap))
|
||||||
@ -429,24 +433,22 @@ final class ClassifierCreator {
|
|||||||
let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
|
let command = "magick convert \(inputUrl.path) -quality 70% -resize 100x100 \(output.path)"
|
||||||
let (code, output) = try safeShell(command)
|
let (code, output) = try safeShell(command)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
print("Failed to create thumbnail for cap \(cap): \(output)")
|
throw TrainingError.failedToCreateThumbnail(cap, output)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
print("Failed to read created thumbnail for cap \(cap): \(error)")
|
throw TrainingError.failedToCreateThumbnail(cap, "\(error)")
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let data: Data
|
let data: Data
|
||||||
do {
|
do {
|
||||||
data = try Data(contentsOf: output)
|
data = try Data(contentsOf: output)
|
||||||
} catch {
|
} catch {
|
||||||
print("Failed to read created thumbnail for cap \(cap): \(error)")
|
throw TrainingError.failedToReadCreatedThumbnail(cap, error)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
guard await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data) else {
|
do {
|
||||||
print("Failed to upload thumbnail for cap \(cap)")
|
try await post(url: server.appendingPathComponent("thumbnails/\(cap)"), body: data)
|
||||||
return
|
} catch {
|
||||||
|
throw TrainingError.failedToUploadCreatedThumbnail(cap, error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -472,57 +474,41 @@ final class ClassifierCreator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private func createFolderIfMissing(_ folder: URL) -> Bool {
|
private func createFolderIfMissing(_ folder: URL) throws {
|
||||||
guard !FileManager.default.fileExists(atPath: folder.path) else {
|
guard !FileManager.default.fileExists(atPath: folder.path) else {
|
||||||
return true
|
return
|
||||||
}
|
}
|
||||||
do {
|
|
||||||
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
|
try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
|
||||||
return true
|
|
||||||
} catch {
|
|
||||||
print("[ERROR] Failed to create directory \(folder.path): \(error)")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// MARK: Requests
|
// MARK: Requests
|
||||||
|
|
||||||
private func post(url: URL, body: Data) async -> Bool {
|
private func post(url: URL, body: Data) async throws {
|
||||||
var request = URLRequest(url: url)
|
var request = URLRequest(url: url)
|
||||||
request.httpMethod = "POST"
|
request.httpMethod = "POST"
|
||||||
request.httpBody = body
|
request.httpBody = body
|
||||||
request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
|
request.addValue(configuration.authenticationToken, forHTTPHeaderField: "key")
|
||||||
return await perform(request) != nil
|
_ = try await perform(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
private func perform(_ request: URLRequest) async -> Data? {
|
private func perform(_ request: URLRequest) async throws -> Data {
|
||||||
let data: Data
|
let (data, response) = try await URLSession.shared.data(for: request)
|
||||||
let response: URLResponse
|
|
||||||
do {
|
|
||||||
(data, response) = try await URLSession.shared.data(for: request)
|
|
||||||
} catch {
|
|
||||||
print("[ERROR] Request to \(request.url!.absoluteString) failed: \(error)")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
let code = (response as! HTTPURLResponse).statusCode
|
let code = (response as! HTTPURLResponse).statusCode
|
||||||
guard code == 200 else {
|
guard code == 200 else {
|
||||||
print("[ERROR] Request to \(request.url!.absoluteString): Invalid response \(code)")
|
throw TrainingError.invalidResponse(request.url!, code)
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
private func get(_ url: URL) async -> Data? {
|
private func get(_ url: URL) async throws -> Data {
|
||||||
await perform(URLRequest(url: url))
|
try await perform(URLRequest(url: url))
|
||||||
}
|
}
|
||||||
|
|
||||||
private func get(_ url: URL) async -> String? {
|
private func get(_ url: URL) async throws -> String {
|
||||||
guard let data: Data = await get(url) else {
|
let data: Data = try await get(url)
|
||||||
return nil
|
|
||||||
}
|
|
||||||
guard let string = String(data: data, encoding: .utf8) else {
|
guard let string = String(data: data, encoding: .utf8) else {
|
||||||
print("[ERROR] Invalid string response \(data)")
|
throw TrainingError.invalidGetResponseData(data.count)
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return string
|
return string
|
||||||
}
|
}
|
||||||
|
@ -15,8 +15,7 @@ extension Configuration {
|
|||||||
|
|
||||||
func serverUrl() throws -> URL {
|
func serverUrl() throws -> URL {
|
||||||
guard let serverUrl = URL(string: serverPath) else {
|
guard let serverUrl = URL(string: serverPath) else {
|
||||||
print("[ERROR] Configuration: Invalid server path \(serverPath)")
|
throw TrainingError.invalidServerPath(serverPath)
|
||||||
throw TrainingError.invalidServerPath
|
|
||||||
}
|
}
|
||||||
return serverUrl
|
return serverUrl
|
||||||
}
|
}
|
||||||
|
@ -19,21 +19,18 @@ extension ConfigurationFile {
|
|||||||
|
|
||||||
init(at url: URL) throws {
|
init(at url: URL) throws {
|
||||||
guard FileManager.default.fileExists(atPath: url.path) else {
|
guard FileManager.default.fileExists(atPath: url.path) else {
|
||||||
print("[ERROR] No configuration at \(url.absoluteURL.path)")
|
throw TrainingError.configurationFileMissing(url)
|
||||||
throw TrainingError.configurationFileMissing
|
|
||||||
}
|
}
|
||||||
let data: Data
|
let data: Data
|
||||||
do {
|
do {
|
||||||
data = try Data(contentsOf: url)
|
data = try Data(contentsOf: url)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to load configuration data at \(url.absoluteURL.path): \(error)")
|
throw TrainingError.configurationFileUnreadable(url, error)
|
||||||
throw TrainingError.configurationFileUnreadable
|
|
||||||
}
|
}
|
||||||
do {
|
do {
|
||||||
self = try JSONDecoder().decode(ConfigurationFile.self, from: data)
|
self = try JSONDecoder().decode(ConfigurationFile.self, from: data)
|
||||||
} catch {
|
} catch {
|
||||||
print("[ERROR] Failed to decode configuration at \(url.absoluteURL.path): \(error)")
|
throw TrainingError.configurationFileDecodingFailed(url, error)
|
||||||
throw TrainingError.configurationFileDecodingFailed
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,29 +1,132 @@
|
|||||||
import Foundation
|
import Foundation
|
||||||
|
|
||||||
enum TrainingError: Error {
|
enum TrainingError: Error {
|
||||||
|
case invalidGetResponseData(Int)
|
||||||
|
case invalidResponse(URL, Int)
|
||||||
|
|
||||||
case missingArguments
|
case missingArguments
|
||||||
|
|
||||||
case configurationFileMissing
|
case configurationFileMissing(URL)
|
||||||
case configurationFileUnreadable
|
case configurationFileUnreadable(URL, Error)
|
||||||
case configurationFileDecodingFailed
|
case configurationFileDecodingFailed(URL, Error)
|
||||||
case invalidServerPath
|
case invalidServerPath(String)
|
||||||
case mainImageFolderNotCreated
|
case mainImageFolderNotCreated(Error)
|
||||||
case failedToLoadImages
|
|
||||||
|
|
||||||
case failedToGetListOfImageFolders
|
case failedToGetCapDatabase(Error)
|
||||||
case failedToRemoveImageFolder
|
case failedToDecodeCapDatabase(Error)
|
||||||
|
|
||||||
case failedToGetClassifierVersion
|
case failedToLoadImage(CapImage, Error)
|
||||||
|
case failedToCreateImageFolder(Int, Error)
|
||||||
|
case invalidImageRequestResponse(CapImage, Int)
|
||||||
|
case failedToSaveImage(CapImage, Error)
|
||||||
|
case failedToLoadImages(expected: Int, loaded: Int)
|
||||||
|
|
||||||
|
case failedToGetChangedImagesList(Error)
|
||||||
|
case invalidEntryInChangeList(String)
|
||||||
|
|
||||||
|
|
||||||
|
case failedToGetListOfImageFolders(Error)
|
||||||
|
case failedToRemoveImageFolder(String, Error)
|
||||||
|
|
||||||
|
case failedToGetClassifierVersion(Error)
|
||||||
case invalidClassifierVersion(String)
|
case invalidClassifierVersion(String)
|
||||||
|
|
||||||
case failedToCreateClassifier
|
case failedToCreateClassifier(String)
|
||||||
case failedToWriteClassifier(Error)
|
case failedToWriteClassifier(Error)
|
||||||
|
|
||||||
case failedToGetChangedImagesList
|
|
||||||
|
|
||||||
case failedToReadClassifierData(Error)
|
case failedToReadClassifierData(Error)
|
||||||
case failedToUploadClassifier
|
case failedToUploadClassifier(Error)
|
||||||
|
|
||||||
case failedToUploadClassifierClasses
|
case failedToUploadClassifierClasses(Error)
|
||||||
|
|
||||||
|
case magickDependencyCheckFailed(Error)
|
||||||
|
case magickDependencyNotFound
|
||||||
|
case failedToCreateThumbnailFolder(Error)
|
||||||
|
case failedToGetMissingThumbnailIds(Error)
|
||||||
|
case missingMainImage(Int)
|
||||||
|
case failedToCreateThumbnail(Int, String)
|
||||||
|
case failedToReadCreatedThumbnail(Int, Error)
|
||||||
|
case failedToUploadCreatedThumbnail(Int, Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
extension TrainingError: CustomStringConvertible {
|
||||||
|
|
||||||
|
var description: String {
|
||||||
|
switch self {
|
||||||
|
case .missingArguments:
|
||||||
|
return "Missing arguments"
|
||||||
|
case .configurationFileMissing(let url):
|
||||||
|
return "No configuration at \(url.absoluteURL.path)"
|
||||||
|
case .configurationFileUnreadable(let url, let error):
|
||||||
|
return "Failed to load configuration data at \(url.absoluteURL.path): \(error)"
|
||||||
|
case .configurationFileDecodingFailed(let url, let error):
|
||||||
|
return "Failed to decode configuration at \(url.absoluteURL.path): \(error)"
|
||||||
|
case .invalidServerPath(let path):
|
||||||
|
return "Configuration: Invalid server path \(path)"
|
||||||
|
case .mainImageFolderNotCreated(let error):
|
||||||
|
return "Failed to create main image folder: \(error)"
|
||||||
|
|
||||||
|
case .failedToGetCapDatabase(let error):
|
||||||
|
return "Failed to get cap database from server: \(error)"
|
||||||
|
case .failedToDecodeCapDatabase(let error):
|
||||||
|
return "Failed to decode cap database: \(error)"
|
||||||
|
|
||||||
|
case .failedToGetListOfImageFolders(let error):
|
||||||
|
return "Failed to get list of image folders: \(error)"
|
||||||
|
case .failedToRemoveImageFolder(let folder, let error):
|
||||||
|
return "Failed to delete unused image folder \(folder): \(error)"
|
||||||
|
case .failedToCreateImageFolder(let cap, let error):
|
||||||
|
return "Failed to create image folder for cap \(cap): \(error)"
|
||||||
|
case .failedToLoadImage(let image, let error):
|
||||||
|
return "Failed to load \(image): \(error)"
|
||||||
|
case .invalidImageRequestResponse(let image, let responseCode):
|
||||||
|
return "Failed to load \(image): Response \(responseCode)"
|
||||||
|
case .failedToSaveImage(let image, let error):
|
||||||
|
return "Failed to save \(image): \(error)"
|
||||||
|
case .failedToLoadImages(let expected, let loaded):
|
||||||
|
return "Only \(expected) of \(loaded) images loaded"
|
||||||
|
|
||||||
|
case .failedToGetChangedImagesList(let error):
|
||||||
|
return "Failed to get list of changed images: \(error)"
|
||||||
|
case .invalidEntryInChangeList(let entry):
|
||||||
|
return "Invalid change list entry: '\(entry)'"
|
||||||
|
|
||||||
|
case .failedToGetClassifierVersion(let error):
|
||||||
|
return "Failed to get classifier version: \(error)"
|
||||||
|
case .invalidClassifierVersion(let string):
|
||||||
|
return "Invalid classifier version \(string)"
|
||||||
|
case .failedToCreateClassifier(let result):
|
||||||
|
return "Failed to create classifier: \(result)"
|
||||||
|
case .failedToWriteClassifier(let error):
|
||||||
|
return "Failed to save model to file: \(error)"
|
||||||
|
case .failedToReadClassifierData(let error):
|
||||||
|
return "Failed to read classifier data: \(error)"
|
||||||
|
case .failedToUploadClassifier(let error):
|
||||||
|
return "Failed to upload classifier: \(error)"
|
||||||
|
case .failedToUploadClassifierClasses(let error):
|
||||||
|
return "Failed to upload classifier classes: \(error)"
|
||||||
|
|
||||||
|
case .magickDependencyCheckFailed(let error):
|
||||||
|
return "Failed to get version of magick: (\(error))"
|
||||||
|
case .magickDependencyNotFound:
|
||||||
|
return "Magick not found, install using 'brew install imagemagick'"
|
||||||
|
case .failedToCreateThumbnailFolder(let error):
|
||||||
|
return "Failed to create folder for thumbnails: \(error)"
|
||||||
|
case .failedToGetMissingThumbnailIds(let error):
|
||||||
|
return "Failed to get missing thumbnails ids: \(error)"
|
||||||
|
case .missingMainImage(let cap):
|
||||||
|
return "Local main image not found for cap \(cap)"
|
||||||
|
case .failedToCreateThumbnail(let cap, let message):
|
||||||
|
return "Failed to read created thumbnail for cap \(cap): \(message)"
|
||||||
|
case .failedToReadCreatedThumbnail(let cap, let error):
|
||||||
|
return "Failed to read created thumbnail for cap \(cap): \(error)"
|
||||||
|
case .failedToUploadCreatedThumbnail(let cap, let error):
|
||||||
|
return "Failed to upload thumbnail for cap \(cap): \(error)"
|
||||||
|
|
||||||
|
case .invalidGetResponseData(let count):
|
||||||
|
return "Response (\(count) bytes) is not a valid string"
|
||||||
|
case .invalidResponse(let url, let code):
|
||||||
|
return "Invalid response \(code) to \(url.absoluteString)"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user