From a8b328efcead27928f9e9075462170a04c2aa2ce Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Sun, 4 Dec 2022 23:10:44 +0100 Subject: [PATCH] Write all logs to disk --- Sources/Generator/Processing/FileData.swift | 2 +- .../Generator/Processing/FileGenerator.swift | 33 ++++++- .../Processing/GenerationResultsHandler.swift | 9 +- .../Generator/Processing/ImageGenerator.swift | 91 +++++++++++++------ .../Processing/MetadataInfoLogger.swift | 53 +++++------ Sources/Generator/Templates/Template.swift | 9 +- Sources/Generator/run.swift | 44 ++++++--- 7 files changed, 159 insertions(+), 82 deletions(-) diff --git a/Sources/Generator/Processing/FileData.swift b/Sources/Generator/Processing/FileData.swift index 80bc5d2..1f40c48 100644 --- a/Sources/Generator/Processing/FileData.swift +++ b/Sources/Generator/Processing/FileData.swift @@ -3,7 +3,7 @@ import Foundation struct FileData { ///The files marked as expected, i.e. they exist after the generation is completed. (`key`: file path, `value`: the file providing the link) - var expected: [String : String] = [:] + var expected: [String : String] = [:] /// All files which should be copied to the output folder (`key`: The file path, `value`: The source requiring the file) var toCopy: [String : String] = [:] diff --git a/Sources/Generator/Processing/FileGenerator.swift b/Sources/Generator/Processing/FileGenerator.swift index b335702..98e1d66 100644 --- a/Sources/Generator/Processing/FileGenerator.swift +++ b/Sources/Generator/Processing/FileGenerator.swift @@ -157,9 +157,10 @@ final class FileGenerator { command = "cleancss \(url.path) -o \(tempFile.path)" } - try? tempFile.delete() - - defer { didMinifyFile() } + defer { + didMinifyFile() + try? tempFile.delete() + } let output: String do { @@ -256,4 +257,30 @@ final class FileGenerator { } return false } + + func writeResults(to file: URL) { + var lines: [String] = [] + func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { + let elements = property.map { " " + convert($0) }.sorted() + guard !elements.isEmpty else { + return + } + lines.append("\(name):") + lines.append(contentsOf: elements) + } + add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Failed minifications", failedMinifications) { "\($0.file) (required by \($0.source)): \($0.message)" } + add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" } + add("Copied files", copiedFiles) { $0 } + add("Minified files", minifiedFiles) { $0 } + add("Expected files", files.expected) { "\($0.key) (required by \($0.value))" } + + let data = lines.joined(separator: "\n").data(using: .utf8)! + do { + try data.createFolderAndWrite(to: file) + } catch { + print(" Failed to save log: \(error)") + } + } } diff --git a/Sources/Generator/Processing/GenerationResultsHandler.swift b/Sources/Generator/Processing/GenerationResultsHandler.swift index 4d74a74..184271d 100644 --- a/Sources/Generator/Processing/GenerationResultsHandler.swift +++ b/Sources/Generator/Processing/GenerationResultsHandler.swift @@ -441,7 +441,7 @@ final class GenerationResultsHandler { print("") } - func writeResultsToFile(file: URL) throws { + func writeResults(to file: URL) { var lines: [String] = [] func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { let elements = property.map { " " + convert($0) }.sorted() @@ -459,9 +459,12 @@ final class GenerationResultsHandler { add("Drafts", draftPages) { $0 } add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" } add("Generated files", generatedFiles) { $0 } - let data = lines.joined(separator: "\n").data(using: .utf8)! - try data.createFolderAndWrite(to: file) + do { + try data.createFolderAndWrite(to: file) + } catch { + print(" Failed to save log: \(error)") + } } } diff --git a/Sources/Generator/Processing/ImageGenerator.swift b/Sources/Generator/Processing/ImageGenerator.swift index 5b855fd..010e278 100644 --- a/Sources/Generator/Processing/ImageGenerator.swift +++ b/Sources/Generator/Processing/ImageGenerator.swift @@ -24,18 +24,24 @@ final class ImageGenerator { /** All warnings produced for images during generation */ - private var imageWarnings: Set = [] + private var imageWarnings: [String] - /** - All images modified or created during this generator run. - */ + /// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path) + private var missingImages: [String : String] + + /// Images which could not be read (`key`: file path relative to content, `value`: source element path) + private var unreadableImages: [String : String] + + private var unhandledImages: [String: String] = [:] + + /// All images modified or created during this generator run. private var generatedImages: Set = [] - /** - The images optimized by ImageOptim - */ + /// The images optimized by ImageOptim private var optimizedImages: Set = [] + private var failedImages: [(path: String, message: String)] = [] + private var numberOfGeneratedImages = 0 private let numberOfTotalImages: Int @@ -71,6 +77,9 @@ final class ImageGenerator { self.images = images self.numberOfTotalImages = images.jobs.reduce(0) { $0 + $1.value.count } + images.multiJobs.count * 2 + self.imageWarnings = images.warnings + self.missingImages = images.missing + self.unreadableImages = images.unreadable } func generateImages() { @@ -81,15 +90,10 @@ final class ImageGenerator { } notes.append("\(count) \(name)") } - addIfNotZero(images.missing.count, "missing images") - addIfNotZero(images.unreadable.count, "unreadable images") print(" Changed sources: \(jobs.count)/\(images.jobs.count)") print(" Total images: \(numberOfTotalImages) (\(numberOfTotalImages - imageReader.numberOfFilesAccessed) versions)") - print(" Warnings: \(images.warnings.count)") - if !notes.isEmpty { - print(" Notes: " + notes.joined(separator: ", ")) - } + for (source, jobs) in jobs { create(images: jobs, from: source) } @@ -99,11 +103,18 @@ final class ImageGenerator { print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)") optimizeImages() print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)") + + addIfNotZero(missingImages.count, "missing images") + addIfNotZero(unreadableImages.count, "unreadable images") + print(" Warnings: \(imageWarnings.count)") + if !notes.isEmpty { + print(" Notes: " + notes.joined(separator: ", ")) + } } private func create(images: [ImageJob], from source: String) { guard let image = imageReader.getImage(atPath: source) else { - // TODO: Add to failed images + unreadableImages[source] = images.first!.destination didGenerateImage(count: images.count) return } @@ -137,7 +148,7 @@ final class ImageGenerator { let destinationExtension = destinationUrl.pathExtension.lowercased() guard let type = ImageType(fileExtension: destinationExtension)?.fileType else { - addWarning("Invalid image extension \(destinationExtension)", job: job) + unhandledImages[source] = job.destination return } @@ -168,25 +179,20 @@ final class ImageGenerator { // Get NSData, and save it guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else { - addWarning("Failed to get data", job: job) + markImageAsFailed(source, error: "Failed to get data") return } do { try data.createFolderAndWrite(to: destinationUrl) } catch { - addWarning("Failed to write image (\(error))", job: job) + markImageAsFailed(job.destination, error: "Failed to write image (\(error))") return } generatedImages.insert(job.destination) } - private func addWarning(_ message: String, destination: String, path: String) { - let warning = " \(destination): \(message) required by \(path)" - imageWarnings.insert(warning) - } - - private func addWarning(_ message: String, job: ImageJob) { - addWarning(message, destination: job.destination, path: job.path) + private func markImageAsFailed(_ source: String, error: String) { + failedImages.append((source, error)) } private func createMultiImages(from source: String, path: String) { @@ -198,7 +204,7 @@ final class ImageGenerator { let sourceUrl = output.appendingPathComponent(source) let sourcePath = sourceUrl.path guard sourceUrl.exists else { - addWarning("No image at path \(sourcePath)", destination: source, path: path) + missingImages[source] = path didGenerateImage(count: 2) return } @@ -222,7 +228,7 @@ final class ImageGenerator { do { _ = try safeShell(command) } catch { - addWarning("Failed to create AVIF image", destination: destination, path: destination) + markImageAsFailed(destination, error: "Failed to create AVIF image") } } @@ -231,7 +237,7 @@ final class ImageGenerator { do { _ = try safeShell(command) } catch { - addWarning("Failed to create WEBP image", destination: destination, path: destination) + markImageAsFailed(destination, error: "Failed to create WEBP image") } } @@ -240,7 +246,7 @@ final class ImageGenerator { do { _ = try safeShell(command) } catch { - addWarning("Failed to compress image", destination: destination, path: destination) + markImageAsFailed(destination, error: "Failed to compress image") } } @@ -264,7 +270,9 @@ final class ImageGenerator { _ = try safeShell(command) return true } catch { - addWarning("Failed to optimize images", destination: "", path: "") + for image in batch { + markImageAsFailed(image, error: "Failed to optimize image") + } return false } } @@ -282,4 +290,29 @@ final class ImageGenerator { print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize) \r", terminator: "") fflush(stdout) } + + func writeResults(to file: URL) { + var lines: [String] = [] + func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { + let elements = property.map { " " + convert($0) }.sorted() + guard !elements.isEmpty else { + return + } + lines.append("\(name):") + lines.append(contentsOf: elements) + } + add("Missing images", missingImages) { "\($0.key) (required by \($0.value))" } + add("Unreadable images", unreadableImages) { "\($0.key) (required by \($0.value))" } + add("Failed images", failedImages) { "\($0.path): \($0.message)" } + add("Unhandled images", unhandledImages) { "\($0.value) (from \($0.key))" } + add("Warnings", imageWarnings) { $0 } + add("Generated images", generatedImages) { $0 } + add("Optimized images", optimizedImages) { $0 } + let data = lines.joined(separator: "\n").data(using: .utf8)! + do { + try data.createFolderAndWrite(to: file) + } catch { + print(" Failed to save log: \(error)") + } + } } diff --git a/Sources/Generator/Processing/MetadataInfoLogger.swift b/Sources/Generator/Processing/MetadataInfoLogger.swift index 2584f38..459b529 100644 --- a/Sources/Generator/Processing/MetadataInfoLogger.swift +++ b/Sources/Generator/Processing/MetadataInfoLogger.swift @@ -4,8 +4,6 @@ final class MetadataInfoLogger { private let input: URL - private let runFolder: URL - private var numberOfMetadataFiles = 0 private var unusedProperties: [(name: String, source: String)] = [] @@ -22,13 +20,8 @@ final class MetadataInfoLogger { private var errors: [(source: String, message: String)] = [] - private var logFile: URL { - runFolder.appendingPathComponent("Metadata issues.txt") - } - - init(input: URL, runFolder: URL) { + init(input: URL) { self.input = input - self.runFolder = runFolder } /** @@ -131,7 +124,7 @@ final class MetadataInfoLogger { // MARK: Printing private func printMetadataScanUpdate() { - print(String(format: " Pages found: %4d \r", numberOfMetadataFiles), terminator: "") + print(" Pages found: \(numberOfMetadataFiles) \r", terminator: "") } func printMetadataScanOverview(languages: Int) { @@ -157,31 +150,29 @@ final class MetadataInfoLogger { } } - func writeResultsToFile() throws { + func writeResults(to file: URL) { var lines: [String] = [] - if !errors.isEmpty { - lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }.sorted() - } - if !warnings.isEmpty { - lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }.sorted() - } - if !unreadableMetadata.isEmpty { - lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }.sorted() - } - if !unusedProperties.isEmpty { - lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }.sorted() - } - if !invalidProperties.isEmpty { - lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }.sorted() - } - if !unknownProperties.isEmpty { - lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }.sorted() - } - if !missingProperties.isEmpty { - lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }.sorted() + func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { + let elements = property.map { " " + convert($0) }.sorted() + guard !elements.isEmpty else { + return + } + lines.append("\(name):") + lines.append(contentsOf: elements) } + add("Errors", errors) { "\($0.source): \($0.message)" } + add("Warnings", warnings) { "\($0.source): \($0.message)" } + add("Unreadable files", unreadableMetadata) { "\($0.source): \($0.error)" } + add("Unused properties", unusedProperties) { "\($0.source): \($0.name)" } + add("Invalid properties", invalidProperties) { "\($0.source): \($0.name) (\($0.reason))" } + add("Unknown properties", unknownProperties) { "\($0.source): \($0.name)" } + add("Missing properties", missingProperties) { "\($0.source): \($0.name)" } let data = lines.joined(separator: "\n").data(using: .utf8)! - try data.createFolderAndWrite(to: logFile) + do { + try data.createFolderAndWrite(to: file) + } catch { + print(" Failed to save log: \(error)") + } } } diff --git a/Sources/Generator/Templates/Template.swift b/Sources/Generator/Templates/Template.swift index d913440..6f62919 100644 --- a/Sources/Generator/Templates/Template.swift +++ b/Sources/Generator/Templates/Template.swift @@ -22,8 +22,13 @@ extension Template { } init(from url: URL, results: GenerationResultsHandler) throws { - let raw = try String(contentsOf: url) - self.init(raw: raw, results: results) + do { + let raw = try String(contentsOf: url) + self.init(raw: raw, results: results) + } catch { + results.warning("Failed to load: \(error)", source: "Template \(url.lastPathComponent)") + throw error + } } @discardableResult diff --git a/Sources/Generator/run.swift b/Sources/Generator/run.swift index 0cba544..93dbd36 100644 --- a/Sources/Generator/run.swift +++ b/Sources/Generator/run.swift @@ -36,22 +36,29 @@ private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Eleme print("--- SOURCE FILES -----------------------------------") print(" ") - let log = MetadataInfoLogger(input: folder, runFolder: runFolder) + let log = MetadataInfoLogger(input: folder) let root = Element(atRoot: folder, log: log) - log.printMetadataScanOverview(languages: root?.languages.count ?? 0) - print(" ") - try log.writeResultsToFile() + + let file = runFolder.appendingPathComponent("metadata.txt") + defer { + log.writeResults(to: file) + print(" ") + } guard let root else { + log.printMetadataScanOverview(languages: 0) + print(" Error: No site root loaded, aborting generation") return nil } let ids = root.getContainedIds(log: log) + log.printMetadataScanOverview(languages: root.languages.count) return (root, ids) } -private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], pageCount: Int, runFolder: URL) throws -> (ImageData, FileData) { +private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], runFolder: URL) -> (ImageData, FileData)? { print("--- GENERATION -------------------------------------") print(" ") + let pageCount = ids.count * root.languages.count let results = GenerationResultsHandler( in: configuration.contentDirectory, to: configuration.outputDirectory, @@ -60,11 +67,22 @@ private func generatePages(from root: Element, configuration: Configuration, fil pagePaths: ids, pageCount: pageCount) - let siteGenerator = try SiteGenerator(results: results) + defer { results.printOverview() } + + let siteGenerator: SiteGenerator + do { + siteGenerator = try SiteGenerator(results: results) + } catch { + return nil + } siteGenerator.generate(site: root) - results.printOverview() - let url = runFolder.appendingPathComponent("files.txt") - try results.writeResultsToFile(file: url) + + let url = runFolder.appendingPathComponent("pages.txt") + results.writeResults(to: url) + + if let error = fileUpdates.writeDetectedFileChanges(to: runFolder) { + print(" Hashes not saved: \(error)") + } return (results.images, results.files) } @@ -78,6 +96,8 @@ private func generateImages(_ images: ImageData, configuration: Configuration, r reader: reader, images: images) generator.generateImages() print(" ") + let file = runFolder.appendingPathComponent("images.txt") + generator.writeResults(to: file) } private func copyFiles(files: FileData, configuration: Configuration, runFolder: URL) { @@ -131,11 +151,9 @@ private func generate(configPath: String) throws { } // 3. Generate pages - let pageCount = ids.count * siteRoot.languages.count - let (images, files) = try generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, pageCount: pageCount, runFolder: runFolder) - if let error = fileUpdates.writeDetectedFileChanges(to: runFolder) { - print(error) + guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, runFolder: runFolder) else { + return } // 4. Generate images