From 396e03279fe52ce8e03a35ffdffceb55ed2ee859 Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Sun, 18 Sep 2022 16:48:15 +0200 Subject: [PATCH] Improve image generation prints --- Sources/Generator/Files/FileSystem.swift | 224 +-------------- .../Generator/Files/FileUpdateChecker.swift | 88 ++++++ Sources/Generator/Files/ImageGenerator.swift | 262 ++++++++++++++++++ Sources/Generator/run.swift | 3 +- 4 files changed, 359 insertions(+), 218 deletions(-) create mode 100644 Sources/Generator/Files/FileUpdateChecker.swift create mode 100644 Sources/Generator/Files/ImageGenerator.swift diff --git a/Sources/Generator/Files/FileSystem.swift b/Sources/Generator/Files/FileSystem.swift index d571639..6a8f8b2 100644 --- a/Sources/Generator/Files/FileSystem.swift +++ b/Sources/Generator/Files/FileSystem.swift @@ -6,36 +6,18 @@ final class FileSystem { private static let tempFileName = "temp.bin" - private static let hashesFileName = "hashes.json" - private let input: URL private let output: URL - private let source = "FileChangeMonitor" + private let source = "FileSystem" - private var hashesFile: URL { - input.appendingPathComponent(FileSystem.hashesFileName) - } + private let images: ImageGenerator private var tempFile: URL { input.appendingPathComponent(FileSystem.tempFileName) } - /** - The hashes of all accessed files from the previous run - - The key is the relative path to the file from the source - */ - private var previousFiles: [String : Data] = [:] - - /** - The paths of all files which were accessed, with their new hashes - - This list is used to check if a file was modified, and to write all accessed files back to disk - */ - private var accessedFiles: [String : Data] = [:] - /** All files which should be copied to the output folder */ @@ -87,30 +69,8 @@ final class FileSystem { init(in input: URL, to output: URL) { self.input = input self.output = output + self.images = .init(input: input, output: output) - guard exists(hashesFile) else { - log.add(info: "No file hashes loaded, regarding all content as new", source: source) - return - } - let data: Data - do { - data = try Data(contentsOf: hashesFile) - } catch { - log.add( - warning: "File hashes could not be read, regarding all content as new", - source: source, - error: error) - return - } - do { - self.previousFiles = try JSONDecoder().decode(from: data) - } catch { - log.add( - warning: "File hashes could not be decoded, regarding all content as new", - source: source, - error: error) - return - } } func urlInOutputFolder(_ path: String) -> URL { @@ -121,15 +81,6 @@ final class FileSystem { input.appendingPathComponent(path) } - /** - Get the current hash of file data at a path. - - If the hash has been computed previously during the current run, then this function directly returns it. - */ - private func hash(_ data: Data, at path: String) -> Data { - accessedFiles[path] ?? SHA256.hash(data: data).data - } - private func exists(_ url: URL) -> Bool { FileManager.default.fileExists(atPath: url.path) } @@ -180,178 +131,19 @@ final class FileSystem { } } - private func getData(atPath path: String) -> (data: Data, didChange: Bool)? { - let url = input.appendingPathComponent(path) - guard exists(url) else { - return nil - } - let data: Data - do { - data = try Data(contentsOf: url) - } catch { - log.add(error: "Failed to read data at \(path)", source: source, error: error) - return nil - } - let newHash = hash(data, at: path) - defer { - accessedFiles[path] = newHash - } - guard let oldHash = previousFiles[path] else { - return (data: data, didChange: true) - } - return (data: data, didChange: oldHash != newHash) + func writeDetectedFileChangesToDisk() { + images.writeDetectedFileChangesToDisk() } - - func writeHashes() { - do { - let encoder = JSONEncoder() - encoder.outputFormatting = .prettyPrinted - let data = try encoder.encode(accessedFiles) - try data.write(to: hashesFile) - } catch { - log.add(warning: "Failed to save file hashes", source: source, error: error) - } - } - + // MARK: Images - private func loadImage(atPath path: String) -> (image: NSImage, changed: Bool)? { - guard let (data, changed) = getData(atPath: path) else { - log.add(error: "Failed to load file", source: path) - return nil - } - guard let image = NSImage(data: data) else { - log.add(error: "Failed to read image", source: path) - return nil - } - return (image, changed) - } - @discardableResult - let height = desiredHeight.unwrapped(CGFloat.init) - let sourceUrl = input.appendingPathComponent(source) - let image = ImageOutput(source: source, width: width, desiredHeight: desiredHeight) - - let standardSize = NSSize(width: CGFloat(width), height: height ?? CGFloat(width) / 16 * 9) - guard sourceUrl.exists else { - log.add(error: "Missing file with size (\(width),\(desiredHeight ?? -1))", - source: source) - return standardSize - } - guard let imageSize = loadImage(atPath: image.source)?.image.size else { - log.add(error: "Unreadable image with size (\(width),\(desiredHeight ?? -1))", - source: source) - return standardSize - } - let scaledSize = imageSize.scaledDown(to: CGFloat(width)) - - guard let existing = imageTasks[destination] else { - imageTasks[destination] = image - return scaledSize - } - guard existing.source == source else { - log.add(error: "Multiple sources (\(existing.source),\(source))", - source: destination) - return scaledSize - } - guard existing.hasSimilarRatio(as: image) else { - log.add(error: "Multiple ratios (\(existing.ratio!),\(image.ratio!))", - source: destination) - return scaledSize - } - if image.width > existing.width { - log.add(info: "Increasing size from \(existing.width) to \(width)", - source: destination) - imageTasks[destination] = image - } - return scaledSize func requireImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize { + images.requireImage(at: destination, generatedFrom: source, requiredBy: path, width: width, height: desiredHeight) } func createImages() { - for (destination, image) in imageTasks.sorted(by: { $0.key < $1.key }) { - createImageIfNeeded(image, for: destination) - } - } - - private func createImageIfNeeded(_ image: ImageOutput, for destination: String) { - guard let (sourceImageData, sourceImageChanged) = getData(atPath: image.source) else { - log.add(error: "Failed to open file", source: image.source) - return - } - - let destinationUrl = output.appendingPathComponent(destination) - - // Check if image needs to be updated - guard !destinationUrl.exists || sourceImageChanged else { - return - } - - // Ensure that image file is supported - let ext = destinationUrl.pathExtension.lowercased() - guard ImageType(fileExtension: ext) != nil else { - // TODO: This should never be reached, since extensions are checked before - log.add(info: "Copying image", source: image.source) - do { - let sourceUrl = input.appendingPathComponent(image.source) - try destinationUrl.ensureParentFolderExistence() - try sourceUrl.copy(to: destinationUrl) - } catch { - log.add(error: "Failed to copy image", source: destination) - } - return - } - guard let sourceImage = NSImage(data: sourceImageData) else { - log.add(error: "Failed to read file", source: image.source) - return - } - - let desiredWidth = CGFloat(image.width) - let desiredHeight = image.desiredHeight.unwrapped(CGFloat.init) - - let destinationSize = sourceImage.size.scaledDown(to: desiredWidth) - let scaledImage = sourceImage.scaledDown(to: destinationSize) - let scaledSize = scaledImage.size - - if abs(scaledSize.width - desiredWidth) > 2 { - log.add(warning: "Desired width \(desiredWidth), got \(scaledSize.width)", source: destination) - } - if abs(destinationSize.height - scaledImage.size.height) > 2 { - log.add(warning: "Desired height \(destinationSize.height), got \(scaledSize.height)", source: destination) - } - if let desiredHeight = desiredHeight { - let desiredRatio = desiredHeight / desiredWidth - let adjustedDesiredHeight = scaledSize.width * desiredRatio - if abs(adjustedDesiredHeight - scaledSize.height) > 5 { - log.add(warning: "Desired height \(desiredHeight), got \(scaledSize.height)", source: destination) - return - } - } - if scaledSize.width > desiredWidth { - log.add(warning:" Desired width \(desiredWidth), got \(scaledSize.width)", source: destination) - } - - let destinationExtension = destinationUrl.pathExtension.lowercased() - guard let type = ImageType(fileExtension: destinationExtension)?.fileType else { - log.add(error: "No image type for extension \(destinationExtension)", - source: destination) - return - } - guard let tiff = scaledImage.tiffRepresentation, let tiffData = NSBitmapImageRep(data: tiff) else { - log.add(error: "Failed to get data", source: image.source) - return - } - - guard let data = tiffData.representation(using: type, properties: [.compressionFactor: NSNumber(0.7)]) else { - log.add(error: "Failed to get data", source: image.source) - return - } - do { - try data.createFolderAndWrite(to: destinationUrl) - } catch { - log.add(error: "Failed to write image \(destination)", source: "Image Processor", error: error) - return - } + images.createImages() } // MARK: File copying diff --git a/Sources/Generator/Files/FileUpdateChecker.swift b/Sources/Generator/Files/FileUpdateChecker.swift new file mode 100644 index 0000000..2aa3cff --- /dev/null +++ b/Sources/Generator/Files/FileUpdateChecker.swift @@ -0,0 +1,88 @@ +import Foundation +import CryptoKit + +final class FileUpdateChecker { + + private static let hashesFileName = "hashes.json" + + private let input: URL + + private var hashesFile: URL { + input.appendingPathComponent(FileUpdateChecker.hashesFileName) + } + + /** + The hashes of all accessed files from the previous run + + The key is the relative path to the file from the source + */ + private var previousFiles: [String : Data] = [:] + + /** + The paths of all files which were accessed, with their new hashes + + This list is used to check if a file was modified, and to write all accessed files back to disk + */ + private var accessedFiles: [String : Data] = [:] + + private var source: String { + "FileUpdateChecker" + } + + init(input: URL) { + self.input = input + guard hashesFile.exists else { + log.add(info: "No file hashes loaded, regarding all content as new", source: source) + return + } + let data: Data + do { + data = try Data(contentsOf: hashesFile) + } catch { + log.add( + warning: "File hashes could not be read, regarding all content as new", + source: source, + error: error) + return + } + do { + self.previousFiles = try JSONDecoder().decode(from: data) + } catch { + log.add( + warning: "File hashes could not be decoded, regarding all content as new", + source: source, + error: error) + return + } + } + + func fileHasChanged(at path: String) -> Bool { + guard let oldHash = previousFiles[path] else { + // Image wasn't used last time, so treat as new + return true + } + guard let newHash = accessedFiles[path] else { + // Each image should have been loaded once + // before using this function + fatalError() + } + return oldHash != newHash + } + + func didLoad(_ data: Data, at path: String) { + accessedFiles[path] = SHA256.hash(data: data).data + } + + + func writeDetectedFileChangesToDisk() { + do { + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + let data = try encoder.encode(accessedFiles) + try data.write(to: hashesFile) + } catch { + log.add(warning: "Failed to save file hashes", source: source, error: error) + } + } + +} diff --git a/Sources/Generator/Files/ImageGenerator.swift b/Sources/Generator/Files/ImageGenerator.swift new file mode 100644 index 0000000..0a49bad --- /dev/null +++ b/Sources/Generator/Files/ImageGenerator.swift @@ -0,0 +1,262 @@ +import Foundation +import AppKit +import CryptoKit + +private struct ImageJob { + + let destination: String + + let width: Int + + let path: String +} + +final class ImageGenerator { + + /** + The path to the input folder. + */ + private let input: URL + + /** + The path to the output folder + */ + private let output: URL + + /** + The images to generate. + + The key is the image source path relative to the input folder, and the values are the destination path (relative to the output folder) and the required image width. + */ + private var imageJobs: [String : [ImageJob]] = [:] + + /** + The images which could not be found, but are required for the site. + + The key is the image path, and the value is the page that requires it. + */ + private var missingImages: [String : String] = [:] + + /** + All warnings produced for images during generation + */ + private var imageWarnings: Set = [] + + /** + All images required by the site. + + The values are the destination paths of the images, relative to the output folder + */ + private var requiredImages: Set = [] + + /** + All images modified or created during this generator run. + */ + private var generatedImages: Set = [] + + /** + A cache to get the size of source images, so that files don't have to be loaded multiple times. + + The key is the absolute source path, and the value is the image size + */ + private var imageSizeCache: [String : NSSize] = [:] + + private var fileUpdates: FileUpdateChecker + + init(input: URL, output: URL) { + self.fileUpdates = FileUpdateChecker(input: input) + self.input = input + self.output = output + } + + func writeDetectedFileChangesToDisk() { + fileUpdates.writeDetectedFileChangesToDisk() + } + + private func getImageSize(atPath path: String) -> NSSize? { + if let size = imageSizeCache[path] { + return size + } + guard let image = getImage(atPath: path) else { + return nil + } + let size = image.size + imageSizeCache[path] = size + return size + } + + private func getImage(atPath path: String) -> NSImage? { + guard let data = getData(atPath: path) else { + log.add(error: "Failed to load file", source: path) + return nil + } + guard let image = NSImage(data: data) else { + log.add(error: "Failed to read image", source: path) + return nil + } + return image + } + + private func getData(atPath path: String) -> Data? { + let url = input.appendingPathComponent(path) + guard url.exists else { + return nil + } + do { + let data = try Data(contentsOf: url) + fileUpdates.didLoad(data, at: path) + return data + } catch { + log.add(error: "Failed to read data", source: path, error: error) + return nil + } + } + + func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, width: Int, height: Int?) -> NSSize { + let height = height.unwrapped(CGFloat.init) + let sourceUrl = input.appendingPathComponent(source) + guard sourceUrl.exists else { + missingImages[source] = path + return .zero + } + + guard let imageSize = getImageSize(atPath: source) else { + missingImages[source] = path + return .zero + } + let scaledSize = imageSize.scaledDown(to: CGFloat(width)) + + // Check desired height, then we can forget about it + if let height = height { + let expectedHeight = scaledSize.width / CGFloat(width) * height + if abs(expectedHeight - scaledSize.height) > 2 { + addWarning("Invalid height (\(scaledSize.height) instead of \(expectedHeight))", destination: destination, path: path) + } + } + + let job = ImageJob(destination: destination, width: width, path: path) + + guard let existingSource = imageJobs[source] else { + imageJobs[source] = [job] + return scaledSize + } + + guard let existingJob = existingSource.first(where: { $0.destination == destination}) else { + imageJobs[source] = existingSource + [job] + return scaledSize + } + + if existingJob.width != width { + addWarning("Multiple image widths (\(existingJob.width) and \(width))", destination: destination, path: "\(existingJob.path) and \(path)") + } + return scaledSize + } + + func createImages() { + for (source, jobs) in imageJobs.sorted(by: { $0.key < $1.key }) { + create(images: jobs, from: source) + } + printMissingImages() + printImageWarnings() + } + + private func printMissingImages() { + guard !missingImages.isEmpty else { + return + } + print("\(missingImages.count) missing images:") + for (source, path) in missingImages { + print(" \(source) (required by \(path))") + } + } + + private func printImageWarnings() { + guard !imageWarnings.isEmpty else { + return + } + print("\(imageWarnings.count) image warnings:") + for imageWarning in imageWarnings { + print(imageWarning) + } + } + + private func addWarning(_ message: String, destination: String, path: String) { + let warning = " \(destination): \(message) required by \(path)" + imageWarnings.insert(warning) + } + + private func addWarning(_ message: String, job: ImageJob) { + addWarning(message, destination: job.destination, path: job.path) + } + + private func isMissing(_ job: ImageJob) -> Bool { + !output.appendingPathComponent(job.destination).exists + } + + private func create(images: [ImageJob], from source: String) { + // Only load image if required + let imageHasChanged = fileUpdates.fileHasChanged(at: source) + guard imageHasChanged || images.contains(where: isMissing) else { + return + } + + guard let image = getImage(atPath: source) else { + missingImages[source] = images.first?.path + return + } + if imageHasChanged { + // Update all images + images.forEach { create(job: $0, from: image, source: source) } + } else { + // Update only missing images + images + .filter(isMissing) + .forEach { create(job: $0, from: image, source: source) } + } + } + + private func create(job: ImageJob, from image: NSImage, source: String) { + let destinationUrl = output.appendingPathComponent(job.destination) + + // Ensure that image file is supported + let ext = destinationUrl.pathExtension.lowercased() + guard ImageType(fileExtension: ext) != nil else { + fatalError() + } + + let desiredWidth = CGFloat(image.size.width) + + let destinationSize = image.size.scaledDown(to: desiredWidth) + let scaledImage = image.scaledDown(to: destinationSize) + let scaledSize = scaledImage.size + + if abs(scaledSize.width - desiredWidth) > 2 { + addWarning("Invalid width (\(scaledSize.width) instead of \(desiredWidth))", job: job) + } + + if scaledSize.width > desiredWidth { + addWarning("Invalid width (\(scaledSize.width) instead of \(desiredWidth))", job: job) + } + + let destinationExtension = destinationUrl.pathExtension.lowercased() + guard let type = ImageType(fileExtension: destinationExtension)?.fileType else { + addWarning("Invalid image extension \(destinationExtension)", job: job) + return + } + guard let tiff = scaledImage.tiffRepresentation, let tiffData = NSBitmapImageRep(data: tiff) else { + addWarning("Failed to get data", job: job) + return + } + + guard let data = tiffData.representation(using: type, properties: [.compressionFactor: NSNumber(0.7)]) else { + addWarning("Failed to get data", job: job) + return + } + do { + try data.createFolderAndWrite(to: destinationUrl) + } catch { + addWarning("Failed to write image (\(error))", job: job) + return + } + } +} diff --git a/Sources/Generator/run.swift b/Sources/Generator/run.swift index 7f52d11..4358071 100644 --- a/Sources/Generator/run.swift +++ b/Sources/Generator/run.swift @@ -36,8 +36,7 @@ private func generate(configPath: String) throws { files.printDraftPages() files.createImages() - print("Images generated") files.copyRequiredFiles() files.printExternalFiles() - files.writeHashes() + files.writeDetectedFileChangesToDisk() }