Improve printing and image creation

Sources/Generator/Processing/FileData.swift

@@ -0,0 +1,20 @@
+import Foundation
+
+struct FileData {
+
+    ///The files marked as expected, i.e. they exist after the generation is completed. (`key`: file path, `value`: the file providing the link)
+     var expected: [String : String] = [:]
+
+    /// All files which should be copied to the output folder (`key`: The file path, `value`: The source requiring the file)
+    var toCopy: [String : String] = [:]
+
+    /// The files to minify when copying into output directory. (`key`: the file path relative to the content folder)
+    var toMinify: [String : (source: String, type: MinificationType)] = [:]
+
+    /**
+     The files marked as external in element metadata. (Key: File path, Value: source element)
+
+     Files included here are not generated, since they are assumed to be added separately.
+     */
+    var external: [String : String] = [:]
+}

Sources/Generator/Processing/FileGenerator.swift

@@ -0,0 +1,259 @@
+import Foundation
+
+final class FileGenerator {
+
+    let input: URL
+
+    let outputFolder: URL
+
+    let runFolder: URL
+
+    private let files: FileData
+
+    /// All files copied to the destination.
+    private var copiedFiles: Set<String> = []
+
+    /// Files which could not be read (`key`: file path relative to content)
+    private var unreadableFiles: [String : (source: String, message: String)] = [:]
+
+    /// Files which could not be written (`key`: file path relative to output folder)
+    private var unwritableFiles: [String : (source: String, message: String)] = [:]
+
+    private var failedMinifications: [(file: String, source: String, message: String)] = []
+
+    /// Non-existent files. `key`: file path, `value`: source element
+    private var missingFiles: [String : String] = [:]
+
+    private var minifiedFiles: [String] = []
+
+    private let numberOfFilesToCopy: Int
+
+    private var numberOfCopiedFiles = 0
+
+    private let numberOfFilesToMinify: Int
+
+    private var numberOfMinifiedFiles = 0
+
+    private var numberOfExistingFiles = 0
+
+    private var tempFile: URL {
+        runFolder.appendingPathComponent("temp")
+    }
+
+    init(input: URL, output: URL, runFolder: URL, files: FileData) {
+        self.input = input
+        self.outputFolder = output
+        self.runFolder = runFolder
+        self.files = files
+
+        numberOfFilesToCopy = files.toCopy.count
+        numberOfFilesToMinify = files.toMinify.count
+    }
+
+    func generate() {
+        copy(files: files.toCopy)
+        print("          Copied files: \(copiedFiles.count)/\(numberOfFilesToCopy)       ")
+        minify(files: files.toMinify)
+        print("        Minified files: \(minifiedFiles.count)/\(numberOfFilesToMinify)     ")
+        checkExpected(files: files.expected)
+        print("        Expected files: \(numberOfExistingFiles)/\(files.expected.count)")
+        print("        External files: \(files.external.count)")
+        print("")
+    }
+
+    func writeResultsToFile(file: URL) throws {
+        var lines: [String] = []
+        func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
+            let elements = property.map { "  " + convert($0) }.sorted()
+            guard !elements.isEmpty else {
+                return
+            }
+            lines.append("\(name):")
+            lines.append(contentsOf: elements)
+        }
+        add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
+        add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
+        add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
+        add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
+        add("External files", files.external) { "\($0.key) (from \($0.value))" }
+
+        let data = lines.joined(separator: "\n").data(using: .utf8)!
+        try data.createFolderAndWrite(to: file)
+    }
+
+    private func didCopyFile() {
+        numberOfCopiedFiles += 1
+        print("          Copied files: \(numberOfCopiedFiles)/\(numberOfFilesToCopy)   \r", terminator: "")
+        fflush(stdout)
+    }
+
+    private func didMinifyFile() {
+        numberOfMinifiedFiles += 1
+        print("        Minified files: \(numberOfMinifiedFiles)/\(numberOfFilesToMinify)    \r", terminator: "")
+        fflush(stdout)
+    }
+
+    // MARK: File copies
+
+    private func copy(files: [String : String]) {
+        for (file, source) in files {
+            copyFileIfChanged(file: file, source: source)
+        }
+    }
+
+    private func copyFileIfChanged(file: String, source: String) {
+        let cleanPath = cleanRelativeURL(file)
+        let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
+
+        defer { didCopyFile() }
+        guard copyIfChanged(cleanPath, to: destinationUrl, source: source) else {
+            return
+        }
+        copiedFiles.insert(cleanPath)
+    }
+
+    private func copyIfChanged(_ file: String, to destination: URL, source: String) -> Bool {
+        let url = input.appendingPathComponent(file)
+        do {
+            let data = try Data(contentsOf: url)
+            return writeIfChanged(data, file: file, source: source)
+        } catch {
+            markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
+            return false
+        }
+    }
+
+    @discardableResult
+    func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
+        let url = outputFolder.appendingPathComponent(file)
+        // Only write changed files
+        if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
+            return false
+        }
+        do {
+            try data.createFolderAndWrite(to: url)
+            return true
+        } catch {
+            markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
+            return false
+        }
+    }
+
+    // MARK: File minification
+
+    private func minify(files: [String : (source: String, type: MinificationType)]) {
+        for (file, other) in files {
+            minify(file: file, source: other.source, type: other.type)
+        }
+    }
+
+    private func minify(file: String, source: String, type: MinificationType) {
+        let url = input.appendingPathComponent(file)
+        let command: String
+        switch type {
+        case .js:
+            command = "uglifyjs \(url.path) > \(tempFile.path)"
+        case .css:
+            command = "cleancss \(url.path) -o \(tempFile.path)"
+        }
+
+        try? tempFile.delete()
+
+        defer { didMinifyFile() }
+
+        let output: String
+        do {
+            output = try safeShell(command)
+        } catch {
+            failedMinifications.append((file, source, "Failed to minify with error: \(error)"))
+            return
+        }
+        guard tempFile.exists else {
+            failedMinifications.append((file, source, output))
+            return
+        }
+        let data: Data
+        do {
+            data = try Data(contentsOf: tempFile)
+        } catch {
+            markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
+            return
+        }
+        if writeIfChanged(data, file: file, source: source) {
+            minifiedFiles.append(file)
+        }
+    }
+
+
+    private func cleanRelativeURL(_ raw: String) -> String {
+        let raw = raw.dropAfterLast("#") // Clean links to page content
+        guard raw.contains("..") else {
+            return raw
+        }
+        var result: [String] = []
+        for component in raw.components(separatedBy: "/") {
+            if component == ".." {
+                _ = result.popLast()
+            } else {
+                result.append(component)
+            }
+        }
+        return result.joined(separator: "/")
+    }
+
+    private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
+        unreadableFiles[path] = (source, message)
+    }
+
+    private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
+        unwritableFiles[path] = (source, message)
+    }
+
+    // MARK: File expectationts
+
+    private func checkExpected(files: [String: String]) {
+        for (file, source) in files {
+            guard !isExternal(file: file) else {
+                numberOfExistingFiles += 1
+                continue
+            }
+            let cleanPath = cleanRelativeURL(file)
+            let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
+            guard destinationUrl.exists else {
+                markFileAsMissing(at: cleanPath, requiredBy: source)
+                continue
+            }
+            numberOfExistingFiles += 1
+        }
+    }
+
+    private func markFileAsMissing(at path: String, requiredBy source: String) {
+        missingFiles[path] = source
+    }
+
+    /**
+     Check if a file is marked as external.
+
+     Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
+     then files like `docs/index.html` are also found to be external.
+     - Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
+     */
+    private func isExternal(file: String) -> Bool {
+        // Deconstruct file path
+        var path = ""
+        for part in file.components(separatedBy: "/") {
+            guard part != "" else {
+                continue
+            }
+            if path == "" {
+                path = part
+            } else {
+                path += "/" + part
+            }
+            if files.external[path] != nil {
+                return true
+            }
+        }
+        return false
+    }
+}

Sources/Generator/Processing/GenerationResultsHandler.swift

@@ -0,0 +1,467 @@
+import Foundation
+import CryptoKit
+import AppKit
+
+
+enum MinificationType {
+    case js
+    case css
+}
+
+final class GenerationResultsHandler {
+
+    /// The content folder where the input data is stored
+    let contentFolder: URL
+
+    /// The folder where the site is generated
+    let outputFolder: URL
+
+    private let fileUpdates: FileUpdateChecker
+
+    /**
+     All paths to page element folders, indexed by their unique id.
+
+     This relation is used to generate relative links to pages using the ``Element.id`
+     */
+    private let pagePaths: [String: String]
+
+    private let configuration: Configuration
+
+    private var numberOfProcessedPages = 0
+
+    private let numberOfTotalPages: Int
+
+    init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) {
+        self.contentFolder = input
+        self.fileUpdates = fileUpdates
+        self.outputFolder = output
+        self.pagePaths = pagePaths
+        self.configuration = configuration
+        self.numberOfTotalPages = pageCount
+    }
+
+    // MARK: Internal storage
+
+    /// Non-existent files. `key`: file path, `value`: source element
+    private var missingFiles: [String : String] = [:]
+
+    private(set) var files = FileData()
+
+    /// Files which could not be read (`key`: file path relative to content)
+    private var unreadableFiles: [String : (source: String, message: String)] = [:]
+
+    /// Files which could not be written (`key`: file path relative to output folder)
+    private var unwritableFiles: [String : (source: String, message: String)] = [:]
+
+    /// The paths to all files which were changed (relative to output)
+    private var generatedFiles: Set<String> = []
+
+    /// The referenced pages which do not exist (`key`: page id, `value`: source element path)
+    private var missingLinkedPages: [String : String] = [:]
+
+    /// All pages without content which have been created (`key`: page path, `value`: source element path)
+    private var emptyPages: [String : String] = [:]
+
+    /// All pages which have `status` set to ``PageState.draft``
+    private var draftPages: Set<String> = []
+
+    /// Generic warnings for pages
+    private var pageWarnings: [(message: String, source: String)] = []
+
+    /// A cache to get the size of source images, so that files don't have to be loaded multiple times (`key` absolute source path, `value`: image size)
+    private var imageSizeCache: [String : NSSize] = [:]
+
+    private(set) var images = ImageData()
+
+    // MARK: Generic warnings
+
+    private func warning(_ message: String, destination: String, path: String) {
+        let warning = "  \(destination): \(message) required by \(path)"
+        images.warnings.append(warning)
+    }
+
+    func warning(_ message: String, source: String) {
+        pageWarnings.append((message, source))
+    }
+
+    // MARK: Page data
+
+    func getPagePath(for id: String, source: String) -> String? {
+        guard let pagePath = pagePaths[id] else {
+            missingLinkedPages[id] = source
+            return nil
+        }
+        return pagePath
+    }
+
+    private func markPageAsEmpty(page: String, source: String) {
+        emptyPages[page] = source
+    }
+
+    func markPageAsDraft(page: String) {
+        draftPages.insert(page)
+    }
+
+    // MARK: File actions
+
+    /**
+     Add a file as required, so that it will be copied to the output directory.
+     Special files may be minified.
+     - Parameter file: The file path, relative to the content directory
+     - Parameter source: The path of the source element requiring the file.
+     */
+    func require(file: String, source: String) {
+        guard !isExternal(file: file) else {
+            return
+        }
+        let url = contentFolder.appendingPathComponent(file)
+        guard url.exists else {
+            markFileAsMissing(at: file, requiredBy: source)
+            return
+        }
+        guard url.isDirectory else {
+            markForCopyOrMinification(file: file, source: source)
+            return
+        }
+        do {
+            try FileManager.default
+                .contentsOfDirectory(atPath: url.path)
+                .forEach {
+                    // Recurse into subfolders
+                    require(file: file + "/" + $0, source: source)
+                }
+        } catch {
+            markFileAsUnreadable(at: file, requiredBy: source, message: "Failed to read folder: \(error)")
+        }
+    }
+
+    private func markFileAsMissing(at path: String, requiredBy source: String) {
+        missingFiles[path] = source
+    }
+
+    private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
+        unreadableFiles[path] = (source, message)
+    }
+
+    private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
+        unwritableFiles[path] = (source, message)
+    }
+
+    private func markFileAsGenerated(file: String) {
+        generatedFiles.insert(file)
+    }
+
+    private func markForCopyOrMinification(file: String, source: String) {
+        let ext = file.lastComponentAfter(".")
+        if configuration.minifyCSSandJS, ext == "js" {
+            files.toMinify[file] = (source, .js)
+            return
+        }
+        if configuration.minifyCSSandJS, ext == "css" {
+            files.toMinify[file] = (source, .css)
+            return
+        }
+        files.toCopy[file] = source
+    }
+
+    /**
+     Mark a file as explicitly missing (external).
+
+     This is done for the `externalFiles` entries in metadata,
+     to indicate that these files will be copied to the output folder manually.
+     */
+    func exclude(file: String, source: String) {
+        files.external[file] = source
+    }
+
+    /**
+     Mark a file as expected to be present in the output folder after generation.
+
+     This is done for all links between pages, which only exist after the pages have been generated.
+     */
+    func expect(file: String, source: String) {
+        files.expected[file] = source
+    }
+
+    /**
+     Check if a file is marked as external.
+
+     Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
+     then files like `docs/index.html` are also found to be external.
+     - Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
+     */
+    private func isExternal(file: String) -> Bool {
+        // Deconstruct file path
+        var path = ""
+        for part in file.components(separatedBy: "/") {
+            guard part != "" else {
+                continue
+            }
+            if path == "" {
+                path = part
+            } else {
+                path += "/" + part
+            }
+            if files.external[path] != nil {
+                return true
+            }
+        }
+        return false
+    }
+
+    func getContentOfRequiredFile(at path: String, source: String) -> String? {
+        let url = contentFolder.appendingPathComponent(path)
+        guard url.exists else {
+            markFileAsMissing(at: path, requiredBy: source)
+            return nil
+        }
+        return getContentOfFile(at: url, path: path, source: source)
+    }
+
+    /**
+     Get the content of a file which may not exist.
+     */
+    func getContentOfOptionalFile(at path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
+        let url = contentFolder.appendingPathComponent(path)
+        guard url.exists else {
+            if createEmptyFileIfMissing {
+                writeIfChanged(.init(), file: path, source: source)
+            }
+            return nil
+        }
+        return getContentOfFile(at: url, path: path, source: source)
+    }
+
+    func getContentOfMdFile(at path: String, source: String) -> String? {
+        guard let result = getContentOfOptionalFile(at: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) else {
+            markPageAsEmpty(page: path, source: source)
+            return nil
+        }
+        return result
+    }
+
+    private func getContentOfFile(at url: URL, path: String, source: String) -> String? {
+        do {
+            return try String(contentsOf: url)
+        } catch {
+            markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
+            return nil
+        }
+    }
+
+    @discardableResult
+    func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
+        let url = outputFolder.appendingPathComponent(file)
+
+        defer {
+            didCompletePage()
+        }
+        // Only write changed files
+        if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
+            return false
+        }
+        do {
+            try data.createFolderAndWrite(to: url)
+            markFileAsGenerated(file: file)
+            return true
+        } catch {
+            markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
+            return false
+        }
+    }
+
+    // MARK: Images
+
+    /**
+     Request the creation of an image.
+     - Returns: The final size of the image.
+     */
+    @discardableResult
+    func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
+        requireImage(
+            at: destination,
+            generatedFrom: source,
+            requiredBy: path,
+            quality: 0.7,
+            width: width,
+            height: desiredHeight,
+            alwaysGenerate: false)
+    }
+
+    /**
+     Create images of different types.
+
+     This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
+     - Parameter destination: The path to the destination file
+     */
+    @discardableResult
+    func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
+        // Add @2x version
+        _ = requireScaledMultiImage(
+            source: source,
+            destination: destination.insert("@2x", beforeLast: "."),
+            requiredBy: path,
+            width: width * 2,
+            desiredHeight: desiredHeight.unwrapped { $0 * 2 })
+
+        // Add @1x version
+        return requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
+    }
+
+    func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
+        requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
+    }
+
+    @discardableResult
+    private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
+        let rawDestinationPath = destination.dropAfterLast(".")
+        let avifPath = rawDestinationPath + ".avif"
+        let webpPath = rawDestinationPath + ".webp"
+        let needsGeneration = !outputFolder.appendingPathComponent(avifPath).exists || !outputFolder.appendingPathComponent(webpPath).exists
+
+        let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
+        images.multiJobs[destination] = path
+        return size
+    }
+
+    private func markImageAsMissing(path: String, source: String) {
+        images.missing[source] = path
+    }
+
+    private func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
+        let height = height.unwrapped(CGFloat.init)
+        guard let imageSize = getImageSize(atPath: source, source: path) else {
+            // Image marked as missing here
+            return .zero
+        }
+        let scaledSize = imageSize.scaledDown(to: CGFloat(width))
+
+        // Check desired height, then we can forget about it
+        if let height = height {
+            let expectedHeight = scaledSize.width / CGFloat(width) * height
+            if abs(expectedHeight - scaledSize.height) > 2 {
+                warning("Expected a height of \(expectedHeight) (is \(scaledSize.height))", destination: destination, path: path)
+            }
+        }
+
+        let job = ImageJob(
+            destination: destination,
+            width: width,
+            path: path,
+            quality: quality,
+            alwaysGenerate: alwaysGenerate)
+        insert(job: job, source: source)
+
+        return scaledSize
+    }
+
+    private func getImageSize(atPath path: String, source: String) -> NSSize? {
+        if let size = imageSizeCache[path] {
+            return size
+        }
+        let url = contentFolder.appendingPathComponent(path)
+        guard url.exists else {
+            markImageAsMissing(path: path, source: source)
+            return nil
+        }
+        guard let data = getImageData(at: url, path: path, source: source) else {
+            return nil
+        }
+
+        guard let image = NSImage(data: data) else {
+            images.unreadable[path] = source
+            return nil
+        }
+
+        let size = image.size
+        imageSizeCache[path] = size
+        return size
+    }
+
+    private func getImageData(at url: URL, path: String, source: String) -> Data? {
+        do {
+            let data = try Data(contentsOf: url)
+            fileUpdates.didLoad(data, at: path)
+            return data
+        } catch {
+            markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
+            return nil
+        }
+    }
+
+    private func insert(job: ImageJob, source: String) {
+        guard let existingSource = images.jobs[source] else {
+            images.jobs[source] = [job]
+            return
+        }
+
+        guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
+            images.jobs[source] = existingSource + [job]
+            return
+        }
+
+        guard existingJob.width != job.width else {
+            return
+        }
+        warning("Different width \(existingJob.width) as \(job.path) (width \(job.width))",
+                destination: job.destination, path: existingJob.path)
+    }
+
+    // MARK: Visual output
+
+    func didCompletePage() {
+        numberOfProcessedPages += 1
+        print("       Processed pages: \(numberOfProcessedPages)/\(numberOfTotalPages)    \r", terminator: "")
+        fflush(stdout)
+    }
+
+    func printOverview() {
+        var notes: [String] = []
+        func addIfNotZero(_ count: Int, _ name: String) {
+            guard count > 0 else {
+                return
+            }
+            notes.append("\(count) \(name)")
+        }
+        func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
+            addIfNotZero(sequence.count, name)
+        }
+        addIfNotZero(missingFiles.count, "missing files")
+        addIfNotZero(unreadableFiles.count, "unreadable files")
+        addIfNotZero(unwritableFiles.count, "unwritable files")
+        addIfNotZero(missingLinkedPages.count, "missing linked pages")
+        addIfNotZero(pageWarnings.count, "warnings")
+
+        print("         Updated pages: \(generatedFiles.count)/\(numberOfProcessedPages)     ")
+        print("                Drafts: \(draftPages.count)")
+        print("           Empty pages: \(emptyPages.count)")
+        if !notes.isEmpty {
+            print("                 Notes: " + notes.joined(separator: ", "))
+        }
+        print("")
+    }
+
+    func writeResultsToFile(file: URL) throws {
+        var lines: [String] = []
+        func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
+            let elements = property.map { "  " + convert($0) }.sorted()
+            guard !elements.isEmpty else {
+                return
+            }
+            lines.append("\(name):")
+            lines.append(contentsOf: elements)
+        }
+        add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
+        add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
+        add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
+        add("Missing linked pages", missingLinkedPages) { "\($0.key) (linked by \($0.value))" }
+        add("Warnings", pageWarnings) { "\($0.source): \($0.message)" }
+        add("Drafts", draftPages) { $0 }
+        add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" }
+        add("Generated files", generatedFiles) { $0 }
+
+        let data = lines.joined(separator: "\n").data(using: .utf8)!
+        try data.createFolderAndWrite(to: file)
+    }
+}
+

Sources/Generator/Processing/ImageData.swift

@@ -0,0 +1,19 @@
+import Foundation
+
+struct ImageData {
+
+    /// The images to generate (`key`: the image source path relative to the input folder)
+    var jobs: [String : [ImageJob]] = [:]
+
+    /// The images for which to generate multiple versions (`key`: the source file, `value`: the path of the requiring page)
+    var multiJobs: [String : String] = [:]
+
+    /// All warnings produced for images during generation
+    var warnings: [String] = []
+
+    /// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path)
+    var missing: [String : String] = [:]
+
+    /// Images which could not be read (`key`: file path relative to content, `value`: source element path)
+    var unreadable: [String : String] = [:]
+}

Sources/Generator/Processing/ImageGenerator.swift

@@ -0,0 +1,285 @@
+import Foundation
+import AppKit
+import CryptoKit
+import Darwin.C
+
+final class ImageGenerator {
+
+    private let imageOptimSupportedFileExtensions: Set<String> = ["jpg", "png", "svg"]
+
+    private let imageOptimizationBatchSize = 50
+
+    /**
+     The path to the input folder.
+     */
+    private let input: URL
+
+    /**
+     The path to the output folder
+     */
+    private let output: URL
+
+    private let imageReader: ImageReader
+    
+    /**
+     All warnings produced for images during generation
+     */
+    private var imageWarnings: Set<String> = []
+
+    /**
+     All images modified or created during this generator run.
+     */
+    private var generatedImages: Set<String> = []
+
+    /**
+     The images optimized by ImageOptim
+     */
+    private var optimizedImages: Set<String> = []
+
+    private var numberOfGeneratedImages = 0
+
+    private let numberOfTotalImages: Int
+
+    private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
+
+    private var numberOfImagesToOptimize = 0
+
+    private var numberOfOptimizedImages = 0
+
+    private let images: ImageData
+
+    private lazy var jobs: [(source: String, images: [ImageJob])] = images.jobs
+        .sorted { $0.key < $1.key }
+        .map { (source: $0.key, images: $0.value) }
+        .filter {
+            // Only load image if required
+            let imageHasChanged = imageReader.imageHasChanged(at: $0.source)
+            return imageHasChanged || $0.images.contains { job in
+                job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
+            }
+        }
+
+    private lazy var multiJobs: [String : String] = {
+        let imagesToGenerate: Set<String> = jobs.reduce([]) { $0.union($1.images.map { $0.destination }) }
+        return images.multiJobs.filter { imagesToGenerate.contains($0.key) }
+    }()
+
+    init(input: URL, output: URL, reader: ImageReader, images: ImageData) {
+        self.input = input
+        self.output = output
+        self.imageReader = reader
+        self.images = images
+        self.numberOfTotalImages = images.jobs.reduce(0) { $0 + $1.value.count }
+            + images.multiJobs.count * 2
+    }
+
+    func generateImages() {
+        var notes: [String] = []
+        func addIfNotZero(_ count: Int, _ name: String) {
+            guard count > 0 else {
+                return
+            }
+            notes.append("\(count) \(name)")
+        }
+        addIfNotZero(images.missing.count, "missing images")
+        addIfNotZero(images.unreadable.count, "unreadable images")
+
+        print("       Changed sources: \(jobs.count)/\(images.jobs.count)")
+        print("          Total images: \(numberOfTotalImages) (\(numberOfTotalImages - imageReader.numberOfFilesAccessed) versions)")
+        print("              Warnings: \(images.warnings.count)")
+        if !notes.isEmpty {
+            print("                 Notes: " + notes.joined(separator: ", "))
+        }
+        for (source, jobs) in jobs {
+            create(images: jobs, from: source)
+        }
+        for (baseImage, source) in multiJobs {
+            createMultiImages(from: baseImage, path: source)
+        }
+        print("      Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)")
+        optimizeImages()
+        print("      Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)")
+    }
+
+    private func create(images: [ImageJob], from source: String) {
+        guard let image = imageReader.getImage(atPath: source) else {
+            // TODO: Add to failed images
+            didGenerateImage(count: images.count)
+            return
+        }
+        let jobs = imageReader.imageHasChanged(at: source) ? images : images.filter(isMissing)
+        // Update all images
+        jobs.forEach { job in
+            // Prevent memory overflow due to repeated NSImage operations
+            autoreleasepool {
+                create(job: job, from: image, source: source)
+                didGenerateImage()
+            }
+        }
+    }
+
+    private func isMissing(_ job: ImageJob) -> Bool {
+        job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
+    }
+
+
+    private func create(job: ImageJob, from image: NSImage, source: String) {
+        let destinationUrl = output.appendingPathComponent(job.destination)
+        create(job: job, from: image, source: source, at: destinationUrl)
+    }
+
+    private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
+        // Ensure that image file is supported
+        let ext = destinationUrl.pathExtension.lowercased()
+        guard ImageType(fileExtension: ext) != nil else {
+            fatalError()
+        }
+
+        let destinationExtension = destinationUrl.pathExtension.lowercased()
+        guard let type = ImageType(fileExtension: destinationExtension)?.fileType else {
+            addWarning("Invalid image extension \(destinationExtension)", job: job)
+            return
+        }
+
+        let desiredWidth = CGFloat(job.width)
+
+        let sourceRep = image.representations[0]
+        let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh)
+            .scaledDown(to: desiredWidth)
+
+        // create NSBitmapRep manually, if using cgImage, the resulting size is wrong
+        let rep = NSBitmapImageRep(bitmapDataPlanes: nil,
+                                   pixelsWide: Int(destinationSize.width),
+                                   pixelsHigh: Int(destinationSize.height),
+                                   bitsPerSample: 8,
+                                   samplesPerPixel: 4,
+                                   hasAlpha: true,
+                                   isPlanar: false,
+                                   colorSpaceName: NSColorSpaceName.deviceRGB,
+                                   bytesPerRow: Int(destinationSize.width) * 4,
+                                   bitsPerPixel: 32)!
+
+        let ctx = NSGraphicsContext(bitmapImageRep: rep)
+        NSGraphicsContext.saveGraphicsState()
+        NSGraphicsContext.current = ctx
+        image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height))
+        ctx?.flushGraphics()
+        NSGraphicsContext.restoreGraphicsState()
+
+        // Get NSData, and save it
+        guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
+            addWarning("Failed to get data", job: job)
+            return
+        }
+        do {
+            try data.createFolderAndWrite(to: destinationUrl)
+        } catch {
+            addWarning("Failed to write image (\(error))", job: job)
+            return
+        }
+        generatedImages.insert(job.destination)
+    }
+
+    private func addWarning(_ message: String, destination: String, path: String) {
+        let warning = "  \(destination): \(message) required by \(path)"
+        imageWarnings.insert(warning)
+    }
+
+    private func addWarning(_ message: String, job: ImageJob) {
+        addWarning(message, destination: job.destination, path: job.path)
+    }
+
+    private func createMultiImages(from source: String, path: String) {
+        guard generatedImages.contains(source) else {
+            didGenerateImage(count: 2)
+            return
+        }
+
+        let sourceUrl = output.appendingPathComponent(source)
+        let sourcePath = sourceUrl.path
+        guard sourceUrl.exists else {
+            addWarning("No image at path \(sourcePath)", destination: source, path: path)
+            didGenerateImage(count: 2)
+            return
+        }
+
+        let avifPath = source.dropAfterLast(".") + ".avif"
+        createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
+        generatedImages.insert(avifPath)
+        didGenerateImage()
+
+        let webpPath = source.dropAfterLast(".") + ".webp"
+        createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
+        generatedImages.insert(webpPath)
+        didGenerateImage()
+
+        compress(at: source)
+    }
+
+    private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
+        let folder = destination.dropAfterLast("/")
+        let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
+        do {
+            _ = try safeShell(command)
+        } catch {
+            addWarning("Failed to create AVIF image", destination: destination, path: destination)
+        }
+    }
+
+    private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
+        let command = "cwebp \(source) -q \(quality) -o \(destination)"
+        do {
+            _ = try safeShell(command)
+        } catch {
+            addWarning("Failed to create WEBP image", destination: destination, path: destination)
+        }
+    }
+
+    private func compress(at destination: String, quality: Int = 70) {
+        let command = "magick convert \(destination) -quality \(quality)% \(destination)"
+        do {
+            _ = try safeShell(command)
+        } catch {
+            addWarning("Failed to compress image", destination: destination, path: destination)
+        }
+    }
+
+    private func optimizeImages() {
+        let all = generatedImages
+            .filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
+            .map { output.appendingPathComponent($0).path }
+        for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
+            let endIndex = min(i+imageOptimizationBatchSize, all.count)
+            let batch = all[i..<endIndex]
+            if optimizeImageBatch(batch) {
+                optimizedImages.formUnion(batch)
+            }
+            didOptimizeImage(count: batch.count)
+        }
+    }
+
+    private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
+        let command = "imageoptim " + batch.joined(separator: " ")
+        do {
+            _ = try safeShell(command)
+            return true
+        } catch {
+            addWarning("Failed to optimize images", destination: "", path: "")
+            return false
+        }
+    }
+
+    // MARK: Output
+
+    private func didGenerateImage(count: Int = 1) {
+        numberOfGeneratedImages += count
+        print("      Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)    \r", terminator: "")
+        fflush(stdout)
+    }
+
+    private func didOptimizeImage(count: Int) {
+        numberOfOptimizedImages += count
+        print("      Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)    \r", terminator: "")
+        fflush(stdout)
+    }
+}

Sources/Generator/Processing/MetadataInfoLogger.swift

@@ -4,6 +4,8 @@ final class MetadataInfoLogger {
 
     private let input: URL
 
+    private let runFolder: URL
+
     private var numberOfMetadataFiles = 0
 
     private var unusedProperties: [(name: String, source: String)] = []
@@ -20,8 +22,13 @@ final class MetadataInfoLogger {
 
     private var errors: [(source: String, message: String)] = []
 
-    init(input: URL) {
+    private var logFile: URL {
+        runFolder.appendingPathComponent("Metadata issues.txt")
+    }
+
+    init(input: URL, runFolder: URL) {
         self.input = input
+        self.runFolder = runFolder
     }
 
     /**
@@ -124,10 +131,10 @@ final class MetadataInfoLogger {
     // MARK: Printing
 
     private func printMetadataScanUpdate() {
-        print(String(format: "Scanning source files: %4d pages found     \r", numberOfMetadataFiles), terminator: "")
+        print(String(format: "           Pages found: %4d   \r", numberOfMetadataFiles), terminator: "")
     }
 
-    func printMetadataScanOverview() {
+    func printMetadataScanOverview(languages: Int) {
         var notes = [String]()
         func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
             guard sequence.count > 0 else {
@@ -139,40 +146,42 @@ final class MetadataInfoLogger {
         addIfNotZero(errors, "errors")
         addIfNotZero(unreadableMetadata, "unreadable files")
         addIfNotZero(unusedProperties, "unused properties")
-        addIfNotZero(invalidProperties, "invalidProperties")
-        addIfNotZero(unknownProperties, "unknownProperties")
-        addIfNotZero(missingProperties, "missingProperties")
+        addIfNotZero(invalidProperties, "invalid properties")
+        addIfNotZero(unknownProperties, "unknown properties")
+        addIfNotZero(missingProperties, "missing properties")
 
-        print(" Number of pages: \(numberOfMetadataFiles)")
-        print("           Notes: " + notes.joined(separator: ", "))
+        print("           Pages found: \(numberOfMetadataFiles)        ")
+        print("             Languages: \(languages)")
+        if !notes.isEmpty {
+            print("                 Notes: " + notes.joined(separator: ", "))
+        }
     }
 
-    func writeResultsToFile(in folder: URL) throws {
-        let url = folder.appendingPathComponent("Metadata issues.txt")
+    func writeResultsToFile() throws {
         var lines: [String] = []
         if !errors.isEmpty {
-            lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }
+            lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }.sorted()
         }
         if !warnings.isEmpty {
-            lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }
+            lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }.sorted()
         }
         if !unreadableMetadata.isEmpty {
-            lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }
+            lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }.sorted()
         }
         if !unusedProperties.isEmpty {
-            lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }
+            lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }.sorted()
         }
         if !invalidProperties.isEmpty {
-            lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }
+            lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }.sorted()
         }
         if !unknownProperties.isEmpty {
-            lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }
+            lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }.sorted()
         }
         if !missingProperties.isEmpty {
-            lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }
+            lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }.sorted()
         }
 
-        let data = lines.joined(separator: "\n").data(using: .utf8)
-        try data?.createFolderAndWrite(to: url)
+        let data = lines.joined(separator: "\n").data(using: .utf8)!
+        try data.createFolderAndWrite(to: logFile)
     }
 }

Sources/Generator/Processing/Shell.swift

@@ -0,0 +1,20 @@
+import Foundation
+
+@discardableResult
+func safeShell(_ command: String) throws -> String {
+    let task = Process()
+    let pipe = Pipe()
+
+    task.standardOutput = pipe
+    task.standardError = pipe
+    task.arguments = ["-cl", command]
+    task.executableURL = URL(fileURLWithPath: "/bin/zsh")
+    task.standardInput = nil
+
+    try task.run()
+
+    let data = pipe.fileHandleForReading.readDataToEndOfFile()
+    let output = String(data: data, encoding: .utf8)!
+
+    return output
+}