From 956cfb52c4786d236239e7aa2a5fb5f1dad660aa Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Sun, 4 Dec 2022 19:15:22 +0100 Subject: [PATCH] Improve printing and image creation --- Sources/Generator/Content/Element.swift | 45 +- Sources/Generator/Files/FileSystem.swift | 443 ----------------- .../Generator/Files/FileUpdateChecker.swift | 56 ++- Sources/Generator/Files/ImageGenerator.swift | 456 ----------------- Sources/Generator/Files/ImageJob.swift | 14 + Sources/Generator/Files/ImageReader.swift | 66 +++ .../Generators/OverviewPageGenerator.swift | 13 +- .../Generators/OverviewSectionGenerator.swift | 11 +- .../Generators/PageContentGenerator.swift | 53 +- .../Generator/Generators/PageGenerator.swift | 30 +- .../Generators/PageHeadGenerator.swift | 10 +- .../Generator/Generators/SiteGenerator.swift | 23 +- .../Generators/ThumbnailListGenerator.swift | 7 +- Sources/Generator/Processing/FileData.swift | 20 + .../Generator/Processing/FileGenerator.swift | 259 ++++++++++ .../Processing/GenerationResultsHandler.swift | 467 ++++++++++++++++++ Sources/Generator/Processing/ImageData.swift | 19 + .../Generator/Processing/ImageGenerator.swift | 285 +++++++++++ .../Processing/MetadataInfoLogger.swift | 47 +- Sources/Generator/Processing/Shell.swift | 20 + .../Filled/LocalizedSiteTemplate.swift | 10 +- Sources/Generator/Templates/Template.swift | 8 +- Sources/Generator/run.swift | 136 +++-- 23 files changed, 1421 insertions(+), 1077 deletions(-) delete mode 100644 Sources/Generator/Files/FileSystem.swift delete mode 100644 Sources/Generator/Files/ImageGenerator.swift create mode 100644 Sources/Generator/Files/ImageJob.swift create mode 100644 Sources/Generator/Files/ImageReader.swift create mode 100644 Sources/Generator/Processing/FileData.swift create mode 100644 Sources/Generator/Processing/FileGenerator.swift create mode 100644 Sources/Generator/Processing/GenerationResultsHandler.swift create mode 100644 Sources/Generator/Processing/ImageData.swift create mode 100644 Sources/Generator/Processing/ImageGenerator.swift create mode 100644 Sources/Generator/Processing/Shell.swift diff --git a/Sources/Generator/Content/Element.swift b/Sources/Generator/Content/Element.swift index 176016c..957a468 100644 --- a/Sources/Generator/Content/Element.swift +++ b/Sources/Generator/Content/Element.swift @@ -183,7 +183,7 @@ struct Element { self.sortIndex = log.unused(metadata.sortIndex, "sortIndex", source: source) self.externalFiles = metadata.externalFiles ?? [] self.requiredFiles = metadata.requiredFiles ?? [] // Paths are already relative to root - self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "") } ?? [] + self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "", log: log) } ?? [] self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName self.thumbnailStyle = log.castUnused(metadata.thumbnailStyle, "thumbnailStyle", source: source) self.useManualSorting = log.unused(metadata.useManualSorting, "useManualSorting", source: source) @@ -204,7 +204,7 @@ struct Element { return nil } - files.add(page: path, id: id) + //files.add(page: path, id: id) self.readElements(in: folder, source: nil, log: log) } @@ -239,13 +239,13 @@ struct Element { self.author = metadata.author ?? parent.author self.topBarTitle = log.unused(metadata.topBarTitle, "topBarTitle", source: source) self.date = metadata.date.unwrapped { log.cast($0, "date", source: source) } - self.endDate = metadata.date.unwrapped { log.cast($0, "endDate", source: source) } + self.endDate = metadata.endDate.unwrapped { log.cast($0, "endDate", source: source) } self.state = log.cast(metadata.state, "state", source: source) self.sortIndex = metadata.sortIndex // TODO: Propagate external files from the parent if subpath matches? self.externalFiles = Element.rootPaths(for: metadata.externalFiles, path: path) self.requiredFiles = Element.rootPaths(for: metadata.requiredFiles, path: path) - self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path) } ?? [] + self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path, log: log) } ?? [] self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName self.thumbnailStyle = log.cast(metadata.thumbnailStyle, "thumbnailStyle", source: source) self.useManualSorting = metadata.useManualSorting ?? false @@ -285,9 +285,21 @@ struct Element { return nil } - files.add(page: path, id: id) + //files.add(page: path, id: id) self.readElements(in: folder, source: path, log: log) } + + func getContainedIds(log: MetadataInfoLogger) -> [String : String] { + elements.reduce(into: [id : path]) { dict, element in + element.getContainedIds(log: log).forEach { id, path in + if let existing = dict[id] { + log.error("Conflicting id with \(existing)", source: path) + } else { + dict[id] = path + } + } + } + } } // MARK: Paths @@ -566,21 +578,14 @@ extension Element { extension Element { - private var additionalHeadContentPath: String { + var additionalHeadContentPath: String { path + "/head.html" } - func customHeadContent() -> String? { - files.contentOfOptionalFile(atPath: additionalHeadContentPath, source: path) - } - - private var additionalFooterContentPath: String { + var additionalFooterContentPath: String { path + "/footer.html" } - func customFooterContent() -> String? { - files.contentOfOptionalFile(atPath: additionalFooterContentPath, source: path) - } } // MARK: Debug @@ -607,14 +612,14 @@ extension Element { let desiredHeight: Int? - init?(input: String, path: String) { + init?(input: String, path: String, log: MetadataInfoLogger) { let parts = input.components(separatedBy: " ").filter { !$0.isEmpty } guard parts.count == 3 || parts.count == 4 else { - log.add(error: "Invalid image specification, expected 'source dest width (height)", source: path) + log.error("Invalid image specification, expected 'source dest width (height)", source: path) return nil } guard let width = Int(parts[2]) else { - log.add(error: "Invalid width for image \(parts[0])", source: path) + log.error("Invalid width for image \(parts[0])", source: path) return nil } @@ -626,7 +631,7 @@ extension Element { return } guard let height = Int(parts[3]) else { - log.add(error: "Invalid height for image \(parts[0])", source: path) + log.error("Invalid height for image \(parts[0])", source: path) return nil } self.desiredHeight = height @@ -716,10 +721,6 @@ extension Element { let destination = pathRelativeToRootForContainedInputFile("thumbnail-\(language).\(ext)") return (source, destination) } - let thumbnailUrl = inputFolder.appendingPathComponent(thumbnailPath) - if !thumbnailUrl.exists { - log.add(error: "Missing thumbnail", source: path) - } let source = pathRelativeToRootForContainedInputFile(thumbnailPath) let ext = thumbnailPath.lastComponentAfter(".") let destination = pathRelativeToRootForContainedInputFile("thumbnail.\(ext)") diff --git a/Sources/Generator/Files/FileSystem.swift b/Sources/Generator/Files/FileSystem.swift deleted file mode 100644 index 6bf0ce8..0000000 --- a/Sources/Generator/Files/FileSystem.swift +++ /dev/null @@ -1,443 +0,0 @@ -import Foundation -import CryptoKit -import AppKit - -final class FileSystem { - - private static let tempFileName = "temp.bin" - - private let input: URL - - private let output: URL - - private let source = "FileSystem" - - private let images: ImageGenerator - - private let configuration: Configuration - - private var tempFile: URL { - input.appendingPathComponent(FileSystem.tempFileName) - } - - let generatorInfoFolder: URL - - /** - All files which should be copied to the output folder - */ - private var requiredFiles: Set = [] - - /** - The files marked as external in element metadata. - - Files included here are not generated, since they are assumed to be added separately. - */ - private var externalFiles: Set = [] - - /** - The files marked as expected, i.e. they exist after the generation is completed. - - The key of the dictionary is the file path, the value is the file providing the link - */ - private var expectedFiles: [String : String] = [:] - - /** - All pages without content which have been created - */ - private var emptyPages: Set = [] - - /** - All pages which have `status` set to ``PageState.draft`` - */ - private var draftPages: Set = [] - - /** - All paths to page element folders, indexed by their unique id. - - This relation is used to generate relative links to pages using the ``Element.id` - */ - private var pagePaths: [String: String] = [:] - - /** - The image creation tasks. - - The key is the destination path. - */ - private var imageTasks: [String : ImageOutput] = [:] - - /** - The paths to all pages which were changed - */ - private var generatedPages: Set = [] - - init(in input: URL, to output: URL, configuration: Configuration) { - self.input = input - self.output = output - self.images = .init(input: input, output: output) - self.generatorInfoFolder = input.appendingPathComponent("run") - self.configuration = configuration - } - - func urlInOutputFolder(_ path: String) -> URL { - output.appendingPathComponent(path) - } - - func urlInContentFolder(_ path: String) -> URL { - input.appendingPathComponent(path) - } - - private func exists(_ url: URL) -> Bool { - FileManager.default.fileExists(atPath: url.path) - } - - func dataOfRequiredFile(atPath path: String, source: String) -> Data? { - let url = input.appendingPathComponent(path) - guard exists(url) else { - log.failedToOpen(path, requiredBy: source, error: nil) - return nil - } - - do { - return try Data(contentsOf: url) - } catch { - log.failedToOpen(path, requiredBy: source, error: error) - return nil - } - } - - func contentOfMdFile(atPath path: String, source: String) -> String? { - contentOfOptionalFile(atPath: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) - } - - func contentOfOptionalFile(atPath path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? { - let url = input.appendingPathComponent(path) - guard exists(url) else { - if createEmptyFileIfMissing { - try? Data().write(to: url) - } - return nil - } - - do { - return try String(contentsOf: url) - } catch { - log.failedToOpen(path, requiredBy: source, error: error) - return nil - } - } - - func writeDetectedFileChangesToDisk() { - images.writeDetectedFileChangesToDisk() - } - - // MARK: Images - - @discardableResult - func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize { - images.requireImage( - at: destination, - generatedFrom: source, - requiredBy: path, - quality: 0.7, - width: width, - height: desiredHeight, - alwaysGenerate: false) - } - - /** - Create images of different types. - - This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated. - - Parameter destination: The path to the destination file - */ - @discardableResult - func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize { - images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight) - } - - func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize { - images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil) - } - - func createImages() { - images.createImages() - } - - // MARK: File copying - - /** - Add a file as required, so that it will be copied to the output directory. - */ - func require(file: String) { - let url = input.appendingPathComponent(file) - guard url.exists, url.isDirectory else { - requiredFiles.insert(file) - return - } - do { - try FileManager.default - .contentsOfDirectory(atPath: url.path) - .forEach { - // Recurse into subfolders - require(file: file + "/" + $0) - } - } catch { - log.add(error: "Failed to read folder \(file): \(error)", source: source) - } - } - - /** - Mark a file as explicitly missing. - - This is done for the `externalFiles` entries in metadata, - to indicate that these files will be copied to the output folder manually. - */ - func exclude(file: String) { - externalFiles.insert(file) - } - - /** - Mark a file as expected to be present in the output folder after generation. - - This is done for all links between pages, which only exist after the pages have been generated. - */ - func expect(file: String, source: String) { - expectedFiles[file] = source - } - - func copyRequiredFiles() { - var copiedFiles = Set() - for file in requiredFiles { - let cleanPath = cleanRelativeURL(file) - let sourceUrl = input.appendingPathComponent(cleanPath) - let destinationUrl = output.appendingPathComponent(cleanPath) - guard sourceUrl.exists else { - if !isExternal(file: file) { - log.add(error: "Missing required file", source: cleanPath) - } - continue - } - if copyFileIfChanged(from: sourceUrl, to: destinationUrl) { - copiedFiles.insert(file) - } - } - try? tempFile.delete() - for (file, source) in expectedFiles { - guard !isExternal(file: file) else { - continue - } - let cleanPath = cleanRelativeURL(file) - let destinationUrl = output.appendingPathComponent(cleanPath) - if !destinationUrl.exists { - log.add(error: "Missing \(cleanPath)", source: source) - } - } - guard !copiedFiles.isEmpty else { - print("No required files copied") - return - } - print("\(copiedFiles.count) required files copied:") - for file in copiedFiles.sorted() { - print(" " + file) - } - } - - private func copyFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool { - guard configuration.minifyCSSandJS else { - return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl) - } - switch sourceUrl.pathExtension.lowercased() { - case "js": - return minifyJS(at: sourceUrl, andWriteTo: destinationUrl) - case "css": - return minifyCSS(at: sourceUrl, andWriteTo: destinationUrl) - default: - return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl) - } - } - - private func copyBinaryFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool { - do { - let data = try Data(contentsOf: sourceUrl) - return writeIfChanged(data, to: destinationUrl) - } catch { - log.add(error: "Failed to read data at \(sourceUrl.path)", source: source, error: error) - return false - } - } - - private func minifyJS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool { - let command = "uglifyjs \(sourceUrl.path) > \(tempFile.path)" - do { - _ = try FileSystem.safeShell(command) - return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl) - } catch { - log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source) - return false - } - } - - private func minifyCSS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool { - let command = "cleancss \(sourceUrl.path) -o \(tempFile.path)" - do { - _ = try FileSystem.safeShell(command) - return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl) - } catch { - log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source) - return false - } - } - - private func cleanRelativeURL(_ raw: String) -> String { - let raw = raw.dropAfterLast("#") // Clean links to page content - guard raw.contains("..") else { - return raw - } - var result: [String] = [] - for component in raw.components(separatedBy: "/") { - if component == ".." { - _ = result.popLast() - } else { - result.append(component) - } - } - return result.joined(separator: "/") - } - - /** - Check if a file is marked as external. - - Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external, - then files like `docs/index.html` are also found to be external. - - Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash) - */ - func isExternal(file: String) -> Bool { - // Deconstruct file path - var path = "" - for part in file.components(separatedBy: "/") { - guard part != "" else { - continue - } - if path == "" { - path = part - } else { - path += "/" + part - } - if externalFiles.contains(path) { - return true - } - } - return false - } - - func printExternalFiles() { - guard !externalFiles.isEmpty else { - return - } - print("\(externalFiles.count) external resources needed:") - for file in externalFiles.sorted() { - print(" " + file) - } - } - - // MARK: Pages - - func isEmpty(page: String) { - emptyPages.insert(page) - } - - func printEmptyPages() { - guard !emptyPages.isEmpty else { - return - } - print("\(emptyPages.count) empty pages:") - for page in emptyPages.sorted() { - print(" " + page) - } - } - - func isDraft(path: String) { - draftPages.insert(path) - } - - func printDraftPages() { - guard !draftPages.isEmpty else { - return - } - print("\(draftPages.count) drafts:") - for page in draftPages.sorted() { - print(" " + page) - } - } - - func add(page: String, id: String) { - if let existing = pagePaths[id] { - log.add(error: "Conflicting id with \(existing)", source: page) - } - pagePaths[id] = page - } - - func getPage(for id: String) -> String? { - pagePaths[id] - } - - func generated(page: String) { - generatedPages.insert(page) - } - - func printGeneratedPages() { - guard !generatedPages.isEmpty else { - print("No pages modified") - return - } - print("\(generatedPages.count) pages modified") - for page in generatedPages.sorted() { - print(" " + page) - } - } - - - // MARK: Writing files - - @discardableResult - func writeIfChanged(_ data: Data, to url: URL) -> Bool { - // Only write changed files - if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent { - return false - } - do { - try data.createFolderAndWrite(to: url) - return true - } catch { - log.add(error: "Failed to write file", source: url.path, error: error) - return false - } - } - - @discardableResult - func write(_ string: String, to url: URL) -> Bool { - let data = string.data(using: .utf8)! - return writeIfChanged(data, to: url) - } - - // MARK: Running other tasks - - @discardableResult - static func safeShell(_ command: String) throws -> String { - let task = Process() - let pipe = Pipe() - - task.standardOutput = pipe - task.standardError = pipe - task.arguments = ["-cl", command] - task.executableURL = URL(fileURLWithPath: "/bin/zsh") - task.standardInput = nil - - try task.run() - - let data = pipe.fileHandleForReading.readDataToEndOfFile() - let output = String(data: data, encoding: .utf8)! - - return output - } -} diff --git a/Sources/Generator/Files/FileUpdateChecker.swift b/Sources/Generator/Files/FileUpdateChecker.swift index 2aa3cff..a671b89 100644 --- a/Sources/Generator/Files/FileUpdateChecker.swift +++ b/Sources/Generator/Files/FileUpdateChecker.swift @@ -3,13 +3,9 @@ import CryptoKit final class FileUpdateChecker { - private static let hashesFileName = "hashes.json" + private let hashesFileName = "hashes.json" private let input: URL - - private var hashesFile: URL { - input.appendingPathComponent(FileUpdateChecker.hashesFileName) - } /** The hashes of all accessed files from the previous run @@ -25,35 +21,41 @@ final class FileUpdateChecker { */ private var accessedFiles: [String : Data] = [:] - private var source: String { - "FileUpdateChecker" + var numberOfFilesLoaded: Int { + previousFiles.count + } + + var numberOfFilesAccessed: Int { + accessedFiles.count } init(input: URL) { self.input = input - guard hashesFile.exists else { - log.add(info: "No file hashes loaded, regarding all content as new", source: source) - return + } + + enum LoadResult { + case notLoaded + case loaded + case failed(String) + } + + func loadPreviousRun(from folder: URL) -> LoadResult { + let url = folder.appendingPathComponent(hashesFileName) + guard url.exists else { + return .notLoaded } let data: Data do { - data = try Data(contentsOf: hashesFile) + data = try Data(contentsOf: url) } catch { - log.add( - warning: "File hashes could not be read, regarding all content as new", - source: source, - error: error) - return + return .failed("Failed to read hashes from last run: \(error)") } do { self.previousFiles = try JSONDecoder().decode(from: data) } catch { - log.add( - warning: "File hashes could not be decoded, regarding all content as new", - source: source, - error: error) - return + return .failed("Failed to decode hashes from last run: \(error)") } + return .loaded } func fileHasChanged(at path: String) -> Bool { @@ -73,16 +75,18 @@ final class FileUpdateChecker { accessedFiles[path] = SHA256.hash(data: data).data } - - func writeDetectedFileChangesToDisk() { + func writeDetectedFileChanges(to folder: URL) -> String? { + let url = folder.appendingPathComponent(hashesFileName) do { let encoder = JSONEncoder() encoder.outputFormatting = .prettyPrinted let data = try encoder.encode(accessedFiles) - try data.write(to: hashesFile) + try data.write(to: url) + return nil } catch { - log.add(warning: "Failed to save file hashes", source: source, error: error) + return "Failed to save file hashes: \(error)" } } - } + +var notFound = 0 diff --git a/Sources/Generator/Files/ImageGenerator.swift b/Sources/Generator/Files/ImageGenerator.swift deleted file mode 100644 index 3b4beeb..0000000 --- a/Sources/Generator/Files/ImageGenerator.swift +++ /dev/null @@ -1,456 +0,0 @@ -import Foundation -import AppKit -import CryptoKit -import Darwin.C - -private struct ImageJob { - - let destination: String - - let width: Int - - let path: String - - let quality: Float - - let alwaysGenerate: Bool -} - -final class ImageGenerator { - - private let imageOptimSupportedFileExtensions: Set = ["jpg", "png", "svg"] - - private let imageOptimizationBatchSize = 50 - - /** - The path to the input folder. - */ - private let input: URL - - /** - The path to the output folder - */ - private let output: URL - - /** - The images to generate. - - The key is the image source path relative to the input folder, and the values are the destination path (relative to the output folder) and the required image width. - */ - private var imageJobs: [String : [ImageJob]] = [:] - - /** - The images for which to generate multiple versions - - The key is the source file, the value is the path of the requiring page. - */ - private var multiImageJobs: [String : String] = [:] - - /** - The images which could not be found, but are required for the site. - - The key is the image path, and the value is the page that requires it. - */ - private var missingImages: [String : String] = [:] - - /** - All warnings produced for images during generation - */ - private var imageWarnings: Set = [] - - /** - All images required by the site. - - The values are the destination paths of the images, relative to the output folder - */ - private var requiredImages: Set = [] - - /** - All images modified or created during this generator run. - */ - private var generatedImages: Set = [] - - /** - The images optimized by ImageOptim - */ - private var optimizedImages: Set = [] - - /** - A cache to get the size of source images, so that files don't have to be loaded multiple times. - - The key is the absolute source path, and the value is the image size - */ - private var imageSizeCache: [String : NSSize] = [:] - - private var fileUpdates: FileUpdateChecker - - init(input: URL, output: URL) { - self.fileUpdates = FileUpdateChecker(input: input) - self.input = input - self.output = output - } - - func writeDetectedFileChangesToDisk() { - fileUpdates.writeDetectedFileChangesToDisk() - } - - private func getImageSize(atPath path: String) -> NSSize? { - if let size = imageSizeCache[path] { - return size - } - guard let image = getImage(atPath: path) else { - return nil - } - let size = image.size - imageSizeCache[path] = size - return size - } - - private func getImage(atPath path: String) -> NSImage? { - guard let data = getData(atPath: path) else { - log.add(error: "Failed to load file", source: path) - return nil - } - guard let image = NSImage(data: data) else { - log.add(error: "Failed to read image", source: path) - return nil - } - return image - } - - private func getData(atPath path: String) -> Data? { - let url = input.appendingPathComponent(path) - guard url.exists else { - return nil - } - do { - let data = try Data(contentsOf: url) - fileUpdates.didLoad(data, at: path) - return data - } catch { - log.add(error: "Failed to read data", source: path, error: error) - return nil - } - } - - func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize { - requiredImages.insert(destination) - let height = height.unwrapped(CGFloat.init) - let sourceUrl = input.appendingPathComponent(source) - guard sourceUrl.exists else { - missingImages[source] = path - return .zero - } - - guard let imageSize = getImageSize(atPath: source) else { - missingImages[source] = path - return .zero - } - let scaledSize = imageSize.scaledDown(to: CGFloat(width)) - - // Check desired height, then we can forget about it - if let height = height { - let expectedHeight = scaledSize.width / CGFloat(width) * height - if abs(expectedHeight - scaledSize.height) > 2 { - addWarning("Invalid height (\(scaledSize.height) instead of \(expectedHeight))", destination: destination, path: path) - } - } - - let job = ImageJob( - destination: destination, - width: width, - path: path, - quality: quality, - alwaysGenerate: alwaysGenerate) - insert(job: job, source: source) - - return scaledSize - } - - private func insert(job: ImageJob, source: String) { - guard let existingSource = imageJobs[source] else { - imageJobs[source] = [job] - return - } - - guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else { - imageJobs[source] = existingSource + [job] - return - } - - if existingJob.width != job.width { - addWarning("Multiple image widths (\(existingJob.width) and \(job.width))", destination: job.destination, path: "\(existingJob.path) and \(job.path)") - } - } - - func createImages() { - var count = 0 - for (source, jobs) in imageJobs.sorted(by: { $0.key < $1.key }) { - print(String(format: "Creating images: %4d / %d\r", count, imageJobs.count), terminator: "") - fflush(stdout) - create(images: jobs, from: source) - count += 1 - } - print(" \r", terminator: "") - createMultiImages() - optimizeImages() - printMissingImages() - printImageWarnings() - printGeneratedImages() - printTotalImageCount() - } - - private func printMissingImages() { - guard !missingImages.isEmpty else { - return - } - print("\(missingImages.count) missing images:") - let sort = missingImages.sorted { (a, b) in - a.value < b.value && a.key < b.key - } - for (source, path) in sort { - print(" \(source) (required by \(path))") - } - } - - private func printImageWarnings() { - guard !imageWarnings.isEmpty else { - return - } - print("\(imageWarnings.count) image warnings:") - for imageWarning in imageWarnings { - print(imageWarning) - } - } - - private func printGeneratedImages() { - guard !generatedImages.isEmpty else { - return - } - print("\(generatedImages.count) images generated:") - for image in generatedImages { - print(" " + image) - } - } - - private func printTotalImageCount() { - print("\(requiredImages.count) images") - } - - private func addWarning(_ message: String, destination: String, path: String) { - let warning = " \(destination): \(message) required by \(path)" - imageWarnings.insert(warning) - } - - private func addWarning(_ message: String, job: ImageJob) { - addWarning(message, destination: job.destination, path: job.path) - } - - private func isMissing(_ job: ImageJob) -> Bool { - job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists - } - - private func create(images: [ImageJob], from source: String) { - // Only load image if required - let imageHasChanged = fileUpdates.fileHasChanged(at: source) - guard imageHasChanged || images.contains(where: isMissing) else { - return - } - - guard let image = getImage(atPath: source) else { - missingImages[source] = images.first?.path - return - } - let jobs = imageHasChanged ? images : images.filter(isMissing) - // Update all images - jobs.forEach { job in - // Prevent memory overflow due to repeated NSImage operations - autoreleasepool { - create(job: job, from: image, source: source) - } - } - } - - private func create(job: ImageJob, from image: NSImage, source: String) { - let destinationUrl = output.appendingPathComponent(job.destination) - create(job: job, from: image, source: source, at: destinationUrl) - } - - private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) { - - // Ensure that image file is supported - let ext = destinationUrl.pathExtension.lowercased() - guard ImageType(fileExtension: ext) != nil else { - fatalError() - } - - let destinationExtension = destinationUrl.pathExtension.lowercased() - guard let type = ImageType(fileExtension: destinationExtension)?.fileType else { - addWarning("Invalid image extension \(destinationExtension)", job: job) - return - } - - let desiredWidth = CGFloat(job.width) - - let sourceRep = image.representations[0] - let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh) - .scaledDown(to: desiredWidth) - - // create NSBitmapRep manually, if using cgImage, the resulting size is wrong - let rep = NSBitmapImageRep(bitmapDataPlanes: nil, - pixelsWide: Int(destinationSize.width), - pixelsHigh: Int(destinationSize.height), - bitsPerSample: 8, - samplesPerPixel: 4, - hasAlpha: true, - isPlanar: false, - colorSpaceName: NSColorSpaceName.deviceRGB, - bytesPerRow: Int(destinationSize.width) * 4, - bitsPerPixel: 32)! - - let ctx = NSGraphicsContext(bitmapImageRep: rep) - NSGraphicsContext.saveGraphicsState() - NSGraphicsContext.current = ctx - image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height)) - ctx?.flushGraphics() - NSGraphicsContext.restoreGraphicsState() - - // Get NSData, and save it - guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else { - addWarning("Failed to get data", job: job) - return - } - do { - try data.createFolderAndWrite(to: destinationUrl) - } catch { - addWarning("Failed to write image (\(error))", job: job) - return - } - generatedImages.insert(job.destination) - } - - /** - Create images of different types. - - This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated. - - Parameter destination: The path to the destination file - */ - @discardableResult - func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize { - // Add @1x version - _ = requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight) - - // Add @2x version - return requireScaledMultiImage( - source: source, - destination: destination.insert("@2x", beforeLast: "."), - requiredBy: path, - width: width * 2, - desiredHeight: desiredHeight.unwrapped { $0 * 2 }) - } - - @discardableResult - private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize { - let rawDestinationPath = destination.dropAfterLast(".") - let avifPath = rawDestinationPath + ".avif" - let webpPath = rawDestinationPath + ".webp" - let needsGeneration = !output.appendingPathComponent(avifPath).exists || !output.appendingPathComponent(webpPath).exists - - let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration) - multiImageJobs[destination] = path - return size - } - - private func createMultiImages() { - let sort = multiImageJobs.sorted { $0.value < $1.value && $0.key < $1.key } - var count = 1 - for (baseImage, path) in sort { - print(String(format: "Creating image versions: %4d / %d\r", count, sort.count), terminator: "") - fflush(stdout) - createMultiImages(from: baseImage, path: path) - count += 1 - } - print(" \r", terminator: "") - } - - private func createMultiImages(from source: String, path: String) { - guard generatedImages.contains(source) else { - return - } - - let sourceUrl = output.appendingPathComponent(source) - let sourcePath = sourceUrl.path - guard sourceUrl.exists else { - addWarning("No image at path \(sourcePath)", destination: source, path: path) - missingImages[source] = path - return - } - - let avifPath = source.dropAfterLast(".") + ".avif" - createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath) - generatedImages.insert(avifPath) - - let webpPath = source.dropAfterLast(".") + ".webp" - createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath) - generatedImages.insert(webpPath) - - compress(at: source) - } - - private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) { - let folder = destination.dropAfterLast("/") - let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite" - do { - _ = try FileSystem.safeShell(command) - } catch { - addWarning("Failed to create AVIF image", destination: destination, path: destination) - } - } - - private func createWEBP(at destination: String, from source: String, quality: Int = 75) { - let command = "cwebp \(source) -q \(quality) -o \(destination)" - do { - _ = try FileSystem.safeShell(command) - } catch { - addWarning("Failed to create WEBP image", destination: destination, path: destination) - } - } - - private func compress(at destination: String, quality: Int = 70) { - let command = "magick convert \(destination) -quality \(quality)% \(destination)" - do { - _ = try FileSystem.safeShell(command) - } catch { - addWarning("Failed to compress image", destination: destination, path: destination) - } - } - - private func optimizeImages() { - let all = generatedImages - .filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) } - .map { output.appendingPathComponent($0).path } - for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) { - let endIndex = min(i+imageOptimizationBatchSize, all.count) - let batch = all[i..) -> Bool { - let command = "imageoptim " + batch.joined(separator: " ") - do { - _ = try FileSystem.safeShell(command) - return true - } catch { - addWarning("Failed to optimize images", destination: "", path: "") - return false - } - } -} diff --git a/Sources/Generator/Files/ImageJob.swift b/Sources/Generator/Files/ImageJob.swift new file mode 100644 index 0000000..cd30a61 --- /dev/null +++ b/Sources/Generator/Files/ImageJob.swift @@ -0,0 +1,14 @@ +import Foundation + +struct ImageJob { + + let destination: String + + let width: Int + + let path: String + + let quality: Float + + let alwaysGenerate: Bool +} diff --git a/Sources/Generator/Files/ImageReader.swift b/Sources/Generator/Files/ImageReader.swift new file mode 100644 index 0000000..835e87d --- /dev/null +++ b/Sources/Generator/Files/ImageReader.swift @@ -0,0 +1,66 @@ +import Foundation +import AppKit + +final class ImageReader { + + /// The content folder where the input data is stored + let contentFolder: URL + + private let fileUpdates: FileUpdateChecker + + let runDataFolder: URL + + init(in input: URL, runFolder: URL, fileUpdates: FileUpdateChecker) { + self.contentFolder = input + self.runDataFolder = runFolder + self.fileUpdates = fileUpdates + } + + var numberOfFilesLoaded: Int { + fileUpdates.numberOfFilesLoaded + } + + var numberOfFilesAccessed: Int { + fileUpdates.numberOfFilesAccessed + } + + func loadData() -> FileUpdateChecker.LoadResult { + fileUpdates.loadPreviousRun(from: runDataFolder) + } + + func writeDetectedFileChangesToDisk() -> String? { + fileUpdates.writeDetectedFileChanges(to: runDataFolder) + } + + func imageHasChanged(at path: String) -> Bool { + fileUpdates.fileHasChanged(at: path) + } + + func getImage(atPath path: String) -> NSImage? { + guard let data = getData(atPath: path) else { + // TODO: log.error("Failed to load file", source: path) + return nil + } + guard let image = NSImage(data: data) else { + // TODO: log.error("Failed to read image", source: path) + return nil + } + return image + } + + private func getData(atPath path: String) -> Data? { + let url = contentFolder.appendingPathComponent(path) + guard url.exists else { + return nil + } + do { + let data = try Data(contentsOf: url) + fileUpdates.didLoad(data, at: path) + return data + } catch { + // TODO: log.error("Failed to read data: \(error)", source: path) + return nil + } + } + +} diff --git a/Sources/Generator/Generators/OverviewPageGenerator.swift b/Sources/Generator/Generators/OverviewPageGenerator.swift index 47a5ca7..75c6825 100644 --- a/Sources/Generator/Generators/OverviewPageGenerator.swift +++ b/Sources/Generator/Generators/OverviewPageGenerator.swift @@ -4,15 +4,17 @@ struct OverviewPageGenerator { private let factory: LocalizedSiteTemplate - init(factory: LocalizedSiteTemplate) { + private let results: GenerationResultsHandler + + init(factory: LocalizedSiteTemplate, results: GenerationResultsHandler) { self.factory = factory + self.results = results } func generate( section: Element, language: String) { let path = section.localizedPath(for: language) - let url = files.urlInOutputFolder(path) let metadata = section.localized(for: language) @@ -26,11 +28,8 @@ struct OverviewPageGenerator { content[.contentClass] = "overview" content[.header] = makeHeader(page: section, metadata: metadata, language: language) content[.content] = makeContent(section: section, language: language) - content[.footer] = section.customFooterContent() - guard factory.page.generate(content, to: url) else { - return - } - files.generated(page: path) + content[.footer] = results.getContentOfOptionalFile(at: section.additionalFooterContentPath, source: section.path) + factory.page.generate(content, to: path, source: section.path) } private func makeContent(section: Element, language: String) -> String { diff --git a/Sources/Generator/Generators/OverviewSectionGenerator.swift b/Sources/Generator/Generators/OverviewSectionGenerator.swift index cb733f6..07630b6 100644 --- a/Sources/Generator/Generators/OverviewSectionGenerator.swift +++ b/Sources/Generator/Generators/OverviewSectionGenerator.swift @@ -8,10 +8,10 @@ struct OverviewSectionGenerator { private let generator: ThumbnailListGenerator - init(factory: TemplateFactory) { + init(factory: TemplateFactory, results: GenerationResultsHandler) { self.multipleSectionsTemplate = factory.overviewSection self.singleSectionsTemplate = factory.overviewSectionClean - self.generator = ThumbnailListGenerator(factory: factory) + self.generator = ThumbnailListGenerator(factory: factory, results: results) } func generate(sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String { @@ -25,8 +25,13 @@ struct OverviewSectionGenerator { } private func newsSectionContent(for element: Element, language: String, sectionItemCount: Int) -> String { - let shownElements = element.mostRecentElements(sectionItemCount) + // let shownElements = element.mostRecentElements(sectionItemCount) return "" +// return generator.generateContent( +// items: shownElements, +// parent: element, +// language: language, +// style: element.thumbnailStyle) } private func sectionsContent(_ sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String { diff --git a/Sources/Generator/Generators/PageContentGenerator.swift b/Sources/Generator/Generators/PageContentGenerator.swift index c56dc46..1f6c2d2 100644 --- a/Sources/Generator/Generators/PageContentGenerator.swift +++ b/Sources/Generator/Generators/PageContentGenerator.swift @@ -10,9 +10,12 @@ struct PageContentGenerator { private let siteRoot: Element - init(factory: TemplateFactory, siteRoot: Element) { + private let results: GenerationResultsHandler + + init(factory: TemplateFactory, siteRoot: Element, results: GenerationResultsHandler) { self.factory = factory self.siteRoot = siteRoot + self.results = results } func generate(page: Element, language: String, content: String) -> (content: String, includesCode: Bool) { @@ -44,8 +47,7 @@ struct PageContentGenerator { let file = markdown.between("(", and: ")") if file.hasPrefix("page:") { let pageId = file.replacingOccurrences(of: "page:", with: "") - guard let pagePath = files.getPage(for: pageId) else { - log.add(warning: "Page id '\(pageId)' not found", source: page.path) + guard let pagePath = results.getPagePath(for: pageId, source: page.path) else { // Remove link since the page can't be found return markdown.between("[", and: "]") } @@ -57,13 +59,13 @@ struct PageContentGenerator { if let filePath = page.nonAbsolutePathRelativeToRootForContainedInputFile(file) { // The target of the page link must be present after generation is complete - files.expect(file: filePath, source: page.path) + results.expect(file: filePath, source: page.path) } return html } private func handleHTML(page: Element, language: String, html: String, markdown: Substring) -> String { - #warning("Check HTML code in markdown for required resources") + // TODO: Check HTML code in markdown for required resources //print("[HTML] Found in page \(page.path):") //print(markdown) // Things to check: @@ -123,7 +125,7 @@ struct PageContentGenerator { private func handleImage(page: Element, file: String, rightTitle: String?, leftTitle: String?) -> String { let imagePath = page.pathRelativeToRootForContainedInputFile(file) - let size = files.requireFullSizeMultiVersionImage( + let size = results.requireFullSizeMultiVersionImage( source: imagePath, destination: imagePath, requiredBy: page.path) @@ -140,28 +142,28 @@ struct PageContentGenerator { private func handleVideo(page: Element, file: String, optionString: String?) -> String { let options: [PageVideoTemplate.VideoOption] = optionString.unwrapped { string in - string.components(separatedBy: " ").compactMap { optionText in + string.components(separatedBy: " ").compactMap { optionText -> PageVideoTemplate.VideoOption? in guard let optionText = optionText.trimmed.nonEmpty else { return nil } guard let option = PageVideoTemplate.VideoOption(rawValue: optionText) else { - log.add(warning: "Unknown video option \(optionText)", source: page.path) + results.warning("Unknown video option \(optionText)", source: page.path) return nil } return option } } ?? [] - #warning("Check page folder for alternative video versions") + // TODO: Check page folder for alternative video versions let sources: [PageVideoTemplate.VideoSource] = [(url: file, type: .mp4)] let filePath = page.pathRelativeToRootForContainedInputFile(file) - files.require(file: filePath) + results.require(file: filePath, source: page.path) return factory.video.generate(sources: sources, options: options) } private func handleSvg(page: Element, file: String, area: String?) -> String { let imagePath = page.pathRelativeToRootForContainedInputFile(file) - files.require(file: imagePath) + results.require(file: imagePath, source: page.path) guard let area = area else { return factory.html.svgImage(file: file) @@ -172,7 +174,7 @@ struct PageContentGenerator { let y = Int(parts[1].trimmed), let width = Int(parts[2].trimmed), let height = Int(parts[3].trimmed) else { - log.add(warning: "Invalid area string for svg image", source: page.path) + results.warning("Invalid area string for svg image", source: page.path) return factory.html.svgImage(file: file) } @@ -180,7 +182,7 @@ struct PageContentGenerator { } private func handleFile(page: Element, file: String, fileExtension: String) -> String { - log.add(warning: "Unhandled file \(file) with extension \(fileExtension)", source: page.path) + results.warning("Unhandled file \(file) with extension \(fileExtension)", source: page.path) return "" } @@ -190,7 +192,7 @@ struct PageContentGenerator { .compactMap { button -> (file: String, text: String, downloadName: String?)? in let parts = button.components(separatedBy: ",") guard parts.count == 2 || parts.count == 3 else { - log.add(warning: "Invalid button definition", source: page.path) + results.warning("Invalid button definition", source: page.path) return nil } let file = parts[0].trimmed @@ -199,7 +201,7 @@ struct PageContentGenerator { // Ensure that file is available let filePath = page.pathRelativeToRootForContainedInputFile(file) - files.require(file: filePath) + results.require(file: filePath, source: page.path) return (file, title, downloadName) } @@ -212,7 +214,7 @@ struct PageContentGenerator { .compactMap { button -> (url: String, text: String)? in let parts = button.components(separatedBy: ",") guard parts.count == 2 else { - log.add(warning: "Invalid external link definition", source: page.path) + results.warning("Invalid external link definition", source: page.path) return nil } let url = parts[0].trimmed @@ -224,23 +226,14 @@ struct PageContentGenerator { } private func handleExternalHTML(page: Element, file: String) -> String { - let url = page.inputFolder.appendingPathComponent(file) - guard url.exists else { - log.add(error: "File \(file) not found", source: page.path) - return "" - } - do { - return try String(contentsOf: url) - } catch { - log.add(error: "File \(file) could not be read", source: page.path, error: error) - return "" - } + let path = page.pathRelativeToRootForContainedInputFile(file) + return results.getContentOfRequiredFile(at: path, source: page.path) ?? "" } private func handleSimpleBox(page: Element, content: String) -> String { let parts = content.components(separatedBy: ";") guard parts.count > 1 else { - log.add(error: "Invalid box specification", source: page.path) + results.warning("Invalid box specification", source: page.path) return "" } let title = parts[0] @@ -250,7 +243,8 @@ struct PageContentGenerator { private func handlePageLink(page: Element, language: String, pageId: String) -> String { guard let linkedPage = siteRoot.find(pageId) else { - log.add(warning: "Page id '\(pageId)' not found", source: page.path) + // Checking the page path will add it to the missing pages + _ = results.getPagePath(for: pageId, source: page.path) // Remove link since the page can't be found return "" } @@ -259,6 +253,7 @@ struct PageContentGenerator { content[.title] = linkedPage.title(for: language) let fullThumbnailPath = linkedPage.thumbnailFilePath(for: language).destination + // Note: Here we assume that the thumbnail was already used elsewhere, so already generated let relativeImageUrl = page.relativePathToOtherSiteElement(file: fullThumbnailPath) let metadata = linkedPage.localized(for: language) diff --git a/Sources/Generator/Generators/PageGenerator.swift b/Sources/Generator/Generators/PageGenerator.swift index f01fff1..f4fa6c9 100644 --- a/Sources/Generator/Generators/PageGenerator.swift +++ b/Sources/Generator/Generators/PageGenerator.swift @@ -7,20 +7,24 @@ struct PageGenerator { private let contentGenerator: PageContentGenerator - init(factory: LocalizedSiteTemplate, siteRoot: Element) { + private let results: GenerationResultsHandler + + init(factory: LocalizedSiteTemplate, siteRoot: Element, results: GenerationResultsHandler) { self.factory = factory - self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot) + self.results = results + self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot, results: results) } func generate(page: Element, language: String, previousPage: Element?, nextPage: Element?) { guard !page.isExternalPage else { + results.didCompletePage() return } let path = page.fullPageUrl(for: language) let inputContentPath = page.path + "/\(language).md" let metadata = page.localized(for: language) let nextLanguage = page.nextLanguage(for: language) - let (pageContent, pageIncludesCode, pageIsEmpty) = makeContent( + let (pageContent, pageIncludesCode) = makeContent( page: page, metadata: metadata, language: language, path: inputContentPath) var content = [PageTemplate.Key : String]() @@ -35,23 +39,17 @@ struct PageGenerator { content[.previousPageUrl] = navLink(from: page, to: previousPage, language: language) content[.nextPageLinkText] = nextText(for: nextPage, language: language) content[.nextPageUrl] = navLink(from: page, to: nextPage, language: language) - content[.footer] = page.customFooterContent() + content[.footer] = results.getContentOfOptionalFile(at: page.additionalFooterContentPath, source: page.path) if pageIncludesCode { let highlightCode = factory.factory.html.codeHighlightFooter() content[.footer] = (content[.footer].unwrapped { $0 + "\n" } ?? "") + highlightCode } - let url = files.urlInOutputFolder(path) if page.state == .draft { - files.isDraft(path: page.path) - } else if pageIsEmpty, page.state != .hidden { - files.isEmpty(page: path) + results.markPageAsDraft(page: page.path) } - guard factory.page.generate(content, to: url) else { - return - } - files.generated(page: path) + factory.page.generate(content, to: path, source: page.path) } private func navLink(from element: Element, to destination: Element?, language: String) -> String? { @@ -75,14 +73,14 @@ struct PageGenerator { return factory.factory.html.makeNextText(text) } - private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool, isEmpty: Bool) { - if let raw = files.contentOfMdFile(atPath: path, source: page.path)?.trimmed.nonEmpty { + private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool) { + if let raw = results.getContentOfMdFile(at: path, source: page.path)?.trimmed.nonEmpty { let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: raw) - return (content, includesCode, false) + return (content, includesCode) } else { let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: metadata.placeholderText) let placeholder = factory.factory.makePlaceholder(title: metadata.placeholderTitle, text: content) - return (placeholder, includesCode, true) + return (placeholder, includesCode) } } diff --git a/Sources/Generator/Generators/PageHeadGenerator.swift b/Sources/Generator/Generators/PageHeadGenerator.swift index 87bff73..22b2901 100644 --- a/Sources/Generator/Generators/PageHeadGenerator.swift +++ b/Sources/Generator/Generators/PageHeadGenerator.swift @@ -2,12 +2,16 @@ import Foundation struct PageHeadGenerator { + // TODO: Add to configuration static let linkPreviewDesiredImageWidth = 1600 let factory: TemplateFactory - init(factory: TemplateFactory) { + private let results: GenerationResultsHandler + + init(factory: TemplateFactory, results: GenerationResultsHandler) { self.factory = factory + self.results = results } func generate(page: Element, language: String, includesCode: Bool = false) -> String { @@ -24,14 +28,14 @@ struct PageHeadGenerator { let linkPreviewImageName = "thumbnail-link.\(image.lastComponentAfter("."))" let sourceImagePath = page.pathRelativeToRootForContainedInputFile(image) let destinationImagePath = page.pathRelativeToRootForContainedInputFile(linkPreviewImageName) - files.requireSingleImage( + results.requireSingleImage( source: sourceImagePath, destination: destinationImagePath, requiredBy: page.path, width: PageHeadGenerator.linkPreviewDesiredImageWidth) content[.image] = factory.html.linkPreviewImage(file: linkPreviewImageName) } - content[.customPageContent] = page.customHeadContent() + content[.customPageContent] = results.getContentOfOptionalFile(at: page.additionalHeadContentPath, source: page.path) if includesCode { let scriptPath = "assets/js/highlight.js" let relative = page.relativePathToOtherSiteElement(file: scriptPath) diff --git a/Sources/Generator/Generators/SiteGenerator.swift b/Sources/Generator/Generators/SiteGenerator.swift index 7f81271..cb4f715 100644 --- a/Sources/Generator/Generators/SiteGenerator.swift +++ b/Sources/Generator/Generators/SiteGenerator.swift @@ -6,9 +6,12 @@ struct SiteGenerator { let templates: TemplateFactory - init() throws { - let templatesFolder = files.urlInContentFolder("templates") - self.templates = try TemplateFactory(templateFolder: templatesFolder) + let results: GenerationResultsHandler + + init(results: GenerationResultsHandler) throws { + self.results = results + let templatesFolder = results.contentFolder.appendingPathComponent("templates") + self.templates = try TemplateFactory(templateFolder: templatesFolder, results: results) } func generate(site: Element) { @@ -22,11 +25,12 @@ struct SiteGenerator { let template = LocalizedSiteTemplate( factory: templates, language: language, - site: site) + site: site, + results: results) // Generate sections - let overviewGenerator = OverviewPageGenerator(factory: template) - let pageGenerator = PageGenerator(factory: template, siteRoot: site) + let overviewGenerator = OverviewPageGenerator(factory: template, results: results) + let pageGenerator = PageGenerator(factory: template, siteRoot: site, results: results) var elementsToProcess: [LinkedElement] = [(nil, site, nil)] while let (previous, element, next) = elementsToProcess.popLast() { @@ -34,7 +38,6 @@ struct SiteGenerator { elementsToProcess.append(contentsOf: element.linkedElements) processAllFiles(for: element) - if !element.elements.isEmpty { overviewGenerator.generate(section: element, language: language) } else { @@ -48,10 +51,10 @@ struct SiteGenerator { } private func processAllFiles(for element: Element) { - element.requiredFiles.forEach(files.require) - element.externalFiles.forEach(files.exclude) + element.externalFiles.forEach { results.exclude(file: $0, source: element.path) } + element.requiredFiles.forEach { results.require(file: $0, source: element.path) } element.images.forEach { - files.requireSingleImage( + results.requireSingleImage( source: $0.sourcePath, destination: $0.destinationPath, requiredBy: element.path, diff --git a/Sources/Generator/Generators/ThumbnailListGenerator.swift b/Sources/Generator/Generators/ThumbnailListGenerator.swift index 6de097f..e001ed7 100644 --- a/Sources/Generator/Generators/ThumbnailListGenerator.swift +++ b/Sources/Generator/Generators/ThumbnailListGenerator.swift @@ -4,8 +4,11 @@ struct ThumbnailListGenerator { private let factory: TemplateFactory - init(factory: TemplateFactory) { + private let results: GenerationResultsHandler + + init(factory: TemplateFactory, results: GenerationResultsHandler) { self.factory = factory + self.results = results } func generateContent(items: [Element], parent: Element, language: String, style: ThumbnailStyle) -> String { @@ -37,7 +40,7 @@ struct ThumbnailListGenerator { factory.largeThumbnail.makeCorner(text: $0) } - files.requireMultiVersionImage( + results.requireMultiVersionImage( source: thumbnailSourcePath, destination: thumbnailDestNoExtension + ".jpg", requiredBy: item.path, diff --git a/Sources/Generator/Processing/FileData.swift b/Sources/Generator/Processing/FileData.swift new file mode 100644 index 0000000..80bc5d2 --- /dev/null +++ b/Sources/Generator/Processing/FileData.swift @@ -0,0 +1,20 @@ +import Foundation + +struct FileData { + + ///The files marked as expected, i.e. they exist after the generation is completed. (`key`: file path, `value`: the file providing the link) + var expected: [String : String] = [:] + + /// All files which should be copied to the output folder (`key`: The file path, `value`: The source requiring the file) + var toCopy: [String : String] = [:] + + /// The files to minify when copying into output directory. (`key`: the file path relative to the content folder) + var toMinify: [String : (source: String, type: MinificationType)] = [:] + + /** + The files marked as external in element metadata. (Key: File path, Value: source element) + + Files included here are not generated, since they are assumed to be added separately. + */ + var external: [String : String] = [:] +} diff --git a/Sources/Generator/Processing/FileGenerator.swift b/Sources/Generator/Processing/FileGenerator.swift new file mode 100644 index 0000000..b335702 --- /dev/null +++ b/Sources/Generator/Processing/FileGenerator.swift @@ -0,0 +1,259 @@ +import Foundation + +final class FileGenerator { + + let input: URL + + let outputFolder: URL + + let runFolder: URL + + private let files: FileData + + /// All files copied to the destination. + private var copiedFiles: Set = [] + + /// Files which could not be read (`key`: file path relative to content) + private var unreadableFiles: [String : (source: String, message: String)] = [:] + + /// Files which could not be written (`key`: file path relative to output folder) + private var unwritableFiles: [String : (source: String, message: String)] = [:] + + private var failedMinifications: [(file: String, source: String, message: String)] = [] + + /// Non-existent files. `key`: file path, `value`: source element + private var missingFiles: [String : String] = [:] + + private var minifiedFiles: [String] = [] + + private let numberOfFilesToCopy: Int + + private var numberOfCopiedFiles = 0 + + private let numberOfFilesToMinify: Int + + private var numberOfMinifiedFiles = 0 + + private var numberOfExistingFiles = 0 + + private var tempFile: URL { + runFolder.appendingPathComponent("temp") + } + + init(input: URL, output: URL, runFolder: URL, files: FileData) { + self.input = input + self.outputFolder = output + self.runFolder = runFolder + self.files = files + + numberOfFilesToCopy = files.toCopy.count + numberOfFilesToMinify = files.toMinify.count + } + + func generate() { + copy(files: files.toCopy) + print(" Copied files: \(copiedFiles.count)/\(numberOfFilesToCopy) ") + minify(files: files.toMinify) + print(" Minified files: \(minifiedFiles.count)/\(numberOfFilesToMinify) ") + checkExpected(files: files.expected) + print(" Expected files: \(numberOfExistingFiles)/\(files.expected.count)") + print(" External files: \(files.external.count)") + print("") + } + + func writeResultsToFile(file: URL) throws { + var lines: [String] = [] + func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { + let elements = property.map { " " + convert($0) }.sorted() + guard !elements.isEmpty else { + return + } + lines.append("\(name):") + lines.append(contentsOf: elements) + } + add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" } + add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("External files", files.external) { "\($0.key) (from \($0.value))" } + + let data = lines.joined(separator: "\n").data(using: .utf8)! + try data.createFolderAndWrite(to: file) + } + + private func didCopyFile() { + numberOfCopiedFiles += 1 + print(" Copied files: \(numberOfCopiedFiles)/\(numberOfFilesToCopy) \r", terminator: "") + fflush(stdout) + } + + private func didMinifyFile() { + numberOfMinifiedFiles += 1 + print(" Minified files: \(numberOfMinifiedFiles)/\(numberOfFilesToMinify) \r", terminator: "") + fflush(stdout) + } + + // MARK: File copies + + private func copy(files: [String : String]) { + for (file, source) in files { + copyFileIfChanged(file: file, source: source) + } + } + + private func copyFileIfChanged(file: String, source: String) { + let cleanPath = cleanRelativeURL(file) + let destinationUrl = outputFolder.appendingPathComponent(cleanPath) + + defer { didCopyFile() } + guard copyIfChanged(cleanPath, to: destinationUrl, source: source) else { + return + } + copiedFiles.insert(cleanPath) + } + + private func copyIfChanged(_ file: String, to destination: URL, source: String) -> Bool { + let url = input.appendingPathComponent(file) + do { + let data = try Data(contentsOf: url) + return writeIfChanged(data, file: file, source: source) + } catch { + markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)") + return false + } + } + + @discardableResult + func writeIfChanged(_ data: Data, file: String, source: String) -> Bool { + let url = outputFolder.appendingPathComponent(file) + // Only write changed files + if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent { + return false + } + do { + try data.createFolderAndWrite(to: url) + return true + } catch { + markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)") + return false + } + } + + // MARK: File minification + + private func minify(files: [String : (source: String, type: MinificationType)]) { + for (file, other) in files { + minify(file: file, source: other.source, type: other.type) + } + } + + private func minify(file: String, source: String, type: MinificationType) { + let url = input.appendingPathComponent(file) + let command: String + switch type { + case .js: + command = "uglifyjs \(url.path) > \(tempFile.path)" + case .css: + command = "cleancss \(url.path) -o \(tempFile.path)" + } + + try? tempFile.delete() + + defer { didMinifyFile() } + + let output: String + do { + output = try safeShell(command) + } catch { + failedMinifications.append((file, source, "Failed to minify with error: \(error)")) + return + } + guard tempFile.exists else { + failedMinifications.append((file, source, output)) + return + } + let data: Data + do { + data = try Data(contentsOf: tempFile) + } catch { + markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)") + return + } + if writeIfChanged(data, file: file, source: source) { + minifiedFiles.append(file) + } + } + + + private func cleanRelativeURL(_ raw: String) -> String { + let raw = raw.dropAfterLast("#") // Clean links to page content + guard raw.contains("..") else { + return raw + } + var result: [String] = [] + for component in raw.components(separatedBy: "/") { + if component == ".." { + _ = result.popLast() + } else { + result.append(component) + } + } + return result.joined(separator: "/") + } + + private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) { + unreadableFiles[path] = (source, message) + } + + private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) { + unwritableFiles[path] = (source, message) + } + + // MARK: File expectationts + + private func checkExpected(files: [String: String]) { + for (file, source) in files { + guard !isExternal(file: file) else { + numberOfExistingFiles += 1 + continue + } + let cleanPath = cleanRelativeURL(file) + let destinationUrl = outputFolder.appendingPathComponent(cleanPath) + guard destinationUrl.exists else { + markFileAsMissing(at: cleanPath, requiredBy: source) + continue + } + numberOfExistingFiles += 1 + } + } + + private func markFileAsMissing(at path: String, requiredBy source: String) { + missingFiles[path] = source + } + + /** + Check if a file is marked as external. + + Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external, + then files like `docs/index.html` are also found to be external. + - Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash) + */ + private func isExternal(file: String) -> Bool { + // Deconstruct file path + var path = "" + for part in file.components(separatedBy: "/") { + guard part != "" else { + continue + } + if path == "" { + path = part + } else { + path += "/" + part + } + if files.external[path] != nil { + return true + } + } + return false + } +} diff --git a/Sources/Generator/Processing/GenerationResultsHandler.swift b/Sources/Generator/Processing/GenerationResultsHandler.swift new file mode 100644 index 0000000..4d74a74 --- /dev/null +++ b/Sources/Generator/Processing/GenerationResultsHandler.swift @@ -0,0 +1,467 @@ +import Foundation +import CryptoKit +import AppKit + + +enum MinificationType { + case js + case css +} + +final class GenerationResultsHandler { + + /// The content folder where the input data is stored + let contentFolder: URL + + /// The folder where the site is generated + let outputFolder: URL + + private let fileUpdates: FileUpdateChecker + + /** + All paths to page element folders, indexed by their unique id. + + This relation is used to generate relative links to pages using the ``Element.id` + */ + private let pagePaths: [String: String] + + private let configuration: Configuration + + private var numberOfProcessedPages = 0 + + private let numberOfTotalPages: Int + + init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) { + self.contentFolder = input + self.fileUpdates = fileUpdates + self.outputFolder = output + self.pagePaths = pagePaths + self.configuration = configuration + self.numberOfTotalPages = pageCount + } + + // MARK: Internal storage + + /// Non-existent files. `key`: file path, `value`: source element + private var missingFiles: [String : String] = [:] + + private(set) var files = FileData() + + /// Files which could not be read (`key`: file path relative to content) + private var unreadableFiles: [String : (source: String, message: String)] = [:] + + /// Files which could not be written (`key`: file path relative to output folder) + private var unwritableFiles: [String : (source: String, message: String)] = [:] + + /// The paths to all files which were changed (relative to output) + private var generatedFiles: Set = [] + + /// The referenced pages which do not exist (`key`: page id, `value`: source element path) + private var missingLinkedPages: [String : String] = [:] + + /// All pages without content which have been created (`key`: page path, `value`: source element path) + private var emptyPages: [String : String] = [:] + + /// All pages which have `status` set to ``PageState.draft`` + private var draftPages: Set = [] + + /// Generic warnings for pages + private var pageWarnings: [(message: String, source: String)] = [] + + /// A cache to get the size of source images, so that files don't have to be loaded multiple times (`key` absolute source path, `value`: image size) + private var imageSizeCache: [String : NSSize] = [:] + + private(set) var images = ImageData() + + // MARK: Generic warnings + + private func warning(_ message: String, destination: String, path: String) { + let warning = " \(destination): \(message) required by \(path)" + images.warnings.append(warning) + } + + func warning(_ message: String, source: String) { + pageWarnings.append((message, source)) + } + + // MARK: Page data + + func getPagePath(for id: String, source: String) -> String? { + guard let pagePath = pagePaths[id] else { + missingLinkedPages[id] = source + return nil + } + return pagePath + } + + private func markPageAsEmpty(page: String, source: String) { + emptyPages[page] = source + } + + func markPageAsDraft(page: String) { + draftPages.insert(page) + } + + // MARK: File actions + + /** + Add a file as required, so that it will be copied to the output directory. + Special files may be minified. + - Parameter file: The file path, relative to the content directory + - Parameter source: The path of the source element requiring the file. + */ + func require(file: String, source: String) { + guard !isExternal(file: file) else { + return + } + let url = contentFolder.appendingPathComponent(file) + guard url.exists else { + markFileAsMissing(at: file, requiredBy: source) + return + } + guard url.isDirectory else { + markForCopyOrMinification(file: file, source: source) + return + } + do { + try FileManager.default + .contentsOfDirectory(atPath: url.path) + .forEach { + // Recurse into subfolders + require(file: file + "/" + $0, source: source) + } + } catch { + markFileAsUnreadable(at: file, requiredBy: source, message: "Failed to read folder: \(error)") + } + } + + private func markFileAsMissing(at path: String, requiredBy source: String) { + missingFiles[path] = source + } + + private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) { + unreadableFiles[path] = (source, message) + } + + private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) { + unwritableFiles[path] = (source, message) + } + + private func markFileAsGenerated(file: String) { + generatedFiles.insert(file) + } + + private func markForCopyOrMinification(file: String, source: String) { + let ext = file.lastComponentAfter(".") + if configuration.minifyCSSandJS, ext == "js" { + files.toMinify[file] = (source, .js) + return + } + if configuration.minifyCSSandJS, ext == "css" { + files.toMinify[file] = (source, .css) + return + } + files.toCopy[file] = source + } + + /** + Mark a file as explicitly missing (external). + + This is done for the `externalFiles` entries in metadata, + to indicate that these files will be copied to the output folder manually. + */ + func exclude(file: String, source: String) { + files.external[file] = source + } + + /** + Mark a file as expected to be present in the output folder after generation. + + This is done for all links between pages, which only exist after the pages have been generated. + */ + func expect(file: String, source: String) { + files.expected[file] = source + } + + /** + Check if a file is marked as external. + + Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external, + then files like `docs/index.html` are also found to be external. + - Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash) + */ + private func isExternal(file: String) -> Bool { + // Deconstruct file path + var path = "" + for part in file.components(separatedBy: "/") { + guard part != "" else { + continue + } + if path == "" { + path = part + } else { + path += "/" + part + } + if files.external[path] != nil { + return true + } + } + return false + } + + func getContentOfRequiredFile(at path: String, source: String) -> String? { + let url = contentFolder.appendingPathComponent(path) + guard url.exists else { + markFileAsMissing(at: path, requiredBy: source) + return nil + } + return getContentOfFile(at: url, path: path, source: source) + } + + /** + Get the content of a file which may not exist. + */ + func getContentOfOptionalFile(at path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? { + let url = contentFolder.appendingPathComponent(path) + guard url.exists else { + if createEmptyFileIfMissing { + writeIfChanged(.init(), file: path, source: source) + } + return nil + } + return getContentOfFile(at: url, path: path, source: source) + } + + func getContentOfMdFile(at path: String, source: String) -> String? { + guard let result = getContentOfOptionalFile(at: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) else { + markPageAsEmpty(page: path, source: source) + return nil + } + return result + } + + private func getContentOfFile(at url: URL, path: String, source: String) -> String? { + do { + return try String(contentsOf: url) + } catch { + markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)") + return nil + } + } + + @discardableResult + func writeIfChanged(_ data: Data, file: String, source: String) -> Bool { + let url = outputFolder.appendingPathComponent(file) + + defer { + didCompletePage() + } + // Only write changed files + if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent { + return false + } + do { + try data.createFolderAndWrite(to: url) + markFileAsGenerated(file: file) + return true + } catch { + markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)") + return false + } + } + + // MARK: Images + + /** + Request the creation of an image. + - Returns: The final size of the image. + */ + @discardableResult + func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize { + requireImage( + at: destination, + generatedFrom: source, + requiredBy: path, + quality: 0.7, + width: width, + height: desiredHeight, + alwaysGenerate: false) + } + + /** + Create images of different types. + + This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated. + - Parameter destination: The path to the destination file + */ + @discardableResult + func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize { + // Add @2x version + _ = requireScaledMultiImage( + source: source, + destination: destination.insert("@2x", beforeLast: "."), + requiredBy: path, + width: width * 2, + desiredHeight: desiredHeight.unwrapped { $0 * 2 }) + + // Add @1x version + return requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight) + } + + func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize { + requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil) + } + + @discardableResult + private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize { + let rawDestinationPath = destination.dropAfterLast(".") + let avifPath = rawDestinationPath + ".avif" + let webpPath = rawDestinationPath + ".webp" + let needsGeneration = !outputFolder.appendingPathComponent(avifPath).exists || !outputFolder.appendingPathComponent(webpPath).exists + + let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration) + images.multiJobs[destination] = path + return size + } + + private func markImageAsMissing(path: String, source: String) { + images.missing[source] = path + } + + private func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize { + let height = height.unwrapped(CGFloat.init) + guard let imageSize = getImageSize(atPath: source, source: path) else { + // Image marked as missing here + return .zero + } + let scaledSize = imageSize.scaledDown(to: CGFloat(width)) + + // Check desired height, then we can forget about it + if let height = height { + let expectedHeight = scaledSize.width / CGFloat(width) * height + if abs(expectedHeight - scaledSize.height) > 2 { + warning("Expected a height of \(expectedHeight) (is \(scaledSize.height))", destination: destination, path: path) + } + } + + let job = ImageJob( + destination: destination, + width: width, + path: path, + quality: quality, + alwaysGenerate: alwaysGenerate) + insert(job: job, source: source) + + return scaledSize + } + + private func getImageSize(atPath path: String, source: String) -> NSSize? { + if let size = imageSizeCache[path] { + return size + } + let url = contentFolder.appendingPathComponent(path) + guard url.exists else { + markImageAsMissing(path: path, source: source) + return nil + } + guard let data = getImageData(at: url, path: path, source: source) else { + return nil + } + + guard let image = NSImage(data: data) else { + images.unreadable[path] = source + return nil + } + + let size = image.size + imageSizeCache[path] = size + return size + } + + private func getImageData(at url: URL, path: String, source: String) -> Data? { + do { + let data = try Data(contentsOf: url) + fileUpdates.didLoad(data, at: path) + return data + } catch { + markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)") + return nil + } + } + + private func insert(job: ImageJob, source: String) { + guard let existingSource = images.jobs[source] else { + images.jobs[source] = [job] + return + } + + guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else { + images.jobs[source] = existingSource + [job] + return + } + + guard existingJob.width != job.width else { + return + } + warning("Different width \(existingJob.width) as \(job.path) (width \(job.width))", + destination: job.destination, path: existingJob.path) + } + + // MARK: Visual output + + func didCompletePage() { + numberOfProcessedPages += 1 + print(" Processed pages: \(numberOfProcessedPages)/\(numberOfTotalPages) \r", terminator: "") + fflush(stdout) + } + + func printOverview() { + var notes: [String] = [] + func addIfNotZero(_ count: Int, _ name: String) { + guard count > 0 else { + return + } + notes.append("\(count) \(name)") + } + func addIfNotZero(_ sequence: Array, _ name: String) { + addIfNotZero(sequence.count, name) + } + addIfNotZero(missingFiles.count, "missing files") + addIfNotZero(unreadableFiles.count, "unreadable files") + addIfNotZero(unwritableFiles.count, "unwritable files") + addIfNotZero(missingLinkedPages.count, "missing linked pages") + addIfNotZero(pageWarnings.count, "warnings") + + print(" Updated pages: \(generatedFiles.count)/\(numberOfProcessedPages) ") + print(" Drafts: \(draftPages.count)") + print(" Empty pages: \(emptyPages.count)") + if !notes.isEmpty { + print(" Notes: " + notes.joined(separator: ", ")) + } + print("") + } + + func writeResultsToFile(file: URL) throws { + var lines: [String] = [] + func add(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence { + let elements = property.map { " " + convert($0) }.sorted() + guard !elements.isEmpty else { + return + } + lines.append("\(name):") + lines.append(contentsOf: elements) + } + add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" } + add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" } + add("Missing linked pages", missingLinkedPages) { "\($0.key) (linked by \($0.value))" } + add("Warnings", pageWarnings) { "\($0.source): \($0.message)" } + add("Drafts", draftPages) { $0 } + add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" } + add("Generated files", generatedFiles) { $0 } + + let data = lines.joined(separator: "\n").data(using: .utf8)! + try data.createFolderAndWrite(to: file) + } +} + diff --git a/Sources/Generator/Processing/ImageData.swift b/Sources/Generator/Processing/ImageData.swift new file mode 100644 index 0000000..4fb0562 --- /dev/null +++ b/Sources/Generator/Processing/ImageData.swift @@ -0,0 +1,19 @@ +import Foundation + +struct ImageData { + + /// The images to generate (`key`: the image source path relative to the input folder) + var jobs: [String : [ImageJob]] = [:] + + /// The images for which to generate multiple versions (`key`: the source file, `value`: the path of the requiring page) + var multiJobs: [String : String] = [:] + + /// All warnings produced for images during generation + var warnings: [String] = [] + + /// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path) + var missing: [String : String] = [:] + + /// Images which could not be read (`key`: file path relative to content, `value`: source element path) + var unreadable: [String : String] = [:] +} diff --git a/Sources/Generator/Processing/ImageGenerator.swift b/Sources/Generator/Processing/ImageGenerator.swift new file mode 100644 index 0000000..5b855fd --- /dev/null +++ b/Sources/Generator/Processing/ImageGenerator.swift @@ -0,0 +1,285 @@ +import Foundation +import AppKit +import CryptoKit +import Darwin.C + +final class ImageGenerator { + + private let imageOptimSupportedFileExtensions: Set = ["jpg", "png", "svg"] + + private let imageOptimizationBatchSize = 50 + + /** + The path to the input folder. + */ + private let input: URL + + /** + The path to the output folder + */ + private let output: URL + + private let imageReader: ImageReader + + /** + All warnings produced for images during generation + */ + private var imageWarnings: Set = [] + + /** + All images modified or created during this generator run. + */ + private var generatedImages: Set = [] + + /** + The images optimized by ImageOptim + */ + private var optimizedImages: Set = [] + + private var numberOfGeneratedImages = 0 + + private let numberOfTotalImages: Int + + private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2 + + private var numberOfImagesToOptimize = 0 + + private var numberOfOptimizedImages = 0 + + private let images: ImageData + + private lazy var jobs: [(source: String, images: [ImageJob])] = images.jobs + .sorted { $0.key < $1.key } + .map { (source: $0.key, images: $0.value) } + .filter { + // Only load image if required + let imageHasChanged = imageReader.imageHasChanged(at: $0.source) + return imageHasChanged || $0.images.contains { job in + job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists + } + } + + private lazy var multiJobs: [String : String] = { + let imagesToGenerate: Set = jobs.reduce([]) { $0.union($1.images.map { $0.destination }) } + return images.multiJobs.filter { imagesToGenerate.contains($0.key) } + }() + + init(input: URL, output: URL, reader: ImageReader, images: ImageData) { + self.input = input + self.output = output + self.imageReader = reader + self.images = images + self.numberOfTotalImages = images.jobs.reduce(0) { $0 + $1.value.count } + + images.multiJobs.count * 2 + } + + func generateImages() { + var notes: [String] = [] + func addIfNotZero(_ count: Int, _ name: String) { + guard count > 0 else { + return + } + notes.append("\(count) \(name)") + } + addIfNotZero(images.missing.count, "missing images") + addIfNotZero(images.unreadable.count, "unreadable images") + + print(" Changed sources: \(jobs.count)/\(images.jobs.count)") + print(" Total images: \(numberOfTotalImages) (\(numberOfTotalImages - imageReader.numberOfFilesAccessed) versions)") + print(" Warnings: \(images.warnings.count)") + if !notes.isEmpty { + print(" Notes: " + notes.joined(separator: ", ")) + } + for (source, jobs) in jobs { + create(images: jobs, from: source) + } + for (baseImage, source) in multiJobs { + createMultiImages(from: baseImage, path: source) + } + print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)") + optimizeImages() + print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)") + } + + private func create(images: [ImageJob], from source: String) { + guard let image = imageReader.getImage(atPath: source) else { + // TODO: Add to failed images + didGenerateImage(count: images.count) + return + } + let jobs = imageReader.imageHasChanged(at: source) ? images : images.filter(isMissing) + // Update all images + jobs.forEach { job in + // Prevent memory overflow due to repeated NSImage operations + autoreleasepool { + create(job: job, from: image, source: source) + didGenerateImage() + } + } + } + + private func isMissing(_ job: ImageJob) -> Bool { + job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists + } + + + private func create(job: ImageJob, from image: NSImage, source: String) { + let destinationUrl = output.appendingPathComponent(job.destination) + create(job: job, from: image, source: source, at: destinationUrl) + } + + private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) { + // Ensure that image file is supported + let ext = destinationUrl.pathExtension.lowercased() + guard ImageType(fileExtension: ext) != nil else { + fatalError() + } + + let destinationExtension = destinationUrl.pathExtension.lowercased() + guard let type = ImageType(fileExtension: destinationExtension)?.fileType else { + addWarning("Invalid image extension \(destinationExtension)", job: job) + return + } + + let desiredWidth = CGFloat(job.width) + + let sourceRep = image.representations[0] + let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh) + .scaledDown(to: desiredWidth) + + // create NSBitmapRep manually, if using cgImage, the resulting size is wrong + let rep = NSBitmapImageRep(bitmapDataPlanes: nil, + pixelsWide: Int(destinationSize.width), + pixelsHigh: Int(destinationSize.height), + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: NSColorSpaceName.deviceRGB, + bytesPerRow: Int(destinationSize.width) * 4, + bitsPerPixel: 32)! + + let ctx = NSGraphicsContext(bitmapImageRep: rep) + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = ctx + image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height)) + ctx?.flushGraphics() + NSGraphicsContext.restoreGraphicsState() + + // Get NSData, and save it + guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else { + addWarning("Failed to get data", job: job) + return + } + do { + try data.createFolderAndWrite(to: destinationUrl) + } catch { + addWarning("Failed to write image (\(error))", job: job) + return + } + generatedImages.insert(job.destination) + } + + private func addWarning(_ message: String, destination: String, path: String) { + let warning = " \(destination): \(message) required by \(path)" + imageWarnings.insert(warning) + } + + private func addWarning(_ message: String, job: ImageJob) { + addWarning(message, destination: job.destination, path: job.path) + } + + private func createMultiImages(from source: String, path: String) { + guard generatedImages.contains(source) else { + didGenerateImage(count: 2) + return + } + + let sourceUrl = output.appendingPathComponent(source) + let sourcePath = sourceUrl.path + guard sourceUrl.exists else { + addWarning("No image at path \(sourcePath)", destination: source, path: path) + didGenerateImage(count: 2) + return + } + + let avifPath = source.dropAfterLast(".") + ".avif" + createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath) + generatedImages.insert(avifPath) + didGenerateImage() + + let webpPath = source.dropAfterLast(".") + ".webp" + createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath) + generatedImages.insert(webpPath) + didGenerateImage() + + compress(at: source) + } + + private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) { + let folder = destination.dropAfterLast("/") + let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite" + do { + _ = try safeShell(command) + } catch { + addWarning("Failed to create AVIF image", destination: destination, path: destination) + } + } + + private func createWEBP(at destination: String, from source: String, quality: Int = 75) { + let command = "cwebp \(source) -q \(quality) -o \(destination)" + do { + _ = try safeShell(command) + } catch { + addWarning("Failed to create WEBP image", destination: destination, path: destination) + } + } + + private func compress(at destination: String, quality: Int = 70) { + let command = "magick convert \(destination) -quality \(quality)% \(destination)" + do { + _ = try safeShell(command) + } catch { + addWarning("Failed to compress image", destination: destination, path: destination) + } + } + + private func optimizeImages() { + let all = generatedImages + .filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) } + .map { output.appendingPathComponent($0).path } + for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) { + let endIndex = min(i+imageOptimizationBatchSize, all.count) + let batch = all[i..) -> Bool { + let command = "imageoptim " + batch.joined(separator: " ") + do { + _ = try safeShell(command) + return true + } catch { + addWarning("Failed to optimize images", destination: "", path: "") + return false + } + } + + // MARK: Output + + private func didGenerateImage(count: Int = 1) { + numberOfGeneratedImages += count + print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate) \r", terminator: "") + fflush(stdout) + } + + private func didOptimizeImage(count: Int) { + numberOfOptimizedImages += count + print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize) \r", terminator: "") + fflush(stdout) + } +} diff --git a/Sources/Generator/Processing/MetadataInfoLogger.swift b/Sources/Generator/Processing/MetadataInfoLogger.swift index d5baa78..2584f38 100644 --- a/Sources/Generator/Processing/MetadataInfoLogger.swift +++ b/Sources/Generator/Processing/MetadataInfoLogger.swift @@ -4,6 +4,8 @@ final class MetadataInfoLogger { private let input: URL + private let runFolder: URL + private var numberOfMetadataFiles = 0 private var unusedProperties: [(name: String, source: String)] = [] @@ -20,8 +22,13 @@ final class MetadataInfoLogger { private var errors: [(source: String, message: String)] = [] - init(input: URL) { + private var logFile: URL { + runFolder.appendingPathComponent("Metadata issues.txt") + } + + init(input: URL, runFolder: URL) { self.input = input + self.runFolder = runFolder } /** @@ -124,10 +131,10 @@ final class MetadataInfoLogger { // MARK: Printing private func printMetadataScanUpdate() { - print(String(format: "Scanning source files: %4d pages found \r", numberOfMetadataFiles), terminator: "") + print(String(format: " Pages found: %4d \r", numberOfMetadataFiles), terminator: "") } - func printMetadataScanOverview() { + func printMetadataScanOverview(languages: Int) { var notes = [String]() func addIfNotZero(_ sequence: Array, _ name: String) { guard sequence.count > 0 else { @@ -139,40 +146,42 @@ final class MetadataInfoLogger { addIfNotZero(errors, "errors") addIfNotZero(unreadableMetadata, "unreadable files") addIfNotZero(unusedProperties, "unused properties") - addIfNotZero(invalidProperties, "invalidProperties") - addIfNotZero(unknownProperties, "unknownProperties") - addIfNotZero(missingProperties, "missingProperties") + addIfNotZero(invalidProperties, "invalid properties") + addIfNotZero(unknownProperties, "unknown properties") + addIfNotZero(missingProperties, "missing properties") - print(" Number of pages: \(numberOfMetadataFiles)") - print(" Notes: " + notes.joined(separator: ", ")) + print(" Pages found: \(numberOfMetadataFiles) ") + print(" Languages: \(languages)") + if !notes.isEmpty { + print(" Notes: " + notes.joined(separator: ", ")) + } } - func writeResultsToFile(in folder: URL) throws { - let url = folder.appendingPathComponent("Metadata issues.txt") + func writeResultsToFile() throws { var lines: [String] = [] if !errors.isEmpty { - lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" } + lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }.sorted() } if !warnings.isEmpty { - lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" } + lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }.sorted() } if !unreadableMetadata.isEmpty { - lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" } + lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }.sorted() } if !unusedProperties.isEmpty { - lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" } + lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }.sorted() } if !invalidProperties.isEmpty { - lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" } + lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }.sorted() } if !unknownProperties.isEmpty { - lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" } + lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }.sorted() } if !missingProperties.isEmpty { - lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" } + lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }.sorted() } - let data = lines.joined(separator: "\n").data(using: .utf8) - try data?.createFolderAndWrite(to: url) + let data = lines.joined(separator: "\n").data(using: .utf8)! + try data.createFolderAndWrite(to: logFile) } } diff --git a/Sources/Generator/Processing/Shell.swift b/Sources/Generator/Processing/Shell.swift new file mode 100644 index 0000000..b464dd2 --- /dev/null +++ b/Sources/Generator/Processing/Shell.swift @@ -0,0 +1,20 @@ +import Foundation + +@discardableResult +func safeShell(_ command: String) throws -> String { + let task = Process() + let pipe = Pipe() + + task.standardOutput = pipe + task.standardError = pipe + task.arguments = ["-cl", command] + task.executableURL = URL(fileURLWithPath: "/bin/zsh") + task.standardInput = nil + + try task.run() + + let data = pipe.fileHandleForReading.readDataToEndOfFile() + let output = String(data: data, encoding: .utf8)! + + return output +} diff --git a/Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift b/Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift index e34f88e..baaf68c 100644 --- a/Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift +++ b/Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift @@ -35,7 +35,7 @@ struct LocalizedSiteTemplate { factory.page } - init(factory: TemplateFactory, language: String, site: Element, log: GenerationResultsHandler) { + init(factory: TemplateFactory, language: String, site: Element, results: GenerationResultsHandler) { self.author = site.author self.factory = factory @@ -66,12 +66,8 @@ struct LocalizedSiteTemplate { language: language, sections: sections, topBarWebsiteTitle: site.topBarTitle) - self.pageHead = PageHeadGenerator( - factory: factory, - log: log) - self.overviewSection = OverviewSectionGenerator( - factory: factory, - log: log) + self.pageHead = PageHeadGenerator(factory: factory, results: results) + self.overviewSection = OverviewSectionGenerator(factory: factory, results: results) } // MARK: Content diff --git a/Sources/Generator/Templates/Template.swift b/Sources/Generator/Templates/Template.swift index fbdc740..d913440 100644 --- a/Sources/Generator/Templates/Template.swift +++ b/Sources/Generator/Templates/Template.swift @@ -26,10 +26,10 @@ extension Template { self.init(raw: raw, results: results) } - func generate(_ content: [Key : String], to url: URL) -> Bool { - let content = generate(content) - #warning("log.write(content, to: url, file: )") - return true + @discardableResult + func generate(_ content: [Key : String], to file: String, source: String) -> Bool { + let content = generate(content).data(using: .utf8)! + return results.writeIfChanged(content, file: file, source: source) } func generate(_ content: [Key : String], shouldIndent: Bool = false) -> String { diff --git a/Sources/Generator/run.swift b/Sources/Generator/run.swift index d6d9d49..0cba544 100644 --- a/Sources/Generator/run.swift +++ b/Sources/Generator/run.swift @@ -1,8 +1,6 @@ import Foundation import ArgumentParser -let log = ValidationLog() -var files: FileSystem! @main struct CHGenerator: ParsableCommand { @@ -15,20 +13,10 @@ struct CHGenerator: ParsableCommand { } } -private func loadSiteData(in folder: URL) throws -> Element? { - let log = MetadataInfoLogger(input: folder) - print("--- SOURCE FILES -----------------------------------") - let root = Element(atRoot: folder, log: log) - print(" ") - log.printMetadataScanOverview() - print(" ") - try log.writeResultsToFile(in: files.generatorInfoFolder) - return root -} - private func loadConfiguration(at configPath: String) -> Configuration? { + print(" ") print("--- CONFIGURATION ----------------------------------") - print("") + print(" ") print(" Configuration file: \(configPath)") let configUrl = URL(fileURLWithPath: configPath) let config: Configuration @@ -36,7 +24,7 @@ private func loadConfiguration(at configPath: String) -> Configuration? { let data = try Data(contentsOf: configUrl) config = try JSONDecoder().decode(from: data) } catch { - print(" Configuration error: \(error)") + print(" Configuration error: \(error)") return nil } config.printOverview() @@ -44,30 +32,118 @@ private func loadConfiguration(at configPath: String) -> Configuration? { return config } +private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, ids: [String : String])? { + print("--- SOURCE FILES -----------------------------------") + print(" ") + + let log = MetadataInfoLogger(input: folder, runFolder: runFolder) + let root = Element(atRoot: folder, log: log) + log.printMetadataScanOverview(languages: root?.languages.count ?? 0) + print(" ") + try log.writeResultsToFile() + guard let root else { + return nil + } + let ids = root.getContainedIds(log: log) + return (root, ids) +} + +private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], pageCount: Int, runFolder: URL) throws -> (ImageData, FileData) { + print("--- GENERATION -------------------------------------") + print(" ") + + let results = GenerationResultsHandler( + in: configuration.contentDirectory, + to: configuration.outputDirectory, + configuration: configuration, + fileUpdates: fileUpdates, + pagePaths: ids, + pageCount: pageCount) + + let siteGenerator = try SiteGenerator(results: results) + siteGenerator.generate(site: root) + results.printOverview() + let url = runFolder.appendingPathComponent("files.txt") + try results.writeResultsToFile(file: url) + return (results.images, results.files) +} + +private func generateImages(_ images: ImageData, configuration: Configuration, runFolder: URL, fileUpdates: FileUpdateChecker) { + print("--- IMAGES -----------------------------------------") + print(" ") + let reader = ImageReader(in: configuration.contentDirectory, runFolder: runFolder, fileUpdates: fileUpdates) + let generator = ImageGenerator( + input: configuration.contentDirectory, + output: configuration.outputDirectory, + reader: reader, images: images) + generator.generateImages() + print(" ") +} + +private func copyFiles(files: FileData, configuration: Configuration, runFolder: URL) { + print("--- FILES ------------------------------------------") + print(" ") + let generator = FileGenerator( + input: configuration.contentDirectory, + output: configuration.outputDirectory, + runFolder: runFolder, + files: files) + generator.generate() +} + +private func finish(start: Date) { + print("----------------------------------------------------") + print(" ") + let duration = Int(-start.timeIntervalSinceNow.rounded()) + if duration < 60 { + print(" Duration: \(duration) s") + } else if duration < 3600 { + print(String(format: " Duration: %d:%02d", duration / 60, duration % 60)) + } else { + print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60)) + } + print("") +} + private func generate(configPath: String) throws { + let start = Date() + + // 1. Load configuration guard let configuration = loadConfiguration(at: configPath) else { return } - files = .init( - in: configuration.contentDirectory, - to: configuration.outputDirectory, - configuration: configuration) - + let runFolder = configuration.contentDirectory.appendingPathComponent("run") // 2. Scan site elements - guard let siteRoot = try loadSiteData(in: configuration.contentDirectory) else { + guard let (siteRoot, ids) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else { return } - let siteGenerator = try SiteGenerator() - siteGenerator.generate(site: siteRoot) - files.printGeneratedPages() - files.printEmptyPages() - files.printDraftPages() + let fileUpdates = FileUpdateChecker(input: configuration.contentDirectory) + switch fileUpdates.loadPreviousRun(from: runFolder) { + case .notLoaded: + print("Regarding all files as new (no hashes loaded)") + case .loaded: + break + case .failed(let error): + print("Regarding all files as new (\(error))") + } - files.createImages() - files.copyRequiredFiles() - files.printExternalFiles() - files.writeDetectedFileChangesToDisk() + // 3. Generate pages + let pageCount = ids.count * siteRoot.languages.count + let (images, files) = try generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, pageCount: pageCount, runFolder: runFolder) + + if let error = fileUpdates.writeDetectedFileChanges(to: runFolder) { + print(error) + } + + // 4. Generate images + generateImages(images, configuration: configuration, runFolder: runFolder, fileUpdates: fileUpdates) + + // 5. Copy/minify files + copyFiles(files: files, configuration: configuration, runFolder: runFolder) + + // 6. Print summary + finish(start: start) }