ch/CHGenerator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: ch:6e24c27fdc69702ac1d19dbd036f37554eb94e3a
Branches Tags
ch:main
...
compare: ch:a8b328efcead27928f9e9075462170a04c2aa2ce
Branches Tags
ch:main

3 Commits
6e24c27fdc ... a8b328efce

Author SHA1 Message Date
Christoph Hagen
a8b328efce Write all logs to disk 2022-12-04 23:10:44 +01:00
Christoph Hagen
956cfb52c4 Improve printing and image creation 2022-12-04 19:15:22 +01:00
Christoph Hagen
6a52f62402 Add result handler to templates 2022-12-02 10:25:54 +01:00
36 changed files with 1564 additions and 1108 deletions
Show Stats Expand all files Collapse all files

45
Sources/Generator/Content/Element.swift
View File

@ -183,7 +183,7 @@ struct Element {
self.sortIndex = log.unused(metadata.sortIndex, "sortIndex", source: source) self.sortIndex = log.unused(metadata.sortIndex, "sortIndex", source: source)
self.externalFiles = metadata.externalFiles ?? [] self.externalFiles = metadata.externalFiles ?? []
self.requiredFiles = metadata.requiredFiles ?? [] // Paths are already relative to root self.requiredFiles = metadata.requiredFiles ?? [] // Paths are already relative to root
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "") } ?? [] self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "", log: log) } ?? []
self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName
self.thumbnailStyle = log.castUnused(metadata.thumbnailStyle, "thumbnailStyle", source: source) self.thumbnailStyle = log.castUnused(metadata.thumbnailStyle, "thumbnailStyle", source: source)
self.useManualSorting = log.unused(metadata.useManualSorting, "useManualSorting", source: source) self.useManualSorting = log.unused(metadata.useManualSorting, "useManualSorting", source: source)
@ -204,7 +204,7 @@ struct Element {
return nil return nil
} }
files.add(page: path, id: id) //files.add(page: path, id: id)
self.readElements(in: folder, source: nil, log: log) self.readElements(in: folder, source: nil, log: log)
} }
@ -239,13 +239,13 @@ struct Element {
self.author = metadata.author ?? parent.author self.author = metadata.author ?? parent.author
self.topBarTitle = log.unused(metadata.topBarTitle, "topBarTitle", source: source) self.topBarTitle = log.unused(metadata.topBarTitle, "topBarTitle", source: source)
self.date = metadata.date.unwrapped { log.cast($0, "date", source: source) } self.date = metadata.date.unwrapped { log.cast($0, "date", source: source) }
self.endDate = metadata.date.unwrapped { log.cast($0, "endDate", source: source) } self.endDate = metadata.endDate.unwrapped { log.cast($0, "endDate", source: source) }
self.state = log.cast(metadata.state, "state", source: source) self.state = log.cast(metadata.state, "state", source: source)
self.sortIndex = metadata.sortIndex self.sortIndex = metadata.sortIndex
// TODO: Propagate external files from the parent if subpath matches? // TODO: Propagate external files from the parent if subpath matches?
self.externalFiles = Element.rootPaths(for: metadata.externalFiles, path: path) self.externalFiles = Element.rootPaths(for: metadata.externalFiles, path: path)
self.requiredFiles = Element.rootPaths(for: metadata.requiredFiles, path: path) self.requiredFiles = Element.rootPaths(for: metadata.requiredFiles, path: path)
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path) } ?? [] self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path, log: log) } ?? []
self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName
self.thumbnailStyle = log.cast(metadata.thumbnailStyle, "thumbnailStyle", source: source) self.thumbnailStyle = log.cast(metadata.thumbnailStyle, "thumbnailStyle", source: source)
self.useManualSorting = metadata.useManualSorting ?? false self.useManualSorting = metadata.useManualSorting ?? false
@ -285,9 +285,21 @@ struct Element {
return nil return nil
} }
files.add(page: path, id: id) //files.add(page: path, id: id)
self.readElements(in: folder, source: path, log: log) self.readElements(in: folder, source: path, log: log)
} }
func getContainedIds(log: MetadataInfoLogger) -> [String : String] {
elements.reduce(into: [id : path]) { dict, element in
element.getContainedIds(log: log).forEach { id, path in
if let existing = dict[id] {
log.error("Conflicting id with \(existing)", source: path)
} else {
dict[id] = path
}
}
}
}
} }
// MARK: Paths // MARK: Paths
@ -566,21 +578,14 @@ extension Element {
extension Element { extension Element {
private var additionalHeadContentPath: String { var additionalHeadContentPath: String {
path + "/head.html" path + "/head.html"
} }
func customHeadContent() -> String? { var additionalFooterContentPath: String {
files.contentOfOptionalFile(atPath: additionalHeadContentPath, source: path)
}
private var additionalFooterContentPath: String {
path + "/footer.html" path + "/footer.html"
} }
func customFooterContent() -> String? {
files.contentOfOptionalFile(atPath: additionalFooterContentPath, source: path)
}
} }
// MARK: Debug // MARK: Debug
@ -607,14 +612,14 @@ extension Element {
let desiredHeight: Int? let desiredHeight: Int?
init?(input: String, path: String) { init?(input: String, path: String, log: MetadataInfoLogger) {
let parts = input.components(separatedBy: " ").filter { !$0.isEmpty } let parts = input.components(separatedBy: " ").filter { !$0.isEmpty }
guard parts.count == 3 || parts.count == 4 else { guard parts.count == 3 || parts.count == 4 else {
log.add(error: "Invalid image specification, expected 'source dest width (height)", source: path) log.error("Invalid image specification, expected 'source dest width (height)", source: path)
return nil return nil
} }
guard let width = Int(parts[2]) else { guard let width = Int(parts[2]) else {
log.add(error: "Invalid width for image \(parts[0])", source: path) log.error("Invalid width for image \(parts[0])", source: path)
return nil return nil
} }
@ -626,7 +631,7 @@ extension Element {
return return
} }
guard let height = Int(parts[3]) else { guard let height = Int(parts[3]) else {
log.add(error: "Invalid height for image \(parts[0])", source: path) log.error("Invalid height for image \(parts[0])", source: path)
return nil return nil
} }
self.desiredHeight = height self.desiredHeight = height
@ -716,10 +721,6 @@ extension Element {
let destination = pathRelativeToRootForContainedInputFile("thumbnail-\(language).\(ext)") let destination = pathRelativeToRootForContainedInputFile("thumbnail-\(language).\(ext)")
return (source, destination) return (source, destination)
} }
let thumbnailUrl = inputFolder.appendingPathComponent(thumbnailPath)
if !thumbnailUrl.exists {
log.add(error: "Missing thumbnail", source: path)
}
let source = pathRelativeToRootForContainedInputFile(thumbnailPath) let source = pathRelativeToRootForContainedInputFile(thumbnailPath)
let ext = thumbnailPath.lastComponentAfter(".") let ext = thumbnailPath.lastComponentAfter(".")
let destination = pathRelativeToRootForContainedInputFile("thumbnail.\(ext)") let destination = pathRelativeToRootForContainedInputFile("thumbnail.\(ext)")

443
Sources/Generator/Files/FileSystem.swift
View File

@ -1,443 +0,0 @@
import Foundation
import CryptoKit
import AppKit
final class FileSystem {
private static let tempFileName = "temp.bin"
private let input: URL
private let output: URL
private let source = "FileSystem"
private let images: ImageGenerator
private let configuration: Configuration
private var tempFile: URL {
input.appendingPathComponent(FileSystem.tempFileName)
}
let generatorInfoFolder: URL
/**
All files which should be copied to the output folder
*/
private var requiredFiles: Set<String> = []
/**
The files marked as external in element metadata.
Files included here are not generated, since they are assumed to be added separately.
*/
private var externalFiles: Set<String> = []
/**
The files marked as expected, i.e. they exist after the generation is completed.
The key of the dictionary is the file path, the value is the file providing the link
*/
private var expectedFiles: [String : String] = [:]
/**
All pages without content which have been created
*/
private var emptyPages: Set<String> = []
/**
All pages which have `status` set to ``PageState.draft``
*/
private var draftPages: Set<String> = []
/**
All paths to page element folders, indexed by their unique id.
This relation is used to generate relative links to pages using the ``Element.id`
*/
private var pagePaths: [String: String] = [:]
/**
The image creation tasks.
The key is the destination path.
*/
private var imageTasks: [String : ImageOutput] = [:]
/**
The paths to all pages which were changed
*/
private var generatedPages: Set<String> = []
init(in input: URL, to output: URL, configuration: Configuration) {
self.input = input
self.output = output
self.images = .init(input: input, output: output)
self.generatorInfoFolder = input.appendingPathComponent("run")
self.configuration = configuration
}
func urlInOutputFolder(_ path: String) -> URL {
output.appendingPathComponent(path)
}
func urlInContentFolder(_ path: String) -> URL {
input.appendingPathComponent(path)
}
private func exists(_ url: URL) -> Bool {
FileManager.default.fileExists(atPath: url.path)
}
func dataOfRequiredFile(atPath path: String, source: String) -> Data? {
let url = input.appendingPathComponent(path)
guard exists(url) else {
log.failedToOpen(path, requiredBy: source, error: nil)
return nil
}
do {
return try Data(contentsOf: url)
} catch {
log.failedToOpen(path, requiredBy: source, error: error)
return nil
}
}
func contentOfMdFile(atPath path: String, source: String) -> String? {
contentOfOptionalFile(atPath: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing)
}
func contentOfOptionalFile(atPath path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
let url = input.appendingPathComponent(path)
guard exists(url) else {
if createEmptyFileIfMissing {
try? Data().write(to: url)
}
return nil
}
do {
return try String(contentsOf: url)
} catch {
log.failedToOpen(path, requiredBy: source, error: error)
return nil
}
}
func writeDetectedFileChangesToDisk() {
images.writeDetectedFileChangesToDisk()
}
// MARK: Images
@discardableResult
func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
images.requireImage(
at: destination,
generatedFrom: source,
requiredBy: path,
quality: 0.7,
width: width,
height: desiredHeight,
alwaysGenerate: false)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
}
func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
}
func createImages() {
images.createImages()
}
// MARK: File copying
/**
Add a file as required, so that it will be copied to the output directory.
*/
func require(file: String) {
let url = input.appendingPathComponent(file)
guard url.exists, url.isDirectory else {
requiredFiles.insert(file)
return
}
do {
try FileManager.default
.contentsOfDirectory(atPath: url.path)
.forEach {
// Recurse into subfolders
require(file: file + "/" + $0)
}
} catch {
log.add(error: "Failed to read folder \(file): \(error)", source: source)
}
}
/**
Mark a file as explicitly missing.
This is done for the `externalFiles` entries in metadata,
to indicate that these files will be copied to the output folder manually.
*/
func exclude(file: String) {
externalFiles.insert(file)
}
/**
Mark a file as expected to be present in the output folder after generation.
This is done for all links between pages, which only exist after the pages have been generated.
*/
func expect(file: String, source: String) {
expectedFiles[file] = source
}
func copyRequiredFiles() {
var copiedFiles = Set<String>()
for file in requiredFiles {
let cleanPath = cleanRelativeURL(file)
let sourceUrl = input.appendingPathComponent(cleanPath)
let destinationUrl = output.appendingPathComponent(cleanPath)
guard sourceUrl.exists else {
if !isExternal(file: file) {
log.add(error: "Missing required file", source: cleanPath)
}
continue
}
if copyFileIfChanged(from: sourceUrl, to: destinationUrl) {
copiedFiles.insert(file)
}
}
try? tempFile.delete()
for (file, source) in expectedFiles {
guard !isExternal(file: file) else {
continue
}
let cleanPath = cleanRelativeURL(file)
let destinationUrl = output.appendingPathComponent(cleanPath)
if !destinationUrl.exists {
log.add(error: "Missing \(cleanPath)", source: source)
}
}
guard !copiedFiles.isEmpty else {
print("No required files copied")
return
}
print("\(copiedFiles.count) required files copied:")
for file in copiedFiles.sorted() {
print(" " + file)
}
}
private func copyFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool {
guard configuration.minifyCSSandJS else {
return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl)
}
switch sourceUrl.pathExtension.lowercased() {
case "js":
return minifyJS(at: sourceUrl, andWriteTo: destinationUrl)
case "css":
return minifyCSS(at: sourceUrl, andWriteTo: destinationUrl)
default:
return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl)
}
}
private func copyBinaryFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool {
do {
let data = try Data(contentsOf: sourceUrl)
return writeIfChanged(data, to: destinationUrl)
} catch {
log.add(error: "Failed to read data at \(sourceUrl.path)", source: source, error: error)
return false
}
}
private func minifyJS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
let command = "uglifyjs \(sourceUrl.path) > \(tempFile.path)"
do {
_ = try FileSystem.safeShell(command)
return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
} catch {
log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
return false
}
}
private func minifyCSS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
let command = "cleancss \(sourceUrl.path) -o \(tempFile.path)"
do {
_ = try FileSystem.safeShell(command)
return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
} catch {
log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
return false
}
}
private func cleanRelativeURL(_ raw: String) -> String {
let raw = raw.dropAfterLast("#") // Clean links to page content
guard raw.contains("..") else {
return raw
}
var result: [String] = []
for component in raw.components(separatedBy: "/") {
if component == ".." {
_ = result.popLast()
} else {
result.append(component)
}
}
return result.joined(separator: "/")
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if externalFiles.contains(path) {
return true
}
}
return false
}
func printExternalFiles() {
guard !externalFiles.isEmpty else {
return
}
print("\(externalFiles.count) external resources needed:")
for file in externalFiles.sorted() {
print(" " + file)
}
}
// MARK: Pages
func isEmpty(page: String) {
emptyPages.insert(page)
}
func printEmptyPages() {
guard !emptyPages.isEmpty else {
return
}
print("\(emptyPages.count) empty pages:")
for page in emptyPages.sorted() {
print(" " + page)
}
}
func isDraft(path: String) {
draftPages.insert(path)
}
func printDraftPages() {
guard !draftPages.isEmpty else {
return
}
print("\(draftPages.count) drafts:")
for page in draftPages.sorted() {
print(" " + page)
}
}
func add(page: String, id: String) {
if let existing = pagePaths[id] {
log.add(error: "Conflicting id with \(existing)", source: page)
}
pagePaths[id] = page
}
func getPage(for id: String) -> String? {
pagePaths[id]
}
func generated(page: String) {
generatedPages.insert(page)
}
func printGeneratedPages() {
guard !generatedPages.isEmpty else {
print("No pages modified")
return
}
print("\(generatedPages.count) pages modified")
for page in generatedPages.sorted() {
print(" " + page)
}
}
// MARK: Writing files
@discardableResult
func writeIfChanged(_ data: Data, to url: URL) -> Bool {
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
return true
} catch {
log.add(error: "Failed to write file", source: url.path, error: error)
return false
}
}
@discardableResult
func write(_ string: String, to url: URL) -> Bool {
let data = string.data(using: .utf8)!
return writeIfChanged(data, to: url)
}
// MARK: Running other tasks
@discardableResult
static func safeShell(_ command: String) throws -> String {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/zsh")
task.standardInput = nil
try task.run()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return output
}
}

56
Sources/Generator/Files/FileUpdateChecker.swift
View File

@ -3,14 +3,10 @@ import CryptoKit
final class FileUpdateChecker { final class FileUpdateChecker {
private static let hashesFileName = "hashes.json" private let hashesFileName = "hashes.json"
private let input: URL private let input: URL
private var hashesFile: URL {
input.appendingPathComponent(FileUpdateChecker.hashesFileName)
}
/** /**
The hashes of all accessed files from the previous run The hashes of all accessed files from the previous run
@ -25,35 +21,41 @@ final class FileUpdateChecker {
*/ */
private var accessedFiles: [String : Data] = [:] private var accessedFiles: [String : Data] = [:]
private var source: String { var numberOfFilesLoaded: Int {
"FileUpdateChecker" previousFiles.count
}
var numberOfFilesAccessed: Int {
accessedFiles.count
} }
init(input: URL) { init(input: URL) {
self.input = input self.input = input
guard hashesFile.exists else { }
log.add(info: "No file hashes loaded, regarding all content as new", source: source)
return enum LoadResult {
case notLoaded
case loaded
case failed(String)
}
func loadPreviousRun(from folder: URL) -> LoadResult {
let url = folder.appendingPathComponent(hashesFileName)
guard url.exists else {
return .notLoaded
} }
let data: Data let data: Data
do { do {
data = try Data(contentsOf: hashesFile) data = try Data(contentsOf: url)
} catch { } catch {
log.add( return .failed("Failed to read hashes from last run: \(error)")
warning: "File hashes could not be read, regarding all content as new",
source: source,
error: error)
return
} }
do { do {
self.previousFiles = try JSONDecoder().decode(from: data) self.previousFiles = try JSONDecoder().decode(from: data)
} catch { } catch {
log.add( return .failed("Failed to decode hashes from last run: \(error)")
warning: "File hashes could not be decoded, regarding all content as new",
source: source,
error: error)
return
} }
return .loaded
} }
func fileHasChanged(at path: String) -> Bool { func fileHasChanged(at path: String) -> Bool {
@ -73,16 +75,18 @@ final class FileUpdateChecker {
accessedFiles[path] = SHA256.hash(data: data).data accessedFiles[path] = SHA256.hash(data: data).data
} }
func writeDetectedFileChanges(to folder: URL) -> String? {
func writeDetectedFileChangesToDisk() { let url = folder.appendingPathComponent(hashesFileName)
do { do {
let encoder = JSONEncoder() let encoder = JSONEncoder()
encoder.outputFormatting = .prettyPrinted encoder.outputFormatting = .prettyPrinted
let data = try encoder.encode(accessedFiles) let data = try encoder.encode(accessedFiles)
try data.write(to: hashesFile) try data.write(to: url)
return nil
} catch { } catch {
log.add(warning: "Failed to save file hashes", source: source, error: error) return "Failed to save file hashes: \(error)"
}
} }
} }
} var notFound = 0

456
Sources/Generator/Files/ImageGenerator.swift
View File

@ -1,456 +0,0 @@
import Foundation
import AppKit
import CryptoKit
import Darwin.C
private struct ImageJob {
let destination: String
let width: Int
let path: String
let quality: Float
let alwaysGenerate: Bool
}
final class ImageGenerator {
private let imageOptimSupportedFileExtensions: Set<String> = ["jpg", "png", "svg"]
private let imageOptimizationBatchSize = 50
/**
The path to the input folder.
*/
private let input: URL
/**
The path to the output folder
*/
private let output: URL
/**
The images to generate.
The key is the image source path relative to the input folder, and the values are the destination path (relative to the output folder) and the required image width.
*/
private var imageJobs: [String : [ImageJob]] = [:]
/**
The images for which to generate multiple versions
The key is the source file, the value is the path of the requiring page.
*/
private var multiImageJobs: [String : String] = [:]
/**
The images which could not be found, but are required for the site.
The key is the image path, and the value is the page that requires it.
*/
private var missingImages: [String : String] = [:]
/**
All warnings produced for images during generation
*/
private var imageWarnings: Set<String> = []
/**
All images required by the site.
The values are the destination paths of the images, relative to the output folder
*/
private var requiredImages: Set<String> = []
/**
All images modified or created during this generator run.
*/
private var generatedImages: Set<String> = []
/**
The images optimized by ImageOptim
*/
private var optimizedImages: Set<String> = []
/**
A cache to get the size of source images, so that files don't have to be loaded multiple times.
The key is the absolute source path, and the value is the image size
*/
private var imageSizeCache: [String : NSSize] = [:]
private var fileUpdates: FileUpdateChecker
init(input: URL, output: URL) {
self.fileUpdates = FileUpdateChecker(input: input)
self.input = input
self.output = output
}
func writeDetectedFileChangesToDisk() {
fileUpdates.writeDetectedFileChangesToDisk()
}
private func getImageSize(atPath path: String) -> NSSize? {
if let size = imageSizeCache[path] {
return size
}
guard let image = getImage(atPath: path) else {
return nil
}
let size = image.size
imageSizeCache[path] = size
return size
}
private func getImage(atPath path: String) -> NSImage? {
guard let data = getData(atPath: path) else {
log.add(error: "Failed to load file", source: path)
return nil
}
guard let image = NSImage(data: data) else {
log.add(error: "Failed to read image", source: path)
return nil
}
return image
}
private func getData(atPath path: String) -> Data? {
let url = input.appendingPathComponent(path)
guard url.exists else {
return nil
}
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
log.add(error: "Failed to read data", source: path, error: error)
return nil
}
}
func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
requiredImages.insert(destination)
let height = height.unwrapped(CGFloat.init)
let sourceUrl = input.appendingPathComponent(source)
guard sourceUrl.exists else {
missingImages[source] = path
return .zero
}
guard let imageSize = getImageSize(atPath: source) else {
missingImages[source] = path
return .zero
}
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
// Check desired height, then we can forget about it
if let height = height {
let expectedHeight = scaledSize.width / CGFloat(width) * height
if abs(expectedHeight - scaledSize.height) > 2 {
addWarning("Invalid height (\(scaledSize.height) instead of \(expectedHeight))", destination: destination, path: path)
}
}
let job = ImageJob(
destination: destination,
width: width,
path: path,
quality: quality,
alwaysGenerate: alwaysGenerate)
insert(job: job, source: source)
return scaledSize
}
private func insert(job: ImageJob, source: String) {
guard let existingSource = imageJobs[source] else {
imageJobs[source] = [job]
return
}
guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
imageJobs[source] = existingSource + [job]
return
}
if existingJob.width != job.width {
addWarning("Multiple image widths (\(existingJob.width) and \(job.width))", destination: job.destination, path: "\(existingJob.path) and \(job.path)")
}
}
func createImages() {
var count = 0
for (source, jobs) in imageJobs.sorted(by: { $0.key < $1.key }) {
print(String(format: "Creating images: %4d / %d\r", count, imageJobs.count), terminator: "")
fflush(stdout)
create(images: jobs, from: source)
count += 1
}
print(" \r", terminator: "")
createMultiImages()
optimizeImages()
printMissingImages()
printImageWarnings()
printGeneratedImages()
printTotalImageCount()
}
private func printMissingImages() {
guard !missingImages.isEmpty else {
return
}
print("\(missingImages.count) missing images:")
let sort = missingImages.sorted { (a, b) in
a.value < b.value && a.key < b.key
}
for (source, path) in sort {
print(" \(source) (required by \(path))")
}
}
private func printImageWarnings() {
guard !imageWarnings.isEmpty else {
return
}
print("\(imageWarnings.count) image warnings:")
for imageWarning in imageWarnings {
print(imageWarning)
}
}
private func printGeneratedImages() {
guard !generatedImages.isEmpty else {
return
}
print("\(generatedImages.count) images generated:")
for image in generatedImages {
print(" " + image)
}
}
private func printTotalImageCount() {
print("\(requiredImages.count) images")
}
private func addWarning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
imageWarnings.insert(warning)
}
private func addWarning(_ message: String, job: ImageJob) {
addWarning(message, destination: job.destination, path: job.path)
}
private func isMissing(_ job: ImageJob) -> Bool {
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
private func create(images: [ImageJob], from source: String) {
// Only load image if required
let imageHasChanged = fileUpdates.fileHasChanged(at: source)
guard imageHasChanged || images.contains(where: isMissing) else {
return
}
guard let image = getImage(atPath: source) else {
missingImages[source] = images.first?.path
return
}
let jobs = imageHasChanged ? images : images.filter(isMissing)
// Update all images
jobs.forEach { job in
// Prevent memory overflow due to repeated NSImage operations
autoreleasepool {
create(job: job, from: image, source: source)
}
}
}
private func create(job: ImageJob, from image: NSImage, source: String) {
let destinationUrl = output.appendingPathComponent(job.destination)
create(job: job, from: image, source: source, at: destinationUrl)
}
private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
// Ensure that image file is supported
let ext = destinationUrl.pathExtension.lowercased()
guard ImageType(fileExtension: ext) != nil else {
fatalError()
}
let destinationExtension = destinationUrl.pathExtension.lowercased()
guard let type = ImageType(fileExtension: destinationExtension)?.fileType else {
addWarning("Invalid image extension \(destinationExtension)", job: job)
return
}
let desiredWidth = CGFloat(job.width)
let sourceRep = image.representations[0]
let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh)
.scaledDown(to: desiredWidth)
// create NSBitmapRep manually, if using cgImage, the resulting size is wrong
let rep = NSBitmapImageRep(bitmapDataPlanes: nil,
pixelsWide: Int(destinationSize.width),
pixelsHigh: Int(destinationSize.height),
bitsPerSample: 8,
samplesPerPixel: 4,
hasAlpha: true,
isPlanar: false,
colorSpaceName: NSColorSpaceName.deviceRGB,
bytesPerRow: Int(destinationSize.width) * 4,
bitsPerPixel: 32)!
let ctx = NSGraphicsContext(bitmapImageRep: rep)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = ctx
image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height))
ctx?.flushGraphics()
NSGraphicsContext.restoreGraphicsState()
// Get NSData, and save it
guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
addWarning("Failed to get data", job: job)
return
}
do {
try data.createFolderAndWrite(to: destinationUrl)
} catch {
addWarning("Failed to write image (\(error))", job: job)
return
}
generatedImages.insert(job.destination)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
// Add @1x version
_ = requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
// Add @2x version
return requireScaledMultiImage(
source: source,
destination: destination.insert("@2x", beforeLast: "."),
requiredBy: path,
width: width * 2,
desiredHeight: desiredHeight.unwrapped { $0 * 2 })
}
@discardableResult
private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
let rawDestinationPath = destination.dropAfterLast(".")
let avifPath = rawDestinationPath + ".avif"
let webpPath = rawDestinationPath + ".webp"
let needsGeneration = !output.appendingPathComponent(avifPath).exists || !output.appendingPathComponent(webpPath).exists
let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
multiImageJobs[destination] = path
return size
}
private func createMultiImages() {
let sort = multiImageJobs.sorted { $0.value < $1.value && $0.key < $1.key }
var count = 1
for (baseImage, path) in sort {
print(String(format: "Creating image versions: %4d / %d\r", count, sort.count), terminator: "")
fflush(stdout)
createMultiImages(from: baseImage, path: path)
count += 1
}
print(" \r", terminator: "")
}
private func createMultiImages(from source: String, path: String) {
guard generatedImages.contains(source) else {
return
}
let sourceUrl = output.appendingPathComponent(source)
let sourcePath = sourceUrl.path
guard sourceUrl.exists else {
addWarning("No image at path \(sourcePath)", destination: source, path: path)
missingImages[source] = path
return
}
let avifPath = source.dropAfterLast(".") + ".avif"
createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
generatedImages.insert(avifPath)
let webpPath = source.dropAfterLast(".") + ".webp"
createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
generatedImages.insert(webpPath)
compress(at: source)
}
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to create AVIF image", destination: destination, path: destination)
}
}
private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to create WEBP image", destination: destination, path: destination)
}
}
private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to compress image", destination: destination, path: destination)
}
}
private func optimizeImages() {
let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path }
for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, all.count)
let batch = all[i..<endIndex]
print(String(format: "Optimizing images: %4d / %d\r", endIndex, all.count), terminator: "")
fflush(stdout)
if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch)
}
}
print(" \r", terminator: "")
fflush(stdout)
print("\(optimizedImages.count) images optimized")
}
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ")
do {
_ = try FileSystem.safeShell(command)
return true
} catch {
addWarning("Failed to optimize images", destination: "", path: "")
return false
}
}
}

14
Sources/Generator/Files/ImageJob.swift Normal file
View File

@ -0,0 +1,14 @@
import Foundation
struct ImageJob {
let destination: String
let width: Int
let path: String
let quality: Float
let alwaysGenerate: Bool
}

66
Sources/Generator/Files/ImageReader.swift Normal file
View File

@ -0,0 +1,66 @@
import Foundation
import AppKit
final class ImageReader {
/// The content folder where the input data is stored
let contentFolder: URL
private let fileUpdates: FileUpdateChecker
let runDataFolder: URL
init(in input: URL, runFolder: URL, fileUpdates: FileUpdateChecker) {
self.contentFolder = input
self.runDataFolder = runFolder
self.fileUpdates = fileUpdates
}
var numberOfFilesLoaded: Int {
fileUpdates.numberOfFilesLoaded
}
var numberOfFilesAccessed: Int {
fileUpdates.numberOfFilesAccessed
}
func loadData() -> FileUpdateChecker.LoadResult {
fileUpdates.loadPreviousRun(from: runDataFolder)
}
func writeDetectedFileChangesToDisk() -> String? {
fileUpdates.writeDetectedFileChanges(to: runDataFolder)
}
func imageHasChanged(at path: String) -> Bool {
fileUpdates.fileHasChanged(at: path)
}
func getImage(atPath path: String) -> NSImage? {
guard let data = getData(atPath: path) else {
// TODO: log.error("Failed to load file", source: path)
return nil
}
guard let image = NSImage(data: data) else {
// TODO: log.error("Failed to read image", source: path)
return nil
}
return image
}
private func getData(atPath path: String) -> Data? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
return nil
}
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
// TODO: log.error("Failed to read data: \(error)", source: path)
return nil
}
}
}

13
Sources/Generator/Generators/OverviewPageGenerator.swift
View File

@ -4,15 +4,17 @@ struct OverviewPageGenerator {
private let factory: LocalizedSiteTemplate private let factory: LocalizedSiteTemplate
init(factory: LocalizedSiteTemplate) { private let results: GenerationResultsHandler
init(factory: LocalizedSiteTemplate, results: GenerationResultsHandler) {
self.factory = factory self.factory = factory
self.results = results
} }
func generate( func generate(
section: Element, section: Element,
language: String) { language: String) {
let path = section.localizedPath(for: language) let path = section.localizedPath(for: language)
let url = files.urlInOutputFolder(path)
let metadata = section.localized(for: language) let metadata = section.localized(for: language)
@ -26,11 +28,8 @@ struct OverviewPageGenerator {
content[.contentClass] = "overview" content[.contentClass] = "overview"
content[.header] = makeHeader(page: section, metadata: metadata, language: language) content[.header] = makeHeader(page: section, metadata: metadata, language: language)
content[.content] = makeContent(section: section, language: language) content[.content] = makeContent(section: section, language: language)
content[.footer] = section.customFooterContent() content[.footer] = results.getContentOfOptionalFile(at: section.additionalFooterContentPath, source: section.path)
guard factory.page.generate(content, to: url) else { factory.page.generate(content, to: path, source: section.path)
return
}
files.generated(page: path)
} }
private func makeContent(section: Element, language: String) -> String { private func makeContent(section: Element, language: String) -> String {

11
Sources/Generator/Generators/OverviewSectionGenerator.swift
View File

@ -8,10 +8,10 @@ struct OverviewSectionGenerator {
private let generator: ThumbnailListGenerator private let generator: ThumbnailListGenerator
init(factory: TemplateFactory) { init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.multipleSectionsTemplate = factory.overviewSection self.multipleSectionsTemplate = factory.overviewSection
self.singleSectionsTemplate = factory.overviewSectionClean self.singleSectionsTemplate = factory.overviewSectionClean
self.generator = ThumbnailListGenerator(factory: factory) self.generator = ThumbnailListGenerator(factory: factory, results: results)
} }
func generate(sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String { func generate(sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String {
@ -25,8 +25,13 @@ struct OverviewSectionGenerator {
} }
private func newsSectionContent(for element: Element, language: String, sectionItemCount: Int) -> String { private func newsSectionContent(for element: Element, language: String, sectionItemCount: Int) -> String {
let shownElements = element.mostRecentElements(sectionItemCount) // let shownElements = element.mostRecentElements(sectionItemCount)
return "" return ""
// return generator.generateContent(
// items: shownElements,
// parent: element,
// language: language,
// style: element.thumbnailStyle)
} }
private func sectionsContent(_ sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String { private func sectionsContent(_ sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String {

53
Sources/Generator/Generators/PageContentGenerator.swift
View File

@ -10,9 +10,12 @@ struct PageContentGenerator {
private let siteRoot: Element private let siteRoot: Element
init(factory: TemplateFactory, siteRoot: Element) { private let results: GenerationResultsHandler
init(factory: TemplateFactory, siteRoot: Element, results: GenerationResultsHandler) {
self.factory = factory self.factory = factory
self.siteRoot = siteRoot self.siteRoot = siteRoot
self.results = results
} }
func generate(page: Element, language: String, content: String) -> (content: String, includesCode: Bool) { func generate(page: Element, language: String, content: String) -> (content: String, includesCode: Bool) {
@ -44,8 +47,7 @@ struct PageContentGenerator {
let file = markdown.between("(", and: ")") let file = markdown.between("(", and: ")")
if file.hasPrefix("page:") { if file.hasPrefix("page:") {
let pageId = file.replacingOccurrences(of: "page:", with: "") let pageId = file.replacingOccurrences(of: "page:", with: "")
guard let pagePath = files.getPage(for: pageId) else { guard let pagePath = results.getPagePath(for: pageId, source: page.path) else {
log.add(warning: "Page id '\(pageId)' not found", source: page.path)
// Remove link since the page can't be found // Remove link since the page can't be found
return markdown.between("[", and: "]") return markdown.between("[", and: "]")
} }
@ -57,13 +59,13 @@ struct PageContentGenerator {
if let filePath = page.nonAbsolutePathRelativeToRootForContainedInputFile(file) { if let filePath = page.nonAbsolutePathRelativeToRootForContainedInputFile(file) {
// The target of the page link must be present after generation is complete // The target of the page link must be present after generation is complete
files.expect(file: filePath, source: page.path) results.expect(file: filePath, source: page.path)
} }
return html return html
} }
private func handleHTML(page: Element, language: String, html: String, markdown: Substring) -> String { private func handleHTML(page: Element, language: String, html: String, markdown: Substring) -> String {
#warning("Check HTML code in markdown for required resources") // TODO: Check HTML code in markdown for required resources
//print("[HTML] Found in page \(page.path):") //print("[HTML] Found in page \(page.path):")
//print(markdown) //print(markdown)
// Things to check: // Things to check:
@ -123,7 +125,7 @@ struct PageContentGenerator {
private func handleImage(page: Element, file: String, rightTitle: String?, leftTitle: String?) -> String { private func handleImage(page: Element, file: String, rightTitle: String?, leftTitle: String?) -> String {
let imagePath = page.pathRelativeToRootForContainedInputFile(file) let imagePath = page.pathRelativeToRootForContainedInputFile(file)
let size = files.requireFullSizeMultiVersionImage( let size = results.requireFullSizeMultiVersionImage(
source: imagePath, source: imagePath,
destination: imagePath, destination: imagePath,
requiredBy: page.path) requiredBy: page.path)
@ -140,28 +142,28 @@ struct PageContentGenerator {
private func handleVideo(page: Element, file: String, optionString: String?) -> String { private func handleVideo(page: Element, file: String, optionString: String?) -> String {
let options: [PageVideoTemplate.VideoOption] = optionString.unwrapped { string in let options: [PageVideoTemplate.VideoOption] = optionString.unwrapped { string in
string.components(separatedBy: " ").compactMap { optionText in string.components(separatedBy: " ").compactMap { optionText -> PageVideoTemplate.VideoOption? in
guard let optionText = optionText.trimmed.nonEmpty else { guard let optionText = optionText.trimmed.nonEmpty else {
return nil return nil
} }
guard let option = PageVideoTemplate.VideoOption(rawValue: optionText) else { guard let option = PageVideoTemplate.VideoOption(rawValue: optionText) else {
log.add(warning: "Unknown video option \(optionText)", source: page.path) results.warning("Unknown video option \(optionText)", source: page.path)
return nil return nil
} }
return option return option
} }
} ?? [] } ?? []
#warning("Check page folder for alternative video versions") // TODO: Check page folder for alternative video versions
let sources: [PageVideoTemplate.VideoSource] = [(url: file, type: .mp4)] let sources: [PageVideoTemplate.VideoSource] = [(url: file, type: .mp4)]
let filePath = page.pathRelativeToRootForContainedInputFile(file) let filePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: filePath) results.require(file: filePath, source: page.path)
return factory.video.generate(sources: sources, options: options) return factory.video.generate(sources: sources, options: options)
} }
private func handleSvg(page: Element, file: String, area: String?) -> String { private func handleSvg(page: Element, file: String, area: String?) -> String {
let imagePath = page.pathRelativeToRootForContainedInputFile(file) let imagePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: imagePath) results.require(file: imagePath, source: page.path)
guard let area = area else { guard let area = area else {
return factory.html.svgImage(file: file) return factory.html.svgImage(file: file)
@ -172,7 +174,7 @@ struct PageContentGenerator {
let y = Int(parts[1].trimmed), let y = Int(parts[1].trimmed),
let width = Int(parts[2].trimmed), let width = Int(parts[2].trimmed),
let height = Int(parts[3].trimmed) else { let height = Int(parts[3].trimmed) else {
log.add(warning: "Invalid area string for svg image", source: page.path) results.warning("Invalid area string for svg image", source: page.path)
return factory.html.svgImage(file: file) return factory.html.svgImage(file: file)
} }
@ -180,7 +182,7 @@ struct PageContentGenerator {
} }
private func handleFile(page: Element, file: String, fileExtension: String) -> String { private func handleFile(page: Element, file: String, fileExtension: String) -> String {
log.add(warning: "Unhandled file \(file) with extension \(fileExtension)", source: page.path) results.warning("Unhandled file \(file) with extension \(fileExtension)", source: page.path)
return "" return ""
} }
@ -190,7 +192,7 @@ struct PageContentGenerator {
.compactMap { button -> (file: String, text: String, downloadName: String?)? in .compactMap { button -> (file: String, text: String, downloadName: String?)? in
let parts = button.components(separatedBy: ",") let parts = button.components(separatedBy: ",")
guard parts.count == 2 || parts.count == 3 else { guard parts.count == 2 || parts.count == 3 else {
log.add(warning: "Invalid button definition", source: page.path) results.warning("Invalid button definition", source: page.path)
return nil return nil
} }
let file = parts[0].trimmed let file = parts[0].trimmed
@ -199,7 +201,7 @@ struct PageContentGenerator {
// Ensure that file is available // Ensure that file is available
let filePath = page.pathRelativeToRootForContainedInputFile(file) let filePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: filePath) results.require(file: filePath, source: page.path)
return (file, title, downloadName) return (file, title, downloadName)
} }
@ -212,7 +214,7 @@ struct PageContentGenerator {
.compactMap { button -> (url: String, text: String)? in .compactMap { button -> (url: String, text: String)? in
let parts = button.components(separatedBy: ",") let parts = button.components(separatedBy: ",")
guard parts.count == 2 else { guard parts.count == 2 else {
log.add(warning: "Invalid external link definition", source: page.path) results.warning("Invalid external link definition", source: page.path)
return nil return nil
} }
let url = parts[0].trimmed let url = parts[0].trimmed
@ -224,23 +226,14 @@ struct PageContentGenerator {
} }
private func handleExternalHTML(page: Element, file: String) -> String { private func handleExternalHTML(page: Element, file: String) -> String {
let url = page.inputFolder.appendingPathComponent(file) let path = page.pathRelativeToRootForContainedInputFile(file)
guard url.exists else { return results.getContentOfRequiredFile(at: path, source: page.path) ?? ""
log.add(error: "File \(file) not found", source: page.path)
return ""
}
do {
return try String(contentsOf: url)
} catch {
log.add(error: "File \(file) could not be read", source: page.path, error: error)
return ""
}
} }
private func handleSimpleBox(page: Element, content: String) -> String { private func handleSimpleBox(page: Element, content: String) -> String {
let parts = content.components(separatedBy: ";") let parts = content.components(separatedBy: ";")
guard parts.count > 1 else { guard parts.count > 1 else {
log.add(error: "Invalid box specification", source: page.path) results.warning("Invalid box specification", source: page.path)
return "" return ""
} }
let title = parts[0] let title = parts[0]
@ -250,7 +243,8 @@ struct PageContentGenerator {
private func handlePageLink(page: Element, language: String, pageId: String) -> String { private func handlePageLink(page: Element, language: String, pageId: String) -> String {
guard let linkedPage = siteRoot.find(pageId) else { guard let linkedPage = siteRoot.find(pageId) else {
log.add(warning: "Page id '\(pageId)' not found", source: page.path) // Checking the page path will add it to the missing pages
_ = results.getPagePath(for: pageId, source: page.path)
// Remove link since the page can't be found // Remove link since the page can't be found
return "" return ""
} }
@ -259,6 +253,7 @@ struct PageContentGenerator {
content[.title] = linkedPage.title(for: language) content[.title] = linkedPage.title(for: language)
let fullThumbnailPath = linkedPage.thumbnailFilePath(for: language).destination let fullThumbnailPath = linkedPage.thumbnailFilePath(for: language).destination
// Note: Here we assume that the thumbnail was already used elsewhere, so already generated
let relativeImageUrl = page.relativePathToOtherSiteElement(file: fullThumbnailPath) let relativeImageUrl = page.relativePathToOtherSiteElement(file: fullThumbnailPath)
let metadata = linkedPage.localized(for: language) let metadata = linkedPage.localized(for: language)

30
Sources/Generator/Generators/PageGenerator.swift
View File

@ -7,20 +7,24 @@ struct PageGenerator {
private let contentGenerator: PageContentGenerator private let contentGenerator: PageContentGenerator
init(factory: LocalizedSiteTemplate, siteRoot: Element) { private let results: GenerationResultsHandler
init(factory: LocalizedSiteTemplate, siteRoot: Element, results: GenerationResultsHandler) {
self.factory = factory self.factory = factory
self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot) self.results = results
self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot, results: results)
} }
func generate(page: Element, language: String, previousPage: Element?, nextPage: Element?) { func generate(page: Element, language: String, previousPage: Element?, nextPage: Element?) {
guard !page.isExternalPage else { guard !page.isExternalPage else {
results.didCompletePage()
return return
} }
let path = page.fullPageUrl(for: language) let path = page.fullPageUrl(for: language)
let inputContentPath = page.path + "/\(language).md" let inputContentPath = page.path + "/\(language).md"
let metadata = page.localized(for: language) let metadata = page.localized(for: language)
let nextLanguage = page.nextLanguage(for: language) let nextLanguage = page.nextLanguage(for: language)
let (pageContent, pageIncludesCode, pageIsEmpty) = makeContent( let (pageContent, pageIncludesCode) = makeContent(
page: page, metadata: metadata, language: language, path: inputContentPath) page: page, metadata: metadata, language: language, path: inputContentPath)
var content = [PageTemplate.Key : String]() var content = [PageTemplate.Key : String]()
@ -35,23 +39,17 @@ struct PageGenerator {
content[.previousPageUrl] = navLink(from: page, to: previousPage, language: language) content[.previousPageUrl] = navLink(from: page, to: previousPage, language: language)
content[.nextPageLinkText] = nextText(for: nextPage, language: language) content[.nextPageLinkText] = nextText(for: nextPage, language: language)
content[.nextPageUrl] = navLink(from: page, to: nextPage, language: language) content[.nextPageUrl] = navLink(from: page, to: nextPage, language: language)
content[.footer] = page.customFooterContent() content[.footer] = results.getContentOfOptionalFile(at: page.additionalFooterContentPath, source: page.path)
if pageIncludesCode { if pageIncludesCode {
let highlightCode = factory.factory.html.codeHighlightFooter() let highlightCode = factory.factory.html.codeHighlightFooter()
content[.footer] = (content[.footer].unwrapped { $0 + "\n" } ?? "") + highlightCode content[.footer] = (content[.footer].unwrapped { $0 + "\n" } ?? "") + highlightCode
} }
let url = files.urlInOutputFolder(path)
if page.state == .draft { if page.state == .draft {
files.isDraft(path: page.path) results.markPageAsDraft(page: page.path)
} else if pageIsEmpty, page.state != .hidden {
files.isEmpty(page: path)
} }
guard factory.page.generate(content, to: url) else { factory.page.generate(content, to: path, source: page.path)
return
}
files.generated(page: path)
} }
private func navLink(from element: Element, to destination: Element?, language: String) -> String? { private func navLink(from element: Element, to destination: Element?, language: String) -> String? {
@ -75,14 +73,14 @@ struct PageGenerator {
return factory.factory.html.makeNextText(text) return factory.factory.html.makeNextText(text)
} }
private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool, isEmpty: Bool) { private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool) {
if let raw = files.contentOfMdFile(atPath: path, source: page.path)?.trimmed.nonEmpty { if let raw = results.getContentOfMdFile(at: path, source: page.path)?.trimmed.nonEmpty {
let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: raw) let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: raw)
return (content, includesCode, false) return (content, includesCode)
} else { } else {
let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: metadata.placeholderText) let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: metadata.placeholderText)
let placeholder = factory.factory.makePlaceholder(title: metadata.placeholderTitle, text: content) let placeholder = factory.factory.makePlaceholder(title: metadata.placeholderTitle, text: content)
return (placeholder, includesCode, true) return (placeholder, includesCode)
} }
} }

10
Sources/Generator/Generators/PageHeadGenerator.swift
View File

@ -2,12 +2,16 @@ import Foundation
struct PageHeadGenerator { struct PageHeadGenerator {
// TODO: Add to configuration
static let linkPreviewDesiredImageWidth = 1600 static let linkPreviewDesiredImageWidth = 1600
let factory: TemplateFactory let factory: TemplateFactory
init(factory: TemplateFactory) { private let results: GenerationResultsHandler
init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.factory = factory self.factory = factory
self.results = results
} }
func generate(page: Element, language: String, includesCode: Bool = false) -> String { func generate(page: Element, language: String, includesCode: Bool = false) -> String {
@ -24,14 +28,14 @@ struct PageHeadGenerator {
let linkPreviewImageName = "thumbnail-link.\(image.lastComponentAfter("."))" let linkPreviewImageName = "thumbnail-link.\(image.lastComponentAfter("."))"
let sourceImagePath = page.pathRelativeToRootForContainedInputFile(image) let sourceImagePath = page.pathRelativeToRootForContainedInputFile(image)
let destinationImagePath = page.pathRelativeToRootForContainedInputFile(linkPreviewImageName) let destinationImagePath = page.pathRelativeToRootForContainedInputFile(linkPreviewImageName)
files.requireSingleImage( results.requireSingleImage(
source: sourceImagePath, source: sourceImagePath,
destination: destinationImagePath, destination: destinationImagePath,
requiredBy: page.path, requiredBy: page.path,
width: PageHeadGenerator.linkPreviewDesiredImageWidth) width: PageHeadGenerator.linkPreviewDesiredImageWidth)
content[.image] = factory.html.linkPreviewImage(file: linkPreviewImageName) content[.image] = factory.html.linkPreviewImage(file: linkPreviewImageName)
} }
content[.customPageContent] = page.customHeadContent() content[.customPageContent] = results.getContentOfOptionalFile(at: page.additionalHeadContentPath, source: page.path)
if includesCode { if includesCode {
let scriptPath = "assets/js/highlight.js" let scriptPath = "assets/js/highlight.js"
let relative = page.relativePathToOtherSiteElement(file: scriptPath) let relative = page.relativePathToOtherSiteElement(file: scriptPath)

23
Sources/Generator/Generators/SiteGenerator.swift
View File

@ -6,9 +6,12 @@ struct SiteGenerator {
let templates: TemplateFactory let templates: TemplateFactory
init() throws { let results: GenerationResultsHandler
let templatesFolder = files.urlInContentFolder("templates")
self.templates = try TemplateFactory(templateFolder: templatesFolder) init(results: GenerationResultsHandler) throws {
self.results = results
let templatesFolder = results.contentFolder.appendingPathComponent("templates")
self.templates = try TemplateFactory(templateFolder: templatesFolder, results: results)
} }
func generate(site: Element) { func generate(site: Element) {
@ -22,11 +25,12 @@ struct SiteGenerator {
let template = LocalizedSiteTemplate( let template = LocalizedSiteTemplate(
factory: templates, factory: templates,
language: language, language: language,
site: site) site: site,
results: results)
// Generate sections // Generate sections
let overviewGenerator = OverviewPageGenerator(factory: template) let overviewGenerator = OverviewPageGenerator(factory: template, results: results)
let pageGenerator = PageGenerator(factory: template, siteRoot: site) let pageGenerator = PageGenerator(factory: template, siteRoot: site, results: results)
var elementsToProcess: [LinkedElement] = [(nil, site, nil)] var elementsToProcess: [LinkedElement] = [(nil, site, nil)]
while let (previous, element, next) = elementsToProcess.popLast() { while let (previous, element, next) = elementsToProcess.popLast() {
@ -34,7 +38,6 @@ struct SiteGenerator {
elementsToProcess.append(contentsOf: element.linkedElements) elementsToProcess.append(contentsOf: element.linkedElements)
processAllFiles(for: element) processAllFiles(for: element)
if !element.elements.isEmpty { if !element.elements.isEmpty {
overviewGenerator.generate(section: element, language: language) overviewGenerator.generate(section: element, language: language)
} else { } else {
@ -48,10 +51,10 @@ struct SiteGenerator {
} }
private func processAllFiles(for element: Element) { private func processAllFiles(for element: Element) {
element.requiredFiles.forEach(files.require) element.externalFiles.forEach { results.exclude(file: $0, source: element.path) }
element.externalFiles.forEach(files.exclude) element.requiredFiles.forEach { results.require(file: $0, source: element.path) }
element.images.forEach { element.images.forEach {
files.requireSingleImage( results.requireSingleImage(
source: $0.sourcePath, source: $0.sourcePath,
destination: $0.destinationPath, destination: $0.destinationPath,
requiredBy: element.path, requiredBy: element.path,

7
Sources/Generator/Generators/ThumbnailListGenerator.swift
View File

@ -4,8 +4,11 @@ struct ThumbnailListGenerator {
private let factory: TemplateFactory private let factory: TemplateFactory
init(factory: TemplateFactory) { private let results: GenerationResultsHandler
init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.factory = factory self.factory = factory
self.results = results
} }
func generateContent(items: [Element], parent: Element, language: String, style: ThumbnailStyle) -> String { func generateContent(items: [Element], parent: Element, language: String, style: ThumbnailStyle) -> String {
@ -37,7 +40,7 @@ struct ThumbnailListGenerator {
factory.largeThumbnail.makeCorner(text: $0) factory.largeThumbnail.makeCorner(text: $0)
} }
files.requireMultiVersionImage( results.requireMultiVersionImage(
source: thumbnailSourcePath, source: thumbnailSourcePath,
destination: thumbnailDestNoExtension + ".jpg", destination: thumbnailDestNoExtension + ".jpg",
requiredBy: item.path, requiredBy: item.path,

20
Sources/Generator/Processing/FileData.swift Normal file
View File

@ -0,0 +1,20 @@
import Foundation
struct FileData {
///The files marked as expected, i.e. they exist after the generation is completed. (`key`: file path, `value`: the file providing the link)
var expected: [String : String] = [:]
/// All files which should be copied to the output folder (`key`: The file path, `value`: The source requiring the file)
var toCopy: [String : String] = [:]
/// The files to minify when copying into output directory. (`key`: the file path relative to the content folder)
var toMinify: [String : (source: String, type: MinificationType)] = [:]
/**
The files marked as external in element metadata. (Key: File path, Value: source element)
Files included here are not generated, since they are assumed to be added separately.
*/
var external: [String : String] = [:]
}

286
Sources/Generator/Processing/FileGenerator.swift Normal file
View File

@ -0,0 +1,286 @@
import Foundation
final class FileGenerator {
let input: URL
let outputFolder: URL
let runFolder: URL
private let files: FileData
/// All files copied to the destination.
private var copiedFiles: Set<String> = []
/// Files which could not be read (`key`: file path relative to content)
private var unreadableFiles: [String : (source: String, message: String)] = [:]
/// Files which could not be written (`key`: file path relative to output folder)
private var unwritableFiles: [String : (source: String, message: String)] = [:]
private var failedMinifications: [(file: String, source: String, message: String)] = []
/// Non-existent files. `key`: file path, `value`: source element
private var missingFiles: [String : String] = [:]
private var minifiedFiles: [String] = []
private let numberOfFilesToCopy: Int
private var numberOfCopiedFiles = 0
private let numberOfFilesToMinify: Int
private var numberOfMinifiedFiles = 0
private var numberOfExistingFiles = 0
private var tempFile: URL {
runFolder.appendingPathComponent("temp")
}
init(input: URL, output: URL, runFolder: URL, files: FileData) {
self.input = input
self.outputFolder = output
self.runFolder = runFolder
self.files = files
numberOfFilesToCopy = files.toCopy.count
numberOfFilesToMinify = files.toMinify.count
}
func generate() {
copy(files: files.toCopy)
print(" Copied files: \(copiedFiles.count)/\(numberOfFilesToCopy) ")
minify(files: files.toMinify)
print(" Minified files: \(minifiedFiles.count)/\(numberOfFilesToMinify) ")
checkExpected(files: files.expected)
print(" Expected files: \(numberOfExistingFiles)/\(files.expected.count)")
print(" External files: \(files.external.count)")
print("")
}
func writeResultsToFile(file: URL) throws {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("External files", files.external) { "\($0.key) (from \($0.value))" }
let data = lines.joined(separator: "\n").data(using: .utf8)!
try data.createFolderAndWrite(to: file)
}
private func didCopyFile() {
numberOfCopiedFiles += 1
print(" Copied files: \(numberOfCopiedFiles)/\(numberOfFilesToCopy) \r", terminator: "")
fflush(stdout)
}
private func didMinifyFile() {
numberOfMinifiedFiles += 1
print(" Minified files: \(numberOfMinifiedFiles)/\(numberOfFilesToMinify) \r", terminator: "")
fflush(stdout)
}
// MARK: File copies
private func copy(files: [String : String]) {
for (file, source) in files {
copyFileIfChanged(file: file, source: source)
}
}
private func copyFileIfChanged(file: String, source: String) {
let cleanPath = cleanRelativeURL(file)
let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
defer { didCopyFile() }
guard copyIfChanged(cleanPath, to: destinationUrl, source: source) else {
return
}
copiedFiles.insert(cleanPath)
}
private func copyIfChanged(_ file: String, to destination: URL, source: String) -> Bool {
let url = input.appendingPathComponent(file)
do {
let data = try Data(contentsOf: url)
return writeIfChanged(data, file: file, source: source)
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
return false
}
}
@discardableResult
func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
let url = outputFolder.appendingPathComponent(file)
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
return true
} catch {
markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
return false
}
}
// MARK: File minification
private func minify(files: [String : (source: String, type: MinificationType)]) {
for (file, other) in files {
minify(file: file, source: other.source, type: other.type)
}
}
private func minify(file: String, source: String, type: MinificationType) {
let url = input.appendingPathComponent(file)
let command: String
switch type {
case .js:
command = "uglifyjs \(url.path) > \(tempFile.path)"
case .css:
command = "cleancss \(url.path) -o \(tempFile.path)"
}
defer {
didMinifyFile()
try? tempFile.delete()
}
let output: String
do {
output = try safeShell(command)
} catch {
failedMinifications.append((file, source, "Failed to minify with error: \(error)"))
return
}
guard tempFile.exists else {
failedMinifications.append((file, source, output))
return
}
let data: Data
do {
data = try Data(contentsOf: tempFile)
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
return
}
if writeIfChanged(data, file: file, source: source) {
minifiedFiles.append(file)
}
}
private func cleanRelativeURL(_ raw: String) -> String {
let raw = raw.dropAfterLast("#") // Clean links to page content
guard raw.contains("..") else {
return raw
}
var result: [String] = []
for component in raw.components(separatedBy: "/") {
if component == ".." {
_ = result.popLast()
} else {
result.append(component)
}
}
return result.joined(separator: "/")
}
private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
unreadableFiles[path] = (source, message)
}
private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
unwritableFiles[path] = (source, message)
}
// MARK: File expectationts
private func checkExpected(files: [String: String]) {
for (file, source) in files {
guard !isExternal(file: file) else {
numberOfExistingFiles += 1
continue
}
let cleanPath = cleanRelativeURL(file)
let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
guard destinationUrl.exists else {
markFileAsMissing(at: cleanPath, requiredBy: source)
continue
}
numberOfExistingFiles += 1
}
}
private func markFileAsMissing(at path: String, requiredBy source: String) {
missingFiles[path] = source
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
private func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if files.external[path] != nil {
return true
}
}
return false
}
func writeResults(to file: URL) {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Failed minifications", failedMinifications) { "\($0.file) (required by \($0.source)): \($0.message)" }
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Copied files", copiedFiles) { $0 }
add("Minified files", minifiedFiles) { $0 }
add("Expected files", files.expected) { "\($0.key) (required by \($0.value))" }
let data = lines.joined(separator: "\n").data(using: .utf8)!
do {
try data.createFolderAndWrite(to: file)
} catch {
print(" Failed to save log: \(error)")
}
}
}

470
Sources/Generator/Processing/GenerationResultsHandler.swift Normal file
View File

@ -0,0 +1,470 @@
import Foundation
import CryptoKit
import AppKit
enum MinificationType {
case js
case css
}
final class GenerationResultsHandler {
/// The content folder where the input data is stored
let contentFolder: URL
/// The folder where the site is generated
let outputFolder: URL
private let fileUpdates: FileUpdateChecker
/**
All paths to page element folders, indexed by their unique id.
This relation is used to generate relative links to pages using the ``Element.id`
*/
private let pagePaths: [String: String]
private let configuration: Configuration
private var numberOfProcessedPages = 0
private let numberOfTotalPages: Int
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) {
self.contentFolder = input
self.fileUpdates = fileUpdates
self.outputFolder = output
self.pagePaths = pagePaths
self.configuration = configuration
self.numberOfTotalPages = pageCount
}
// MARK: Internal storage
/// Non-existent files. `key`: file path, `value`: source element
private var missingFiles: [String : String] = [:]
private(set) var files = FileData()
/// Files which could not be read (`key`: file path relative to content)
private var unreadableFiles: [String : (source: String, message: String)] = [:]
/// Files which could not be written (`key`: file path relative to output folder)
private var unwritableFiles: [String : (source: String, message: String)] = [:]
/// The paths to all files which were changed (relative to output)
private var generatedFiles: Set<String> = []
/// The referenced pages which do not exist (`key`: page id, `value`: source element path)
private var missingLinkedPages: [String : String] = [:]
/// All pages without content which have been created (`key`: page path, `value`: source element path)
private var emptyPages: [String : String] = [:]
/// All pages which have `status` set to ``PageState.draft``
private var draftPages: Set<String> = []
/// Generic warnings for pages
private var pageWarnings: [(message: String, source: String)] = []
/// A cache to get the size of source images, so that files don't have to be loaded multiple times (`key` absolute source path, `value`: image size)
private var imageSizeCache: [String : NSSize] = [:]
private(set) var images = ImageData()
// MARK: Generic warnings
private func warning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
images.warnings.append(warning)
}
func warning(_ message: String, source: String) {
pageWarnings.append((message, source))
}
// MARK: Page data
func getPagePath(for id: String, source: String) -> String? {
guard let pagePath = pagePaths[id] else {
missingLinkedPages[id] = source
return nil
}
return pagePath
}
private func markPageAsEmpty(page: String, source: String) {
emptyPages[page] = source
}
func markPageAsDraft(page: String) {
draftPages.insert(page)
}
// MARK: File actions
/**
Add a file as required, so that it will be copied to the output directory.
Special files may be minified.
- Parameter file: The file path, relative to the content directory
- Parameter source: The path of the source element requiring the file.
*/
func require(file: String, source: String) {
guard !isExternal(file: file) else {
return
}
let url = contentFolder.appendingPathComponent(file)
guard url.exists else {
markFileAsMissing(at: file, requiredBy: source)
return
}
guard url.isDirectory else {
markForCopyOrMinification(file: file, source: source)
return
}
do {
try FileManager.default
.contentsOfDirectory(atPath: url.path)
.forEach {
// Recurse into subfolders
require(file: file + "/" + $0, source: source)
}
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "Failed to read folder: \(error)")
}
}
private func markFileAsMissing(at path: String, requiredBy source: String) {
missingFiles[path] = source
}
private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
unreadableFiles[path] = (source, message)
}
private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
unwritableFiles[path] = (source, message)
}
private func markFileAsGenerated(file: String) {
generatedFiles.insert(file)
}
private func markForCopyOrMinification(file: String, source: String) {
let ext = file.lastComponentAfter(".")
if configuration.minifyCSSandJS, ext == "js" {
files.toMinify[file] = (source, .js)
return
}
if configuration.minifyCSSandJS, ext == "css" {
files.toMinify[file] = (source, .css)
return
}
files.toCopy[file] = source
}
/**
Mark a file as explicitly missing (external).
This is done for the `externalFiles` entries in metadata,
to indicate that these files will be copied to the output folder manually.
*/
func exclude(file: String, source: String) {
files.external[file] = source
}
/**
Mark a file as expected to be present in the output folder after generation.
This is done for all links between pages, which only exist after the pages have been generated.
*/
func expect(file: String, source: String) {
files.expected[file] = source
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
private func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if files.external[path] != nil {
return true
}
}
return false
}
func getContentOfRequiredFile(at path: String, source: String) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markFileAsMissing(at: path, requiredBy: source)
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
/**
Get the content of a file which may not exist.
*/
func getContentOfOptionalFile(at path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
if createEmptyFileIfMissing {
writeIfChanged(.init(), file: path, source: source)
}
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
func getContentOfMdFile(at path: String, source: String) -> String? {
guard let result = getContentOfOptionalFile(at: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) else {
markPageAsEmpty(page: path, source: source)
return nil
}
return result
}
private func getContentOfFile(at url: URL, path: String, source: String) -> String? {
do {
return try String(contentsOf: url)
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
@discardableResult
func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
let url = outputFolder.appendingPathComponent(file)
defer {
didCompletePage()
}
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
markFileAsGenerated(file: file)
return true
} catch {
markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
return false
}
}
// MARK: Images
/**
Request the creation of an image.
- Returns: The final size of the image.
*/
@discardableResult
func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
requireImage(
at: destination,
generatedFrom: source,
requiredBy: path,
quality: 0.7,
width: width,
height: desiredHeight,
alwaysGenerate: false)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
// Add @2x version
_ = requireScaledMultiImage(
source: source,
destination: destination.insert("@2x", beforeLast: "."),
requiredBy: path,
width: width * 2,
desiredHeight: desiredHeight.unwrapped { $0 * 2 })
// Add @1x version
return requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
}
func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
}
@discardableResult
private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
let rawDestinationPath = destination.dropAfterLast(".")
let avifPath = rawDestinationPath + ".avif"
let webpPath = rawDestinationPath + ".webp"
let needsGeneration = !outputFolder.appendingPathComponent(avifPath).exists || !outputFolder.appendingPathComponent(webpPath).exists
let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
images.multiJobs[destination] = path
return size
}
private func markImageAsMissing(path: String, source: String) {
images.missing[source] = path
}
private func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
let height = height.unwrapped(CGFloat.init)
guard let imageSize = getImageSize(atPath: source, source: path) else {
// Image marked as missing here
return .zero
}
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
// Check desired height, then we can forget about it
if let height = height {
let expectedHeight = scaledSize.width / CGFloat(width) * height
if abs(expectedHeight - scaledSize.height) > 2 {
warning("Expected a height of \(expectedHeight) (is \(scaledSize.height))", destination: destination, path: path)
}
}
let job = ImageJob(
destination: destination,
width: width,
path: path,
quality: quality,
alwaysGenerate: alwaysGenerate)
insert(job: job, source: source)
return scaledSize
}
private func getImageSize(atPath path: String, source: String) -> NSSize? {
if let size = imageSizeCache[path] {
return size
}
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markImageAsMissing(path: path, source: source)
return nil
}
guard let data = getImageData(at: url, path: path, source: source) else {
return nil
}
guard let image = NSImage(data: data) else {
images.unreadable[path] = source
return nil
}
let size = image.size
imageSizeCache[path] = size
return size
}
private func getImageData(at url: URL, path: String, source: String) -> Data? {
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
private func insert(job: ImageJob, source: String) {
guard let existingSource = images.jobs[source] else {
images.jobs[source] = [job]
return
}
guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
images.jobs[source] = existingSource + [job]
return
}
guard existingJob.width != job.width else {
return
}
warning("Different width \(existingJob.width) as \(job.path) (width \(job.width))",
destination: job.destination, path: existingJob.path)
}
// MARK: Visual output
func didCompletePage() {
numberOfProcessedPages += 1
print(" Processed pages: \(numberOfProcessedPages)/\(numberOfTotalPages) \r", terminator: "")
fflush(stdout)
}
func printOverview() {
var notes: [String] = []
func addIfNotZero(_ count: Int, _ name: String) {
guard count > 0 else {
return
}
notes.append("\(count) \(name)")
}
func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
addIfNotZero(sequence.count, name)
}
addIfNotZero(missingFiles.count, "missing files")
addIfNotZero(unreadableFiles.count, "unreadable files")
addIfNotZero(unwritableFiles.count, "unwritable files")
addIfNotZero(missingLinkedPages.count, "missing linked pages")
addIfNotZero(pageWarnings.count, "warnings")
print(" Updated pages: \(generatedFiles.count)/\(numberOfProcessedPages) ")
print(" Drafts: \(draftPages.count)")
print(" Empty pages: \(emptyPages.count)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
print("")
}
func writeResults(to file: URL) {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Missing linked pages", missingLinkedPages) { "\($0.key) (linked by \($0.value))" }
add("Warnings", pageWarnings) { "\($0.source): \($0.message)" }
add("Drafts", draftPages) { $0 }
add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" }
add("Generated files", generatedFiles) { $0 }
let data = lines.joined(separator: "\n").data(using: .utf8)!
do {
try data.createFolderAndWrite(to: file)
} catch {
print(" Failed to save log: \(error)")
}
}
}

19
Sources/Generator/Processing/ImageData.swift Normal file
View File

@ -0,0 +1,19 @@
import Foundation
struct ImageData {
/// The images to generate (`key`: the image source path relative to the input folder)
var jobs: [String : [ImageJob]] = [:]
/// The images for which to generate multiple versions (`key`: the source file, `value`: the path of the requiring page)
var multiJobs: [String : String] = [:]
/// All warnings produced for images during generation
var warnings: [String] = []
/// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path)
var missing: [String : String] = [:]
/// Images which could not be read (`key`: file path relative to content, `value`: source element path)
var unreadable: [String : String] = [:]
}

318
Sources/Generator/Processing/ImageGenerator.swift Normal file
View File

@ -0,0 +1,318 @@
import Foundation
import AppKit
import CryptoKit
import Darwin.C
final class ImageGenerator {
private let imageOptimSupportedFileExtensions: Set<String> = ["jpg", "png", "svg"]
private let imageOptimizationBatchSize = 50
/**
The path to the input folder.
*/
private let input: URL
/**
The path to the output folder
*/
private let output: URL
private let imageReader: ImageReader
/**
All warnings produced for images during generation
*/
private var imageWarnings: [String]
/// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path)
private var missingImages: [String : String]
/// Images which could not be read (`key`: file path relative to content, `value`: source element path)
private var unreadableImages: [String : String]
private var unhandledImages: [String: String] = [:]
/// All images modified or created during this generator run.
private var generatedImages: Set<String> = []
/// The images optimized by ImageOptim
private var optimizedImages: Set<String> = []
private var failedImages: [(path: String, message: String)] = []
private var numberOfGeneratedImages = 0
private let numberOfTotalImages: Int
private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
private var numberOfImagesToOptimize = 0
private var numberOfOptimizedImages = 0
private let images: ImageData
private lazy var jobs: [(source: String, images: [ImageJob])] = images.jobs
.sorted { $0.key < $1.key }
.map { (source: $0.key, images: $0.value) }
.filter {
// Only load image if required
let imageHasChanged = imageReader.imageHasChanged(at: $0.source)
return imageHasChanged || $0.images.contains { job in
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
}
private lazy var multiJobs: [String : String] = {
let imagesToGenerate: Set<String> = jobs.reduce([]) { $0.union($1.images.map { $0.destination }) }
return images.multiJobs.filter { imagesToGenerate.contains($0.key) }
}()
init(input: URL, output: URL, reader: ImageReader, images: ImageData) {
self.input = input
self.output = output
self.imageReader = reader
self.images = images
self.numberOfTotalImages = images.jobs.reduce(0) { $0 + $1.value.count }
+ images.multiJobs.count * 2
self.imageWarnings = images.warnings
self.missingImages = images.missing
self.unreadableImages = images.unreadable
}
func generateImages() {
var notes: [String] = []
func addIfNotZero(_ count: Int, _ name: String) {
guard count > 0 else {
return
}
notes.append("\(count) \(name)")
}
print(" Changed sources: \(jobs.count)/\(images.jobs.count)")
print(" Total images: \(numberOfTotalImages) (\(numberOfTotalImages - imageReader.numberOfFilesAccessed) versions)")
for (source, jobs) in jobs {
create(images: jobs, from: source)
}
for (baseImage, source) in multiJobs {
createMultiImages(from: baseImage, path: source)
}
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)")
optimizeImages()
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)")
addIfNotZero(missingImages.count, "missing images")
addIfNotZero(unreadableImages.count, "unreadable images")
print(" Warnings: \(imageWarnings.count)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
}
private func create(images: [ImageJob], from source: String) {
guard let image = imageReader.getImage(atPath: source) else {
unreadableImages[source] = images.first!.destination
didGenerateImage(count: images.count)
return
}
let jobs = imageReader.imageHasChanged(at: source) ? images : images.filter(isMissing)
// Update all images
jobs.forEach { job in
// Prevent memory overflow due to repeated NSImage operations
autoreleasepool {
create(job: job, from: image, source: source)
didGenerateImage()
}
}
}
private func isMissing(_ job: ImageJob) -> Bool {
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
private func create(job: ImageJob, from image: NSImage, source: String) {
let destinationUrl = output.appendingPathComponent(job.destination)
create(job: job, from: image, source: source, at: destinationUrl)
}
private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
// Ensure that image file is supported
let ext = destinationUrl.pathExtension.lowercased()
guard ImageType(fileExtension: ext) != nil else {
fatalError()
}
let destinationExtension = destinationUrl.pathExtension.lowercased()
guard let type = ImageType(fileExtension: destinationExtension)?.fileType else {
unhandledImages[source] = job.destination
return
}
let desiredWidth = CGFloat(job.width)
let sourceRep = image.representations[0]
let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh)
.scaledDown(to: desiredWidth)
// create NSBitmapRep manually, if using cgImage, the resulting size is wrong
let rep = NSBitmapImageRep(bitmapDataPlanes: nil,
pixelsWide: Int(destinationSize.width),
pixelsHigh: Int(destinationSize.height),
bitsPerSample: 8,
samplesPerPixel: 4,
hasAlpha: true,
isPlanar: false,
colorSpaceName: NSColorSpaceName.deviceRGB,
bytesPerRow: Int(destinationSize.width) * 4,
bitsPerPixel: 32)!
let ctx = NSGraphicsContext(bitmapImageRep: rep)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = ctx
image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height))
ctx?.flushGraphics()
NSGraphicsContext.restoreGraphicsState()
// Get NSData, and save it
guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
markImageAsFailed(source, error: "Failed to get data")
return
}
do {
try data.createFolderAndWrite(to: destinationUrl)
} catch {
markImageAsFailed(job.destination, error: "Failed to write image (\(error))")
return
}
generatedImages.insert(job.destination)
}
private func markImageAsFailed(_ source: String, error: String) {
failedImages.append((source, error))
}
private func createMultiImages(from source: String, path: String) {
guard generatedImages.contains(source) else {
didGenerateImage(count: 2)
return
}
let sourceUrl = output.appendingPathComponent(source)
let sourcePath = sourceUrl.path
guard sourceUrl.exists else {
missingImages[source] = path
didGenerateImage(count: 2)
return
}
let avifPath = source.dropAfterLast(".") + ".avif"
createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
generatedImages.insert(avifPath)
didGenerateImage()
let webpPath = source.dropAfterLast(".") + ".webp"
createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
generatedImages.insert(webpPath)
didGenerateImage()
compress(at: source)
}
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do {
_ = try safeShell(command)
} catch {
markImageAsFailed(destination, error: "Failed to create AVIF image")
}
}
private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)"
do {
_ = try safeShell(command)
} catch {
markImageAsFailed(destination, error: "Failed to create WEBP image")
}
}
private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do {
_ = try safeShell(command)
} catch {
markImageAsFailed(destination, error: "Failed to compress image")
}
}
private func optimizeImages() {
let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path }
for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, all.count)
let batch = all[i..<endIndex]
if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch)
}
didOptimizeImage(count: batch.count)
}
}
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ")
do {
_ = try safeShell(command)
return true
} catch {
for image in batch {
markImageAsFailed(image, error: "Failed to optimize image")
}
return false
}
}
// MARK: Output
private func didGenerateImage(count: Int = 1) {
numberOfGeneratedImages += count
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate) \r", terminator: "")
fflush(stdout)
}
private func didOptimizeImage(count: Int) {
numberOfOptimizedImages += count
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize) \r", terminator: "")
fflush(stdout)
}
func writeResults(to file: URL) {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing images", missingImages) { "\($0.key) (required by \($0.value))" }
add("Unreadable images", unreadableImages) { "\($0.key) (required by \($0.value))" }
add("Failed images", failedImages) { "\($0.path): \($0.message)" }
add("Unhandled images", unhandledImages) { "\($0.value) (from \($0.key))" }
add("Warnings", imageWarnings) { $0 }
add("Generated images", generatedImages) { $0 }
add("Optimized images", optimizedImages) { $0 }
let data = lines.joined(separator: "\n").data(using: .utf8)!
do {
try data.createFolderAndWrite(to: file)
} catch {
print(" Failed to save log: \(error)")
}
}
}

58
Sources/Generator/Processing/MetadataInfoLogger.swift
View File

@ -124,10 +124,10 @@ final class MetadataInfoLogger {
// MARK: Printing // MARK: Printing
private func printMetadataScanUpdate() { private func printMetadataScanUpdate() {
print(String(format: "Scanning source files: %4d pages found \r", numberOfMetadataFiles), terminator: "") print(" Pages found: \(numberOfMetadataFiles) \r", terminator: "")
} }
func printMetadataScanOverview() { func printMetadataScanOverview(languages: Int) {
var notes = [String]() var notes = [String]()
func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) { func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
guard sequence.count > 0 else { guard sequence.count > 0 else {
@ -139,40 +139,40 @@ final class MetadataInfoLogger {
addIfNotZero(errors, "errors") addIfNotZero(errors, "errors")
addIfNotZero(unreadableMetadata, "unreadable files") addIfNotZero(unreadableMetadata, "unreadable files")
addIfNotZero(unusedProperties, "unused properties") addIfNotZero(unusedProperties, "unused properties")
addIfNotZero(invalidProperties, "invalidProperties") addIfNotZero(invalidProperties, "invalid properties")
addIfNotZero(unknownProperties, "unknownProperties") addIfNotZero(unknownProperties, "unknown properties")
addIfNotZero(missingProperties, "missingProperties") addIfNotZero(missingProperties, "missing properties")
print(" Number of pages: \(numberOfMetadataFiles)") print(" Pages found: \(numberOfMetadataFiles) ")
print(" Languages: \(languages)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", ")) print(" Notes: " + notes.joined(separator: ", "))
} }
}
func writeResultsToFile(in folder: URL) throws { func writeResults(to file: URL) {
let url = folder.appendingPathComponent("Metadata issues.txt")
var lines: [String] = [] var lines: [String] = []
if !errors.isEmpty { func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" } let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
} }
if !warnings.isEmpty { lines.append("\(name):")
lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" } lines.append(contentsOf: elements)
}
if !unreadableMetadata.isEmpty {
lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }
}
if !unusedProperties.isEmpty {
lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }
}
if !invalidProperties.isEmpty {
lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }
}
if !unknownProperties.isEmpty {
lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }
}
if !missingProperties.isEmpty {
lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }
} }
add("Errors", errors) { "\($0.source): \($0.message)" }
add("Warnings", warnings) { "\($0.source): \($0.message)" }
add("Unreadable files", unreadableMetadata) { "\($0.source): \($0.error)" }
add("Unused properties", unusedProperties) { "\($0.source): \($0.name)" }
add("Invalid properties", invalidProperties) { "\($0.source): \($0.name) (\($0.reason))" }
add("Unknown properties", unknownProperties) { "\($0.source): \($0.name)" }
add("Missing properties", missingProperties) { "\($0.source): \($0.name)" }
let data = lines.joined(separator: "\n").data(using: .utf8) let data = lines.joined(separator: "\n").data(using: .utf8)!
try data?.createFolderAndWrite(to: url) do {
try data.createFolderAndWrite(to: file)
} catch {
print(" Failed to save log: \(error)")
}
} }
} }

20
Sources/Generator/Processing/Shell.swift Normal file
View File

@ -0,0 +1,20 @@
import Foundation
@discardableResult
func safeShell(_ command: String) throws -> String {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/zsh")
task.standardInput = nil
try task.run()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return output
}

2
Sources/Generator/Templates/Elements/BackNavigationTemplate.swift
View File

@ -10,4 +10,6 @@ struct BackNavigationTemplate: Template {
static let templateName = "back.html" static let templateName = "back.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/BoxTemplate.swift
View File

@ -10,4 +10,6 @@ struct BoxTemplate: Template {
static let templateName = "box.html" static let templateName = "box.html"
var raw: String var raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/OverviewSectionCleanTemplate.swift
View File

@ -9,4 +9,6 @@ struct OverviewSectionCleanTemplate: Template {
static let templateName = "overview-section-clean.html" static let templateName = "overview-section-clean.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/OverviewSectionTemplate.swift
View File

@ -12,4 +12,6 @@ struct OverviewSectionTemplate: Template {
static let templateName = "overview-section.html" static let templateName = "overview-section.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/PageHeadTemplate.swift
View File

@ -12,5 +12,7 @@ struct PageHeadTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
static let templateName = "head.html" static let templateName = "head.html"
} }

2
Sources/Generator/Templates/Elements/PageImageTemplate.swift
View File

@ -15,4 +15,6 @@ struct PageImageTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/PageLinkTemplate.swift
View File

@ -15,6 +15,8 @@ struct PageLinkTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
func makePath(components: [String]) -> String { func makePath(components: [String]) -> String {
components.joined(separator: " » ") // &ensp;&raquo;&ensp;") components.joined(separator: " » ") // &ensp;&raquo;&ensp;")
} }

2
Sources/Generator/Templates/Elements/PageVideoTemplate.swift
View File

@ -23,6 +23,8 @@ struct PageVideoTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
func generate<T>(sources: [VideoSource], options: T) -> String where T: Sequence, T.Element == VideoOption { func generate<T>(sources: [VideoSource], options: T) -> String where T: Sequence, T.Element == VideoOption {
let sourcesCode = sources.map(makeSource).joined(separator: "\n") let sourcesCode = sources.map(makeSource).joined(separator: "\n")
let optionCode = options.map { $0.rawValue }.joined(separator: " ") let optionCode = options.map { $0.rawValue }.joined(separator: " ")

6
Sources/Generator/Templates/Elements/ThumbnailTemplate.swift
View File

@ -20,6 +20,8 @@ struct LargeThumbnailTemplate: Template, ThumbnailTemplate {
let raw: String let raw: String
let results: GenerationResultsHandler
func makeCorner(text: String) -> String { func makeCorner(text: String) -> String {
"<span class=\"corner\"><span>\(text)</span></span>" "<span class=\"corner\"><span>\(text)</span></span>"
} }
@ -36,6 +38,8 @@ struct SquareThumbnailTemplate: Template, ThumbnailTemplate {
static let templateName = "thumbnail-square.html" static let templateName = "thumbnail-square.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }
struct SmallThumbnailTemplate: Template, ThumbnailTemplate { struct SmallThumbnailTemplate: Template, ThumbnailTemplate {
@ -45,5 +49,7 @@ struct SmallThumbnailTemplate: Template, ThumbnailTemplate {
static let templateName = "thumbnail-small.html" static let templateName = "thumbnail-small.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }

2
Sources/Generator/Templates/Elements/TopBarTemplate.swift
View File

@ -12,4 +12,6 @@ struct TopBarTemplate: Template {
static let templateName = "bar.html" static let templateName = "bar.html"
var raw: String var raw: String
let results: GenerationResultsHandler
} }

8
Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift
View File

@ -35,7 +35,7 @@ struct LocalizedSiteTemplate {
factory.page factory.page
} }
init(factory: TemplateFactory, language: String, site: Element) { init(factory: TemplateFactory, language: String, site: Element, results: GenerationResultsHandler) {
self.author = site.author self.author = site.author
self.factory = factory self.factory = factory
@ -66,10 +66,8 @@ struct LocalizedSiteTemplate {
language: language, language: language,
sections: sections, sections: sections,
topBarWebsiteTitle: site.topBarTitle) topBarWebsiteTitle: site.topBarTitle)
self.pageHead = PageHeadGenerator( self.pageHead = PageHeadGenerator(factory: factory, results: results)
factory: factory) self.overviewSection = OverviewSectionGenerator(factory: factory, results: results)
self.overviewSection = OverviewSectionGenerator(
factory: factory)
} }
// MARK: Content // MARK: Content

4
Sources/Generator/Templates/Pages/HeaderTemplate.swift
View File

@ -19,6 +19,8 @@ struct CenteredHeaderTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
static let templateName = "header-center.html" static let templateName = "header-center.html"
} }
@ -28,5 +30,7 @@ struct LeftHeaderTemplate: Template {
let raw: String let raw: String
let results: GenerationResultsHandler
static let templateName = "header-left.html" static let templateName = "header-left.html"
} }

2
Sources/Generator/Templates/Pages/PageTemplate.swift
View File

@ -18,4 +18,6 @@ struct PageTemplate: Template {
static let templateName = "page.html" static let templateName = "page.html"
let raw: String let raw: String
let results: GenerationResultsHandler
} }

24
Sources/Generator/Templates/Template.swift
View File

@ -8,25 +8,33 @@ protocol Template {
var raw: String { get } var raw: String { get }
init(raw: String) var results: GenerationResultsHandler { get }
init(raw: String, results: GenerationResultsHandler)
} }
extension Template { extension Template {
init(in folder: URL) throws { init(in folder: URL, results: GenerationResultsHandler) throws {
let url = folder.appendingPathComponent(Self.templateName) let url = folder.appendingPathComponent(Self.templateName)
try self.init(from: url) try self.init(from: url, results: results)
} }
init(from url: URL) throws { init(from url: URL, results: GenerationResultsHandler) throws {
do {
let raw = try String(contentsOf: url) let raw = try String(contentsOf: url)
self.init(raw: raw) self.init(raw: raw, results: results)
} catch {
results.warning("Failed to load: \(error)", source: "Template \(url.lastPathComponent)")
throw error
}
} }
func generate(_ content: [Key : String], to url: URL) -> Bool { @discardableResult
let content = generate(content) func generate(_ content: [Key : String], to file: String, source: String) -> Bool {
return files.write(content, to: url) let content = generate(content).data(using: .utf8)!
return results.writeIfChanged(content, file: file, source: source)
} }
func generate(_ content: [Key : String], shouldIndent: Bool = false) -> String { func generate(_ content: [Key : String], shouldIndent: Bool = false) -> String {

32
Sources/Generator/Templates/TemplateFactory.swift
View File

@ -59,23 +59,23 @@ final class TemplateFactory {
// MARK: Init // MARK: Init
init(templateFolder: URL) throws { init(templateFolder: URL, results: GenerationResultsHandler) throws {
self.templateFolder = templateFolder self.templateFolder = templateFolder
self.backNavigation = try .init(in: templateFolder) self.backNavigation = try .init(in: templateFolder, results: results)
self.pageHead = try .init(in: templateFolder) self.pageHead = try .init(in: templateFolder, results: results)
self.topBar = try .init(in: templateFolder) self.topBar = try .init(in: templateFolder, results: results)
self.overviewSection = try .init(in: templateFolder) self.overviewSection = try .init(in: templateFolder, results: results)
self.overviewSectionClean = try .init(in: templateFolder) self.overviewSectionClean = try .init(in: templateFolder, results: results)
self.box = try .init(in: templateFolder) self.box = try .init(in: templateFolder, results: results)
self.pageLink = try .init(in: templateFolder) self.pageLink = try .init(in: templateFolder, results: results)
self.largeThumbnail = try .init(in: templateFolder) self.largeThumbnail = try .init(in: templateFolder, results: results)
self.squareThumbnail = try .init(in: templateFolder) self.squareThumbnail = try .init(in: templateFolder, results: results)
self.smallThumbnail = try .init(in: templateFolder) self.smallThumbnail = try .init(in: templateFolder, results: results)
self.leftHeader = try .init(in: templateFolder) self.leftHeader = try .init(in: templateFolder, results: results)
self.centeredHeader = try .init(in: templateFolder) self.centeredHeader = try .init(in: templateFolder, results: results)
self.page = try .init(in: templateFolder) self.page = try .init(in: templateFolder, results: results)
self.image = try .init(in: templateFolder) self.image = try .init(in: templateFolder, results: results)
self.video = try .init(in: templateFolder) self.video = try .init(in: templateFolder, results: results)
self.html = .init() self.html = .init()
} }

152
Sources/Generator/run.swift
View File

@ -1,8 +1,6 @@
import Foundation import Foundation
import ArgumentParser import ArgumentParser
let log = ValidationLog()
var files: FileSystem!
@main @main
struct CHGenerator: ParsableCommand { struct CHGenerator: ParsableCommand {
@ -15,18 +13,8 @@ struct CHGenerator: ParsableCommand {
} }
} }
private func loadSiteData(in folder: URL) throws -> Element? {
let log = MetadataInfoLogger(input: folder)
print("--- SOURCE FILES -----------------------------------")
let root = Element(atRoot: folder, log: log)
print(" ")
log.printMetadataScanOverview()
print(" ")
try log.writeResultsToFile(in: files.generatorInfoFolder)
return root
}
private func loadConfiguration(at configPath: String) -> Configuration? { private func loadConfiguration(at configPath: String) -> Configuration? {
print(" ")
print("--- CONFIGURATION ----------------------------------") print("--- CONFIGURATION ----------------------------------")
print(" ") print(" ")
print(" Configuration file: \(configPath)") print(" Configuration file: \(configPath)")
@ -44,30 +32,136 @@ private func loadConfiguration(at configPath: String) -> Configuration? {
return config return config
} }
private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, ids: [String : String])? {
print("--- SOURCE FILES -----------------------------------")
print(" ")
let log = MetadataInfoLogger(input: folder)
let root = Element(atRoot: folder, log: log)
let file = runFolder.appendingPathComponent("metadata.txt")
defer {
log.writeResults(to: file)
print(" ")
}
guard let root else {
log.printMetadataScanOverview(languages: 0)
print(" Error: No site root loaded, aborting generation")
return nil
}
let ids = root.getContainedIds(log: log)
log.printMetadataScanOverview(languages: root.languages.count)
return (root, ids)
}
private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], runFolder: URL) -> (ImageData, FileData)? {
print("--- GENERATION -------------------------------------")
print(" ")
let pageCount = ids.count * root.languages.count
let results = GenerationResultsHandler(
in: configuration.contentDirectory,
to: configuration.outputDirectory,
configuration: configuration,
fileUpdates: fileUpdates,
pagePaths: ids,
pageCount: pageCount)
defer { results.printOverview() }
let siteGenerator: SiteGenerator
do {
siteGenerator = try SiteGenerator(results: results)
} catch {
return nil
}
siteGenerator.generate(site: root)
let url = runFolder.appendingPathComponent("pages.txt")
results.writeResults(to: url)
if let error = fileUpdates.writeDetectedFileChanges(to: runFolder) {
print(" Hashes not saved: \(error)")
}
return (results.images, results.files)
}
private func generateImages(_ images: ImageData, configuration: Configuration, runFolder: URL, fileUpdates: FileUpdateChecker) {
print("--- IMAGES -----------------------------------------")
print(" ")
let reader = ImageReader(in: configuration.contentDirectory, runFolder: runFolder, fileUpdates: fileUpdates)
let generator = ImageGenerator(
input: configuration.contentDirectory,
output: configuration.outputDirectory,
reader: reader, images: images)
generator.generateImages()
print(" ")
let file = runFolder.appendingPathComponent("images.txt")
generator.writeResults(to: file)
}
private func copyFiles(files: FileData, configuration: Configuration, runFolder: URL) {
print("--- FILES ------------------------------------------")
print(" ")
let generator = FileGenerator(
input: configuration.contentDirectory,
output: configuration.outputDirectory,
runFolder: runFolder,
files: files)
generator.generate()
}
private func finish(start: Date) {
print("----------------------------------------------------")
print(" ")
let duration = Int(-start.timeIntervalSinceNow.rounded())
if duration < 60 {
print(" Duration: \(duration) s")
} else if duration < 3600 {
print(String(format: " Duration: %d:%02d", duration / 60, duration % 60))
} else {
print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60))
}
print("")
}
private func generate(configPath: String) throws { private func generate(configPath: String) throws {
let start = Date()
// 1. Load configuration
guard let configuration = loadConfiguration(at: configPath) else { guard let configuration = loadConfiguration(at: configPath) else {
return return
} }
files = .init( let runFolder = configuration.contentDirectory.appendingPathComponent("run")
in: configuration.contentDirectory,
to: configuration.outputDirectory,
configuration: configuration)
// 2. Scan site elements // 2. Scan site elements
guard let siteRoot = try loadSiteData(in: configuration.contentDirectory) else { guard let (siteRoot, ids) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else {
return return
} }
let siteGenerator = try SiteGenerator()
siteGenerator.generate(site: siteRoot)
files.printGeneratedPages() let fileUpdates = FileUpdateChecker(input: configuration.contentDirectory)
files.printEmptyPages() switch fileUpdates.loadPreviousRun(from: runFolder) {
files.printDraftPages() case .notLoaded:
print("Regarding all files as new (no hashes loaded)")
files.createImages() case .loaded:
files.copyRequiredFiles() break
files.printExternalFiles() case .failed(let error):
files.writeDetectedFileChangesToDisk() print("Regarding all files as new (\(error))")
}
// 3. Generate pages
guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, runFolder: runFolder) else {
return
}
// 4. Generate images
generateImages(images, configuration: configuration, runFolder: runFolder, fileUpdates: fileUpdates)
// 5. Copy/minify files
copyFiles(files: files, configuration: configuration, runFolder: runFolder)
// 6. Print summary
finish(start: start)
} }