ch/CHGenerator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve printing and image creation

Browse Source
This commit is contained in:
Christoph Hagen 2022-12-04 19:15:22 +01:00
parent 6a52f62402
commit 956cfb52c4
23 changed files with 1421 additions and 1077 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Sources/Generator/Content/Element.swift
View File

@ -183,7 +183,7 @@ struct Element {
self.sortIndex = log.unused(metadata.sortIndex, "sortIndex", source: source)
self.externalFiles = metadata.externalFiles ?? []
self.requiredFiles = metadata.requiredFiles ?? [] // Paths are already relative to root
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "") } ?? []
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: "", log: log) } ?? []
self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName
self.thumbnailStyle = log.castUnused(metadata.thumbnailStyle, "thumbnailStyle", source: source)
self.useManualSorting = log.unused(metadata.useManualSorting, "useManualSorting", source: source)
@ -204,7 +204,7 @@ struct Element {
return nil
}
files.add(page: path, id: id)
//files.add(page: path, id: id)
self.readElements(in: folder, source: nil, log: log)
}
@ -239,13 +239,13 @@ struct Element {
self.author = metadata.author ?? parent.author
self.topBarTitle = log.unused(metadata.topBarTitle, "topBarTitle", source: source)
self.date = metadata.date.unwrapped { log.cast($0, "date", source: source) }
self.endDate = metadata.date.unwrapped { log.cast($0, "endDate", source: source) }
self.endDate = metadata.endDate.unwrapped { log.cast($0, "endDate", source: source) }
self.state = log.cast(metadata.state, "state", source: source)
self.sortIndex = metadata.sortIndex
// TODO: Propagate external files from the parent if subpath matches?
self.externalFiles = Element.rootPaths(for: metadata.externalFiles, path: path)
self.requiredFiles = Element.rootPaths(for: metadata.requiredFiles, path: path)
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path) } ?? []
self.images = metadata.images?.compactMap { ManualImage(input: $0, path: path, log: log) } ?? []
self.thumbnailPath = metadata.thumbnailPath ?? Element.defaultThumbnailName
self.thumbnailStyle = log.cast(metadata.thumbnailStyle, "thumbnailStyle", source: source)
self.useManualSorting = metadata.useManualSorting ?? false
@ -285,9 +285,21 @@ struct Element {
return nil
}
files.add(page: path, id: id)
//files.add(page: path, id: id)
self.readElements(in: folder, source: path, log: log)
}
func getContainedIds(log: MetadataInfoLogger) -> [String : String] {
elements.reduce(into: [id : path]) { dict, element in
element.getContainedIds(log: log).forEach { id, path in
if let existing = dict[id] {
log.error("Conflicting id with \(existing)", source: path)
} else {
dict[id] = path
}
}
}
}
}
// MARK: Paths
@ -566,21 +578,14 @@ extension Element {
extension Element {
private var additionalHeadContentPath: String {
var additionalHeadContentPath: String {
path + "/head.html"
}
func customHeadContent() -> String? {
files.contentOfOptionalFile(atPath: additionalHeadContentPath, source: path)
}
private var additionalFooterContentPath: String {
var additionalFooterContentPath: String {
path + "/footer.html"
}
func customFooterContent() -> String? {
files.contentOfOptionalFile(atPath: additionalFooterContentPath, source: path)
}
}
// MARK: Debug
@ -607,14 +612,14 @@ extension Element {
let desiredHeight: Int?
init?(input: String, path: String) {
init?(input: String, path: String, log: MetadataInfoLogger) {
let parts = input.components(separatedBy: " ").filter { !$0.isEmpty }
guard parts.count == 3 || parts.count == 4 else {
log.add(error: "Invalid image specification, expected 'source dest width (height)", source: path)
log.error("Invalid image specification, expected 'source dest width (height)", source: path)
return nil
}
guard let width = Int(parts[2]) else {
log.add(error: "Invalid width for image \(parts[0])", source: path)
log.error("Invalid width for image \(parts[0])", source: path)
return nil
}
@ -626,7 +631,7 @@ extension Element {
return
}
guard let height = Int(parts[3]) else {
log.add(error: "Invalid height for image \(parts[0])", source: path)
log.error("Invalid height for image \(parts[0])", source: path)
return nil
}
self.desiredHeight = height
@ -716,10 +721,6 @@ extension Element {
let destination = pathRelativeToRootForContainedInputFile("thumbnail-\(language).\(ext)")
return (source, destination)
}
let thumbnailUrl = inputFolder.appendingPathComponent(thumbnailPath)
if !thumbnailUrl.exists {
log.add(error: "Missing thumbnail", source: path)
}
let source = pathRelativeToRootForContainedInputFile(thumbnailPath)
let ext = thumbnailPath.lastComponentAfter(".")
let destination = pathRelativeToRootForContainedInputFile("thumbnail.\(ext)")

443
Sources/Generator/Files/FileSystem.swift
View File

@ -1,443 +0,0 @@
import Foundation
import CryptoKit
import AppKit
final class FileSystem {
private static let tempFileName = "temp.bin"
private let input: URL
private let output: URL
private let source = "FileSystem"
private let images: ImageGenerator
private let configuration: Configuration
private var tempFile: URL {
input.appendingPathComponent(FileSystem.tempFileName)
}
let generatorInfoFolder: URL
/**
All files which should be copied to the output folder
*/
private var requiredFiles: Set<String> = []
/**
The files marked as external in element metadata.
Files included here are not generated, since they are assumed to be added separately.
*/
private var externalFiles: Set<String> = []
/**
The files marked as expected, i.e. they exist after the generation is completed.
The key of the dictionary is the file path, the value is the file providing the link
*/
private var expectedFiles: [String : String] = [:]
/**
All pages without content which have been created
*/
private var emptyPages: Set<String> = []
/**
All pages which have `status` set to ``PageState.draft``
*/
private var draftPages: Set<String> = []
/**
All paths to page element folders, indexed by their unique id.
This relation is used to generate relative links to pages using the ``Element.id`
*/
private var pagePaths: [String: String] = [:]
/**
The image creation tasks.
The key is the destination path.
*/
private var imageTasks: [String : ImageOutput] = [:]
/**
The paths to all pages which were changed
*/
private var generatedPages: Set<String> = []
init(in input: URL, to output: URL, configuration: Configuration) {
self.input = input
self.output = output
self.images = .init(input: input, output: output)
self.generatorInfoFolder = input.appendingPathComponent("run")
self.configuration = configuration
}
func urlInOutputFolder(_ path: String) -> URL {
output.appendingPathComponent(path)
}
func urlInContentFolder(_ path: String) -> URL {
input.appendingPathComponent(path)
}
private func exists(_ url: URL) -> Bool {
FileManager.default.fileExists(atPath: url.path)
}
func dataOfRequiredFile(atPath path: String, source: String) -> Data? {
let url = input.appendingPathComponent(path)
guard exists(url) else {
log.failedToOpen(path, requiredBy: source, error: nil)
return nil
}
do {
return try Data(contentsOf: url)
} catch {
log.failedToOpen(path, requiredBy: source, error: error)
return nil
}
}
func contentOfMdFile(atPath path: String, source: String) -> String? {
contentOfOptionalFile(atPath: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing)
}
func contentOfOptionalFile(atPath path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
let url = input.appendingPathComponent(path)
guard exists(url) else {
if createEmptyFileIfMissing {
try? Data().write(to: url)
}
return nil
}
do {
return try String(contentsOf: url)
} catch {
log.failedToOpen(path, requiredBy: source, error: error)
return nil
}
}
func writeDetectedFileChangesToDisk() {
images.writeDetectedFileChangesToDisk()
}
// MARK: Images
@discardableResult
func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
images.requireImage(
at: destination,
generatedFrom: source,
requiredBy: path,
quality: 0.7,
width: width,
height: desiredHeight,
alwaysGenerate: false)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
}
func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
}
func createImages() {
images.createImages()
}
// MARK: File copying
/**
Add a file as required, so that it will be copied to the output directory.
*/
func require(file: String) {
let url = input.appendingPathComponent(file)
guard url.exists, url.isDirectory else {
requiredFiles.insert(file)
return
}
do {
try FileManager.default
.contentsOfDirectory(atPath: url.path)
.forEach {
// Recurse into subfolders
require(file: file + "/" + $0)
}
} catch {
log.add(error: "Failed to read folder \(file): \(error)", source: source)
}
}
/**
Mark a file as explicitly missing.
This is done for the `externalFiles` entries in metadata,
to indicate that these files will be copied to the output folder manually.
*/
func exclude(file: String) {
externalFiles.insert(file)
}
/**
Mark a file as expected to be present in the output folder after generation.
This is done for all links between pages, which only exist after the pages have been generated.
*/
func expect(file: String, source: String) {
expectedFiles[file] = source
}
func copyRequiredFiles() {
var copiedFiles = Set<String>()
for file in requiredFiles {
let cleanPath = cleanRelativeURL(file)
let sourceUrl = input.appendingPathComponent(cleanPath)
let destinationUrl = output.appendingPathComponent(cleanPath)
guard sourceUrl.exists else {
if !isExternal(file: file) {
log.add(error: "Missing required file", source: cleanPath)
}
continue
}
if copyFileIfChanged(from: sourceUrl, to: destinationUrl) {
copiedFiles.insert(file)
}
}
try? tempFile.delete()
for (file, source) in expectedFiles {
guard !isExternal(file: file) else {
continue
}
let cleanPath = cleanRelativeURL(file)
let destinationUrl = output.appendingPathComponent(cleanPath)
if !destinationUrl.exists {
log.add(error: "Missing \(cleanPath)", source: source)
}
}
guard !copiedFiles.isEmpty else {
print("No required files copied")
return
}
print("\(copiedFiles.count) required files copied:")
for file in copiedFiles.sorted() {
print(" " + file)
}
}
private func copyFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool {
guard configuration.minifyCSSandJS else {
return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl)
}
switch sourceUrl.pathExtension.lowercased() {
case "js":
return minifyJS(at: sourceUrl, andWriteTo: destinationUrl)
case "css":
return minifyCSS(at: sourceUrl, andWriteTo: destinationUrl)
default:
return copyBinaryFileIfChanged(from: sourceUrl, to: destinationUrl)
}
}
private func copyBinaryFileIfChanged(from sourceUrl: URL, to destinationUrl: URL) -> Bool {
do {
let data = try Data(contentsOf: sourceUrl)
return writeIfChanged(data, to: destinationUrl)
} catch {
log.add(error: "Failed to read data at \(sourceUrl.path)", source: source, error: error)
return false
}
}
private func minifyJS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
let command = "uglifyjs \(sourceUrl.path) > \(tempFile.path)"
do {
_ = try FileSystem.safeShell(command)
return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
} catch {
log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
return false
}
}
private func minifyCSS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
let command = "cleancss \(sourceUrl.path) -o \(tempFile.path)"
do {
_ = try FileSystem.safeShell(command)
return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
} catch {
log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
return false
}
}
private func cleanRelativeURL(_ raw: String) -> String {
let raw = raw.dropAfterLast("#") // Clean links to page content
guard raw.contains("..") else {
return raw
}
var result: [String] = []
for component in raw.components(separatedBy: "/") {
if component == ".." {
_ = result.popLast()
} else {
result.append(component)
}
}
return result.joined(separator: "/")
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if externalFiles.contains(path) {
return true
}
}
return false
}
func printExternalFiles() {
guard !externalFiles.isEmpty else {
return
}
print("\(externalFiles.count) external resources needed:")
for file in externalFiles.sorted() {
print(" " + file)
}
}
// MARK: Pages
func isEmpty(page: String) {
emptyPages.insert(page)
}
func printEmptyPages() {
guard !emptyPages.isEmpty else {
return
}
print("\(emptyPages.count) empty pages:")
for page in emptyPages.sorted() {
print(" " + page)
}
}
func isDraft(path: String) {
draftPages.insert(path)
}
func printDraftPages() {
guard !draftPages.isEmpty else {
return
}
print("\(draftPages.count) drafts:")
for page in draftPages.sorted() {
print(" " + page)
}
}
func add(page: String, id: String) {
if let existing = pagePaths[id] {
log.add(error: "Conflicting id with \(existing)", source: page)
}
pagePaths[id] = page
}
func getPage(for id: String) -> String? {
pagePaths[id]
}
func generated(page: String) {
generatedPages.insert(page)
}
func printGeneratedPages() {
guard !generatedPages.isEmpty else {
print("No pages modified")
return
}
print("\(generatedPages.count) pages modified")
for page in generatedPages.sorted() {
print(" " + page)
}
}
// MARK: Writing files
@discardableResult
func writeIfChanged(_ data: Data, to url: URL) -> Bool {
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
return true
} catch {
log.add(error: "Failed to write file", source: url.path, error: error)
return false
}
}
@discardableResult
func write(_ string: String, to url: URL) -> Bool {
let data = string.data(using: .utf8)!
return writeIfChanged(data, to: url)
}
// MARK: Running other tasks
@discardableResult
static func safeShell(_ command: String) throws -> String {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/zsh")
task.standardInput = nil
try task.run()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return output
}
}

56
Sources/Generator/Files/FileUpdateChecker.swift
View File

@ -3,14 +3,10 @@ import CryptoKit
final class FileUpdateChecker {
private static let hashesFileName = "hashes.json"
private let hashesFileName = "hashes.json"
private let input: URL
private var hashesFile: URL {
input.appendingPathComponent(FileUpdateChecker.hashesFileName)
}
/**
The hashes of all accessed files from the previous run
@ -25,35 +21,41 @@ final class FileUpdateChecker {
*/
private var accessedFiles: [String : Data] = [:]
private var source: String {
"FileUpdateChecker"
var numberOfFilesLoaded: Int {
previousFiles.count
}
var numberOfFilesAccessed: Int {
accessedFiles.count
}
init(input: URL) {
self.input = input
guard hashesFile.exists else {
log.add(info: "No file hashes loaded, regarding all content as new", source: source)
return
}
enum LoadResult {
case notLoaded
case loaded
case failed(String)
}
func loadPreviousRun(from folder: URL) -> LoadResult {
let url = folder.appendingPathComponent(hashesFileName)
guard url.exists else {
return .notLoaded
}
let data: Data
do {
data = try Data(contentsOf: hashesFile)
data = try Data(contentsOf: url)
} catch {
log.add(
warning: "File hashes could not be read, regarding all content as new",
source: source,
error: error)
return
return .failed("Failed to read hashes from last run: \(error)")
}
do {
self.previousFiles = try JSONDecoder().decode(from: data)
} catch {
log.add(
warning: "File hashes could not be decoded, regarding all content as new",
source: source,
error: error)
return
return .failed("Failed to decode hashes from last run: \(error)")
}
return .loaded
}
func fileHasChanged(at path: String) -> Bool {
@ -73,16 +75,18 @@ final class FileUpdateChecker {
accessedFiles[path] = SHA256.hash(data: data).data
}
func writeDetectedFileChangesToDisk() {
func writeDetectedFileChanges(to folder: URL) -> String? {
let url = folder.appendingPathComponent(hashesFileName)
do {
let encoder = JSONEncoder()
encoder.outputFormatting = .prettyPrinted
let data = try encoder.encode(accessedFiles)
try data.write(to: hashesFile)
try data.write(to: url)
return nil
} catch {
log.add(warning: "Failed to save file hashes", source: source, error: error)
return "Failed to save file hashes: \(error)"
}
}
}
}
var notFound = 0

456
Sources/Generator/Files/ImageGenerator.swift
View File

@ -1,456 +0,0 @@
import Foundation
import AppKit
import CryptoKit
import Darwin.C
private struct ImageJob {
let destination: String
let width: Int
let path: String
let quality: Float
let alwaysGenerate: Bool
}
final class ImageGenerator {
private let imageOptimSupportedFileExtensions: Set<String> = ["jpg", "png", "svg"]
private let imageOptimizationBatchSize = 50
/**
The path to the input folder.
*/
private let input: URL
/**
The path to the output folder
*/
private let output: URL
/**
The images to generate.
The key is the image source path relative to the input folder, and the values are the destination path (relative to the output folder) and the required image width.
*/
private var imageJobs: [String : [ImageJob]] = [:]
/**
The images for which to generate multiple versions
The key is the source file, the value is the path of the requiring page.
*/
private var multiImageJobs: [String : String] = [:]
/**
The images which could not be found, but are required for the site.
The key is the image path, and the value is the page that requires it.
*/
private var missingImages: [String : String] = [:]
/**
All warnings produced for images during generation
*/
private var imageWarnings: Set<String> = []
/**
All images required by the site.
The values are the destination paths of the images, relative to the output folder
*/
private var requiredImages: Set<String> = []
/**
All images modified or created during this generator run.
*/
private var generatedImages: Set<String> = []
/**
The images optimized by ImageOptim
*/
private var optimizedImages: Set<String> = []
/**
A cache to get the size of source images, so that files don't have to be loaded multiple times.
The key is the absolute source path, and the value is the image size
*/
private var imageSizeCache: [String : NSSize] = [:]
private var fileUpdates: FileUpdateChecker
init(input: URL, output: URL) {
self.fileUpdates = FileUpdateChecker(input: input)
self.input = input
self.output = output
}
func writeDetectedFileChangesToDisk() {
fileUpdates.writeDetectedFileChangesToDisk()
}
private func getImageSize(atPath path: String) -> NSSize? {
if let size = imageSizeCache[path] {
return size
}
guard let image = getImage(atPath: path) else {
return nil
}
let size = image.size
imageSizeCache[path] = size
return size
}
private func getImage(atPath path: String) -> NSImage? {
guard let data = getData(atPath: path) else {
log.add(error: "Failed to load file", source: path)
return nil
}
guard let image = NSImage(data: data) else {
log.add(error: "Failed to read image", source: path)
return nil
}
return image
}
private func getData(atPath path: String) -> Data? {
let url = input.appendingPathComponent(path)
guard url.exists else {
return nil
}
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
log.add(error: "Failed to read data", source: path, error: error)
return nil
}
}
func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
requiredImages.insert(destination)
let height = height.unwrapped(CGFloat.init)
let sourceUrl = input.appendingPathComponent(source)
guard sourceUrl.exists else {
missingImages[source] = path
return .zero
}
guard let imageSize = getImageSize(atPath: source) else {
missingImages[source] = path
return .zero
}
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
// Check desired height, then we can forget about it
if let height = height {
let expectedHeight = scaledSize.width / CGFloat(width) * height
if abs(expectedHeight - scaledSize.height) > 2 {
addWarning("Invalid height (\(scaledSize.height) instead of \(expectedHeight))", destination: destination, path: path)
}
}
let job = ImageJob(
destination: destination,
width: width,
path: path,
quality: quality,
alwaysGenerate: alwaysGenerate)
insert(job: job, source: source)
return scaledSize
}
private func insert(job: ImageJob, source: String) {
guard let existingSource = imageJobs[source] else {
imageJobs[source] = [job]
return
}
guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
imageJobs[source] = existingSource + [job]
return
}
if existingJob.width != job.width {
addWarning("Multiple image widths (\(existingJob.width) and \(job.width))", destination: job.destination, path: "\(existingJob.path) and \(job.path)")
}
}
func createImages() {
var count = 0
for (source, jobs) in imageJobs.sorted(by: { $0.key < $1.key }) {
print(String(format: "Creating images: %4d / %d\r", count, imageJobs.count), terminator: "")
fflush(stdout)
create(images: jobs, from: source)
count += 1
}
print(" \r", terminator: "")
createMultiImages()
optimizeImages()
printMissingImages()
printImageWarnings()
printGeneratedImages()
printTotalImageCount()
}
private func printMissingImages() {
guard !missingImages.isEmpty else {
return
}
print("\(missingImages.count) missing images:")
let sort = missingImages.sorted { (a, b) in
a.value < b.value && a.key < b.key
}
for (source, path) in sort {
print(" \(source) (required by \(path))")
}
}
private func printImageWarnings() {
guard !imageWarnings.isEmpty else {
return
}
print("\(imageWarnings.count) image warnings:")
for imageWarning in imageWarnings {
print(imageWarning)
}
}
private func printGeneratedImages() {
guard !generatedImages.isEmpty else {
return
}
print("\(generatedImages.count) images generated:")
for image in generatedImages {
print(" " + image)
}
}
private func printTotalImageCount() {
print("\(requiredImages.count) images")
}
private func addWarning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
imageWarnings.insert(warning)
}
private func addWarning(_ message: String, job: ImageJob) {
addWarning(message, destination: job.destination, path: job.path)
}
private func isMissing(_ job: ImageJob) -> Bool {
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
private func create(images: [ImageJob], from source: String) {
// Only load image if required
let imageHasChanged = fileUpdates.fileHasChanged(at: source)
guard imageHasChanged || images.contains(where: isMissing) else {
return
}
guard let image = getImage(atPath: source) else {
missingImages[source] = images.first?.path
return
}
let jobs = imageHasChanged ? images : images.filter(isMissing)
// Update all images
jobs.forEach { job in
// Prevent memory overflow due to repeated NSImage operations
autoreleasepool {
create(job: job, from: image, source: source)
}
}
}
private func create(job: ImageJob, from image: NSImage, source: String) {
let destinationUrl = output.appendingPathComponent(job.destination)
create(job: job, from: image, source: source, at: destinationUrl)
}
private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
// Ensure that image file is supported
let ext = destinationUrl.pathExtension.lowercased()
guard ImageType(fileExtension: ext) != nil else {
fatalError()
}
let destinationExtension = destinationUrl.pathExtension.lowercased()
guard let type = ImageType(fileExtension: destinationExtension)?.fileType else {
addWarning("Invalid image extension \(destinationExtension)", job: job)
return
}
let desiredWidth = CGFloat(job.width)
let sourceRep = image.representations[0]
let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh)
.scaledDown(to: desiredWidth)
// create NSBitmapRep manually, if using cgImage, the resulting size is wrong
let rep = NSBitmapImageRep(bitmapDataPlanes: nil,
pixelsWide: Int(destinationSize.width),
pixelsHigh: Int(destinationSize.height),
bitsPerSample: 8,
samplesPerPixel: 4,
hasAlpha: true,
isPlanar: false,
colorSpaceName: NSColorSpaceName.deviceRGB,
bytesPerRow: Int(destinationSize.width) * 4,
bitsPerPixel: 32)!
let ctx = NSGraphicsContext(bitmapImageRep: rep)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = ctx
image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height))
ctx?.flushGraphics()
NSGraphicsContext.restoreGraphicsState()
// Get NSData, and save it
guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
addWarning("Failed to get data", job: job)
return
}
do {
try data.createFolderAndWrite(to: destinationUrl)
} catch {
addWarning("Failed to write image (\(error))", job: job)
return
}
generatedImages.insert(job.destination)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
// Add @1x version
_ = requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
// Add @2x version
return requireScaledMultiImage(
source: source,
destination: destination.insert("@2x", beforeLast: "."),
requiredBy: path,
width: width * 2,
desiredHeight: desiredHeight.unwrapped { $0 * 2 })
}
@discardableResult
private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
let rawDestinationPath = destination.dropAfterLast(".")
let avifPath = rawDestinationPath + ".avif"
let webpPath = rawDestinationPath + ".webp"
let needsGeneration = !output.appendingPathComponent(avifPath).exists || !output.appendingPathComponent(webpPath).exists
let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
multiImageJobs[destination] = path
return size
}
private func createMultiImages() {
let sort = multiImageJobs.sorted { $0.value < $1.value && $0.key < $1.key }
var count = 1
for (baseImage, path) in sort {
print(String(format: "Creating image versions: %4d / %d\r", count, sort.count), terminator: "")
fflush(stdout)
createMultiImages(from: baseImage, path: path)
count += 1
}
print(" \r", terminator: "")
}
private func createMultiImages(from source: String, path: String) {
guard generatedImages.contains(source) else {
return
}
let sourceUrl = output.appendingPathComponent(source)
let sourcePath = sourceUrl.path
guard sourceUrl.exists else {
addWarning("No image at path \(sourcePath)", destination: source, path: path)
missingImages[source] = path
return
}
let avifPath = source.dropAfterLast(".") + ".avif"
createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
generatedImages.insert(avifPath)
let webpPath = source.dropAfterLast(".") + ".webp"
createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
generatedImages.insert(webpPath)
compress(at: source)
}
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to create AVIF image", destination: destination, path: destination)
}
}
private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to create WEBP image", destination: destination, path: destination)
}
}
private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do {
_ = try FileSystem.safeShell(command)
} catch {
addWarning("Failed to compress image", destination: destination, path: destination)
}
}
private func optimizeImages() {
let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path }
for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, all.count)
let batch = all[i..<endIndex]
print(String(format: "Optimizing images: %4d / %d\r", endIndex, all.count), terminator: "")
fflush(stdout)
if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch)
}
}
print(" \r", terminator: "")
fflush(stdout)
print("\(optimizedImages.count) images optimized")
}
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ")
do {
_ = try FileSystem.safeShell(command)
return true
} catch {
addWarning("Failed to optimize images", destination: "", path: "")
return false
}
}
}

14
Sources/Generator/Files/ImageJob.swift Normal file
View File

@ -0,0 +1,14 @@
import Foundation
struct ImageJob {
let destination: String
let width: Int
let path: String
let quality: Float
let alwaysGenerate: Bool
}

66
Sources/Generator/Files/ImageReader.swift Normal file
View File

@ -0,0 +1,66 @@
import Foundation
import AppKit
final class ImageReader {
/// The content folder where the input data is stored
let contentFolder: URL
private let fileUpdates: FileUpdateChecker
let runDataFolder: URL
init(in input: URL, runFolder: URL, fileUpdates: FileUpdateChecker) {
self.contentFolder = input
self.runDataFolder = runFolder
self.fileUpdates = fileUpdates
}
var numberOfFilesLoaded: Int {
fileUpdates.numberOfFilesLoaded
}
var numberOfFilesAccessed: Int {
fileUpdates.numberOfFilesAccessed
}
func loadData() -> FileUpdateChecker.LoadResult {
fileUpdates.loadPreviousRun(from: runDataFolder)
}
func writeDetectedFileChangesToDisk() -> String? {
fileUpdates.writeDetectedFileChanges(to: runDataFolder)
}
func imageHasChanged(at path: String) -> Bool {
fileUpdates.fileHasChanged(at: path)
}
func getImage(atPath path: String) -> NSImage? {
guard let data = getData(atPath: path) else {
// TODO: log.error("Failed to load file", source: path)
return nil
}
guard let image = NSImage(data: data) else {
// TODO: log.error("Failed to read image", source: path)
return nil
}
return image
}
private func getData(atPath path: String) -> Data? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
return nil
}
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
// TODO: log.error("Failed to read data: \(error)", source: path)
return nil
}
}
}

13
Sources/Generator/Generators/OverviewPageGenerator.swift
View File

@ -4,15 +4,17 @@ struct OverviewPageGenerator {
private let factory: LocalizedSiteTemplate
init(factory: LocalizedSiteTemplate) {
private let results: GenerationResultsHandler
init(factory: LocalizedSiteTemplate, results: GenerationResultsHandler) {
self.factory = factory
self.results = results
}
func generate(
section: Element,
language: String) {
let path = section.localizedPath(for: language)
let url = files.urlInOutputFolder(path)
let metadata = section.localized(for: language)
@ -26,11 +28,8 @@ struct OverviewPageGenerator {
content[.contentClass] = "overview"
content[.header] = makeHeader(page: section, metadata: metadata, language: language)
content[.content] = makeContent(section: section, language: language)
content[.footer] = section.customFooterContent()
guard factory.page.generate(content, to: url) else {
return
}
files.generated(page: path)
content[.footer] = results.getContentOfOptionalFile(at: section.additionalFooterContentPath, source: section.path)
factory.page.generate(content, to: path, source: section.path)
}
private func makeContent(section: Element, language: String) -> String {

11
Sources/Generator/Generators/OverviewSectionGenerator.swift
View File

@ -8,10 +8,10 @@ struct OverviewSectionGenerator {
private let generator: ThumbnailListGenerator
init(factory: TemplateFactory) {
init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.multipleSectionsTemplate = factory.overviewSection
self.singleSectionsTemplate = factory.overviewSectionClean
self.generator = ThumbnailListGenerator(factory: factory)
self.generator = ThumbnailListGenerator(factory: factory, results: results)
}
func generate(sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String {
@ -25,8 +25,13 @@ struct OverviewSectionGenerator {
}
private func newsSectionContent(for element: Element, language: String, sectionItemCount: Int) -> String {
let shownElements = element.mostRecentElements(sectionItemCount)
// let shownElements = element.mostRecentElements(sectionItemCount)
return ""
// return generator.generateContent(
// items: shownElements,
// parent: element,
// language: language,
// style: element.thumbnailStyle)
}
private func sectionsContent(_ sections: [Element], in parent: Element, language: String, sectionItemCount: Int) -> String {

53
Sources/Generator/Generators/PageContentGenerator.swift
View File

@ -10,9 +10,12 @@ struct PageContentGenerator {
private let siteRoot: Element
init(factory: TemplateFactory, siteRoot: Element) {
private let results: GenerationResultsHandler
init(factory: TemplateFactory, siteRoot: Element, results: GenerationResultsHandler) {
self.factory = factory
self.siteRoot = siteRoot
self.results = results
}
func generate(page: Element, language: String, content: String) -> (content: String, includesCode: Bool) {
@ -44,8 +47,7 @@ struct PageContentGenerator {
let file = markdown.between("(", and: ")")
if file.hasPrefix("page:") {
let pageId = file.replacingOccurrences(of: "page:", with: "")
guard let pagePath = files.getPage(for: pageId) else {
log.add(warning: "Page id '\(pageId)' not found", source: page.path)
guard let pagePath = results.getPagePath(for: pageId, source: page.path) else {
// Remove link since the page can't be found
return markdown.between("[", and: "]")
}
@ -57,13 +59,13 @@ struct PageContentGenerator {
if let filePath = page.nonAbsolutePathRelativeToRootForContainedInputFile(file) {
// The target of the page link must be present after generation is complete
files.expect(file: filePath, source: page.path)
results.expect(file: filePath, source: page.path)
}
return html
}
private func handleHTML(page: Element, language: String, html: String, markdown: Substring) -> String {
#warning("Check HTML code in markdown for required resources")
// TODO: Check HTML code in markdown for required resources
//print("[HTML] Found in page \(page.path):")
//print(markdown)
// Things to check:
@ -123,7 +125,7 @@ struct PageContentGenerator {
private func handleImage(page: Element, file: String, rightTitle: String?, leftTitle: String?) -> String {
let imagePath = page.pathRelativeToRootForContainedInputFile(file)
let size = files.requireFullSizeMultiVersionImage(
let size = results.requireFullSizeMultiVersionImage(
source: imagePath,
destination: imagePath,
requiredBy: page.path)
@ -140,28 +142,28 @@ struct PageContentGenerator {
private func handleVideo(page: Element, file: String, optionString: String?) -> String {
let options: [PageVideoTemplate.VideoOption] = optionString.unwrapped { string in
string.components(separatedBy: " ").compactMap { optionText in
string.components(separatedBy: " ").compactMap { optionText -> PageVideoTemplate.VideoOption? in
guard let optionText = optionText.trimmed.nonEmpty else {
return nil
}
guard let option = PageVideoTemplate.VideoOption(rawValue: optionText) else {
log.add(warning: "Unknown video option \(optionText)", source: page.path)
results.warning("Unknown video option \(optionText)", source: page.path)
return nil
}
return option
}
} ?? []
#warning("Check page folder for alternative video versions")
// TODO: Check page folder for alternative video versions
let sources: [PageVideoTemplate.VideoSource] = [(url: file, type: .mp4)]
let filePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: filePath)
results.require(file: filePath, source: page.path)
return factory.video.generate(sources: sources, options: options)
}
private func handleSvg(page: Element, file: String, area: String?) -> String {
let imagePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: imagePath)
results.require(file: imagePath, source: page.path)
guard let area = area else {
return factory.html.svgImage(file: file)
@ -172,7 +174,7 @@ struct PageContentGenerator {
let y = Int(parts[1].trimmed),
let width = Int(parts[2].trimmed),
let height = Int(parts[3].trimmed) else {
log.add(warning: "Invalid area string for svg image", source: page.path)
results.warning("Invalid area string for svg image", source: page.path)
return factory.html.svgImage(file: file)
}
@ -180,7 +182,7 @@ struct PageContentGenerator {
}
private func handleFile(page: Element, file: String, fileExtension: String) -> String {
log.add(warning: "Unhandled file \(file) with extension \(fileExtension)", source: page.path)
results.warning("Unhandled file \(file) with extension \(fileExtension)", source: page.path)
return ""
}
@ -190,7 +192,7 @@ struct PageContentGenerator {
.compactMap { button -> (file: String, text: String, downloadName: String?)? in
let parts = button.components(separatedBy: ",")
guard parts.count == 2 || parts.count == 3 else {
log.add(warning: "Invalid button definition", source: page.path)
results.warning("Invalid button definition", source: page.path)
return nil
}
let file = parts[0].trimmed
@ -199,7 +201,7 @@ struct PageContentGenerator {
// Ensure that file is available
let filePath = page.pathRelativeToRootForContainedInputFile(file)
files.require(file: filePath)
results.require(file: filePath, source: page.path)
return (file, title, downloadName)
}
@ -212,7 +214,7 @@ struct PageContentGenerator {
.compactMap { button -> (url: String, text: String)? in
let parts = button.components(separatedBy: ",")
guard parts.count == 2 else {
log.add(warning: "Invalid external link definition", source: page.path)
results.warning("Invalid external link definition", source: page.path)
return nil
}
let url = parts[0].trimmed
@ -224,23 +226,14 @@ struct PageContentGenerator {
}
private func handleExternalHTML(page: Element, file: String) -> String {
let url = page.inputFolder.appendingPathComponent(file)
guard url.exists else {
log.add(error: "File \(file) not found", source: page.path)
return ""
}
do {
return try String(contentsOf: url)
} catch {
log.add(error: "File \(file) could not be read", source: page.path, error: error)
return ""
}
let path = page.pathRelativeToRootForContainedInputFile(file)
return results.getContentOfRequiredFile(at: path, source: page.path) ?? ""
}
private func handleSimpleBox(page: Element, content: String) -> String {
let parts = content.components(separatedBy: ";")
guard parts.count > 1 else {
log.add(error: "Invalid box specification", source: page.path)
results.warning("Invalid box specification", source: page.path)
return ""
}
let title = parts[0]
@ -250,7 +243,8 @@ struct PageContentGenerator {
private func handlePageLink(page: Element, language: String, pageId: String) -> String {
guard let linkedPage = siteRoot.find(pageId) else {
log.add(warning: "Page id '\(pageId)' not found", source: page.path)
// Checking the page path will add it to the missing pages
_ = results.getPagePath(for: pageId, source: page.path)
// Remove link since the page can't be found
return ""
}
@ -259,6 +253,7 @@ struct PageContentGenerator {
content[.title] = linkedPage.title(for: language)
let fullThumbnailPath = linkedPage.thumbnailFilePath(for: language).destination
// Note: Here we assume that the thumbnail was already used elsewhere, so already generated
let relativeImageUrl = page.relativePathToOtherSiteElement(file: fullThumbnailPath)
let metadata = linkedPage.localized(for: language)

30
Sources/Generator/Generators/PageGenerator.swift
View File

@ -7,20 +7,24 @@ struct PageGenerator {
private let contentGenerator: PageContentGenerator
init(factory: LocalizedSiteTemplate, siteRoot: Element) {
private let results: GenerationResultsHandler
init(factory: LocalizedSiteTemplate, siteRoot: Element, results: GenerationResultsHandler) {
self.factory = factory
self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot)
self.results = results
self.contentGenerator = PageContentGenerator(factory: factory.factory, siteRoot: siteRoot, results: results)
}
func generate(page: Element, language: String, previousPage: Element?, nextPage: Element?) {
guard !page.isExternalPage else {
results.didCompletePage()
return
}
let path = page.fullPageUrl(for: language)
let inputContentPath = page.path + "/\(language).md"
let metadata = page.localized(for: language)
let nextLanguage = page.nextLanguage(for: language)
let (pageContent, pageIncludesCode, pageIsEmpty) = makeContent(
let (pageContent, pageIncludesCode) = makeContent(
page: page, metadata: metadata, language: language, path: inputContentPath)
var content = [PageTemplate.Key : String]()
@ -35,23 +39,17 @@ struct PageGenerator {
content[.previousPageUrl] = navLink(from: page, to: previousPage, language: language)
content[.nextPageLinkText] = nextText(for: nextPage, language: language)
content[.nextPageUrl] = navLink(from: page, to: nextPage, language: language)
content[.footer] = page.customFooterContent()
content[.footer] = results.getContentOfOptionalFile(at: page.additionalFooterContentPath, source: page.path)
if pageIncludesCode {
let highlightCode = factory.factory.html.codeHighlightFooter()
content[.footer] = (content[.footer].unwrapped { $0 + "\n" } ?? "") + highlightCode
}
let url = files.urlInOutputFolder(path)
if page.state == .draft {
files.isDraft(path: page.path)
} else if pageIsEmpty, page.state != .hidden {
files.isEmpty(page: path)
results.markPageAsDraft(page: page.path)
}
guard factory.page.generate(content, to: url) else {
return
}
files.generated(page: path)
factory.page.generate(content, to: path, source: page.path)
}
private func navLink(from element: Element, to destination: Element?, language: String) -> String? {
@ -75,14 +73,14 @@ struct PageGenerator {
return factory.factory.html.makeNextText(text)
}
private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool, isEmpty: Bool) {
if let raw = files.contentOfMdFile(atPath: path, source: page.path)?.trimmed.nonEmpty {
private func makeContent(page: Element, metadata: Element.LocalizedMetadata, language: String, path: String) -> (content: String, includesCode: Bool) {
if let raw = results.getContentOfMdFile(at: path, source: page.path)?.trimmed.nonEmpty {
let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: raw)
return (content, includesCode, false)
return (content, includesCode)
} else {
let (content, includesCode) = contentGenerator.generate(page: page, language: language, content: metadata.placeholderText)
let placeholder = factory.factory.makePlaceholder(title: metadata.placeholderTitle, text: content)
return (placeholder, includesCode, true)
return (placeholder, includesCode)
}
}

10
Sources/Generator/Generators/PageHeadGenerator.swift
View File

@ -2,12 +2,16 @@ import Foundation
struct PageHeadGenerator {
// TODO: Add to configuration
static let linkPreviewDesiredImageWidth = 1600
let factory: TemplateFactory
init(factory: TemplateFactory) {
private let results: GenerationResultsHandler
init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.factory = factory
self.results = results
}
func generate(page: Element, language: String, includesCode: Bool = false) -> String {
@ -24,14 +28,14 @@ struct PageHeadGenerator {
let linkPreviewImageName = "thumbnail-link.\(image.lastComponentAfter("."))"
let sourceImagePath = page.pathRelativeToRootForContainedInputFile(image)
let destinationImagePath = page.pathRelativeToRootForContainedInputFile(linkPreviewImageName)
files.requireSingleImage(
results.requireSingleImage(
source: sourceImagePath,
destination: destinationImagePath,
requiredBy: page.path,
width: PageHeadGenerator.linkPreviewDesiredImageWidth)
content[.image] = factory.html.linkPreviewImage(file: linkPreviewImageName)
}
content[.customPageContent] = page.customHeadContent()
content[.customPageContent] = results.getContentOfOptionalFile(at: page.additionalHeadContentPath, source: page.path)
if includesCode {
let scriptPath = "assets/js/highlight.js"
let relative = page.relativePathToOtherSiteElement(file: scriptPath)

23
Sources/Generator/Generators/SiteGenerator.swift
View File

@ -6,9 +6,12 @@ struct SiteGenerator {
let templates: TemplateFactory
init() throws {
let templatesFolder = files.urlInContentFolder("templates")
self.templates = try TemplateFactory(templateFolder: templatesFolder)
let results: GenerationResultsHandler
init(results: GenerationResultsHandler) throws {
self.results = results
let templatesFolder = results.contentFolder.appendingPathComponent("templates")
self.templates = try TemplateFactory(templateFolder: templatesFolder, results: results)
}
func generate(site: Element) {
@ -22,11 +25,12 @@ struct SiteGenerator {
let template = LocalizedSiteTemplate(
factory: templates,
language: language,
site: site)
site: site,
results: results)
// Generate sections
let overviewGenerator = OverviewPageGenerator(factory: template)
let pageGenerator = PageGenerator(factory: template, siteRoot: site)
let overviewGenerator = OverviewPageGenerator(factory: template, results: results)
let pageGenerator = PageGenerator(factory: template, siteRoot: site, results: results)
var elementsToProcess: [LinkedElement] = [(nil, site, nil)]
while let (previous, element, next) = elementsToProcess.popLast() {
@ -34,7 +38,6 @@ struct SiteGenerator {
elementsToProcess.append(contentsOf: element.linkedElements)
processAllFiles(for: element)
if !element.elements.isEmpty {
overviewGenerator.generate(section: element, language: language)
} else {
@ -48,10 +51,10 @@ struct SiteGenerator {
}
private func processAllFiles(for element: Element) {
element.requiredFiles.forEach(files.require)
element.externalFiles.forEach(files.exclude)
element.externalFiles.forEach { results.exclude(file: $0, source: element.path) }
element.requiredFiles.forEach { results.require(file: $0, source: element.path) }
element.images.forEach {
files.requireSingleImage(
results.requireSingleImage(
source: $0.sourcePath,
destination: $0.destinationPath,
requiredBy: element.path,

7
Sources/Generator/Generators/ThumbnailListGenerator.swift
View File

@ -4,8 +4,11 @@ struct ThumbnailListGenerator {
private let factory: TemplateFactory
init(factory: TemplateFactory) {
private let results: GenerationResultsHandler
init(factory: TemplateFactory, results: GenerationResultsHandler) {
self.factory = factory
self.results = results
}
func generateContent(items: [Element], parent: Element, language: String, style: ThumbnailStyle) -> String {
@ -37,7 +40,7 @@ struct ThumbnailListGenerator {
factory.largeThumbnail.makeCorner(text: $0)
}
files.requireMultiVersionImage(
results.requireMultiVersionImage(
source: thumbnailSourcePath,
destination: thumbnailDestNoExtension + ".jpg",
requiredBy: item.path,

20
Sources/Generator/Processing/FileData.swift Normal file
View File

@ -0,0 +1,20 @@
import Foundation
struct FileData {
///The files marked as expected, i.e. they exist after the generation is completed. (`key`: file path, `value`: the file providing the link)
var expected: [String : String] = [:]
/// All files which should be copied to the output folder (`key`: The file path, `value`: The source requiring the file)
var toCopy: [String : String] = [:]
/// The files to minify when copying into output directory. (`key`: the file path relative to the content folder)
var toMinify: [String : (source: String, type: MinificationType)] = [:]
/**
The files marked as external in element metadata. (Key: File path, Value: source element)
Files included here are not generated, since they are assumed to be added separately.
*/
var external: [String : String] = [:]
}

259
Sources/Generator/Processing/FileGenerator.swift Normal file
View File

@ -0,0 +1,259 @@
import Foundation
final class FileGenerator {
let input: URL
let outputFolder: URL
let runFolder: URL
private let files: FileData
/// All files copied to the destination.
private var copiedFiles: Set<String> = []
/// Files which could not be read (`key`: file path relative to content)
private var unreadableFiles: [String : (source: String, message: String)] = [:]
/// Files which could not be written (`key`: file path relative to output folder)
private var unwritableFiles: [String : (source: String, message: String)] = [:]
private var failedMinifications: [(file: String, source: String, message: String)] = []
/// Non-existent files. `key`: file path, `value`: source element
private var missingFiles: [String : String] = [:]
private var minifiedFiles: [String] = []
private let numberOfFilesToCopy: Int
private var numberOfCopiedFiles = 0
private let numberOfFilesToMinify: Int
private var numberOfMinifiedFiles = 0
private var numberOfExistingFiles = 0
private var tempFile: URL {
runFolder.appendingPathComponent("temp")
}
init(input: URL, output: URL, runFolder: URL, files: FileData) {
self.input = input
self.outputFolder = output
self.runFolder = runFolder
self.files = files
numberOfFilesToCopy = files.toCopy.count
numberOfFilesToMinify = files.toMinify.count
}
func generate() {
copy(files: files.toCopy)
print(" Copied files: \(copiedFiles.count)/\(numberOfFilesToCopy) ")
minify(files: files.toMinify)
print(" Minified files: \(minifiedFiles.count)/\(numberOfFilesToMinify) ")
checkExpected(files: files.expected)
print(" Expected files: \(numberOfExistingFiles)/\(files.expected.count)")
print(" External files: \(files.external.count)")
print("")
}
func writeResultsToFile(file: URL) throws {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("External files", files.external) { "\($0.key) (from \($0.value))" }
let data = lines.joined(separator: "\n").data(using: .utf8)!
try data.createFolderAndWrite(to: file)
}
private func didCopyFile() {
numberOfCopiedFiles += 1
print(" Copied files: \(numberOfCopiedFiles)/\(numberOfFilesToCopy) \r", terminator: "")
fflush(stdout)
}
private func didMinifyFile() {
numberOfMinifiedFiles += 1
print(" Minified files: \(numberOfMinifiedFiles)/\(numberOfFilesToMinify) \r", terminator: "")
fflush(stdout)
}
// MARK: File copies
private func copy(files: [String : String]) {
for (file, source) in files {
copyFileIfChanged(file: file, source: source)
}
}
private func copyFileIfChanged(file: String, source: String) {
let cleanPath = cleanRelativeURL(file)
let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
defer { didCopyFile() }
guard copyIfChanged(cleanPath, to: destinationUrl, source: source) else {
return
}
copiedFiles.insert(cleanPath)
}
private func copyIfChanged(_ file: String, to destination: URL, source: String) -> Bool {
let url = input.appendingPathComponent(file)
do {
let data = try Data(contentsOf: url)
return writeIfChanged(data, file: file, source: source)
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
return false
}
}
@discardableResult
func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
let url = outputFolder.appendingPathComponent(file)
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
return true
} catch {
markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
return false
}
}
// MARK: File minification
private func minify(files: [String : (source: String, type: MinificationType)]) {
for (file, other) in files {
minify(file: file, source: other.source, type: other.type)
}
}
private func minify(file: String, source: String, type: MinificationType) {
let url = input.appendingPathComponent(file)
let command: String
switch type {
case .js:
command = "uglifyjs \(url.path) > \(tempFile.path)"
case .css:
command = "cleancss \(url.path) -o \(tempFile.path)"
}
try? tempFile.delete()
defer { didMinifyFile() }
let output: String
do {
output = try safeShell(command)
} catch {
failedMinifications.append((file, source, "Failed to minify with error: \(error)"))
return
}
guard tempFile.exists else {
failedMinifications.append((file, source, output))
return
}
let data: Data
do {
data = try Data(contentsOf: tempFile)
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "\(error)")
return
}
if writeIfChanged(data, file: file, source: source) {
minifiedFiles.append(file)
}
}
private func cleanRelativeURL(_ raw: String) -> String {
let raw = raw.dropAfterLast("#") // Clean links to page content
guard raw.contains("..") else {
return raw
}
var result: [String] = []
for component in raw.components(separatedBy: "/") {
if component == ".." {
_ = result.popLast()
} else {
result.append(component)
}
}
return result.joined(separator: "/")
}
private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
unreadableFiles[path] = (source, message)
}
private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
unwritableFiles[path] = (source, message)
}
// MARK: File expectationts
private func checkExpected(files: [String: String]) {
for (file, source) in files {
guard !isExternal(file: file) else {
numberOfExistingFiles += 1
continue
}
let cleanPath = cleanRelativeURL(file)
let destinationUrl = outputFolder.appendingPathComponent(cleanPath)
guard destinationUrl.exists else {
markFileAsMissing(at: cleanPath, requiredBy: source)
continue
}
numberOfExistingFiles += 1
}
}
private func markFileAsMissing(at path: String, requiredBy source: String) {
missingFiles[path] = source
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
private func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if files.external[path] != nil {
return true
}
}
return false
}
}

467
Sources/Generator/Processing/GenerationResultsHandler.swift Normal file
View File

@ -0,0 +1,467 @@
import Foundation
import CryptoKit
import AppKit
enum MinificationType {
case js
case css
}
final class GenerationResultsHandler {
/// The content folder where the input data is stored
let contentFolder: URL
/// The folder where the site is generated
let outputFolder: URL
private let fileUpdates: FileUpdateChecker
/**
All paths to page element folders, indexed by their unique id.
This relation is used to generate relative links to pages using the ``Element.id`
*/
private let pagePaths: [String: String]
private let configuration: Configuration
private var numberOfProcessedPages = 0
private let numberOfTotalPages: Int
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) {
self.contentFolder = input
self.fileUpdates = fileUpdates
self.outputFolder = output
self.pagePaths = pagePaths
self.configuration = configuration
self.numberOfTotalPages = pageCount
}
// MARK: Internal storage
/// Non-existent files. `key`: file path, `value`: source element
private var missingFiles: [String : String] = [:]
private(set) var files = FileData()
/// Files which could not be read (`key`: file path relative to content)
private var unreadableFiles: [String : (source: String, message: String)] = [:]
/// Files which could not be written (`key`: file path relative to output folder)
private var unwritableFiles: [String : (source: String, message: String)] = [:]
/// The paths to all files which were changed (relative to output)
private var generatedFiles: Set<String> = []
/// The referenced pages which do not exist (`key`: page id, `value`: source element path)
private var missingLinkedPages: [String : String] = [:]
/// All pages without content which have been created (`key`: page path, `value`: source element path)
private var emptyPages: [String : String] = [:]
/// All pages which have `status` set to ``PageState.draft``
private var draftPages: Set<String> = []
/// Generic warnings for pages
private var pageWarnings: [(message: String, source: String)] = []
/// A cache to get the size of source images, so that files don't have to be loaded multiple times (`key` absolute source path, `value`: image size)
private var imageSizeCache: [String : NSSize] = [:]
private(set) var images = ImageData()
// MARK: Generic warnings
private func warning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
images.warnings.append(warning)
}
func warning(_ message: String, source: String) {
pageWarnings.append((message, source))
}
// MARK: Page data
func getPagePath(for id: String, source: String) -> String? {
guard let pagePath = pagePaths[id] else {
missingLinkedPages[id] = source
return nil
}
return pagePath
}
private func markPageAsEmpty(page: String, source: String) {
emptyPages[page] = source
}
func markPageAsDraft(page: String) {
draftPages.insert(page)
}
// MARK: File actions
/**
Add a file as required, so that it will be copied to the output directory.
Special files may be minified.
- Parameter file: The file path, relative to the content directory
- Parameter source: The path of the source element requiring the file.
*/
func require(file: String, source: String) {
guard !isExternal(file: file) else {
return
}
let url = contentFolder.appendingPathComponent(file)
guard url.exists else {
markFileAsMissing(at: file, requiredBy: source)
return
}
guard url.isDirectory else {
markForCopyOrMinification(file: file, source: source)
return
}
do {
try FileManager.default
.contentsOfDirectory(atPath: url.path)
.forEach {
// Recurse into subfolders
require(file: file + "/" + $0, source: source)
}
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "Failed to read folder: \(error)")
}
}
private func markFileAsMissing(at path: String, requiredBy source: String) {
missingFiles[path] = source
}
private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
unreadableFiles[path] = (source, message)
}
private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
unwritableFiles[path] = (source, message)
}
private func markFileAsGenerated(file: String) {
generatedFiles.insert(file)
}
private func markForCopyOrMinification(file: String, source: String) {
let ext = file.lastComponentAfter(".")
if configuration.minifyCSSandJS, ext == "js" {
files.toMinify[file] = (source, .js)
return
}
if configuration.minifyCSSandJS, ext == "css" {
files.toMinify[file] = (source, .css)
return
}
files.toCopy[file] = source
}
/**
Mark a file as explicitly missing (external).
This is done for the `externalFiles` entries in metadata,
to indicate that these files will be copied to the output folder manually.
*/
func exclude(file: String, source: String) {
files.external[file] = source
}
/**
Mark a file as expected to be present in the output folder after generation.
This is done for all links between pages, which only exist after the pages have been generated.
*/
func expect(file: String, source: String) {
files.expected[file] = source
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
private func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if files.external[path] != nil {
return true
}
}
return false
}
func getContentOfRequiredFile(at path: String, source: String) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markFileAsMissing(at: path, requiredBy: source)
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
/**
Get the content of a file which may not exist.
*/
func getContentOfOptionalFile(at path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
if createEmptyFileIfMissing {
writeIfChanged(.init(), file: path, source: source)
}
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
func getContentOfMdFile(at path: String, source: String) -> String? {
guard let result = getContentOfOptionalFile(at: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) else {
markPageAsEmpty(page: path, source: source)
return nil
}
return result
}
private func getContentOfFile(at url: URL, path: String, source: String) -> String? {
do {
return try String(contentsOf: url)
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
@discardableResult
func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
let url = outputFolder.appendingPathComponent(file)
defer {
didCompletePage()
}
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
markFileAsGenerated(file: file)
return true
} catch {
markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
return false
}
}
// MARK: Images
/**
Request the creation of an image.
- Returns: The final size of the image.
*/
@discardableResult
func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
requireImage(
at: destination,
generatedFrom: source,
requiredBy: path,
quality: 0.7,
width: width,
height: desiredHeight,
alwaysGenerate: false)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
// Add @2x version
_ = requireScaledMultiImage(
source: source,
destination: destination.insert("@2x", beforeLast: "."),
requiredBy: path,
width: width * 2,
desiredHeight: desiredHeight.unwrapped { $0 * 2 })
// Add @1x version
return requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
}
func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
}
@discardableResult
private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
let rawDestinationPath = destination.dropAfterLast(".")
let avifPath = rawDestinationPath + ".avif"
let webpPath = rawDestinationPath + ".webp"
let needsGeneration = !outputFolder.appendingPathComponent(avifPath).exists || !outputFolder.appendingPathComponent(webpPath).exists
let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
images.multiJobs[destination] = path
return size
}
private func markImageAsMissing(path: String, source: String) {
images.missing[source] = path
}
private func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
let height = height.unwrapped(CGFloat.init)
guard let imageSize = getImageSize(atPath: source, source: path) else {
// Image marked as missing here
return .zero
}
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
// Check desired height, then we can forget about it
if let height = height {
let expectedHeight = scaledSize.width / CGFloat(width) * height
if abs(expectedHeight - scaledSize.height) > 2 {
warning("Expected a height of \(expectedHeight) (is \(scaledSize.height))", destination: destination, path: path)
}
}
let job = ImageJob(
destination: destination,
width: width,
path: path,
quality: quality,
alwaysGenerate: alwaysGenerate)
insert(job: job, source: source)
return scaledSize
}
private func getImageSize(atPath path: String, source: String) -> NSSize? {
if let size = imageSizeCache[path] {
return size
}
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markImageAsMissing(path: path, source: source)
return nil
}
guard let data = getImageData(at: url, path: path, source: source) else {
return nil
}
guard let image = NSImage(data: data) else {
images.unreadable[path] = source
return nil
}
let size = image.size
imageSizeCache[path] = size
return size
}
private func getImageData(at url: URL, path: String, source: String) -> Data? {
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
private func insert(job: ImageJob, source: String) {
guard let existingSource = images.jobs[source] else {
images.jobs[source] = [job]
return
}
guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
images.jobs[source] = existingSource + [job]
return
}
guard existingJob.width != job.width else {
return
}
warning("Different width \(existingJob.width) as \(job.path) (width \(job.width))",
destination: job.destination, path: existingJob.path)
}
// MARK: Visual output
func didCompletePage() {
numberOfProcessedPages += 1
print(" Processed pages: \(numberOfProcessedPages)/\(numberOfTotalPages) \r", terminator: "")
fflush(stdout)
}
func printOverview() {
var notes: [String] = []
func addIfNotZero(_ count: Int, _ name: String) {
guard count > 0 else {
return
}
notes.append("\(count) \(name)")
}
func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
addIfNotZero(sequence.count, name)
}
addIfNotZero(missingFiles.count, "missing files")
addIfNotZero(unreadableFiles.count, "unreadable files")
addIfNotZero(unwritableFiles.count, "unwritable files")
addIfNotZero(missingLinkedPages.count, "missing linked pages")
addIfNotZero(pageWarnings.count, "warnings")
print(" Updated pages: \(generatedFiles.count)/\(numberOfProcessedPages) ")
print(" Drafts: \(draftPages.count)")
print(" Empty pages: \(emptyPages.count)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
print("")
}
func writeResultsToFile(file: URL) throws {
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Missing linked pages", missingLinkedPages) { "\($0.key) (linked by \($0.value))" }
add("Warnings", pageWarnings) { "\($0.source): \($0.message)" }
add("Drafts", draftPages) { $0 }
add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" }
add("Generated files", generatedFiles) { $0 }
let data = lines.joined(separator: "\n").data(using: .utf8)!
try data.createFolderAndWrite(to: file)
}
}

19
Sources/Generator/Processing/ImageData.swift Normal file
View File

@ -0,0 +1,19 @@
import Foundation
struct ImageData {
/// The images to generate (`key`: the image source path relative to the input folder)
var jobs: [String : [ImageJob]] = [:]
/// The images for which to generate multiple versions (`key`: the source file, `value`: the path of the requiring page)
var multiJobs: [String : String] = [:]
/// All warnings produced for images during generation
var warnings: [String] = []
/// The images which could not be found, but are required for the site (`key`: image path, `value`: source element path)
var missing: [String : String] = [:]
/// Images which could not be read (`key`: file path relative to content, `value`: source element path)
var unreadable: [String : String] = [:]
}

285
Sources/Generator/Processing/ImageGenerator.swift Normal file
View File

@ -0,0 +1,285 @@
import Foundation
import AppKit
import CryptoKit
import Darwin.C
final class ImageGenerator {
private let imageOptimSupportedFileExtensions: Set<String> = ["jpg", "png", "svg"]
private let imageOptimizationBatchSize = 50
/**
The path to the input folder.
*/
private let input: URL
/**
The path to the output folder
*/
private let output: URL
private let imageReader: ImageReader
/**
All warnings produced for images during generation
*/
private var imageWarnings: Set<String> = []
/**
All images modified or created during this generator run.
*/
private var generatedImages: Set<String> = []
/**
The images optimized by ImageOptim
*/
private var optimizedImages: Set<String> = []
private var numberOfGeneratedImages = 0
private let numberOfTotalImages: Int
private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
private var numberOfImagesToOptimize = 0
private var numberOfOptimizedImages = 0
private let images: ImageData
private lazy var jobs: [(source: String, images: [ImageJob])] = images.jobs
.sorted { $0.key < $1.key }
.map { (source: $0.key, images: $0.value) }
.filter {
// Only load image if required
let imageHasChanged = imageReader.imageHasChanged(at: $0.source)
return imageHasChanged || $0.images.contains { job in
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
}
private lazy var multiJobs: [String : String] = {
let imagesToGenerate: Set<String> = jobs.reduce([]) { $0.union($1.images.map { $0.destination }) }
return images.multiJobs.filter { imagesToGenerate.contains($0.key) }
}()
init(input: URL, output: URL, reader: ImageReader, images: ImageData) {
self.input = input
self.output = output
self.imageReader = reader
self.images = images
self.numberOfTotalImages = images.jobs.reduce(0) { $0 + $1.value.count }
+ images.multiJobs.count * 2
}
func generateImages() {
var notes: [String] = []
func addIfNotZero(_ count: Int, _ name: String) {
guard count > 0 else {
return
}
notes.append("\(count) \(name)")
}
addIfNotZero(images.missing.count, "missing images")
addIfNotZero(images.unreadable.count, "unreadable images")
print(" Changed sources: \(jobs.count)/\(images.jobs.count)")
print(" Total images: \(numberOfTotalImages) (\(numberOfTotalImages - imageReader.numberOfFilesAccessed) versions)")
print(" Warnings: \(images.warnings.count)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
for (source, jobs) in jobs {
create(images: jobs, from: source)
}
for (baseImage, source) in multiJobs {
createMultiImages(from: baseImage, path: source)
}
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)")
optimizeImages()
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)")
}
private func create(images: [ImageJob], from source: String) {
guard let image = imageReader.getImage(atPath: source) else {
// TODO: Add to failed images
didGenerateImage(count: images.count)
return
}
let jobs = imageReader.imageHasChanged(at: source) ? images : images.filter(isMissing)
// Update all images
jobs.forEach { job in
// Prevent memory overflow due to repeated NSImage operations
autoreleasepool {
create(job: job, from: image, source: source)
didGenerateImage()
}
}
}
private func isMissing(_ job: ImageJob) -> Bool {
job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
}
private func create(job: ImageJob, from image: NSImage, source: String) {
let destinationUrl = output.appendingPathComponent(job.destination)
create(job: job, from: image, source: source, at: destinationUrl)
}
private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
// Ensure that image file is supported
let ext = destinationUrl.pathExtension.lowercased()
guard ImageType(fileExtension: ext) != nil else {
fatalError()
}
let destinationExtension = destinationUrl.pathExtension.lowercased()
guard let type = ImageType(fileExtension: destinationExtension)?.fileType else {
addWarning("Invalid image extension \(destinationExtension)", job: job)
return
}
let desiredWidth = CGFloat(job.width)
let sourceRep = image.representations[0]
let destinationSize = NSSize(width: sourceRep.pixelsWide, height: sourceRep.pixelsHigh)
.scaledDown(to: desiredWidth)
// create NSBitmapRep manually, if using cgImage, the resulting size is wrong
let rep = NSBitmapImageRep(bitmapDataPlanes: nil,
pixelsWide: Int(destinationSize.width),
pixelsHigh: Int(destinationSize.height),
bitsPerSample: 8,
samplesPerPixel: 4,
hasAlpha: true,
isPlanar: false,
colorSpaceName: NSColorSpaceName.deviceRGB,
bytesPerRow: Int(destinationSize.width) * 4,
bitsPerPixel: 32)!
let ctx = NSGraphicsContext(bitmapImageRep: rep)
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = ctx
image.draw(in: NSMakeRect(0, 0, destinationSize.width, destinationSize.height))
ctx?.flushGraphics()
NSGraphicsContext.restoreGraphicsState()
// Get NSData, and save it
guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
addWarning("Failed to get data", job: job)
return
}
do {
try data.createFolderAndWrite(to: destinationUrl)
} catch {
addWarning("Failed to write image (\(error))", job: job)
return
}
generatedImages.insert(job.destination)
}
private func addWarning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
imageWarnings.insert(warning)
}
private func addWarning(_ message: String, job: ImageJob) {
addWarning(message, destination: job.destination, path: job.path)
}
private func createMultiImages(from source: String, path: String) {
guard generatedImages.contains(source) else {
didGenerateImage(count: 2)
return
}
let sourceUrl = output.appendingPathComponent(source)
let sourcePath = sourceUrl.path
guard sourceUrl.exists else {
addWarning("No image at path \(sourcePath)", destination: source, path: path)
didGenerateImage(count: 2)
return
}
let avifPath = source.dropAfterLast(".") + ".avif"
createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
generatedImages.insert(avifPath)
didGenerateImage()
let webpPath = source.dropAfterLast(".") + ".webp"
createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
generatedImages.insert(webpPath)
didGenerateImage()
compress(at: source)
}
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do {
_ = try safeShell(command)
} catch {
addWarning("Failed to create AVIF image", destination: destination, path: destination)
}
}
private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)"
do {
_ = try safeShell(command)
} catch {
addWarning("Failed to create WEBP image", destination: destination, path: destination)
}
}
private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do {
_ = try safeShell(command)
} catch {
addWarning("Failed to compress image", destination: destination, path: destination)
}
}
private func optimizeImages() {
let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path }
for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, all.count)
let batch = all[i..<endIndex]
if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch)
}
didOptimizeImage(count: batch.count)
}
}
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ")
do {
_ = try safeShell(command)
return true
} catch {
addWarning("Failed to optimize images", destination: "", path: "")
return false
}
}
// MARK: Output
private func didGenerateImage(count: Int = 1) {
numberOfGeneratedImages += count
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate) \r", terminator: "")
fflush(stdout)
}
private func didOptimizeImage(count: Int) {
numberOfOptimizedImages += count
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize) \r", terminator: "")
fflush(stdout)
}
}

45
Sources/Generator/Processing/MetadataInfoLogger.swift
View File

@ -4,6 +4,8 @@ final class MetadataInfoLogger {
private let input: URL
private let runFolder: URL
private var numberOfMetadataFiles = 0
private var unusedProperties: [(name: String, source: String)] = []
@ -20,8 +22,13 @@ final class MetadataInfoLogger {
private var errors: [(source: String, message: String)] = []
init(input: URL) {
private var logFile: URL {
runFolder.appendingPathComponent("Metadata issues.txt")
}
init(input: URL, runFolder: URL) {
self.input = input
self.runFolder = runFolder
}
/**
@ -124,10 +131,10 @@ final class MetadataInfoLogger {
// MARK: Printing
private func printMetadataScanUpdate() {
print(String(format: "Scanning source files: %4d pages found \r", numberOfMetadataFiles), terminator: "")
print(String(format: " Pages found: %4d \r", numberOfMetadataFiles), terminator: "")
}
func printMetadataScanOverview() {
func printMetadataScanOverview(languages: Int) {
var notes = [String]()
func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
guard sequence.count > 0 else {
@ -139,40 +146,42 @@ final class MetadataInfoLogger {
addIfNotZero(errors, "errors")
addIfNotZero(unreadableMetadata, "unreadable files")
addIfNotZero(unusedProperties, "unused properties")
addIfNotZero(invalidProperties, "invalidProperties")
addIfNotZero(unknownProperties, "unknownProperties")
addIfNotZero(missingProperties, "missingProperties")
addIfNotZero(invalidProperties, "invalid properties")
addIfNotZero(unknownProperties, "unknown properties")
addIfNotZero(missingProperties, "missing properties")
print(" Number of pages: \(numberOfMetadataFiles)")
print(" Pages found: \(numberOfMetadataFiles) ")
print(" Languages: \(languages)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
}
func writeResultsToFile(in folder: URL) throws {
let url = folder.appendingPathComponent("Metadata issues.txt")
func writeResultsToFile() throws {
var lines: [String] = []
if !errors.isEmpty {
lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }
lines += ["Errors:"] + errors.map { "\($0.source): \($0.message)" }.sorted()
}
if !warnings.isEmpty {
lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }
lines += ["Warnings:"] + warnings.map { "\($0.source): \($0.message)" }.sorted()
}
if !unreadableMetadata.isEmpty {
lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }
lines += ["Unreadable files:"] + unreadableMetadata.map { "\($0.source): \($0.error)" }.sorted()
}
if !unusedProperties.isEmpty {
lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }
lines += ["Unused properties:"] + unusedProperties.map { "\($0.source): \($0.name)" }.sorted()
}
if !invalidProperties.isEmpty {
lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }
lines += ["Invalid properties:"] + invalidProperties.map { "\($0.source): \($0.name) (\($0.reason))" }.sorted()
}
if !unknownProperties.isEmpty {
lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }
lines += ["Unknown properties:"] + unknownProperties.map { "\($0.source): \($0.name)" }.sorted()
}
if !missingProperties.isEmpty {
lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }
lines += ["Missing properties:"] + missingProperties.map { "\($0.source): \($0.name)" }.sorted()
}
let data = lines.joined(separator: "\n").data(using: .utf8)
try data?.createFolderAndWrite(to: url)
let data = lines.joined(separator: "\n").data(using: .utf8)!
try data.createFolderAndWrite(to: logFile)
}
}

20
Sources/Generator/Processing/Shell.swift Normal file
View File

@ -0,0 +1,20 @@
import Foundation
@discardableResult
func safeShell(_ command: String) throws -> String {
let task = Process()
let pipe = Pipe()
task.standardOutput = pipe
task.standardError = pipe
task.arguments = ["-cl", command]
task.executableURL = URL(fileURLWithPath: "/bin/zsh")
task.standardInput = nil
try task.run()
let data = pipe.fileHandleForReading.readDataToEndOfFile()
let output = String(data: data, encoding: .utf8)!
return output
}

10
Sources/Generator/Templates/Filled/LocalizedSiteTemplate.swift
View File

@ -35,7 +35,7 @@ struct LocalizedSiteTemplate {
factory.page
}
init(factory: TemplateFactory, language: String, site: Element, log: GenerationResultsHandler) {
init(factory: TemplateFactory, language: String, site: Element, results: GenerationResultsHandler) {
self.author = site.author
self.factory = factory
@ -66,12 +66,8 @@ struct LocalizedSiteTemplate {
language: language,
sections: sections,
topBarWebsiteTitle: site.topBarTitle)
self.pageHead = PageHeadGenerator(
factory: factory,
log: log)
self.overviewSection = OverviewSectionGenerator(
factory: factory,
log: log)
self.pageHead = PageHeadGenerator(factory: factory, results: results)
self.overviewSection = OverviewSectionGenerator(factory: factory, results: results)
}
// MARK: Content

8
Sources/Generator/Templates/Template.swift
View File

@ -26,10 +26,10 @@ extension Template {
self.init(raw: raw, results: results)
}
func generate(_ content: [Key : String], to url: URL) -> Bool {
let content = generate(content)
#warning("log.write(content, to: url, file: )")
return true
@discardableResult
func generate(_ content: [Key : String], to file: String, source: String) -> Bool {
let content = generate(content).data(using: .utf8)!
return results.writeIfChanged(content, file: file, source: source)
}
func generate(_ content: [Key : String], shouldIndent: Bool = false) -> String {

134
Sources/Generator/run.swift
View File

@ -1,8 +1,6 @@
import Foundation
import ArgumentParser
let log = ValidationLog()
var files: FileSystem!
@main
struct CHGenerator: ParsableCommand {
@ -15,18 +13,8 @@ struct CHGenerator: ParsableCommand {
}
}
private func loadSiteData(in folder: URL) throws -> Element? {
let log = MetadataInfoLogger(input: folder)
print("--- SOURCE FILES -----------------------------------")
let root = Element(atRoot: folder, log: log)
print(" ")
log.printMetadataScanOverview()
print(" ")
try log.writeResultsToFile(in: files.generatorInfoFolder)
return root
}
private func loadConfiguration(at configPath: String) -> Configuration? {
print(" ")
print("--- CONFIGURATION ----------------------------------")
print(" ")
print(" Configuration file: \(configPath)")
@ -44,30 +32,118 @@ private func loadConfiguration(at configPath: String) -> Configuration? {
return config
}
private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, ids: [String : String])? {
print("--- SOURCE FILES -----------------------------------")
print(" ")
let log = MetadataInfoLogger(input: folder, runFolder: runFolder)
let root = Element(atRoot: folder, log: log)
log.printMetadataScanOverview(languages: root?.languages.count ?? 0)
print(" ")
try log.writeResultsToFile()
guard let root else {
return nil
}
let ids = root.getContainedIds(log: log)
return (root, ids)
}
private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], pageCount: Int, runFolder: URL) throws -> (ImageData, FileData) {
print("--- GENERATION -------------------------------------")
print(" ")
let results = GenerationResultsHandler(
in: configuration.contentDirectory,
to: configuration.outputDirectory,
configuration: configuration,
fileUpdates: fileUpdates,
pagePaths: ids,
pageCount: pageCount)
let siteGenerator = try SiteGenerator(results: results)
siteGenerator.generate(site: root)
results.printOverview()
let url = runFolder.appendingPathComponent("files.txt")
try results.writeResultsToFile(file: url)
return (results.images, results.files)
}
private func generateImages(_ images: ImageData, configuration: Configuration, runFolder: URL, fileUpdates: FileUpdateChecker) {
print("--- IMAGES -----------------------------------------")
print(" ")
let reader = ImageReader(in: configuration.contentDirectory, runFolder: runFolder, fileUpdates: fileUpdates)
let generator = ImageGenerator(
input: configuration.contentDirectory,
output: configuration.outputDirectory,
reader: reader, images: images)
generator.generateImages()
print(" ")
}
private func copyFiles(files: FileData, configuration: Configuration, runFolder: URL) {
print("--- FILES ------------------------------------------")
print(" ")
let generator = FileGenerator(
input: configuration.contentDirectory,
output: configuration.outputDirectory,
runFolder: runFolder,
files: files)
generator.generate()
}
private func finish(start: Date) {
print("----------------------------------------------------")
print(" ")
let duration = Int(-start.timeIntervalSinceNow.rounded())
if duration < 60 {
print(" Duration: \(duration) s")
} else if duration < 3600 {
print(String(format: " Duration: %d:%02d", duration / 60, duration % 60))
} else {
print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60))
}
print("")
}
private func generate(configPath: String) throws {
let start = Date()
// 1. Load configuration
guard let configuration = loadConfiguration(at: configPath) else {
return
}
files = .init(
in: configuration.contentDirectory,
to: configuration.outputDirectory,
configuration: configuration)
let runFolder = configuration.contentDirectory.appendingPathComponent("run")
// 2. Scan site elements
guard let siteRoot = try loadSiteData(in: configuration.contentDirectory) else {
guard let (siteRoot, ids) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else {
return
}
let siteGenerator = try SiteGenerator()
siteGenerator.generate(site: siteRoot)
files.printGeneratedPages()
files.printEmptyPages()
files.printDraftPages()
files.createImages()
files.copyRequiredFiles()
files.printExternalFiles()
files.writeDetectedFileChangesToDisk()
let fileUpdates = FileUpdateChecker(input: configuration.contentDirectory)
switch fileUpdates.loadPreviousRun(from: runFolder) {
case .notLoaded:
print("Regarding all files as new (no hashes loaded)")
case .loaded:
break
case .failed(let error):
print("Regarding all files as new (\(error))")
}
// 3. Generate pages
let pageCount = ids.count * siteRoot.languages.count
let (images, files) = try generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, pageCount: pageCount, runFolder: runFolder)
if let error = fileUpdates.writeDetectedFileChanges(to: runFolder) {
print(error)
}
// 4. Generate images
generateImages(images, configuration: configuration, runFolder: runFolder, fileUpdates: fileUpdates)
// 5. Copy/minify files
copyFiles(files: files, configuration: configuration, runFolder: runFolder)
// 6. Print summary
finish(start: start)
}