CHGenerator/Sources/Generator/Processing/GenerationResultsHandler.swift

484 lines
17 KiB
Swift
Raw Normal View History

2022-12-04 19:15:22 +01:00
import Foundation
import CryptoKit
import AppKit
enum MinificationType {
case js
case css
}
2022-12-05 17:25:07 +01:00
typealias PageMap = [(language: String, pages: [String : String])]
2022-12-04 19:15:22 +01:00
final class GenerationResultsHandler {
/// The content folder where the input data is stored
let contentFolder: URL
/// The folder where the site is generated
let outputFolder: URL
private let fileUpdates: FileUpdateChecker
/**
All paths to page element folders, indexed by their unique id.
This relation is used to generate relative links to pages using the ``Element.id`
*/
2022-12-05 17:25:07 +01:00
private let pageMap: PageMap
2022-12-04 19:15:22 +01:00
private let configuration: Configuration
private var numberOfProcessedPages = 0
private let numberOfTotalPages: Int
2022-12-05 17:25:07 +01:00
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pageMap: PageMap, pageCount: Int) {
2022-12-04 19:15:22 +01:00
self.contentFolder = input
self.fileUpdates = fileUpdates
self.outputFolder = output
2022-12-05 17:25:07 +01:00
self.pageMap = pageMap
2022-12-04 19:15:22 +01:00
self.configuration = configuration
self.numberOfTotalPages = pageCount
}
// MARK: Internal storage
/// Non-existent files. `key`: file path, `value`: source element
private var missingFiles: [String : String] = [:]
private(set) var files = FileData()
/// Files which could not be read (`key`: file path relative to content)
private var unreadableFiles: [String : (source: String, message: String)] = [:]
/// Files which could not be written (`key`: file path relative to output folder)
private var unwritableFiles: [String : (source: String, message: String)] = [:]
/// The paths to all files which were changed (relative to output)
private var generatedFiles: Set<String> = []
/// The referenced pages which do not exist (`key`: page id, `value`: source element path)
private var missingLinkedPages: [String : String] = [:]
/// All pages without content which have been created (`key`: page path, `value`: source element path)
private var emptyPages: [String : String] = [:]
/// All pages which have `status` set to ``PageState.draft``
private var draftPages: Set<String> = []
/// Generic warnings for pages
private var pageWarnings: [(message: String, source: String)] = []
/// A cache to get the size of source images, so that files don't have to be loaded multiple times (`key` absolute source path, `value`: image size)
private var imageSizeCache: [String : NSSize] = [:]
private(set) var images = ImageData()
// MARK: Generic warnings
private func warning(_ message: String, destination: String, path: String) {
let warning = " \(destination): \(message) required by \(path)"
images.warnings.append(warning)
}
func warning(_ message: String, source: String) {
pageWarnings.append((message, source))
}
// MARK: Page data
2022-12-05 17:25:07 +01:00
func getPagePath(for id: String, source: String, language: String) -> String? {
guard let pagePath = pageMap.first(where: { $0.language == language})?.pages[id] else {
2022-12-04 19:15:22 +01:00
missingLinkedPages[id] = source
return nil
}
return pagePath
}
private func markPageAsEmpty(page: String, source: String) {
emptyPages[page] = source
}
func markPageAsDraft(page: String) {
draftPages.insert(page)
}
// MARK: File actions
/**
Add a file as required, so that it will be copied to the output directory.
Special files may be minified.
- Parameter file: The file path, relative to the content directory
- Parameter source: The path of the source element requiring the file.
*/
func require(file: String, source: String) {
guard !isExternal(file: file) else {
return
}
let url = contentFolder.appendingPathComponent(file)
guard url.exists else {
markFileAsMissing(at: file, requiredBy: source)
return
}
guard url.isDirectory else {
markForCopyOrMinification(file: file, source: source)
return
}
do {
try FileManager.default
.contentsOfDirectory(atPath: url.path)
.forEach {
// Recurse into subfolders
require(file: file + "/" + $0, source: source)
}
} catch {
markFileAsUnreadable(at: file, requiredBy: source, message: "Failed to read folder: \(error)")
}
}
private func markFileAsMissing(at path: String, requiredBy source: String) {
missingFiles[path] = source
}
private func markFileAsUnreadable(at path: String, requiredBy source: String, message: String) {
unreadableFiles[path] = (source, message)
}
private func markFileAsUnwritable(at path: String, requiredBy source: String, message: String) {
unwritableFiles[path] = (source, message)
}
private func markFileAsGenerated(file: String) {
generatedFiles.insert(file)
}
private func markForCopyOrMinification(file: String, source: String) {
let ext = file.lastComponentAfter(".")
2022-12-10 22:28:39 +01:00
if configuration.minifyJavaScript, ext == "js" {
2022-12-04 19:15:22 +01:00
files.toMinify[file] = (source, .js)
return
}
2022-12-10 22:28:39 +01:00
if configuration.minifyCSS, ext == "css" {
2022-12-04 19:15:22 +01:00
files.toMinify[file] = (source, .css)
return
}
files.toCopy[file] = source
}
/**
Mark a file as explicitly missing (external).
This is done for the `externalFiles` entries in metadata,
to indicate that these files will be copied to the output folder manually.
*/
func exclude(file: String, source: String) {
files.external[file] = source
}
/**
Mark a file as expected to be present in the output folder after generation.
This is done for all links between pages, which only exist after the pages have been generated.
*/
func expect(file: String, source: String) {
files.expected[file] = source
}
/**
Check if a file is marked as external.
Also checks for sub-paths of the file, e.g if the folder `docs` is marked as external,
then files like `docs/index.html` are also found to be external.
- Note: All paths are either relative to root (no leading slash) or absolute paths of the domain (leading slash)
*/
private func isExternal(file: String) -> Bool {
// Deconstruct file path
var path = ""
for part in file.components(separatedBy: "/") {
guard part != "" else {
continue
}
if path == "" {
path = part
} else {
path += "/" + part
}
if files.external[path] != nil {
return true
}
}
return false
}
func getContentOfRequiredFile(at path: String, source: String) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markFileAsMissing(at: path, requiredBy: source)
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
/**
Get the content of a file which may not exist.
*/
func getContentOfOptionalFile(at path: String, source: String, createEmptyFileIfMissing: Bool = false) -> String? {
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
if createEmptyFileIfMissing {
writeIfChanged(.init(), file: path, source: source)
}
return nil
}
return getContentOfFile(at: url, path: path, source: source)
}
func getContentOfMdFile(at path: String, source: String) -> String? {
guard let result = getContentOfOptionalFile(at: path, source: source, createEmptyFileIfMissing: configuration.createMdFilesIfMissing) else {
markPageAsEmpty(page: path, source: source)
return nil
}
return result
}
private func getContentOfFile(at url: URL, path: String, source: String) -> String? {
do {
return try String(contentsOf: url)
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
@discardableResult
func writeIfChanged(_ data: Data, file: String, source: String) -> Bool {
let url = outputFolder.appendingPathComponent(file)
defer {
didCompletePage()
}
// Only write changed files
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
return false
}
do {
try data.createFolderAndWrite(to: url)
markFileAsGenerated(file: file)
return true
} catch {
markFileAsUnwritable(at: file, requiredBy: source, message: "Failed to write file: \(error)")
return false
}
}
// MARK: Images
/**
Request the creation of an image.
- Returns: The final size of the image.
*/
@discardableResult
func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
requireImage(
at: destination,
generatedFrom: source,
requiredBy: path,
quality: 0.7,
width: width,
height: desiredHeight,
alwaysGenerate: false)
}
/**
Create images of different types.
This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
- Parameter destination: The path to the destination file
*/
@discardableResult
func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
// Add @2x version
_ = requireScaledMultiImage(
source: source,
destination: destination.insert("@2x", beforeLast: "."),
requiredBy: path,
width: width * 2,
desiredHeight: desiredHeight.unwrapped { $0 * 2 })
// Add @1x version
return requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
}
func requireFullSizeMultiVersionImage(source: String, destination: String, requiredBy path: String) -> NSSize {
requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: configuration.pageImageWidth, desiredHeight: nil)
}
2022-12-08 17:16:54 +01:00
func requireOriginalSizeImages(
source: String,
destination: String,
requiredBy path: String) {
_ = requireScaledMultiImage(
source: source,
destination: destination.insert("@full", beforeLast: "."),
requiredBy: path,
width: configuration.fullScreenImageWidth,
desiredHeight: nil)
}
2022-12-04 19:15:22 +01:00
private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
let rawDestinationPath = destination.dropAfterLast(".")
let avifPath = rawDestinationPath + ".avif"
let webpPath = rawDestinationPath + ".webp"
let needsGeneration = !outputFolder.appendingPathComponent(avifPath).exists || !outputFolder.appendingPathComponent(webpPath).exists
let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
images.multiJobs[destination] = path
return size
}
private func markImageAsMissing(path: String, source: String) {
2022-12-14 09:51:46 +01:00
images.missing[path] = source
2022-12-04 19:15:22 +01:00
}
private func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
let height = height.unwrapped(CGFloat.init)
guard let imageSize = getImageSize(atPath: source, source: path) else {
// Image marked as missing here
return .zero
}
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
// Check desired height, then we can forget about it
if let height = height {
let expectedHeight = scaledSize.width / CGFloat(width) * height
if abs(expectedHeight - scaledSize.height) > 2 {
warning("Expected a height of \(expectedHeight) (is \(scaledSize.height))", destination: destination, path: path)
}
}
let job = ImageJob(
destination: destination,
width: width,
path: path,
quality: quality,
alwaysGenerate: alwaysGenerate)
insert(job: job, source: source)
return scaledSize
}
2022-12-19 23:31:06 +01:00
func getImageSize(atPath path: String, source: String) -> NSSize? {
2022-12-04 19:15:22 +01:00
if let size = imageSizeCache[path] {
return size
}
let url = contentFolder.appendingPathComponent(path)
guard url.exists else {
markImageAsMissing(path: path, source: source)
return nil
}
guard let data = getImageData(at: url, path: path, source: source) else {
return nil
}
guard let image = NSImage(data: data) else {
images.unreadable[path] = source
return nil
}
let size = image.size
imageSizeCache[path] = size
return size
}
private func getImageData(at url: URL, path: String, source: String) -> Data? {
do {
let data = try Data(contentsOf: url)
fileUpdates.didLoad(data, at: path)
return data
} catch {
markFileAsUnreadable(at: path, requiredBy: source, message: "\(error)")
return nil
}
}
private func insert(job: ImageJob, source: String) {
guard let existingSource = images.jobs[source] else {
images.jobs[source] = [job]
return
}
guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
images.jobs[source] = existingSource + [job]
return
}
guard existingJob.width != job.width else {
return
}
warning("Different width \(existingJob.width) as \(job.path) (width \(job.width))",
destination: job.destination, path: existingJob.path)
}
// MARK: Visual output
func didCompletePage() {
numberOfProcessedPages += 1
print(" Processed pages: \(numberOfProcessedPages)/\(numberOfTotalPages) \r", terminator: "")
fflush(stdout)
}
func printOverview() {
var notes: [String] = []
func addIfNotZero(_ count: Int, _ name: String) {
guard count > 0 else {
return
}
notes.append("\(count) \(name)")
}
func addIfNotZero<S>(_ sequence: Array<S>, _ name: String) {
addIfNotZero(sequence.count, name)
}
addIfNotZero(missingFiles.count, "missing files")
addIfNotZero(unreadableFiles.count, "unreadable files")
addIfNotZero(unwritableFiles.count, "unwritable files")
addIfNotZero(missingLinkedPages.count, "missing linked pages")
addIfNotZero(pageWarnings.count, "warnings")
print(" Updated pages: \(generatedFiles.count)/\(numberOfProcessedPages) ")
print(" Drafts: \(draftPages.count)")
print(" Empty pages: \(emptyPages.count)")
if !notes.isEmpty {
print(" Notes: " + notes.joined(separator: ", "))
}
print("")
}
2022-12-04 23:10:44 +01:00
func writeResults(to file: URL) {
2022-12-04 19:15:22 +01:00
var lines: [String] = []
func add<S>(_ name: String, _ property: S, convert: (S.Element) -> String) where S: Sequence {
let elements = property.map { " " + convert($0) }.sorted()
guard !elements.isEmpty else {
return
}
lines.append("\(name):")
lines.append(contentsOf: elements)
}
add("Missing files", missingFiles) { "\($0.key) (required by \($0.value))" }
add("Unreadable files", unreadableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Unwritable files", unwritableFiles) { "\($0.key) (required by \($0.value.source)): \($0.value.message)" }
add("Missing linked pages", missingLinkedPages) { "\($0.key) (linked by \($0.value))" }
add("Warnings", pageWarnings) { "\($0.source): \($0.message)" }
2022-12-19 23:31:06 +01:00
add("Generated files", generatedFiles) { $0 }
2022-12-04 19:15:22 +01:00
add("Drafts", draftPages) { $0 }
add("Empty pages", emptyPages) { "\($0.key) (from \($0.value))" }
let data = lines.joined(separator: "\n").data(using: .utf8)!
2022-12-04 23:10:44 +01:00
do {
try data.createFolderAndWrite(to: file)
} catch {
print(" Failed to save log: \(error)")
}
2022-12-04 19:15:22 +01:00
}
}