Update generation
- Move to global objects for files and validation - Only write changed files - Check images for changes before scaling - Simplify code
This commit is contained in:
Christoph Hagen
27
WebsiteGenerator/Files/ContentError.swift
Normal file
27
WebsiteGenerator/Files/ContentError.swift
Normal file
@ -0,0 +1,27 @@
|
||||
import Foundation
|
||||
|
||||
struct ContentError: Error {
|
||||
|
||||
let reason: String
|
||||
|
||||
let source: String
|
||||
|
||||
let error: Error?
|
||||
}
|
||||
|
||||
extension Optional {
|
||||
|
||||
func unwrapped(or error: ContentError) throws -> Wrapped {
|
||||
guard case let .some(value) = self else {
|
||||
throw error
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func unwrapOrFail(_ reason: String, source: String, error: Error? = nil) throws -> Wrapped {
|
||||
guard case let .some(value) = self else {
|
||||
throw ContentError(reason: reason, source: source, error: error)
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
378
WebsiteGenerator/Files/FileSystem.swift
Normal file
378
WebsiteGenerator/Files/FileSystem.swift
Normal file
@ -0,0 +1,378 @@
|
||||
import Foundation
|
||||
import CryptoKit
|
||||
import AppKit
|
||||
|
||||
typealias SourceFile = (data: Data, didChange: Bool)
|
||||
typealias SourceTextFile = (content: String, didChange: Bool)
|
||||
|
||||
final class FileSystem {
|
||||
|
||||
private static let hashesFileName = "hashes.json"
|
||||
|
||||
private let input: URL
|
||||
|
||||
private let output: URL
|
||||
|
||||
private let source = "FileChangeMonitor"
|
||||
|
||||
private var hashesFile: URL {
|
||||
input.appendingPathComponent(FileSystem.hashesFileName)
|
||||
}
|
||||
|
||||
/**
|
||||
The hashes of all accessed files from the previous run
|
||||
|
||||
The key is the relative path to the file from the source
|
||||
*/
|
||||
private var previousFiles: [String : Data] = [:]
|
||||
|
||||
/**
|
||||
The paths of all files which were accessed, with their new hashes
|
||||
|
||||
This list is used to check if a file was modified, and to write all accessed files back to disk
|
||||
*/
|
||||
private var accessedFiles: [String : Data] = [:]
|
||||
|
||||
/**
|
||||
All files which should be copied to the output folder
|
||||
*/
|
||||
private var requiredFiles: Set<String> = []
|
||||
|
||||
/**
|
||||
The image creation tasks.
|
||||
|
||||
The key is the destination path.
|
||||
*/
|
||||
private var imageTasks: [String : ImageOutput] = [:]
|
||||
|
||||
init(in input: URL, to output: URL) {
|
||||
self.input = input
|
||||
self.output = output
|
||||
|
||||
guard exists(hashesFile) else {
|
||||
log.add(info: "No file hashes loaded, regarding all content as new", source: source)
|
||||
return
|
||||
}
|
||||
let data: Data
|
||||
do {
|
||||
data = try Data(contentsOf: hashesFile)
|
||||
} catch {
|
||||
log.add(
|
||||
warning: "File hashes could not be read, regarding all content as new",
|
||||
source: source,
|
||||
error: error)
|
||||
return
|
||||
}
|
||||
do {
|
||||
self.previousFiles = try JSONDecoder().decode(from: data)
|
||||
} catch {
|
||||
log.add(
|
||||
warning: "File hashes could not be decoded, regarding all content as new",
|
||||
source: source,
|
||||
error: error)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func urlInOutputFolder(_ path: String) -> URL {
|
||||
output.appendingPathComponent(path)
|
||||
}
|
||||
|
||||
func urlInContentFolder(_ path: String) -> URL {
|
||||
input.appendingPathComponent(path)
|
||||
}
|
||||
|
||||
/**
|
||||
Get the current hash of file data at a path.
|
||||
|
||||
If the hash has been computed previously during the current run, then this function directly returns it.
|
||||
*/
|
||||
private func hash(_ data: Data, at path: String) -> Data {
|
||||
accessedFiles[path] ?? SHA256.hash(data: data).data
|
||||
}
|
||||
|
||||
private func exists(_ url: URL) -> Bool {
|
||||
FileManager.default.fileExists(atPath: url.path)
|
||||
}
|
||||
|
||||
func dataOfRequiredFile(atPath path: String, source: String) -> Data? {
|
||||
let url = input.appendingPathComponent(path)
|
||||
guard exists(url) else {
|
||||
log.failedToOpen(path, requiredBy: source, error: nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
do {
|
||||
return try Data(contentsOf: url)
|
||||
} catch {
|
||||
log.failedToOpen(path, requiredBy: source, error: error)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func dataOfOptionalFile(atPath path: String, source: String) -> Data? {
|
||||
let url = input.appendingPathComponent(path)
|
||||
guard exists(url) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
do {
|
||||
return try Data(contentsOf: url)
|
||||
} catch {
|
||||
log.failedToOpen(path, requiredBy: source, error: error)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func contentOfOptionalFile(atPath path: String, source: String) -> String? {
|
||||
let url = input.appendingPathComponent(path)
|
||||
guard exists(url) else {
|
||||
return nil
|
||||
}
|
||||
|
||||
do {
|
||||
return try String(contentsOf: url)
|
||||
} catch {
|
||||
log.failedToOpen(path, requiredBy: source, error: error)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private func getData(atPath path: String) -> SourceFile? {
|
||||
let url = input.appendingPathComponent(path)
|
||||
guard exists(url) else {
|
||||
return nil
|
||||
}
|
||||
let data: Data
|
||||
do {
|
||||
data = try Data(contentsOf: url)
|
||||
} catch {
|
||||
log.add(error: "Failed to read data at \(path)", source: source, error: error)
|
||||
return nil
|
||||
}
|
||||
let newHash = hash(data, at: path)
|
||||
defer {
|
||||
accessedFiles[path] = newHash
|
||||
}
|
||||
guard let oldHash = previousFiles[path] else {
|
||||
return (data: data, didChange: true)
|
||||
}
|
||||
return (data: data, didChange: oldHash != newHash)
|
||||
}
|
||||
|
||||
func writeHashes() {
|
||||
do {
|
||||
let encoder = JSONEncoder()
|
||||
encoder.outputFormatting = .prettyPrinted
|
||||
let data = try encoder.encode(accessedFiles)
|
||||
try data.write(to: hashesFile)
|
||||
} catch {
|
||||
log.add(warning: "Failed to save file hashes", source: source, error: error)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: Images
|
||||
|
||||
private func loadImage(atPath path: String) -> (image: NSImage, changed: Bool)? {
|
||||
guard let (data, changed) = getData(atPath: path) else {
|
||||
print("Failed to load image data \(path)")
|
||||
return nil
|
||||
}
|
||||
guard let image = NSImage(data: data) else {
|
||||
print("Failed to read image \(path)")
|
||||
return nil
|
||||
}
|
||||
return (image, changed)
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func requireImage(source: String, destination: String, width: Int, desiredHeight: Int? = nil, createDoubleVersion: Bool = false) -> NSSize {
|
||||
let height = desiredHeight.unwrapped(CGFloat.init)
|
||||
let sourceUrl = input.appendingPathComponent(source)
|
||||
let image = ImageOutput(source: source, width: width, desiredHeight: desiredHeight)
|
||||
|
||||
let standardSize = NSSize(width: CGFloat(width), height: height ?? CGFloat(width) / 16 * 9)
|
||||
guard sourceUrl.exists else {
|
||||
log.add(error: "Missing image \(source) with size (\(width),\(desiredHeight ?? -1)",
|
||||
source: "Image Processor")
|
||||
return standardSize
|
||||
}
|
||||
guard let imageSize = loadImage(atPath: image.source)?.image.size else {
|
||||
log.add(error: "Unreadable image \(source) with size (\(width),\(desiredHeight ?? -1)",
|
||||
source: "Image Processor")
|
||||
return standardSize
|
||||
}
|
||||
let scaledSize = imageSize.scaledDown(to: CGFloat(width))
|
||||
|
||||
guard let existing = imageTasks[destination] else {
|
||||
//print("Image(\(image.width),\(image.desiredHeight ?? -1)) requested for \(destination)")
|
||||
imageTasks[destination] = image
|
||||
return scaledSize
|
||||
}
|
||||
guard existing.source == source else {
|
||||
log.add(error: "Multiple sources (\(existing.source),\(source)) for image \(destination)",
|
||||
source: "Image Processor")
|
||||
return scaledSize
|
||||
}
|
||||
guard existing.hasSimilarRatio(as: image) else {
|
||||
log.add(error: "Multiple ratios (\(existing.ratio!),\(image.ratio!)) for image \(destination)",
|
||||
source: "Image Processor")
|
||||
return scaledSize
|
||||
}
|
||||
if image.width > existing.width {
|
||||
log.add(info: "Increasing size of image \(destination) from \(existing.width) to \(width)",
|
||||
source: "Image Processor")
|
||||
imageTasks[destination] = image
|
||||
}
|
||||
return scaledSize
|
||||
}
|
||||
|
||||
#warning("Implement image functions")
|
||||
func createImages() {
|
||||
for (destination, image) in imageTasks.sorted(by: { $0.key < $1.key }) {
|
||||
createImageIfNeeded(image, for: destination)
|
||||
}
|
||||
}
|
||||
|
||||
private func createImageIfNeeded(_ image: ImageOutput, for destination: String) {
|
||||
guard let (sourceImageData, sourceImageChanged) = getData(atPath: image.source) else {
|
||||
log.add(error: "Failed to open image \(image.source)", source: "Image Processor")
|
||||
return
|
||||
}
|
||||
|
||||
let destinationUrl = output.appendingPathComponent(destination)
|
||||
|
||||
// Check if image needs to be updated
|
||||
guard !destinationUrl.exists || sourceImageChanged else {
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that image file is supported
|
||||
let ext = destinationUrl.pathExtension.lowercased()
|
||||
guard MediaType.isProcessableImage(ext) else {
|
||||
log.add(info: "Copying image \(image.source)", source: "Image Processor")
|
||||
do {
|
||||
let sourceUrl = input.appendingPathComponent(image.source)
|
||||
try destinationUrl.ensureParentFolderExistence()
|
||||
try sourceUrl.copy(to: destinationUrl)
|
||||
} catch {
|
||||
log.add(error: "Failed to copy image \(image.source) to \(destination)", source: "Image Processor")
|
||||
}
|
||||
return
|
||||
}
|
||||
guard let sourceImage = NSImage(data: sourceImageData) else {
|
||||
print("Failed to read image \(image.source)")
|
||||
return
|
||||
}
|
||||
|
||||
let desiredWidth = CGFloat(image.width)
|
||||
let desiredHeight = image.desiredHeight.unwrapped(CGFloat.init)
|
||||
|
||||
let destinationSize = sourceImage.size.scaledDown(to: desiredWidth)
|
||||
let scaledImage = sourceImage.scaledDown(to: destinationSize)
|
||||
let scaledSize = scaledImage.size
|
||||
|
||||
if abs(scaledImage.size.width - desiredWidth) > 2 {
|
||||
print("[WARN] Image \(destination) scaled incorrectly (wanted width \(desiredWidth), is \(scaledSize.width))")
|
||||
}
|
||||
if abs(destinationSize.height - scaledImage.size.height) > 2 {
|
||||
print("[WARN] Image \(destination) scaled incorrectly (wanted height \(destinationSize.height), is \(scaledSize.height))")
|
||||
}
|
||||
if let desiredHeight = desiredHeight {
|
||||
let desiredRatio = desiredHeight / desiredWidth
|
||||
let adjustedDesiredHeight = scaledSize.width * desiredRatio
|
||||
if abs(adjustedDesiredHeight - scaledSize.height) > 5 {
|
||||
log.add(warning: "Image \(image.source): Desired height \(adjustedDesiredHeight) (actually \(desiredHeight)), got \(scaledSize.height) after reduction", source: "Image Processor")
|
||||
return
|
||||
}
|
||||
}
|
||||
if scaledSize.width > desiredWidth {
|
||||
print("[WARN] Image \(image.source) is too large (expected width \(desiredWidth), got \(scaledSize.width)")
|
||||
}
|
||||
|
||||
let destinationExtension = destinationUrl.pathExtension.lowercased()
|
||||
guard let type = MediaType.supportedImage(destinationExtension) else {
|
||||
log.add(error: "No image type for \(destination) with extension \(destinationExtension)",
|
||||
source: "Image Processor")
|
||||
return
|
||||
}
|
||||
guard let tiff = scaledImage.tiffRepresentation, let tiffData = NSBitmapImageRep(data: tiff) else {
|
||||
log.add(error: "Failed to get data for image \(image.source)", source: "Image Processor")
|
||||
return
|
||||
}
|
||||
|
||||
guard let data = tiffData.representation(using: type, properties: [.compressionFactor: NSNumber(0.7)]) else {
|
||||
log.add(error: "Failed to get data for image \(image.source)", source: "Image Processor")
|
||||
return
|
||||
}
|
||||
do {
|
||||
try data.createFolderAndWrite(to: destinationUrl)
|
||||
} catch {
|
||||
log.add(error: "Failed to write image \(destination)", source: "Image Processor", error: error)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: File copying
|
||||
|
||||
/**
|
||||
Add a file as required, so that it will be copied to the output directory.
|
||||
*/
|
||||
func require(file: String) {
|
||||
requiredFiles.insert(file)
|
||||
}
|
||||
|
||||
func copyRequiredFiles() {
|
||||
var missingFiles = [String]()
|
||||
for file in requiredFiles {
|
||||
let sourceUrl = input.appendingPathComponent(file)
|
||||
guard sourceUrl.exists else {
|
||||
missingFiles.append(file)
|
||||
continue
|
||||
}
|
||||
let data: Data
|
||||
do {
|
||||
data = try Data(contentsOf: sourceUrl)
|
||||
} catch {
|
||||
log.add(error: "Failed to read data at \(sourceUrl.path)", source: source, error: error)
|
||||
continue
|
||||
}
|
||||
let destinationUrl = output.appendingPathComponent(file)
|
||||
write(data, to: destinationUrl)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: Writing files
|
||||
|
||||
@discardableResult
|
||||
func write(_ data: Data, to url: URL) -> Bool {
|
||||
// Only write changed files
|
||||
if url.exists, let oldContent = try? Data(contentsOf: url), data == oldContent {
|
||||
return false
|
||||
}
|
||||
do {
|
||||
try data.createFolderAndWrite(to: url)
|
||||
return true
|
||||
} catch {
|
||||
log.add(error: "Failed to write file", source: url.path, error: error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func write(_ string: String, to url: URL) -> Bool {
|
||||
let data = string.data(using: .utf8)!
|
||||
return write(data, to: url)
|
||||
}
|
||||
}
|
||||
|
||||
private extension Digest {
|
||||
|
||||
var bytes: [UInt8] { Array(makeIterator()) }
|
||||
|
||||
var data: Data { Data(bytes) }
|
||||
|
||||
var hexStr: String {
|
||||
bytes.map { String(format: "%02X", $0) }.joined()
|
||||
}
|
||||
}
|
||||
24
WebsiteGenerator/Files/ImageOutput.swift
Normal file
24
WebsiteGenerator/Files/ImageOutput.swift
Normal file
@ -0,0 +1,24 @@
|
||||
import Foundation
|
||||
|
||||
struct ImageOutput: Hashable {
|
||||
|
||||
let source: String
|
||||
|
||||
let width: Int
|
||||
|
||||
let desiredHeight: Int?
|
||||
|
||||
var ratio: Float? {
|
||||
guard let desiredHeight = desiredHeight else {
|
||||
return nil
|
||||
}
|
||||
return Float(desiredHeight) / Float(width)
|
||||
}
|
||||
|
||||
func hasSimilarRatio(as other: ImageOutput) -> Bool {
|
||||
guard let other = other.ratio, let ratio = ratio else {
|
||||
return true
|
||||
}
|
||||
return abs(other - ratio) < 0.1
|
||||
}
|
||||
}
|
||||
36
WebsiteGenerator/Files/MediaType.swift
Normal file
36
WebsiteGenerator/Files/MediaType.swift
Normal file
@ -0,0 +1,36 @@
|
||||
import Foundation
|
||||
import AppKit
|
||||
|
||||
private let supportedImageExtensions: [String : NSBitmapImageRep.FileType] = [
|
||||
"jpg" : .jpeg,
|
||||
"jpeg" : .jpeg,
|
||||
"png" : .png,
|
||||
]
|
||||
|
||||
private let supportedVideoExtensions: Set<String> = [
|
||||
"mp4", "mov"
|
||||
]
|
||||
|
||||
enum MediaType {
|
||||
case image
|
||||
case video
|
||||
case file
|
||||
|
||||
init(fileExtension: String) {
|
||||
if supportedImageExtensions[fileExtension] != nil {
|
||||
self = .image
|
||||
} else if supportedVideoExtensions.contains(fileExtension) {
|
||||
self = .video
|
||||
} else {
|
||||
self = .file
|
||||
}
|
||||
}
|
||||
|
||||
static func isProcessableImage(_ fileExtension: String) -> Bool {
|
||||
supportedImage(fileExtension) != nil
|
||||
}
|
||||
|
||||
static func supportedImage(_ fileExtension: String) -> NSBitmapImageRep.FileType? {
|
||||
supportedImageExtensions[fileExtension]
|
||||
}
|
||||
}
|
||||
153
WebsiteGenerator/Files/ValidationLog.swift
Normal file
153
WebsiteGenerator/Files/ValidationLog.swift
Normal file
@ -0,0 +1,153 @@
|
||||
import Foundation
|
||||
|
||||
final class ValidationLog {
|
||||
|
||||
private enum LogLevel: String {
|
||||
case error = "ERROR"
|
||||
case warning = "WARNING"
|
||||
case info = "INFO"
|
||||
}
|
||||
|
||||
init() {
|
||||
|
||||
}
|
||||
|
||||
private func add(_ type: LogLevel, item: ContentError) {
|
||||
let errorText: String
|
||||
if let err = item.error {
|
||||
errorText = ", Error: \(err.localizedDescription)"
|
||||
} else {
|
||||
errorText = ""
|
||||
}
|
||||
print("[\(type.rawValue)] \(item.reason), Source: \(item.source)\(errorText)")
|
||||
}
|
||||
|
||||
func add(error: ContentError) {
|
||||
add(.error, item: error)
|
||||
}
|
||||
|
||||
func add(error reason: String, source: String, error: Error? = nil) {
|
||||
add(error: .init(reason: reason, source: source, error: error))
|
||||
}
|
||||
|
||||
func add(warning: ContentError) {
|
||||
add(.warning, item: warning)
|
||||
}
|
||||
|
||||
func add(warning reason: String, source: String, error: Error? = nil) {
|
||||
add(warning: .init(reason: reason, source: source, error: error))
|
||||
}
|
||||
|
||||
func add(info: ContentError) {
|
||||
add(.info, item: info)
|
||||
}
|
||||
|
||||
func add(info reason: String, source: String, error: Error? = nil) {
|
||||
add(info: .init(reason: reason, source: source, error: error))
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
func unused<T, R>(_ value: Optional<T>, _ name: String, source: String) -> Optional<R> {
|
||||
if value != nil {
|
||||
add(info: "Unused property '\(name)'", source: source)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func unknown(property: String, source: String) {
|
||||
add(info: "Unknown property '\(property)'", source: source)
|
||||
}
|
||||
|
||||
func required<T>(_ value: Optional<T>, name: String, source: String) -> Optional<T> {
|
||||
guard let value = value else {
|
||||
add(error: "Missing property '\(name)'", source: source)
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func unexpected<T>(_ value: Optional<T>, name: String, source: String) -> Optional<T> {
|
||||
if let value = value {
|
||||
add(error: "Unexpected property '\(name)' = '\(value)'", source: source)
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func missing(_ file: String, requiredBy source: String) {
|
||||
print("[ERROR] Missing file '\(file)' required by \(source)")
|
||||
}
|
||||
|
||||
func failedToOpen(_ file: String, requiredBy source: String, error: Error?) {
|
||||
print("[ERROR] Failed to open file '\(file)' required by \(source): \(error?.localizedDescription ?? "No error provided")")
|
||||
}
|
||||
|
||||
func state(_ raw: String?, source: String) -> PageState {
|
||||
guard let raw = raw else {
|
||||
return .standard
|
||||
}
|
||||
guard let state = PageState(rawValue: raw) else {
|
||||
add(warning: "Invalid 'state' '\(raw)', using 'standard'", source: source)
|
||||
return .standard
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func thumbnailStyle(_ raw: String?, source: String) -> ThumbnailStyle {
|
||||
guard let raw = raw else {
|
||||
return .large
|
||||
}
|
||||
guard let style = ThumbnailStyle(rawValue: raw) else {
|
||||
add(warning: "Invalid 'thumbnailStyle' '\(raw)', using 'large'", source: source)
|
||||
return .large
|
||||
}
|
||||
return style
|
||||
}
|
||||
|
||||
func linkPreviewThumbnail(customFile: String?, for language: String, in folder: URL, source: String) -> String? {
|
||||
if let customFile = customFile {
|
||||
#warning("Allow absolute urls for link preview thumbnails")
|
||||
let customFileUrl = folder.appendingPathComponent(customFile)
|
||||
guard customFileUrl.exists else {
|
||||
missing(customFile, requiredBy: "property 'linkPreviewImage' in metadata of \(source)")
|
||||
return nil
|
||||
}
|
||||
return customFile
|
||||
}
|
||||
guard let thumbnail = Element.findThumbnail(for: language, in: folder) else {
|
||||
// Link preview images are not necessarily required
|
||||
return nil
|
||||
}
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
func moreLinkText(_ elementText: String?, parent parentText: String?, source: String) -> String {
|
||||
if let elementText = elementText {
|
||||
return elementText
|
||||
}
|
||||
let standard = Element.LocalizedMetadata.moreLinkDefaultText
|
||||
guard let parentText = parentText, parentText != standard else {
|
||||
add(error: "Missing property 'moreLinkText'", source: source)
|
||||
return standard
|
||||
}
|
||||
|
||||
return parentText
|
||||
}
|
||||
|
||||
func date(from string: String?, property: String, source: String) -> Date? {
|
||||
guard let string = string else {
|
||||
return nil
|
||||
}
|
||||
guard let date = ValidationLog.metadataDate.date(from: string) else {
|
||||
add(warning: "Invalid date string '\(string)' for property '\(property)'", source: source)
|
||||
return nil
|
||||
}
|
||||
return date
|
||||
}
|
||||
|
||||
private static let metadataDate: DateFormatter = {
|
||||
let df = DateFormatter()
|
||||
df.dateFormat = "dd.MM.yy"
|
||||
return df
|
||||
}()
|
||||
}
|
||||
12
WebsiteGenerator/Files/VideoType.swift
Normal file
12
WebsiteGenerator/Files/VideoType.swift
Normal file
@ -0,0 +1,12 @@
|
||||
import Foundation
|
||||
|
||||
enum VideoType: String {
|
||||
case mp4
|
||||
|
||||
var htmlType: String {
|
||||
switch self {
|
||||
case .mp4:
|
||||
return "video/mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user