ch/CHGenerator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: ch:deb7e6187e2269c6a44c78488e85c98f116314d7
Branches Tags
ch:main
..
compare: ch:a8b328efcead27928f9e9075462170a04c2aa2ce
Branches Tags
ch:main

No commits in common. "deb7e6187e2269c6a44c78488e85c98f116314d7" and "a8b328efcead27928f9e9075462170a04c2aa2ce" have entirely different histories.
deb7e6187e ... a8b328efce

6 changed files with 44 additions and 162 deletions
Show Stats Expand all files Collapse all files

22
Sources/Generator/Content/Element.swift
View File

@ -289,23 +289,16 @@ struct Element {
self.readElements(in: folder, source: path, log: log) self.readElements(in: folder, source: path, log: log)
} }
func getExternalPageMap(language: String) -> [String : String] { func getContainedIds(log: MetadataInfoLogger) -> [String : String] {
var result = [String : String]() elements.reduce(into: [id : path]) { dict, element in
if let ext = getExternalLink(for: language) { element.getContainedIds(log: log).forEach { id, path in
result[id] = ext if let existing = dict[id] {
log.error("Conflicting id with \(existing)", source: path)
} else { } else {
result[id] = path + Element.htmlPagePathAddition(for: language) dict[id] = path
}
elements.forEach { element in
element.getExternalPageMap(language: language).forEach { key, value in
result[key] = value
} }
} }
return result
} }
private func getExternalLink(for language: String) -> String? {
languages.first { $0.language == language }?.externalUrl
} }
} }
@ -388,9 +381,6 @@ extension Element {
- Returns: The relative url from a localized page of the element to the target file. - Returns: The relative url from a localized page of the element to the target file.
*/ */
func relativePathToOtherSiteElement(file: String) -> String { func relativePathToOtherSiteElement(file: String) -> String {
guard !file.hasPrefix("/") else {
return file
}
// Note: The element `path` is missing the last component // Note: The element `path` is missing the last component
// i.e. travel/alps instead of travel/alps/en.html // i.e. travel/alps instead of travel/alps/en.html
let ownParts = path.components(separatedBy: "/") let ownParts = path.components(separatedBy: "/")

7
Sources/Generator/Generators/PageContentGenerator.swift
View File

@ -47,12 +47,13 @@ struct PageContentGenerator {
let file = markdown.between("(", and: ")") let file = markdown.between("(", and: ")")
if file.hasPrefix("page:") { if file.hasPrefix("page:") {
let pageId = file.replacingOccurrences(of: "page:", with: "") let pageId = file.replacingOccurrences(of: "page:", with: "")
guard let pagePath = results.getPagePath(for: pageId, source: page.path, language: language) else { guard let pagePath = results.getPagePath(for: pageId, source: page.path) else {
// Remove link since the page can't be found // Remove link since the page can't be found
return markdown.between("[", and: "]") return markdown.between("[", and: "]")
} }
let fullPath = pagePath + Element.htmlPagePathAddition(for: language)
// Adjust file path to get the page url // Adjust file path to get the page url
let url = page.relativePathToOtherSiteElement(file: pagePath) let url = page.relativePathToOtherSiteElement(file: fullPath)
return html.replacingOccurrences(of: file, with: url) return html.replacingOccurrences(of: file, with: url)
} }
@ -243,7 +244,7 @@ struct PageContentGenerator {
private func handlePageLink(page: Element, language: String, pageId: String) -> String { private func handlePageLink(page: Element, language: String, pageId: String) -> String {
guard let linkedPage = siteRoot.find(pageId) else { guard let linkedPage = siteRoot.find(pageId) else {
// Checking the page path will add it to the missing pages // Checking the page path will add it to the missing pages
_ = results.getPagePath(for: pageId, source: page.path, language: language) _ = results.getPagePath(for: pageId, source: page.path)
// Remove link since the page can't be found // Remove link since the page can't be found
return "" return ""
} }

75
Sources/Generator/Processing/DependencyCheck.swift
View File

@ -1,75 +0,0 @@
import Foundation
func checkDependencies() -> Bool {
print("--- DEPENDENCIES -----------------------------------")
print(" ")
defer { print(" ") }
return checkImageOptimAvailability() && checkMagickAvailability() && checkCwebpAvailability() && checkAvifAvailability()
}
private func checkImageOptimAvailability() -> Bool {
do {
let output = try safeShell("imageoptim --version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" ImageOptim: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" ImageOptim: Not found")
return false
}
} catch {
print(" ImageOptim: Failed to get version (\(error))")
return false
}
return true
}
private func checkMagickAvailability() -> Bool {
do {
let output = try safeShell("magick --version")
guard let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
.components(separatedBy: " ").first else {
print(" Magick: Not found")
return false
}
print(" Magick: \(version)")
} catch {
print(" Magick: Failed to get version (\(error))")
return false
}
return true
}
private func checkCwebpAvailability() -> Bool {
do {
let output = try safeShell("cwebp -version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" cwebp: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" cwebp: Not found")
return false
}
} catch {
print(" cwebp: Failed to get version (\(error))")
return false
}
return true
}
private func checkAvifAvailability() -> Bool {
do {
let output = try safeShell("npx avif --version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" avif: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" avif: Not found")
return false
}
} catch {
print(" avif: Failed to get version (\(error))")
return false
}
return true
}

12
Sources/Generator/Processing/GenerationResultsHandler.swift
View File

@ -8,8 +8,6 @@ enum MinificationType {
case css case css
} }
typealias PageMap = [(language: String, pages: [String : String])]
final class GenerationResultsHandler { final class GenerationResultsHandler {
/// The content folder where the input data is stored /// The content folder where the input data is stored
@ -25,7 +23,7 @@ final class GenerationResultsHandler {
This relation is used to generate relative links to pages using the ``Element.id` This relation is used to generate relative links to pages using the ``Element.id`
*/ */
private let pageMap: PageMap private let pagePaths: [String: String]
private let configuration: Configuration private let configuration: Configuration
@ -33,11 +31,11 @@ final class GenerationResultsHandler {
private let numberOfTotalPages: Int private let numberOfTotalPages: Int
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pageMap: PageMap, pageCount: Int) { init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) {
self.contentFolder = input self.contentFolder = input
self.fileUpdates = fileUpdates self.fileUpdates = fileUpdates
self.outputFolder = output self.outputFolder = output
self.pageMap = pageMap self.pagePaths = pagePaths
self.configuration = configuration self.configuration = configuration
self.numberOfTotalPages = pageCount self.numberOfTotalPages = pageCount
} }
@ -88,8 +86,8 @@ final class GenerationResultsHandler {
// MARK: Page data // MARK: Page data
func getPagePath(for id: String, source: String, language: String) -> String? { func getPagePath(for id: String, source: String) -> String? {
guard let pagePath = pageMap.first(where: { $0.language == language})?.pages[id] else { guard let pagePath = pagePaths[id] else {
missingLinkedPages[id] = source missingLinkedPages[id] = source
return nil return nil
} }

46
Sources/Generator/Processing/ImageGenerator.swift
View File

@ -46,7 +46,7 @@ final class ImageGenerator {
private let numberOfTotalImages: Int private let numberOfTotalImages: Int
private lazy var numberOfImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2 private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
private var numberOfImagesToOptimize = 0 private var numberOfImagesToOptimize = 0
@ -100,7 +100,7 @@ final class ImageGenerator {
for (baseImage, source) in multiJobs { for (baseImage, source) in multiJobs {
createMultiImages(from: baseImage, path: source) createMultiImages(from: baseImage, path: source)
} }
print(" Generated images: \(numberOfGeneratedImages)/\(numberOfImagesToCreate)") print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)")
optimizeImages() optimizeImages()
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)") print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)")
@ -219,18 +219,14 @@ final class ImageGenerator {
generatedImages.insert(webpPath) generatedImages.insert(webpPath)
didGenerateImage() didGenerateImage()
compress(at: sourcePath) compress(at: source)
} }
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) { private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/") let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite" let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do { do {
let output = try safeShell(command) _ = try safeShell(command)
if output == "" {
return
}
markImageAsFailed(destination, error: "Failed to create AVIF image: \(output)")
} catch { } catch {
markImageAsFailed(destination, error: "Failed to create AVIF image") markImageAsFailed(destination, error: "Failed to create AVIF image")
} }
@ -239,26 +235,18 @@ final class ImageGenerator {
private func createWEBP(at destination: String, from source: String, quality: Int = 75) { private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)" let command = "cwebp \(source) -q \(quality) -o \(destination)"
do { do {
let output = try safeShell(command) _ = try safeShell(command)
if !output.contains("Error") {
return
}
markImageAsFailed(destination, error: "Failed to create WEBP image: \(output)")
} catch { } catch {
markImageAsFailed(destination, error: "Failed to create WEBP image: \(error)") markImageAsFailed(destination, error: "Failed to create WEBP image")
} }
} }
private func compress(at destination: String, quality: Int = 70) { private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)" let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do { do {
let output = try safeShell(command) _ = try safeShell(command)
if output == "" {
return
}
markImageAsFailed(destination, error: "Failed to compress image: \(output)")
} catch { } catch {
markImageAsFailed(destination, error: "Failed to compress image: \(error)") markImageAsFailed(destination, error: "Failed to compress image")
} }
} }
@ -266,9 +254,8 @@ final class ImageGenerator {
let all = generatedImages let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) } .filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path } .map { output.appendingPathComponent($0).path }
numberOfImagesToOptimize = all.count for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
for i in stride(from: 0, to: numberOfImagesToOptimize, by: imageOptimizationBatchSize) { let endIndex = min(i+imageOptimizationBatchSize, all.count)
let endIndex = min(i+imageOptimizationBatchSize, numberOfImagesToOptimize)
let batch = all[i..<endIndex] let batch = all[i..<endIndex]
if optimizeImageBatch(batch) { if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch) optimizedImages.formUnion(batch)
@ -280,18 +267,11 @@ final class ImageGenerator {
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool { private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ") let command = "imageoptim " + batch.joined(separator: " ")
do { do {
let output = try safeShell(command) _ = try safeShell(command)
if output.contains("Finished") {
return true
}
for image in batch {
markImageAsFailed(image, error: "Failed to optimize image: \(output)")
}
return true return true
} catch { } catch {
for image in batch { for image in batch {
markImageAsFailed(image, error: "Failed to optimize image: \(error)") markImageAsFailed(image, error: "Failed to optimize image")
} }
return false return false
} }
@ -301,7 +281,7 @@ final class ImageGenerator {
private func didGenerateImage(count: Int = 1) { private func didGenerateImage(count: Int = 1) {
numberOfGeneratedImages += count numberOfGeneratedImages += count
print(" Generated images: \(numberOfGeneratedImages)/\(numberOfImagesToCreate) \r", terminator: "") print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate) \r", terminator: "")
fflush(stdout) fflush(stdout)
} }

40
Sources/Generator/run.swift
View File

@ -14,6 +14,7 @@ struct CHGenerator: ParsableCommand {
} }
private func loadConfiguration(at configPath: String) -> Configuration? { private func loadConfiguration(at configPath: String) -> Configuration? {
print(" ")
print("--- CONFIGURATION ----------------------------------") print("--- CONFIGURATION ----------------------------------")
print(" ") print(" ")
print(" Configuration file: \(configPath)") print(" Configuration file: \(configPath)")
@ -31,7 +32,7 @@ private func loadConfiguration(at configPath: String) -> Configuration? {
return config return config
} }
private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, pageMap: PageMap)? { private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, ids: [String : String])? {
print("--- SOURCE FILES -----------------------------------") print("--- SOURCE FILES -----------------------------------")
print(" ") print(" ")
@ -48,22 +49,22 @@ private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Eleme
print(" Error: No site root loaded, aborting generation") print(" Error: No site root loaded, aborting generation")
return nil return nil
} }
let pageMap = root.languages.map { (language: $0.language, pages: root.getExternalPageMap(language: $0.language)) } let ids = root.getContainedIds(log: log)
log.printMetadataScanOverview(languages: root.languages.count) log.printMetadataScanOverview(languages: root.languages.count)
return (root, pageMap) return (root, ids)
} }
private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, pageMap: PageMap, runFolder: URL) -> (ImageData, FileData)? { private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], runFolder: URL) -> (ImageData, FileData)? {
print("--- GENERATION -------------------------------------") print("--- GENERATION -------------------------------------")
print(" ") print(" ")
let pageCount = pageMap.reduce(0) { $0 + $1.pages.count } let pageCount = ids.count * root.languages.count
let results = GenerationResultsHandler( let results = GenerationResultsHandler(
in: configuration.contentDirectory, in: configuration.contentDirectory,
to: configuration.outputDirectory, to: configuration.outputDirectory,
configuration: configuration, configuration: configuration,
fileUpdates: fileUpdates, fileUpdates: fileUpdates,
pageMap: pageMap, pagePaths: ids,
pageCount: pageCount) pageCount: pageCount)
defer { results.printOverview() } defer { results.printOverview() }
@ -110,8 +111,8 @@ private func copyFiles(files: FileData, configuration: Configuration, runFolder:
generator.generate() generator.generate()
} }
private func finish(start: Date, complete: Bool) { private func finish(start: Date) {
print("--- SUMMARY ----------------------------------------") print("----------------------------------------------------")
print(" ") print(" ")
let duration = Int(-start.timeIntervalSinceNow.rounded()) let duration = Int(-start.timeIntervalSinceNow.rounded())
if duration < 60 { if duration < 60 {
@ -121,25 +122,11 @@ private func finish(start: Date, complete: Bool) {
} else { } else {
print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60)) print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60))
} }
print(" Complete: \(complete ? "Yes" : "No")") print("")
print(" ")
print("----------------------------------------------------")
} }
private func generate(configPath: String) throws { private func generate(configPath: String) throws {
let start = Date() let start = Date()
var complete = false
defer {
// 6. Print summary
finish(start: start, complete: complete)
}
print(" ")
guard checkDependencies() else {
return
}
// 1. Load configuration // 1. Load configuration
guard let configuration = loadConfiguration(at: configPath) else { guard let configuration = loadConfiguration(at: configPath) else {
@ -149,7 +136,7 @@ private func generate(configPath: String) throws {
let runFolder = configuration.contentDirectory.appendingPathComponent("run") let runFolder = configuration.contentDirectory.appendingPathComponent("run")
// 2. Scan site elements // 2. Scan site elements
guard let (siteRoot, pageMap) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else { guard let (siteRoot, ids) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else {
return return
} }
@ -165,7 +152,7 @@ private func generate(configPath: String) throws {
// 3. Generate pages // 3. Generate pages
guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, pageMap: pageMap, runFolder: runFolder) else { guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, runFolder: runFolder) else {
return return
} }
@ -175,5 +162,6 @@ private func generate(configPath: String) throws {
// 5. Copy/minify files // 5. Copy/minify files
copyFiles(files: files, configuration: configuration, runFolder: runFolder) copyFiles(files: files, configuration: configuration, runFolder: runFolder)
complete = true // 6. Print summary
finish(start: start)
} }