Compare commits

...

4 Commits

Author SHA1 Message Date
Christoph Hagen
deb7e6187e Fix links to external pages 2022-12-05 17:25:07 +01:00
Christoph Hagen
464ece4a03 Improve summary print 2022-12-05 11:51:19 +01:00
Christoph Hagen
225c68ecd1 Fix display of total optimization count 2022-12-05 11:51:04 +01:00
Christoph Hagen
f52c3bc8b9 Check dependencies and outputs 2022-12-05 11:43:30 +01:00
6 changed files with 162 additions and 44 deletions

View File

@ -289,16 +289,23 @@ struct Element {
self.readElements(in: folder, source: path, log: log)
}
func getContainedIds(log: MetadataInfoLogger) -> [String : String] {
elements.reduce(into: [id : path]) { dict, element in
element.getContainedIds(log: log).forEach { id, path in
if let existing = dict[id] {
log.error("Conflicting id with \(existing)", source: path)
} else {
dict[id] = path
}
func getExternalPageMap(language: String) -> [String : String] {
var result = [String : String]()
if let ext = getExternalLink(for: language) {
result[id] = ext
} else {
result[id] = path + Element.htmlPagePathAddition(for: language)
}
elements.forEach { element in
element.getExternalPageMap(language: language).forEach { key, value in
result[key] = value
}
}
return result
}
private func getExternalLink(for language: String) -> String? {
languages.first { $0.language == language }?.externalUrl
}
}
@ -381,6 +388,9 @@ extension Element {
- Returns: The relative url from a localized page of the element to the target file.
*/
func relativePathToOtherSiteElement(file: String) -> String {
guard !file.hasPrefix("/") else {
return file
}
// Note: The element `path` is missing the last component
// i.e. travel/alps instead of travel/alps/en.html
let ownParts = path.components(separatedBy: "/")

View File

@ -47,13 +47,12 @@ struct PageContentGenerator {
let file = markdown.between("(", and: ")")
if file.hasPrefix("page:") {
let pageId = file.replacingOccurrences(of: "page:", with: "")
guard let pagePath = results.getPagePath(for: pageId, source: page.path) else {
guard let pagePath = results.getPagePath(for: pageId, source: page.path, language: language) else {
// Remove link since the page can't be found
return markdown.between("[", and: "]")
}
let fullPath = pagePath + Element.htmlPagePathAddition(for: language)
// Adjust file path to get the page url
let url = page.relativePathToOtherSiteElement(file: fullPath)
let url = page.relativePathToOtherSiteElement(file: pagePath)
return html.replacingOccurrences(of: file, with: url)
}
@ -244,7 +243,7 @@ struct PageContentGenerator {
private func handlePageLink(page: Element, language: String, pageId: String) -> String {
guard let linkedPage = siteRoot.find(pageId) else {
// Checking the page path will add it to the missing pages
_ = results.getPagePath(for: pageId, source: page.path)
_ = results.getPagePath(for: pageId, source: page.path, language: language)
// Remove link since the page can't be found
return ""
}

View File

@ -0,0 +1,75 @@
import Foundation
func checkDependencies() -> Bool {
print("--- DEPENDENCIES -----------------------------------")
print(" ")
defer { print(" ") }
return checkImageOptimAvailability() && checkMagickAvailability() && checkCwebpAvailability() && checkAvifAvailability()
}
private func checkImageOptimAvailability() -> Bool {
do {
let output = try safeShell("imageoptim --version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" ImageOptim: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" ImageOptim: Not found")
return false
}
} catch {
print(" ImageOptim: Failed to get version (\(error))")
return false
}
return true
}
private func checkMagickAvailability() -> Bool {
do {
let output = try safeShell("magick --version")
guard let version = output.components(separatedBy: "ImageMagick ").dropFirst().first?
.components(separatedBy: " ").first else {
print(" Magick: Not found")
return false
}
print(" Magick: \(version)")
} catch {
print(" Magick: Failed to get version (\(error))")
return false
}
return true
}
private func checkCwebpAvailability() -> Bool {
do {
let output = try safeShell("cwebp -version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" cwebp: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" cwebp: Not found")
return false
}
} catch {
print(" cwebp: Failed to get version (\(error))")
return false
}
return true
}
private func checkAvifAvailability() -> Bool {
do {
let output = try safeShell("npx avif --version")
let version = output.components(separatedBy: ".").compactMap { Int($0.trimmed) }
if version.count > 1 {
print(" avif: \(version.map { "\($0)" }.joined(separator: "."))")
} else {
print(" avif: Not found")
return false
}
} catch {
print(" avif: Failed to get version (\(error))")
return false
}
return true
}

View File

@ -8,6 +8,8 @@ enum MinificationType {
case css
}
typealias PageMap = [(language: String, pages: [String : String])]
final class GenerationResultsHandler {
/// The content folder where the input data is stored
@ -23,7 +25,7 @@ final class GenerationResultsHandler {
This relation is used to generate relative links to pages using the ``Element.id`
*/
private let pagePaths: [String: String]
private let pageMap: PageMap
private let configuration: Configuration
@ -31,11 +33,11 @@ final class GenerationResultsHandler {
private let numberOfTotalPages: Int
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pagePaths: [String: String], pageCount: Int) {
init(in input: URL, to output: URL, configuration: Configuration, fileUpdates: FileUpdateChecker, pageMap: PageMap, pageCount: Int) {
self.contentFolder = input
self.fileUpdates = fileUpdates
self.outputFolder = output
self.pagePaths = pagePaths
self.pageMap = pageMap
self.configuration = configuration
self.numberOfTotalPages = pageCount
}
@ -86,8 +88,8 @@ final class GenerationResultsHandler {
// MARK: Page data
func getPagePath(for id: String, source: String) -> String? {
guard let pagePath = pagePaths[id] else {
func getPagePath(for id: String, source: String, language: String) -> String? {
guard let pagePath = pageMap.first(where: { $0.language == language})?.pages[id] else {
missingLinkedPages[id] = source
return nil
}

View File

@ -46,7 +46,7 @@ final class ImageGenerator {
private let numberOfTotalImages: Int
private lazy var numberImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
private lazy var numberOfImagesToCreate = jobs.reduce(0) { $0 + $1.images.count } + multiJobs.count * 2
private var numberOfImagesToOptimize = 0
@ -100,7 +100,7 @@ final class ImageGenerator {
for (baseImage, source) in multiJobs {
createMultiImages(from: baseImage, path: source)
}
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate)")
print(" Generated images: \(numberOfGeneratedImages)/\(numberOfImagesToCreate)")
optimizeImages()
print(" Optimized images: \(numberOfOptimizedImages)/\(numberOfImagesToOptimize)")
@ -219,14 +219,18 @@ final class ImageGenerator {
generatedImages.insert(webpPath)
didGenerateImage()
compress(at: source)
compress(at: sourcePath)
}
private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
let folder = destination.dropAfterLast("/")
let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
do {
_ = try safeShell(command)
let output = try safeShell(command)
if output == "" {
return
}
markImageAsFailed(destination, error: "Failed to create AVIF image: \(output)")
} catch {
markImageAsFailed(destination, error: "Failed to create AVIF image")
}
@ -235,18 +239,26 @@ final class ImageGenerator {
private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
let command = "cwebp \(source) -q \(quality) -o \(destination)"
do {
_ = try safeShell(command)
let output = try safeShell(command)
if !output.contains("Error") {
return
}
markImageAsFailed(destination, error: "Failed to create WEBP image: \(output)")
} catch {
markImageAsFailed(destination, error: "Failed to create WEBP image")
markImageAsFailed(destination, error: "Failed to create WEBP image: \(error)")
}
}
private func compress(at destination: String, quality: Int = 70) {
let command = "magick convert \(destination) -quality \(quality)% \(destination)"
do {
_ = try safeShell(command)
let output = try safeShell(command)
if output == "" {
return
}
markImageAsFailed(destination, error: "Failed to compress image: \(output)")
} catch {
markImageAsFailed(destination, error: "Failed to compress image")
markImageAsFailed(destination, error: "Failed to compress image: \(error)")
}
}
@ -254,8 +266,9 @@ final class ImageGenerator {
let all = generatedImages
.filter { imageOptimSupportedFileExtensions.contains($0.lastComponentAfter(".")) }
.map { output.appendingPathComponent($0).path }
for i in stride(from: 0, to: all.count, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, all.count)
numberOfImagesToOptimize = all.count
for i in stride(from: 0, to: numberOfImagesToOptimize, by: imageOptimizationBatchSize) {
let endIndex = min(i+imageOptimizationBatchSize, numberOfImagesToOptimize)
let batch = all[i..<endIndex]
if optimizeImageBatch(batch) {
optimizedImages.formUnion(batch)
@ -267,11 +280,18 @@ final class ImageGenerator {
private func optimizeImageBatch(_ batch: ArraySlice<String>) -> Bool {
let command = "imageoptim " + batch.joined(separator: " ")
do {
_ = try safeShell(command)
let output = try safeShell(command)
if output.contains("Finished") {
return true
}
for image in batch {
markImageAsFailed(image, error: "Failed to optimize image: \(output)")
}
return true
} catch {
for image in batch {
markImageAsFailed(image, error: "Failed to optimize image")
markImageAsFailed(image, error: "Failed to optimize image: \(error)")
}
return false
}
@ -281,7 +301,7 @@ final class ImageGenerator {
private func didGenerateImage(count: Int = 1) {
numberOfGeneratedImages += count
print(" Generated images: \(numberOfGeneratedImages)/\(numberImagesToCreate) \r", terminator: "")
print(" Generated images: \(numberOfGeneratedImages)/\(numberOfImagesToCreate) \r", terminator: "")
fflush(stdout)
}

View File

@ -14,7 +14,6 @@ struct CHGenerator: ParsableCommand {
}
private func loadConfiguration(at configPath: String) -> Configuration? {
print(" ")
print("--- CONFIGURATION ----------------------------------")
print(" ")
print(" Configuration file: \(configPath)")
@ -32,7 +31,7 @@ private func loadConfiguration(at configPath: String) -> Configuration? {
return config
}
private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, ids: [String : String])? {
private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Element, pageMap: PageMap)? {
print("--- SOURCE FILES -----------------------------------")
print(" ")
@ -49,22 +48,22 @@ private func loadSiteData(in folder: URL, runFolder: URL) throws -> (root: Eleme
print(" Error: No site root loaded, aborting generation")
return nil
}
let ids = root.getContainedIds(log: log)
let pageMap = root.languages.map { (language: $0.language, pages: root.getExternalPageMap(language: $0.language)) }
log.printMetadataScanOverview(languages: root.languages.count)
return (root, ids)
return (root, pageMap)
}
private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, ids: [String: String], runFolder: URL) -> (ImageData, FileData)? {
private func generatePages(from root: Element, configuration: Configuration, fileUpdates: FileUpdateChecker, pageMap: PageMap, runFolder: URL) -> (ImageData, FileData)? {
print("--- GENERATION -------------------------------------")
print(" ")
let pageCount = ids.count * root.languages.count
let pageCount = pageMap.reduce(0) { $0 + $1.pages.count }
let results = GenerationResultsHandler(
in: configuration.contentDirectory,
to: configuration.outputDirectory,
configuration: configuration,
fileUpdates: fileUpdates,
pagePaths: ids,
pageMap: pageMap,
pageCount: pageCount)
defer { results.printOverview() }
@ -111,8 +110,8 @@ private func copyFiles(files: FileData, configuration: Configuration, runFolder:
generator.generate()
}
private func finish(start: Date) {
print("----------------------------------------------------")
private func finish(start: Date, complete: Bool) {
print("--- SUMMARY ----------------------------------------")
print(" ")
let duration = Int(-start.timeIntervalSinceNow.rounded())
if duration < 60 {
@ -122,11 +121,25 @@ private func finish(start: Date) {
} else {
print(String(format: " Duration: %d:%02d:%02d", duration / 3600, (duration / 60) % 60, duration % 60))
}
print("")
print(" Complete: \(complete ? "Yes" : "No")")
print(" ")
print("----------------------------------------------------")
}
private func generate(configPath: String) throws {
let start = Date()
var complete = false
defer {
// 6. Print summary
finish(start: start, complete: complete)
}
print(" ")
guard checkDependencies() else {
return
}
// 1. Load configuration
guard let configuration = loadConfiguration(at: configPath) else {
@ -136,7 +149,7 @@ private func generate(configPath: String) throws {
let runFolder = configuration.contentDirectory.appendingPathComponent("run")
// 2. Scan site elements
guard let (siteRoot, ids) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else {
guard let (siteRoot, pageMap) = try loadSiteData(in: configuration.contentDirectory, runFolder: runFolder) else {
return
}
@ -152,7 +165,7 @@ private func generate(configPath: String) throws {
// 3. Generate pages
guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, ids: ids, runFolder: runFolder) else {
guard let (images, files) = generatePages(from: siteRoot, configuration: configuration, fileUpdates: fileUpdates, pageMap: pageMap, runFolder: runFolder) else {
return
}
@ -162,6 +175,5 @@ private func generate(configPath: String) throws {
// 5. Copy/minify files
copyFiles(files: files, configuration: configuration, runFolder: runFolder)
// 6. Print summary
finish(start: start)
complete = true
}