Generate avif and webp image versions

Sources/Generator/Extensions/String+Extensions.swift

@@ -20,6 +20,11 @@ extension String {
             .joined(separator: "\n")
     }
 
+    /**
+     Remove the part after the last occurence of the separator (including the separator itself).
+
+     The string is left unchanges, if it does not contain the separator.
+     */
     func dropAfterLast(_ separator: String) -> String {
         guard contains(separator) else {
             return self

Sources/Generator/Files/FileSystem.swift

@@ -138,8 +138,26 @@ final class FileSystem {
     // MARK: Images
 
     @discardableResult
-    func requireImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil)  -> NSSize {
+    func requireSingleImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil) -> NSSize {
-        images.requireImage(at: destination, generatedFrom: source, requiredBy: path, width: width, height: desiredHeight)
+        images.requireImage(
+            at: destination,
+            generatedFrom: source,
+            requiredBy: path,
+            quality: 0.7,
+            width: width,
+            height: desiredHeight,
+            alwaysGenerate: false)
+    }
+
+    /**
+     Create images of different types.
+
+     This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
+     - Parameter destination: The path to the destination file
+     */
+    @discardableResult
+    func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int? = nil)  -> NSSize {
+        images.requireMultiVersionImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
     }
 
     func createImages() {
@@ -252,7 +270,7 @@ final class FileSystem {
     private func minifyJS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
         let command = "uglifyjs \(sourceUrl.path) > \(tempFile.path)"
         do {
-            _ = try safeShell(command)
+            _ = try FileSystem.safeShell(command)
             return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
         } catch {
             log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
@@ -263,7 +281,7 @@ final class FileSystem {
     private func minifyCSS(at sourceUrl: URL, andWriteTo destinationUrl: URL) -> Bool {
         let command = "cleancss \(sourceUrl.path) -o \(tempFile.path)"
         do {
-            _ = try safeShell(command)
+            _ = try FileSystem.safeShell(command)
             return copyBinaryFileIfChanged(from: tempFile, to: destinationUrl)
         } catch {
             log.add(error: "Failed to minify \(sourceUrl.path): \(error)", source: source)
@@ -406,7 +424,7 @@ final class FileSystem {
     // MARK: Running other tasks
 
     @discardableResult
-    func safeShell(_ command: String) throws -> String {
+    static func safeShell(_ command: String) throws -> String {
         let task = Process()
         let pipe = Pipe()
 

Sources/Generator/Files/ImageGenerator.swift

@@ -1,6 +1,7 @@
 import Foundation
 import AppKit
 import CryptoKit
+import Darwin.C
 
 private struct ImageJob {
 
@@ -9,6 +10,10 @@ private struct ImageJob {
     let width: Int
 
     let path: String
+
+    let quality: Float
+
+    let alwaysGenerate: Bool
 }
 
 final class ImageGenerator {
@@ -30,6 +35,13 @@ final class ImageGenerator {
      */
     private var imageJobs: [String : [ImageJob]] = [:]
 
+    /**
+     The images for which to generate multiple versions
+
+     The key is the source file, the value is the path of the requiring page.
+     */
+    private var multiImageJobs: [String : String] = [:]
+
     /**
      The images which could not be found, but are required for the site.
 
@@ -112,7 +124,7 @@ final class ImageGenerator {
         }
     }
 
-    func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, width: Int, height: Int?) -> NSSize {
+    func requireImage(at destination: String, generatedFrom source: String, requiredBy path: String, quality: Float, width: Int, height: Int?, alwaysGenerate: Bool) -> NSSize {
         requiredImages.insert(destination)
         let height = height.unwrapped(CGFloat.init)
         let sourceUrl = input.appendingPathComponent(source)
@@ -135,28 +147,43 @@ final class ImageGenerator {
             }
         }
 
-        let job = ImageJob(destination: destination, width: width, path: path)
+        let job = ImageJob(
+            destination: destination,
+            width: width,
+            path: path,
+            quality: quality,
+            alwaysGenerate: alwaysGenerate)
+        insert(job: job, source: source)
 
+        return scaledSize
+    }
+
+    private func insert(job: ImageJob, source: String) {
         guard let existingSource = imageJobs[source] else {
             imageJobs[source] = [job]
-            return scaledSize
+            return
         }
 
-        guard let existingJob = existingSource.first(where: { $0.destination == destination}) else {
+        guard let existingJob = existingSource.first(where: { $0.destination == job.destination }) else {
             imageJobs[source] = existingSource + [job]
-            return scaledSize
+            return
         }
 
-        if existingJob.width != width {
+        if existingJob.width != job.width {
-            addWarning("Multiple image widths (\(existingJob.width) and \(width))", destination: destination, path: "\(existingJob.path) and \(path)")
+            addWarning("Multiple image widths (\(existingJob.width) and \(job.width))", destination: job.destination, path: "\(existingJob.path) and \(job.path)")
         }
-        return scaledSize
     }
 
     func createImages() {
+        var count = 0
         for (source, jobs) in imageJobs.sorted(by: { $0.key < $1.key }) {
+            print(String(format: "Creating images: %4d / %d\r", count, imageJobs.count), terminator: "")
+            fflush(stdout)
             create(images: jobs, from: source)
+            count += 1
         }
+        print(String(format: "                                     \r", count, imageJobs.count), terminator: "")
+        createMultiImages()
         printMissingImages()
         printImageWarnings()
         printGeneratedImages()
@@ -168,7 +195,10 @@ final class ImageGenerator {
             return
         }
         print("\(missingImages.count) missing images:")
-        for (source, path) in missingImages {
+        let sort = missingImages.sorted { (a, b) in
+            a.value < b.value && a.key < b.key
+        }
+        for (source, path) in sort {
             print("  \(source) (required by \(path))")
         }
     }
@@ -207,7 +237,7 @@ final class ImageGenerator {
     }
 
     private func isMissing(_ job: ImageJob) -> Bool {
-        !output.appendingPathComponent(job.destination).exists
+        job.alwaysGenerate || !output.appendingPathComponent(job.destination).exists
     }
 
     private func create(images: [ImageJob], from source: String) {
@@ -233,6 +263,10 @@ final class ImageGenerator {
 
     private func create(job: ImageJob, from image: NSImage, source: String) {
         let destinationUrl = output.appendingPathComponent(job.destination)
+        create(job: job, from: image, source: source, at: destinationUrl)
+    }
+
+    private func create(job: ImageJob, from image: NSImage, source: String, at destinationUrl: URL) {
 
         // Ensure that image file is supported
         let ext = destinationUrl.pathExtension.lowercased()
@@ -272,7 +306,7 @@ final class ImageGenerator {
         NSGraphicsContext.restoreGraphicsState()
 
         // Get NSData, and save it
-        guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(0.7)]) else {
+        guard let data = rep.representation(using: type, properties: [.compressionFactor: NSNumber(value: job.quality)]) else {
             addWarning("Failed to get data", job: job)
             return
         }
@@ -284,4 +318,100 @@ final class ImageGenerator {
         }
         generatedImages.insert(job.destination)
     }
+
+    /**
+     Create images of different types.
+
+     This function generates versions for the given image, including png/jpg, avif, and webp. Different pixel density versions (1x and 2x) are also generated.
+     - Parameter destination: The path to the destination file
+     */
+    @discardableResult
+    func requireMultiVersionImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
+        // Add @1x version
+        _ = requireScaledMultiImage(source: source, destination: destination, requiredBy: path, width: width, desiredHeight: desiredHeight)
+
+        // Add @2x version
+        return requireScaledMultiImage(
+            source: source,
+            destination: destination.insert("@2x", beforeLast: "."),
+            requiredBy: path,
+            width: width * 2,
+            desiredHeight: desiredHeight.unwrapped { $0 * 2 })
+    }
+
+    @discardableResult
+    private func requireScaledMultiImage(source: String, destination: String, requiredBy path: String, width: Int, desiredHeight: Int?) -> NSSize {
+        let rawDestinationPath = destination.dropAfterLast(".")
+        let avifPath = rawDestinationPath + ".avif"
+        let webpPath = rawDestinationPath + ".webp"
+        let needsGeneration = !output.appendingPathComponent(avifPath).exists || !output.appendingPathComponent(webpPath).exists
+
+        let size = requireImage(at: destination, generatedFrom: source, requiredBy: path, quality: 1.0, width: width, height: desiredHeight, alwaysGenerate: needsGeneration)
+        multiImageJobs[destination] = path
+        return size
+    }
+
+    private func createMultiImages() {
+        let sort = multiImageJobs.sorted { $0.value < $1.value && $0.key < $1.key }
+        var count = 1
+        for (baseImage, path) in sort {
+            print(String(format: "Creating image versions: %4d / %d\r", count, sort.count), terminator: "")
+            fflush(stdout)
+            createMultiImages(from: baseImage, path: path)
+            count += 1
+        }
+        print(String(format: "                                     \r", count, sort.count), terminator: "")
+    }
+
+    private func createMultiImages(from source: String, path: String) {
+        guard generatedImages.contains(source) else {
+            return
+        }
+
+        let sourceUrl = output.appendingPathComponent(source)
+        let sourcePath = sourceUrl.path
+        guard sourceUrl.exists else {
+            addWarning("No image at path \(sourcePath)", destination: source, path: path)
+            missingImages[source] = path
+            return
+        }
+
+        let avifPath = source.dropAfterLast(".") + ".avif"
+        createAVIF(at: output.appendingPathComponent(avifPath).path, from: sourcePath)
+        generatedImages.insert(avifPath)
+
+        let webpPath = source.dropAfterLast(".") + ".webp"
+        createWEBP(at: output.appendingPathComponent(webpPath).path, from: sourcePath)
+        generatedImages.insert(webpPath)
+
+        compress(at: source)
+    }
+
+    private func createAVIF(at destination: String, from source: String, quality: Int = 55, effort: Int = 5) {
+        let folder = destination.dropAfterLast("/")
+        let command = "npx avif --input=\(source) --quality=\(quality) --effort=\(effort) --output=\(folder) --overwrite"
+        do {
+            _ = try FileSystem.safeShell(command)
+        } catch {
+            addWarning("Failed to create AVIF image", destination: destination, path: destination)
+        }
+    }
+
+    private func createWEBP(at destination: String, from source: String, quality: Int = 75) {
+        let command = "cwebp \(source) -q \(quality) -o \(destination)"
+        do {
+            _ = try FileSystem.safeShell(command)
+        } catch {
+            addWarning("Failed to create WEBP image", destination: destination, path: destination)
+        }
+    }
+
+    private func compress(at destination: String, quality: Int = 70) {
+        let command = "magick convert \(destination) -quality \(quality)% \(destination)"
+        do {
+            _ = try FileSystem.safeShell(command)
+        } catch {
+            addWarning("Failed to compress image", destination: destination, path: destination)
+        }
+    }
 }

Sources/Generator/Files/ImageType.swift

@@ -4,6 +4,8 @@ import AppKit
 enum ImageType: CaseIterable {
     case jpg
     case png
+    case avif
+    case webp
 
     init?(fileExtension: String) {
         switch fileExtension {
@@ -11,6 +13,10 @@ enum ImageType: CaseIterable {
             self = .jpg
         case "png":
             self = .png
+        case "avif":
+            self = .avif
+        case "webp":
+            self = .webp
         default:
             return nil
         }
@@ -20,7 +26,7 @@ enum ImageType: CaseIterable {
         switch self {
         case .jpg:
             return .jpeg
-        case .png:
+        case .png, .avif, .webp:
             return .png
         }
     }

Sources/Generator/Generators/MarkdownProcessor.swift

@@ -120,23 +120,15 @@ struct PageContentGenerator {
     private func handleImage(page: Element, file: String, rightTitle: String?, leftTitle: String?) -> String {
         let imagePath = page.pathRelativeToRootForContainedInputFile(file)
 
-        let size = files.requireImage(
+        let size = files.requireMultiVersionImage(
             source: imagePath,
             destination: imagePath,
             requiredBy: page.path,
             width: configuration.pageImageWidth)
 
-        let imagePath2x = imagePath.insert("@2x", beforeLast: ".")
-        let file2x = file.insert("@2x", beforeLast: ".")
-        files.requireImage(
-            source: imagePath,
-            destination: imagePath2x,
-            requiredBy: page.path,
-            width: 2 * configuration.pageImageWidth)
-
         let content: [PageImageTemplate.Key : String] = [
-            .image: file,
+            .image: file.dropAfterLast("."),
-            .image2x: file2x,
+            .imageExtension: file.lastComponentAfter("."),
             .width: "\(Int(size.width))",
             .height: "\(Int(size.height))",
             .leftText: leftTitle ?? "",

Sources/Generator/Generators/PageHeadGenerator.swift

@@ -24,7 +24,7 @@ struct PageHeadGenerator {
             let linkPreviewImageName = "thumbnail-link.\(image.lastComponentAfter("."))"
             let sourceImagePath = page.pathRelativeToRootForContainedInputFile(image)
             let destinationImagePath = page.pathRelativeToRootForContainedInputFile(linkPreviewImageName)
-            files.requireImage(
+            files.requireSingleImage(
                 source: sourceImagePath,
                 destination: destinationImagePath,
                 requiredBy: page.path,

Sources/Generator/Generators/SiteGenerator.swift

@@ -51,7 +51,7 @@ struct SiteGenerator {
         element.requiredFiles.forEach(files.require)
         element.externalFiles.forEach(files.exclude)
         element.images.forEach {
-            files.requireImage(
+            files.requireSingleImage(
                 source: $0.sourcePath,
                 destination: $0.destinationPath,
                 requiredBy: element.path,

Sources/Generator/Generators/ThumbnailListGenerator.swift

@@ -14,8 +14,7 @@ struct ThumbnailListGenerator {
     }
 
     private func itemContent(_ item: Element, parent: Element, language: String, style: ThumbnailStyle) -> String {
-        let (thumbnailSourcePath, thumbnailDestPath) = item.thumbnailFilePath(for: language)
+
-        let relativeImageUrl = parent.relativePathToFileWithPath(thumbnailDestPath)
         let metadata = item.localized(for: language)
         var content = [ThumbnailKey : String]()
 
@@ -25,32 +24,26 @@ struct ThumbnailListGenerator {
             content[.url] = "href=\"\(relativePageUrl)\""
         }
 
-        content[.image] = relativeImageUrl
+        let (thumbnailSourcePath, thumbnailDestPath) = item.thumbnailFilePath(for: language)
+        let thumbnailDestNoExtension = thumbnailDestPath.dropAfterLast(".")
+        content[.image] = parent.relativePathToFileWithPath(thumbnailDestNoExtension)
+
         if style == .large, let suffix = metadata.thumbnailSuffix {
             content[.title] = factory.html.make(title: metadata.title, suffix: suffix)
         } else {
             content[.title] = metadata.title
         }
-        content[.image2x] = relativeImageUrl.insert("@2x", beforeLast: ".")
         content[.corner] = item.cornerText(for: language).unwrapped {
             factory.largeThumbnail.makeCorner(text: $0)
         }
 
-        files.requireImage(
+        files.requireMultiVersionImage(
             source: thumbnailSourcePath,
-            destination: thumbnailDestPath,
+            destination: thumbnailDestNoExtension + ".jpg",
             requiredBy: item.path,
             width: style.width,
             desiredHeight: style.height)
 
-        // Create image version for high-resolution screens
-        files.requireImage(
-            source: thumbnailSourcePath,
-            destination: thumbnailDestPath.insert("@2x", beforeLast: "."),
-            requiredBy: item.path,
-            width: style.width * 2,
-            desiredHeight: style.height * 2)
-
         return factory.thumbnail(style: style).generate(content, shouldIndent: false)
     }
 }

Sources/Generator/Templates/Elements/PageImageTemplate.swift

@@ -4,7 +4,7 @@ struct PageImageTemplate: Template {
 
     enum Key: String, CaseIterable {
         case image = "IMAGE"
-        case image2x = "IMAGE_2X"
+        case imageExtension = "IMAGE_EXT"
         case width = "WIDTH"
         case height = "HEIGHT"
         case leftText = "LEFT_TEXT"

Sources/Generator/Templates/Elements/ThumbnailTemplate.swift

@@ -8,7 +8,6 @@ protocol ThumbnailTemplate {
 enum ThumbnailKey: String, CaseIterable {
     case url = "URL"
     case image = "IMAGE"
-    case image2x = "IMAGE_2X"
     case title = "TITLE"
     case corner = "CORNER"
 }