External files, improve page generation

This commit is contained in:
Christoph Hagen
2024-12-10 15:21:28 +01:00
parent 8183bc4903
commit efc9234917
50 changed files with 1069 additions and 424 deletions

View File

@ -7,6 +7,8 @@ final class GenerationResultsHandler {
/// Generic warnings for pages
private var pageWarnings: [(message: String, source: String)] = []
private var missingPages: [String : [String]] = [:]
func warning(_ message: String, page: Page) {
pageWarnings.append((message, page.id))
print("Page: \(page.id): \(message)")
@ -15,4 +17,8 @@ final class GenerationResultsHandler {
func addRequiredVideoFile(fileId: String) {
requiredVideoFiles.insert(fileId)
}
func missing(page: String, linkedBy source: String) {
missingPages[page, default: []].append(source)
}
}

View File

@ -54,6 +54,9 @@ final class ImageGenerator {
}
func runJobs(callback: (String) -> Void) -> Bool {
guard !jobs.isEmpty else {
return true
}
print("Generating \(jobs.count) images...")
for job in jobs {
callback("Generating image \(job.version)")
@ -80,7 +83,7 @@ final class ImageGenerator {
return "\(prefix).\(type.fileExtension)"
}
func generateImageSet(for image: String, maxWidth: CGFloat, maxHeight: CGFloat, altText: String) -> FeedEntryData.Image {
func generateImageSet(for image: String, maxWidth: CGFloat, maxHeight: CGFloat) {
let type = ImageFileType(fileExtension: image.fileExtension!)!
let width2x = maxWidth * 2
@ -94,12 +97,6 @@ final class ImageGenerator {
_ = generateVersion(for: image, type: type, maximumWidth: maxWidth, maximumHeight: maxHeight)
_ = generateVersion(for: image, type: type, maximumWidth: width2x, maximumHeight: height2x)
let path = "/" + relativeImageOutputPath + "/" + image
return .init(rawImagePath: path,
width: Int(maxWidth),
height: Int(maxHeight),
altText: altText)
}
func generateVersion(for image: String, type: ImageFileType, maximumWidth: CGFloat, maximumHeight: CGFloat) -> String {
@ -133,6 +130,10 @@ final class ImageGenerator {
return versions.contains(version)
}
private func exists(imageVersion version: String) -> Bool {
inOutputImagesFolder { $0.appendingPathComponent(version).exists }
}
private func hasNowGenerated(version: String, for image: String) {
guard var versions = generatedImages[image] else {
generatedImages[image] = [version]
@ -149,7 +150,8 @@ final class ImageGenerator {
// MARK: Image operations
private func generate(job: ImageJob) -> Bool {
if hasPreviouslyGenerated(version: job.version, for: job.image), exists(job.version) {
if hasPreviouslyGenerated(version: job.version, for: job.image), exists(job.version),
exists(imageVersion: job.version) {
return true
}
@ -201,7 +203,7 @@ final class ImageGenerator {
let url = folder.appendingPathComponent(job.version)
if job.type == .avif {
let out = url.path()
let input = out.replacingOccurrences(of: ".avif", with: ".jpg")
let input = url.deletingPathExtension().appendingPathExtension(job.image.fileExtension!).path()
print("avifenc -q 70 \(input) \(out)")
return true
}

View File

@ -1,6 +1,6 @@
import Foundation
final class WebsiteGenerator {
final class LocalizedWebsiteGenerator {
let language: ContentLanguage
@ -38,7 +38,7 @@ final class WebsiteGenerator {
self.localizedSettings = content.settings.localized(in: language)
self.imageGenerator = ImageGenerator(
storage: content.storage,
relativeImageOutputPath: "images")
relativeImageOutputPath: "images") // TODO: Get from settings
}
func generateWebsite(callback: (String) -> Void) -> Bool {
@ -48,6 +48,7 @@ final class WebsiteGenerator {
guard createMainPostFeedPages() else {
return false
}
#warning("Generate content pages")
guard generateTagPages() else {
return false
}
@ -127,12 +128,17 @@ final class WebsiteGenerator {
let pageGenerator = PageGenerator(content: content, imageGenerator: imageGenerator, navigationBarData: navigationBarData)
let content: String
let results: PageGenerationResults
do {
content = try pageGenerator.generate(page: page, language: language)
(content, results) = try pageGenerator.generate(page: page, language: language)
} catch {
print("Failed to generate page \(page.id) in language \(language): \(error)")
return false
}
guard !content.trimmed.isEmpty else {
#warning("Generate page with placeholder content")
return true
}
let path = self.content.pageLink(page, language: language) + ".html"
guard save(content, to: path) else {
@ -142,22 +148,32 @@ final class WebsiteGenerator {
guard imageGenerator.runJobs(callback: { _ in }) else {
return false
}
guard copy(requiredVideoFiles: pageGenerator.results.requiredVideoFiles) else {
guard copy(requiredFiles: results.files) else {
return false
}
return true
}
private func copy(requiredVideoFiles: Set<String>) -> Bool {
print("Copying \(requiredVideoFiles.count) videos...")
for fileId in requiredVideoFiles {
guard let outputPath = content.pathToFile(fileId) else {
return false
private func copy(requiredFiles: Set<FileResource>) -> Bool {
//print("Copying \(requiredVideoFiles.count) files...")
for file in requiredFiles {
guard !file.isExternallyStored else {
continue
}
let outputPath: String
switch file.type {
case .video:
outputPath = content.pathToVideo(file)
case .image:
outputPath = content.pathToImage(file)
default:
outputPath = content.pathToFile(file)
}
do {
try content.storage.copy(file: fileId, to: outputPath)
try content.storage.copy(file: file.id, to: outputPath)
} catch {
print("Failed to copy video file: \(error)")
print("Failed to copy file \(file.id): \(error)")
return false
}
}

View File

@ -8,68 +8,86 @@ final class PageContentParser {
private let pageLinkMarker = "page:"
private let largeImageIndicator = "*large*"
private let swift = SyntaxHighlighter(format: HTMLOutputFormat())
private let results: GenerationResultsHandler
let results = PageGenerationResults()
private let content: Content
private let imageGenerator: ImageGenerator
private let page: Page
private let language: ContentLanguage
private var largeImageCount: Int = 0
init(page: Page, content: Content, language: ContentLanguage, results: GenerationResultsHandler, imageGenerator: ImageGenerator) {
self.page = page
var largeImageWidth: Int {
content.settings.pages.largeImageWidth
}
var thumbnailWidth: Int {
content.settings.pages.contentWidth
}
init(content: Content, language: ContentLanguage) {
self.content = content
self.language = language
self.results = results
self.imageGenerator = imageGenerator
}
func requestImages(_ generator: ImageGenerator) {
let thumbnailWidth = CGFloat(thumbnailWidth)
let largeImageWidth = CGFloat(largeImageWidth)
for image in results.files {
guard case .image = image.type else {
continue
}
generator.generateImageSet(
for: image.id,
maxWidth: thumbnailWidth, maxHeight: thumbnailWidth)
generator.generateImageSet(
for: image.id,
maxWidth: largeImageWidth, maxHeight: largeImageWidth)
}
}
func reset() {
results.reset()
largeImageCount = 0
}
func generatePage(from content: String) -> String {
let imageModifier = Modifier(target: .images) { html, markdown in
self.processMarkdownImage(markdown: markdown, html: html)
}
let codeModifier = Modifier(target: .codeBlocks) { html, markdown in
if markdown.starts(with: "```swift") {
let code = markdown.between("```swift", and: "```").trimmed
return "<pre><code>" + self.swift.highlight(code) + "</pre></code>"
}
return html
}
let linkModifier = Modifier(target: .links) { html, markdown in
self.handleLink(html: html, markdown: markdown)
}
let htmlModifier = Modifier(target: .html) { html, markdown in
self.handleHTML(html: html, markdown: markdown)
}
let headlinesModifier = Modifier(target: .headings) { html, markdown in
self.handleHeadlines(html: html, markdown: markdown)
}
let parser = MarkdownParser(modifiers: [imageModifier, codeModifier, linkModifier, htmlModifier, headlinesModifier])
reset()
let parser = MarkdownParser(modifiers: [
Modifier(target: .images, closure: processMarkdownImage),
Modifier(target: .codeBlocks, closure: handleCode),
Modifier(target: .links, closure: handleLink),
Modifier(target: .html, closure: handleHTML),
Modifier(target: .headings, closure: handleHeadlines)
])
return parser.html(from: content)
}
private func handleCode(html: String, markdown: Substring) -> String {
guard markdown.starts(with: "```swift") else {
return html // Just use normal code highlighting
}
// Highlight swift code using Splash
let code = markdown.between("```swift", and: "```").trimmed
return "<pre><code>" + swift.highlight(code) + "</pre></code>"
}
private func handleLink(html: String, markdown: Substring) -> String {
let file = markdown.between("(", and: ")")
if file.hasPrefix(pageLinkMarker) {
// Retain links pointing to elements within a page
let textToChange = file.dropAfterFirst("#")
let pageId = textToChange.replacingOccurrences(of: pageLinkMarker, with: "")
guard let pagePath = content.pageLink(pageId: pageId, language: language) else {
guard let page = content.page(pageId) else {
results.missingPages.insert(pageId)
// Remove link since the page can't be found
return markdown.between("[", and: "]")
}
// Adjust file path to get the page url
// TODO: Calculate relative links to make pages more portable
results.linkedPages.insert(page)
let pagePath = content.pageLink(page, language: language)
return html.replacingOccurrences(of: textToChange, with: pagePath)
}
@ -92,6 +110,13 @@ final class PageContentParser {
return html
}
/**
Modify headlines by extracting an id from the headline and adding it into the html element
Format: ##<title>#<id>
The id is created by lowercasing the string, removing all special characters, and replacing spaces with scores
*/
private func handleHeadlines(html: String, markdown: Substring) -> String {
let id = markdown
.last(after: "#")
@ -101,18 +126,18 @@ final class PageContentParser {
.components(separatedBy: " ")
.filter { $0 != "" }
.joined(separator: "-")
let parts = html.components(separatedBy: ">")
let parts = html.components(separatedBy: ">")
return parts[0] + " id=\"\(id)\">" + parts.dropFirst().joined(separator: ">")
}
private func processMarkdownImage(markdown: Substring, html: String) -> String {
private func processMarkdownImage(html: String, markdown: Substring) -> String {
// First, check the content type, then parse the remaining arguments
// Notation:
// <abc?> -> Optional argument
// <abc...> -> Repeated argument (0 or more)
// ![url](<url>;<text>)
// ![image](<imageId>;<caption?>]
// ![video](<fileId>;<alt>;<option1...>]
// ![video](<fileId>;<option1...>]
// ![svg](<fileId>;<<x>;<y>;<width>;<height>?>)
// ![download](<<fileId>,<text>,<download-filename?>;...)
// ![box](<title>;<body>)
@ -120,12 +145,11 @@ final class PageContentParser {
// ![page](<pageId>)
// ![external](<<url>;<text>...>
// ![html](<fileId>)
guard let argumentList = markdown.between(first: "](", andLast: ")").removingPercentEncoding else {
results.warning("Invalid percent encoding for markdown image", page: page)
return ""
}
let argumentList = markdown.between(first: "](", andLast: ")").removingPercentEncoding ?? markdown.between(first: "](", andLast: ")")
let arguments = argumentList.components(separatedBy: ";")
let rawCommand = markdown.between("![", and: "]").trimmed
guard rawCommand != "" else {
return handleImage(arguments)
@ -134,7 +158,7 @@ final class PageContentParser {
guard let convertedCommand = rawCommand.removingPercentEncoding,
let command = ShorthandMarkdownKey(rawValue: convertedCommand) else {
// Treat unknown commands as normal links
print("Unknown markdown command: \(rawCommand)")
results.warnings.append("Unknown markdown command '\(rawCommand)'")
return html
}
@ -147,12 +171,9 @@ final class PageContentParser {
return handleDownloadButtons(arguments)
case .video:
return handleVideo(arguments)
default:
print("Unhandled markdown command: \(command)")
return ""
/*
case .externalLink:
return handleExternalButtons(content: content)
return handleExternalButtons(arguments)
/*
case .includedHtml:
return handleExternalHTML(file: content)
case .box:
@ -162,35 +183,42 @@ final class PageContentParser {
case .model:
return handle3dModel(content: content)
*/
default:
results.warnings.append("Unhandled command '\(command.rawValue)'")
return ""
}
}
private func handleImage(_ arguments: [String]) -> String {
// [image](<imageId>;<caption?>]
guard (1...2).contains(arguments.count) else {
results.warning("Invalid image arguments: \(arguments)", page: page)
results.invalidCommandArguments.append((.image , arguments))
return ""
}
let imageId = arguments[0]
guard let image = content.image(imageId) else {
results.warning("Missing image \(imageId)", page: page)
results.missingFiles.insert(imageId)
return ""
}
results.files.insert(image)
let caption = arguments.count == 2 ? arguments[1] : nil
let altText = image.getDescription(for: language)
let thumbnailWidth = CGFloat(content.settings.pages.contentWidth)
let thumbnail = imageGenerator.generateImageSet(
for: imageId,
maxWidth: thumbnailWidth, maxHeight: thumbnailWidth,
let path = content.pathToImage(image)
let thumbnail = FeedEntryData.Image(
rawImagePath: path,
width: thumbnailWidth,
height: thumbnailWidth,
altText: altText)
let largeImageWidth = CGFloat(1200) // TODO: Move to settings
let largeImage = imageGenerator.generateImageSet(
for: imageId,
maxWidth: largeImageWidth, maxHeight: largeImageWidth,
let largeImage = FeedEntryData.Image(
rawImagePath: path,
width: largeImageWidth,
height: largeImageWidth,
altText: altText)
return PageImage(
@ -202,7 +230,7 @@ final class PageContentParser {
private func handleHikingStatistics(_ arguments: [String]) -> String {
guard (1...5).contains(arguments.count) else {
results.warning("Invalid hiking statistic arguments: \(arguments)", page: page)
results.invalidCommandArguments.append((.hikingStatistics, arguments))
return ""
}
@ -222,10 +250,11 @@ final class PageContentParser {
}
private func handleDownloadButtons(_ arguments: [String]) -> String {
let buttons: [DownloadButtons.Item] = arguments.compactMap { button in
// ![download](<<fileId>,<text>,<download-filename?>;...)
let buttons: [ContentButtons.Item] = arguments.compactMap { button in
let parts = button.components(separatedBy: ",")
guard (2...3).contains(parts.count) else {
results.warning("Invalid download definition with \(parts)", page: page)
results.invalidCommandArguments.append((.downloadButtons, parts))
return nil
}
let file = parts[0].trimmed
@ -234,44 +263,36 @@ final class PageContentParser {
// Ensure that file is available
guard let filePath = content.pathToFile(file) else {
results.warning("Missing download file \(file)", page: page)
results.missingFiles.insert(file)
return nil
}
return DownloadButtons.Item(filePath: filePath, text: title, downloadFileName: downloadName)
return ContentButtons.Item(icon: .download, filePath: filePath, text: title, downloadFileName: downloadName)
}
return DownloadButtons(items: buttons).content
return ContentButtons(items: buttons).content
}
private func handleVideo(_ arguments: [String]) -> String {
// ![video](<fileId>;<option1...>]
guard arguments.count >= 1 else {
results.invalidCommandArguments.append((.video, arguments))
return ""
}
let fileId = arguments[0].trimmed
let options: [VideoOption] = arguments.dropFirst().compactMap { optionText in
guard let optionText = optionText.trimmed.nonEmpty else {
return nil
}
guard let option = VideoOption(rawValue: optionText) else {
results.warning("Unknown video option \(optionText)", page: page)
return nil
}
return option
}
let options = arguments.dropFirst().compactMap(convertVideoOption)
guard let filePath = content.pathToFile(fileId),
let file = content.file(id: fileId) else {
results.warning("Missing video file \(fileId)", page: page)
guard let file = content.file(id: fileId) else {
results.missingFiles.insert(fileId)
return ""
}
results.files.insert(file)
guard let videoType = file.type.videoType?.htmlType else {
results.warning("Unknown video file type for \(fileId)", page: page)
results.warnings.append("Unknown video file type for \(fileId)")
return ""
}
results.addRequiredVideoFile(fileId: fileId)
let filePath = content.pathToFile(file)
return ContentPageVideo(
filePath: filePath,
videoType: videoType,
@ -279,6 +300,40 @@ final class PageContentParser {
.content
}
private func convertVideoOption(_ videoOption: String) -> VideoOption? {
guard let optionText = videoOption.trimmed.nonEmpty else {
return nil
}
guard let option = VideoOption(rawValue: optionText) else {
results.invalidCommandArguments.append((.video, [optionText]))
return nil
}
if case let .poster(imageId) = option {
if let image = content.image(imageId) {
results.files.insert(image)
let link = content.pathToImage(image)
let width = 2*thumbnailWidth
let fullLink = WebsiteImage.imagePath(source: link, width: width, height: width)
return .poster(image: fullLink)
} else {
results.missingFiles.insert(imageId)
return nil // Image file not present, so skip the option
}
}
if case let .src(videoId) = option {
if let video = content.video(videoId) {
results.files.insert(video)
let link = content.pathToVideo(video)
// TODO: Set correct video path?
return .src(link)
} else {
results.missingFiles.insert(videoId)
return nil // Video file not present, so skip the option
}
}
return option
}
/*
private func handleGif(file: String, altText: String) -> String {
@ -334,27 +389,34 @@ final class PageContentParser {
results.warning("Unhandled file \(file) with extension \(fileExtension)", source: page.path)
return ""
}
private func handleExternalButtons(content: String) -> String {
let buttons = content
.components(separatedBy: ";")
.compactMap { button -> (url: String, text: String)? in
let parts = button.components(separatedBy: ",")
guard parts.count == 2 else {
results.warning("Invalid external link definition", page: page)
return nil
}
guard let url = parts[0].trimmed.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) else {
results.warning("Invalid external link \(parts[0].trimmed)", source: page.path)
return nil
}
let title = parts[1].trimmed
return (url, title)
*/
private func handleExternalButtons(_ arguments: [String]) -> String {
// ![external](<<url>;<text>...>
guard arguments.count >= 1 else {
results.invalidCommandArguments.append((.externalLink, arguments))
return ""
}
let buttons: [ContentButtons.Item] = arguments.compactMap { button in
let parts = button.components(separatedBy: ",")
guard parts.count == 2 else {
results.invalidCommandArguments.append((.externalLink, parts))
return nil
}
return factory.html.externalButtons(buttons)
}
let rawUrl = parts[0].trimmed
guard let url = rawUrl.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) else {
results.invalidCommandArguments.append((.externalLink, parts))
return nil
}
let title = parts[1].trimmed
return .init(
icon: .externalLink,
filePath: url,
text: title)
}
return ContentButtons(items: buttons).content
}
/*
private func handleExternalHTML(file: String) -> String {
let path = page.pathRelativeToRootForContainedInputFile(file)
return results.getContentOfRequiredFile(at: path, source: page.path) ?? ""

View File

@ -0,0 +1,32 @@
import Foundation
final class PageGenerationResults: ObservableObject {
@Published
var linkedPages: Set<Page> = []
@Published
var files: Set<FileResource> = []
@Published
var missingPages: Set<String> = []
@Published
var missingFiles: Set<String> = []
@Published
var invalidCommandArguments: [(command: ShorthandMarkdownKey, arguments: [String])] = []
@Published
var warnings: [String] = []
func reset() {
linkedPages = []
files = []
missingPages = []
missingFiles = []
invalidCommandArguments = []
warnings = []
}
}

View File

@ -6,26 +6,23 @@ final class PageGenerator {
private let navigationBarData: NavigationBarData
let results = GenerationResultsHandler()
init(content: Content, imageGenerator: ImageGenerator, navigationBarData: NavigationBarData) {
self.content = content
self.imageGenerator = imageGenerator
self.navigationBarData = navigationBarData
}
func generate(page: Page, language: ContentLanguage) throws -> String {
func generate(page: Page, language: ContentLanguage) throws -> (page: String, results: PageGenerationResults) {
let contentGenerator = PageContentParser(
page: page,
content: content,
language: language,
results: results,
imageGenerator: imageGenerator)
language: language)
let rawPageContent = try content.storage.pageContent(for: page.id, language: language)
let pageContent = contentGenerator.generatePage(from: rawPageContent)
contentGenerator.requestImages(imageGenerator)
let localized = page.localized(in: language)
let tags: [FeedEntryData.Tag] = page.tags.map { tag in
@ -33,7 +30,7 @@ final class PageGenerator {
url: content.tagLink(tag, language: language))
}
return ContentPage(
let fullPage = ContentPage(
language: language,
dateString: page.dateText(in: language),
title: localized.title,
@ -43,5 +40,7 @@ final class PageGenerator {
navigationBarData: navigationBarData,
pageContent: pageContent)
.content
return (fullPage, contentGenerator.results)
}
}

View File

@ -66,7 +66,7 @@ final class PostListPageGenerator {
text: language == .english ? "View" : "Anzeigen") // TODO: Add to settings
}
let tags: [FeedEntryData.Tag] = post.tags.map { tag in
let tags: [FeedEntryData.Tag] = post.tags.filter { $0.isVisible }.map { tag in
.init(name: tag.localized(in: language).name,
url: content.tagLink(tag, language: language))
}
@ -102,7 +102,11 @@ final class PostListPageGenerator {
imageGenerator.generateImageSet(
for: image.id,
maxWidth: mainContentMaximumWidth,
maxHeight: mainContentMaximumWidth,
maxHeight: mainContentMaximumWidth)
return .init(
rawImagePath: content.pathToImage(image),
width: Int(mainContentMaximumWidth),
height: Int(mainContentMaximumWidth),
altText: image.getDescription(for: language))
}

View File

@ -18,7 +18,7 @@ enum ShorthandMarkdownKey: String {
case hikingStatistics = "hiking-stats"
/// A video
/// Format: `![video](<fileId>;<alt>;<option1...>]`
/// Format: `![video](<fileId>;<option1...>]`
case video
/// An SVG image

View File

@ -1,11 +1,123 @@
/// HTML video options
enum VideoOption: String {
enum VideoOption {
/// Specifies that video controls should be displayed (such as a play/pause button etc).
case controls
/// Specifies that the video will start playing as soon as it is ready
case autoplay
case muted
/// Specifies that the video will start over again, every time it is finished
case loop
/// Specifies that the audio output of the video should be muted
case muted
/// Mobile browsers will play the video right where it is instead of the default, which is to open it up fullscreen while it plays
case playsinline
case poster
case preload
/// Sets the height of the video player
case height(Int)
/// Sets the width of the video player
case width(Int)
/// Specifies if and how the author thinks the video should be loaded when the page loads
case preload(VideoPreloadOption)
/// Specifies an image to be shown while the video is downloading, or until the user hits the play button
case poster(image: String)
/// Specifies the URL of the video file
case src(String)
init?(rawValue: String) {
switch rawValue {
case "controls":
self = .controls
return
case "autoplay":
self = .autoplay
return
case "muted":
self = .muted
return
case "loop":
self = .loop
return
case "playsinline":
self = .playsinline
return
default: break
}
let parts = rawValue.components(separatedBy: "=")
guard parts.count == 2 else {
return nil
}
let optionName = parts[0]
let value = parts[1].removingSurroundingQuotes
switch optionName {
case "height":
guard let height = Int(value) else {
return nil
}
self = .height(height)
case "width":
guard let width = Int(value) else {
return nil
}
self = .width(width)
case "preload":
guard let preloadOption = VideoPreloadOption(rawValue: value) else {
return nil
}
self = .preload(preloadOption)
case "poster":
self = .poster(image: value)
case "src":
self = .src(value)
default:
return nil
}
return
}
var rawValue: String {
switch self {
case .controls: return "controls"
case .autoplay: return "autoplay"
case .muted: return "muted"
case .loop: return "loop"
case .playsinline: return "playsinline"
case .height(let height): return "height='\(height)'"
case .width(let width): return "width='\(width)'"
case .preload(let option): return "preload='\(option)'"
case .poster(let image): return "poster='\(image)'"
case .src(let url): return "src='\(url)'"
}
}
}
/**
The `preload` attribute specifies if and how the author thinks that the video should be loaded when the page loads.
The `preload` attribute allows the author to provide a hint to the browser about what he/she thinks will lead to the best user experience.
This attribute may be ignored in some instances.
Note: The `preload` attribute is ignored if `autoplay` is present.
*/
enum VideoPreloadOption: String {
/// The author thinks that the browser should load the entire video when the page loads
case auto
/// The author thinks that the browser should load only metadata when the page loads
case metadata
/// The author thinks that the browser should NOT load the video when the page loads
case none
}