Full generation, file type cleanup

This commit is contained in:
Christoph Hagen
2024-12-25 18:06:05 +01:00
parent 41887a1401
commit 1e4682dad1
56 changed files with 1577 additions and 1103 deletions

View File

@@ -1,5 +1,5 @@
enum PageContentAnomaly {
enum GenerationAnomaly {
case failedToLoadContent
case failedToParseContent
case missingFile(file: String, markdown: String)
@@ -9,7 +9,7 @@ enum PageContentAnomaly {
case warning(String)
}
extension PageContentAnomaly: Identifiable {
extension GenerationAnomaly: Identifiable {
var id: String {
switch self {
@@ -31,21 +31,21 @@ extension PageContentAnomaly: Identifiable {
}
}
extension PageContentAnomaly: Equatable {
extension GenerationAnomaly: Equatable {
static func == (lhs: PageContentAnomaly, rhs: PageContentAnomaly) -> Bool {
static func == (lhs: GenerationAnomaly, rhs: GenerationAnomaly) -> Bool {
lhs.id == rhs.id
}
}
extension PageContentAnomaly: Hashable {
extension GenerationAnomaly: Hashable {
func hash(into hasher: inout Hasher) {
hasher.combine(id)
}
}
extension PageContentAnomaly {
extension GenerationAnomaly {
enum Severity: String, CaseIterable {
case warning
@@ -62,7 +62,7 @@ extension PageContentAnomaly {
}
}
extension PageContentAnomaly: CustomStringConvertible {
extension GenerationAnomaly: CustomStringConvertible {
var description: String {
switch self {

View File

@@ -0,0 +1,199 @@
import Foundation
struct LocalizedPageId: Hashable {
let language: ContentLanguage
let pageId: String
}
final class GenerationResults: ObservableObject {
/// The files that could not be accessed
@Published
var inaccessibleFiles: Set<FileResource> = []
/// The files that could not be parsed, with the error message produced
@Published
var unparsableFiles: Set<FileResource> = []
@Published
var missingFiles: Set<String> = []
@Published
var missingTags: Set<String> = []
@Published
var missingPages: Set<String> = []
@Published
var externalLinks: Set<String> = []
@Published
var requiredFiles: Set<FileResource> = []
@Published
var imagesToGenerate: Set<ImageGenerationJob> = []
@Published
var invalidCommands: Set<String> = []
@Published
var warnings: Set<String> = []
@Published
var unsavedOutputFiles: Set<String> = []
@Published
var failedImages: Set<ImageGenerationJob> = []
@Published
var emptyPages: Set<LocalizedPageId> = []
/// The cache of previously used GenerationResults
private var cache: [ItemId : PageGenerationResults] = [:]
private(set) var general: PageGenerationResults!
var resultCount: Int {
cache.count
}
// MARK: Life cycle
init() {
let id = ItemId(language: .english, itemType: .general)
let general = PageGenerationResults(itemId: id, delegate: self)
self.general = general
cache[id] = general
}
func makeResults(_ itemId: ItemId) -> PageGenerationResults {
guard let result = cache[itemId] else {
let result = PageGenerationResults(itemId: itemId, delegate: self)
cache[itemId] = result
return result
}
return result
}
func makeResults(for type: ItemType, in language: ContentLanguage) -> PageGenerationResults {
let itemId = ItemId(language: language, itemType: type)
return makeResults(itemId)
}
func makeResults(for page: Page, in language: ContentLanguage) -> PageGenerationResults {
let itemId = ItemId(language: language, itemType: .page(page))
return makeResults(itemId)
}
func makeResults(for tag: Tag, in language: ContentLanguage) -> PageGenerationResults {
let itemId = ItemId(language: language, itemType: .tagPage(tag))
return makeResults(itemId)
}
func recalculate() {
let inaccessibleFiles = cache.values.map { $0.inaccessibleFiles }.union()
update { self.inaccessibleFiles = inaccessibleFiles }
let unparsableFiles = cache.values.map { $0.unparsableFiles.keys }.union()
update { self.unparsableFiles = unparsableFiles }
let missingFiles = cache.values.map { $0.missingFiles.keys }.union()
update { self.missingFiles = missingFiles }
let missingTags = cache.values.map { $0.missingLinkedTags.keys }.union()
update { self.missingTags = missingTags }
let missingPages = cache.values.map { $0.missingLinkedPages.keys }.union()
update { self.missingPages = missingPages }
let externalLinks = cache.values.map { $0.externalLinks }.union()
update { self.externalLinks = externalLinks }
let requiredFiles = cache.values.map { $0.requiredFiles }.union()
update { self.requiredFiles = requiredFiles }
let imagesToGenerate = cache.values.map { $0.imagesToGenerate }.union()
update { self.imagesToGenerate = imagesToGenerate }
let invalidCommands = cache.values.map { $0.invalidCommands.map { $0.markdown }}.union()
update { self.invalidCommands = invalidCommands }
let warnings = cache.values.map { $0.warnings }.union()
update { self.warnings = warnings }
let unsavedOutputFiles = cache.values.map { $0.unsavedOutputFiles.keys }.union()
update { self.unsavedOutputFiles = unsavedOutputFiles }
}
private func update(_ operation: @escaping () -> Void) {
DispatchQueue.main.async {
operation()
}
}
// MARK: Adding entries
func inaccessibleContent(file: FileResource) {
update { self.inaccessibleFiles.insert(file) }
}
func unparsable(file: FileResource) {
update { self.unparsableFiles.insert(file) }
}
func missing(file: String) {
update { self.missingFiles.insert(file) }
}
func missing(tag: String) {
update { self.missingTags.insert(tag) }
}
func missing(page: String) {
update { self.missingPages.insert(page) }
}
func externalLink(_ url: String) {
update { self.externalLinks.insert(url) }
}
func require(file: FileResource) {
update { self.requiredFiles.insert(file) }
}
func require<S>(files: S) where S: Sequence, S.Element == FileResource {
update { self.requiredFiles.formUnion(files) }
}
func generate(_ image: ImageGenerationJob) {
update { self.imagesToGenerate.insert(image) }
}
func generate<S>(_ images: S) where S: Sequence, S.Element == ImageGenerationJob {
update { self.imagesToGenerate.formUnion(images) }
}
func invalidCommand(_ markdown: String) {
update { self.invalidCommands.insert(markdown) }
}
func warning(_ warning: String) {
update { self.warnings.insert(warning) }
}
func failed(image: ImageGenerationJob) {
update { self.failedImages.insert(image) }
}
func unsaved(_ path: String) {
update { self.unsavedOutputFiles.insert(path) }
}
}
private extension Dictionary where Value == Set<ItemId> {
mutating func remove<S>(keys: S, of item: ItemId) where S: Sequence, S.Element == Key {
for key in keys {
guard var value = self[key] else { continue }
value.remove(item)
if value.isEmpty {
self[key] = nil
} else {
self[key] = value
}
}
}
}

View File

@@ -97,14 +97,14 @@ extension HeaderElement {
var content: String {
switch self {
case .icon(let file, let size, let rel):
return "<link rel='\(rel)' sizes='\(size)x\(size)' href='\(file.assetUrl)'>"
return "<link rel='\(rel)' sizes='\(size)x\(size)' href='\(file.absoluteUrl)'>"
case .css(let file, _):
return "<link rel='stylesheet' href='\(file.assetUrl)' />"
return "<link rel='stylesheet' href='\(file.absoluteUrl)' />"
case .js(let file, let deferred):
let deferText = deferred ? " defer" : ""
return "<script src='\(file.assetUrl)'\(deferText)></script>"
return "<script src='\(file.absoluteUrl)'\(deferText)></script>"
case .jsModule(let file):
return "<script type='module' src='\(file.assetUrl)'></script>"
return "<script type='module' src='\(file.absoluteUrl)'></script>"
case .author(let author):
return "<meta name='author' content='\(author)'>"
case .title(let title):

View File

@@ -11,8 +11,6 @@ final class ImageGenerator {
private var generatedImages: [String : Set<String>] = [:]
private var jobs: [ImageGenerationJob] = []
init(storage: Storage, settings: Settings) {
self.storage = storage
self.settings = settings
@@ -23,20 +21,6 @@ final class ImageGenerator {
settings.paths.imagesOutputFolderPath
}
func runJobs(callback: (String) -> Void) -> Bool {
guard !jobs.isEmpty else {
return true
}
print("Generating \(jobs.count) images...")
while let job = jobs.popLast() {
callback("Generating image \(job.version)")
guard generate(job: job) else {
return false
}
}
return true
}
func save() -> Bool {
guard storage.save(listOfGeneratedImages: generatedImages) else {
print("Failed to save list of generated images")
@@ -45,50 +29,6 @@ final class ImageGenerator {
return true
}
private func versionFileName(image: String, type: ImageFileType, width: CGFloat, height: CGFloat) -> String {
let fileName = image.fileNameAndExtension.fileName
let prefix = "\(fileName)@\(Int(width))x\(Int(height))"
return "\(prefix).\(type.fileExtension)"
}
func generateImageSet(for image: String, maxWidth: CGFloat, maxHeight: CGFloat) {
let type = ImageFileType(fileExtension: image.fileExtension!)!
let width2x = maxWidth * 2
let height2x = maxHeight * 2
generateVersion(for: image, type: .avif, maximumWidth: maxWidth, maximumHeight: maxHeight)
generateVersion(for: image, type: .avif, maximumWidth: width2x, maximumHeight: height2x)
generateVersion(for: image, type: .webp, maximumWidth: maxWidth, maximumHeight: maxHeight)
generateVersion(for: image, type: .webp, maximumWidth: width2x, maximumHeight: height2x)
generateVersion(for: image, type: type, maximumWidth: maxWidth, maximumHeight: maxHeight)
generateVersion(for: image, type: type, maximumWidth: width2x, maximumHeight: height2x)
}
func generateVersion(for image: String, type: ImageFileType, maximumWidth: CGFloat, maximumHeight: CGFloat) {
let version = versionFileName(image: image, type: type, width: maximumWidth, height: maximumHeight)
guard needsToGenerate(version: version, for: image) else {
// Image already present
return
}
guard !jobs.contains(where: { $0.version == version }) else {
// Job already in queue
return
}
let job = ImageGenerationJob(
image: image,
version: version,
maximumWidth: maximumWidth,
maximumHeight: maximumHeight,
quality: 0.7,
type: type)
jobs.append(job)
}
/**
Remove all versions of an image, so that they will be recreated on the next run.
@@ -105,6 +45,9 @@ final class ImageGenerator {
}
private func needsToGenerate(version: String, for image: String) -> Bool {
if exists(version) {
return false
}
guard let versions = generatedImages[image] else {
return true
}
@@ -143,7 +86,7 @@ final class ImageGenerator {
// MARK: Image operations
private func generate(job: ImageGenerationJob) -> Bool {
func generate(job: ImageGenerationJob) -> Bool {
guard needsToGenerate(version: job.version, for: job.image) else {
return true
}
@@ -158,7 +101,7 @@ final class ImageGenerator {
return false
}
let representation = create(image: originalImage, width: job.maximumWidth, height: job.maximumHeight)
let representation = create(image: originalImage, width: CGFloat(job.maximumWidth), height: CGFloat(job.maximumHeight))
guard let data = create(image: representation, type: job.type, quality: job.quality) else {
print("Failed to get data for type \(job.type)")
@@ -209,7 +152,7 @@ final class ImageGenerator {
// MARK: Avif images
private func create(image: NSBitmapImageRep, type: ImageFileType, quality: CGFloat) -> Data? {
private func create(image: NSBitmapImageRep, type: FileType, quality: CGFloat) -> Data? {
switch type {
case .jpg:
return image.representation(using: .jpeg, properties: [.compressionFactor: NSNumber(value: 0.6)])
@@ -225,6 +168,8 @@ final class ImageGenerator {
return nil
case .tiff:
return nil
default:
return nil
}
}

View File

@@ -4,13 +4,70 @@ struct ImageGenerationJob {
let image: String
let version: String
let type: FileType
let maximumWidth: CGFloat
let maximumWidth: Int
let maximumHeight: CGFloat
let maximumHeight: Int
let quality: CGFloat
let type: ImageFileType
init(image: String, type: FileType, maximumWidth: CGFloat, maximumHeight: CGFloat, quality: CGFloat = 0.7) {
self.image = image
self.type = type
self.maximumWidth = Int(maximumWidth)
self.maximumHeight = Int(maximumHeight)
self.quality = quality
}
init(image: String, type: FileType, maximumWidth: Int, maximumHeight: Int, quality: CGFloat = 0.7) {
self.image = image
self.type = type
self.maximumWidth = maximumWidth
self.maximumHeight = maximumHeight
self.quality = quality
}
var version: String {
let fileName = image.fileNameAndExtension.fileName
let prefix = "\(fileName)@\(maximumWidth)x\(maximumHeight)"
return "\(prefix).\(type.fileExtension)"
}
static func imageSet(for image: String, maxWidth: Int, maxHeight: Int, quality: CGFloat = 0.7) -> [ImageGenerationJob] {
let type = FileType(fileExtension: image.fileExtension)
let width2x = maxWidth * 2
let height2x = maxHeight * 2
return [
.init(image: image, type: .avif, maximumWidth: maxWidth, maximumHeight: maxHeight, quality: quality),
.init(image: image, type: .avif, maximumWidth: width2x, maximumHeight: height2x, quality: quality),
.init(image: image, type: .webp, maximumWidth: maxWidth, maximumHeight: maxHeight, quality: quality),
.init(image: image, type: .webp, maximumWidth: width2x, maximumHeight: height2x, quality: quality),
.init(image: image, type: type, maximumWidth: maxWidth, maximumHeight: maxHeight, quality: quality),
.init(image: image, type: type, maximumWidth: width2x, maximumHeight: height2x, quality: quality)
]
}
}
extension ImageGenerationJob: Equatable {
static func == (lhs: ImageGenerationJob, rhs: ImageGenerationJob) -> Bool {
lhs.version == rhs.version
}
}
extension ImageGenerationJob: Hashable {
func hash(into hasher: inout Hasher) {
hasher.combine(version)
}
}
extension ImageGenerationJob: Comparable {
static func < (lhs: ImageGenerationJob, rhs: ImageGenerationJob) -> Bool {
lhs.version < rhs.version
}
}

View File

@@ -1,97 +0,0 @@
import Foundation
final class LocalizedWebsiteGenerator {
private let content: Content
let language: ContentLanguage
private let imageGenerator: ImageGenerator
private let localizedPostSettings: LocalizedPostSettings
init(content: Content, language: ContentLanguage) {
self.language = language
self.content = content
self.localizedPostSettings = content.settings.localized(in: language)
self.imageGenerator = ImageGenerator(
storage: content.storage,
settings: content.settings)
}
private var postsPerPage: Int {
content.settings.posts.postsPerPage
}
private var mainContentMaximumWidth: CGFloat {
CGFloat(content.settings.posts.contentWidth)
}
func generateWebsite(callback: (String) -> Void) -> Bool {
guard createMainPostFeedPages() else {
return false
}
#warning("Generate content pages")
#warning("Generate tag overview page")
guard generateTagPages() else {
return false
}
guard imageGenerator.runJobs(callback: callback) else {
return false
}
return imageGenerator.save()
}
private func createMainPostFeedPages() -> Bool {
let generator = PostListPageGenerator(
language: language,
content: content,
imageGenerator: imageGenerator,
showTitle: false,
pageTitle: localizedPostSettings.title,
pageDescription: localizedPostSettings.description,
pageUrlPrefix: localizedPostSettings.feedUrlPrefix)
return generator.createPages(for: content.posts)
}
private func generateTagPages() -> Bool {
for tag in content.tags {
let posts = content.posts.filter { $0.tags.contains(tag) }
guard posts.count > 0 else { continue }
let localized = tag.localized(in: language)
let urlPrefix = content.absoluteUrlPrefixForTag(tag, language: language)
let generator = PostListPageGenerator(
language: language,
content: content,
imageGenerator: imageGenerator,
showTitle: true,
pageTitle: localized.name,
pageDescription: localized.description ?? "",
pageUrlPrefix: urlPrefix)
guard generator.createPages(for: posts) else {
return false
}
}
return true
}
private func copy(requiredFiles: Set<FileResource>) -> Bool {
//print("Copying \(requiredVideoFiles.count) files...")
for file in requiredFiles {
guard !file.isExternallyStored else {
continue
}
guard content.storage.copy(file: file.id, to: file.absoluteUrl) else {
return false
}
}
return true
}
private func save(_ content: String, to relativePath: String) -> Bool {
self.content.storage.write(content, to: relativePath)
}
}

View File

@@ -27,18 +27,18 @@ struct AudioPlayerCommandProcessor: CommandProcessor {
}
guard let file = content.file(fileId) else {
results.missingFiles.insert(fileId)
results.missing(file: fileId, source: "Audio player song list")
return ""
}
guard let data = file.dataContent() else {
results.issues.insert(.failedToLoadContent)
results.inaccessibleContent(file: file)
return ""
}
let songs: [Song]
do {
songs = try JSONDecoder().decode([Song].self, from: data)
} catch {
results.issues.insert(.failedToParseContent)
results.invalidFormat(file: file, error: "Not valid JSON containing [Song]: \(error)")
return ""
}
@@ -47,12 +47,12 @@ struct AudioPlayerCommandProcessor: CommandProcessor {
for song in songs {
guard let image = content.image(song.cover) else {
results.missing(file: song.cover, markdown: "Missing cover image \(song.cover) in \(file.id)")
results.missing(file: song.cover, containedIn: file)
continue
}
guard let audioFile = content.file(song.file) else {
results.missing(file: song.file, markdown: "Missing audio file \(song.file) in \(file.id)")
results.missing(file: song.cover, containedIn: file)
continue
}
#warning("Check if file is audio")
@@ -79,18 +79,17 @@ struct AudioPlayerCommandProcessor: CommandProcessor {
}
let footerScript = AudioPlayerScript(items: amplitude).content
results.requiredFooters.insert(footerScript)
results.requiredHeaders.insert(.audioPlayerCss)
results.requiredHeaders.insert(.audioPlayerJs)
results.require(footer: footerScript)
results.require(headers: .audioPlayerCss, .audioPlayerJs)
results.requiredIcons.formUnion([
results.require(icons:
.audioPlayerClose,
.audioPlayerPlaylist,
.audioPlayerNext,
.audioPlayerPrevious,
.audioPlayerPlay,
.audioPlayerPause
])
)
return AudioPlayer(playingText: titleText, items: playlist).content
}

View File

@@ -57,11 +57,11 @@ struct ButtonCommandProcessor: CommandProcessor {
let downloadName = arguments.count > 2 ? arguments[2].trimmed : nil
guard let file = content.file(fileId) else {
results.missing(file: fileId, markdown: markdown)
results.missing(file: fileId, source: "Download button")
return nil
}
results.files.insert(file)
results.requiredIcons.insert(.buttonDownload)
results.require(file: file)
results.require(icon: .buttonDownload)
return ContentButtons.Item(
icon: .buttonDownload,
filePath: file.absoluteUrl,
@@ -80,8 +80,8 @@ struct ButtonCommandProcessor: CommandProcessor {
return nil
}
results.externalLinks.insert(rawUrl)
results.requiredIcons.insert(icon)
results.externalLink(to: rawUrl)
results.require(icon: icon)
let title = arguments[1].trimmed
@@ -96,7 +96,7 @@ struct ButtonCommandProcessor: CommandProcessor {
let text = arguments[0].trimmed
let event = arguments[1].trimmed
results.requiredIcons.insert(.buttonPlay)
results.require(icon: .buttonPlay)
return .init(icon: .buttonPlay, filePath: nil, text: text, onClickText: event)
}

View File

@@ -0,0 +1,75 @@
struct InlineLinkProcessor {
private let pageLinkMarker = "page:"
private let tagLinkMarker = "tag:"
private let fileLinkMarker = "file:"
let content: Content
let results: PageGenerationResults
let language: ContentLanguage
func handleLink(html: String, markdown: Substring) -> String {
let url = markdown.between("(", and: ")")
if url.hasPrefix(pageLinkMarker) {
return handleInlinePageLink(url: url, html: html, markdown: markdown)
}
if url.hasPrefix(tagLinkMarker) {
return handleInlineTagLink(url: url, html: html, markdown: markdown)
}
if url.hasPrefix(fileLinkMarker) {
return handleInlineFileLink(url: url, html: html, markdown: markdown)
}
results.externalLink(to: url)
return html
}
private func handleInlinePageLink(url: String, html: String, markdown: Substring) -> String {
// Retain links pointing to elements within a page
let textToChange = url.dropAfterFirst("#")
let pageId = textToChange.replacingOccurrences(of: pageLinkMarker, with: "")
guard let page = content.page(pageId) else {
results.missing(page: pageId, source: "Inline page link")
// Remove link since the page can't be found
return markdown.between("[", and: "]")
}
guard !page.isDraft else {
return markdown.between("[", and: "]")
}
results.linked(to: page)
let pagePath = page.absoluteUrl(in: language)
return html.replacingOccurrences(of: textToChange, with: pagePath)
}
private func handleInlineTagLink(url: String, html: String, markdown: Substring) -> String {
// Retain links pointing to elements within a page
let textToChange = url.dropAfterFirst("#")
let tagId = textToChange.replacingOccurrences(of: tagLinkMarker, with: "")
guard let tag = content.tag(tagId) else {
results.missing(tag: tagId, source: "Inline tag link")
// Remove link since the tag can't be found
return markdown.between("[", and: "]")
}
results.linked(to: tag)
let tagPath = content.absoluteUrlToTag(tag, language: language)
return html.replacingOccurrences(of: textToChange, with: tagPath)
}
private func handleInlineFileLink(url: String, html: String, markdown: Substring) -> String {
// Retain links pointing to elements within a page
let fileId = url.replacingOccurrences(of: fileLinkMarker, with: "")
guard let file = content.file(fileId) else {
results.missing(file: fileId, source: "Inline file link")
// Remove link since the file can't be found
return markdown.between("[", and: "]")
}
results.require(file: file)
let filePath = file.absoluteUrl
return html.replacingOccurrences(of: url, with: filePath)
}
}

View File

@@ -23,7 +23,7 @@ struct LabelsCommandProcessor: CommandProcessor {
results.invalid(command: .labels, markdown)
return nil
}
results.requiredIcons.insert(icon)
results.require(icon: icon)
return .init(icon: icon, value: parts[1])
}
return ContentLabels(labels: labels).content

View File

@@ -3,29 +3,25 @@ import Ink
import Splash
import SwiftSoup
typealias VideoSource = (url: String, type: VideoFileType)
final class PageContentParser {
private let pageLinkMarker = "page:"
private let tagLinkMarker = "tag:"
private static let codeHighlightFooter = "<script>hljs.highlightAll();</script>"
private let swift = SyntaxHighlighter(format: HTMLOutputFormat())
let results = PageGenerationResults()
private let language: ContentLanguage
private let content: Content
private let results: PageGenerationResults
private let buttonHandler: ButtonCommandProcessor
private let labelHandler: LabelsCommandProcessor
private let audioPlayer: AudioPlayerCommandProcessor
let language: ContentLanguage
private let inlineLink: InlineLinkProcessor
var largeImageWidth: Int {
content.settings.pages.largeImageWidth
@@ -35,33 +31,21 @@ final class PageContentParser {
content.settings.pages.contentWidth
}
init(content: Content, language: ContentLanguage) {
init(content: Content, language: ContentLanguage, results: PageGenerationResults) {
self.content = content
self.results = results
self.language = language
self.buttonHandler = .init(content: content, results: results)
self.labelHandler = .init(content: content, results: results)
self.audioPlayer = .init(content: content, results: results)
}
func requestImages(_ generator: ImageGenerator) {
for request in results.imagesToGenerate {
generator.generateImageSet(
for: request.image.id,
maxWidth: CGFloat(request.size),
maxHeight: CGFloat(request.size))
}
}
func reset() {
results.reset()
self.inlineLink = .init(content: content, results: results, language: language)
}
func generatePage(from content: String) -> String {
reset()
let parser = MarkdownParser(modifiers: [
Modifier(target: .images, closure: processMarkdownImage),
Modifier(target: .codeBlocks, closure: handleCode),
Modifier(target: .links, closure: handleLink),
Modifier(target: .links, closure: inlineLink.handleLink),
Modifier(target: .html, closure: handleHTML),
Modifier(target: .headings, closure: handleHeadlines)
])
@@ -70,8 +54,8 @@ final class PageContentParser {
private func handleCode(html: String, markdown: Substring) -> String {
guard markdown.starts(with: "```swift") else {
results.requiredHeaders.insert(.codeHightlighting)
results.requiredFooters.insert(PageContentParser.codeHighlightFooter)
results.require(header: .codeHightlighting)
results.require(footer: PageContentParser.codeHighlightFooter)
return html // Just use normal code highlighting
}
// Highlight swift code using Splash
@@ -79,46 +63,6 @@ final class PageContentParser {
return "<pre><code>" + swift.highlight(code) + "</pre></code>"
}
private func handleLink(html: String, markdown: Substring) -> String {
let file = markdown.between("(", and: ")")
if file.hasPrefix(pageLinkMarker) {
return handlePageLink(file: file, html: html, markdown: markdown)
}
if file.hasPrefix(tagLinkMarker) {
return handleTagLink(file: file, html: html, markdown: markdown)
}
results.externalLinks.insert(file)
return html
}
private func handlePageLink(file: String, html: String, markdown: Substring) -> String {
// Retain links pointing to elements within a page
let textToChange = file.dropAfterFirst("#")
let pageId = textToChange.replacingOccurrences(of: pageLinkMarker, with: "")
guard let page = content.page(pageId) else {
results.missing(page: pageId, markdown: markdown)
// Remove link since the page can't be found
return markdown.between("[", and: "]")
}
results.linkedPages.insert(page)
let pagePath = page.absoluteUrl(in: language)
return html.replacingOccurrences(of: textToChange, with: pagePath)
}
private func handleTagLink(file: String, html: String, markdown: Substring) -> String {
// Retain links pointing to elements within a page
let textToChange = file.dropAfterFirst("#")
let tagId = textToChange.replacingOccurrences(of: tagLinkMarker, with: "")
guard let tag = content.tag(tagId) else {
results.missing(tag: tagId, markdown: markdown)
// Remove link since the tag can't be found
return markdown.between("[", and: "]")
}
results.linkedTags.insert(tag)
let tagPath = content.absoluteUrlToTag(tag, language: language)
return html.replacingOccurrences(of: textToChange, with: tagPath)
}
private func handleHTML(html: String, _: Substring) -> String {
findResourcesInHtml(html: html)
return html
@@ -144,7 +88,7 @@ final class PageContentParser {
.filter { !$0.trimmed.isEmpty }
for src in srcAttributes {
results.issues.insert(.warning("Found image in html: \(src)"))
results.warning("Found image in html: \(src)")
}
} catch {
print("Error parsing HTML: \(error)")
@@ -166,9 +110,9 @@ final class PageContentParser {
for url in srcAttributes {
if url.hasPrefix("http://") || url.hasPrefix("https://") {
results.externalLinks.insert(url)
results.externalLink(to: url)
} else {
results.issues.insert(.warning("Relative link in HTML: \(url)"))
results.warning("Relative link in HTML: \(url)")
}
}
} catch {
@@ -190,7 +134,7 @@ final class PageContentParser {
.filter { !$0.trimmed.isEmpty }
for src in srcsetAttributes {
results.issues.insert(.warning("Found source set in html: \(src)"))
results.warning("Found source set in html: \(src)")
}
let srcAttributes = try linkElements.array()
@@ -199,14 +143,15 @@ final class PageContentParser {
for src in srcAttributes {
guard content.isValidIdForFile(src) else {
results.issues.insert(.warning("Found source in html: \(src)"))
results.warning("Found source in html: \(src)")
continue
}
guard let file = content.file(src) else {
results.issues.insert(.warning("Found source in html: \(src)"))
results.warning("Found source in html: \(src)")
continue
}
results.files.insert(file)
#warning("Either find files by their full path, or replace file id with full path")
results.require(file: file)
}
} catch {
print("Error parsing HTML: \(error)")
@@ -285,7 +230,7 @@ final class PageContentParser {
}
/**
Format: `[image](<imageId>;<caption?>]`
Format: `![image](<imageId>;<caption?>]`
*/
private func handleImage(_ arguments: [String], markdown: Substring) -> String {
guard (1...2).contains(arguments.count) else {
@@ -295,10 +240,10 @@ final class PageContentParser {
let imageId = arguments[0]
guard let image = content.image(imageId) else {
results.missing(file: imageId, markdown: markdown)
results.missing(file: imageId, source: "Image command")
return ""
}
results.files.insert(image)
results.used(file: image)
let caption = arguments.count == 2 ? arguments[1] : nil
let altText = image.localized(in: language)
@@ -314,14 +259,14 @@ final class PageContentParser {
width: thumbnailWidth,
height: thumbnailWidth,
altText: altText)
results.imagesToGenerate.insert(.init(size: thumbnailWidth, image: image))
results.requireImageSet(for: image, size: thumbnailWidth)
let largeImage = FeedEntryData.Image(
rawImagePath: path,
width: largeImageWidth,
height: largeImageWidth,
altText: altText)
results.imagesToGenerate.insert(.init(size: largeImageWidth, image: image))
results.requireImageSet(for: image, size: largeImageWidth)
return PageImage(
imageId: imageId.replacingOccurrences(of: ".", with: "-"),
@@ -343,12 +288,13 @@ final class PageContentParser {
let options = arguments.dropFirst().compactMap { convertVideoOption($0, markdown: markdown) }
guard let file = content.file(fileId) else {
results.missing(file: fileId, markdown: markdown)
results.missing(file: fileId, source: "Video command")
return ""
}
results.files.insert(file)
#warning("Create/specify video alternatives")
results.require(file: file)
guard let videoType = file.type.videoType?.htmlType else {
guard let videoType = file.type.htmlType else {
results.invalid(command: .video, markdown)
return ""
}
@@ -370,23 +316,22 @@ final class PageContentParser {
}
if case let .poster(imageId) = option {
if let image = content.image(imageId) {
results.files.insert(image)
results.used(file: image)
let width = 2*thumbnailWidth
let fullLink = WebsiteImage.imagePath(source: image.absoluteUrl, width: width, height: width)
return .poster(image: fullLink)
} else {
results.missing(file: imageId, markdown: markdown)
results.missing(file: imageId, source: "Video command poster")
return nil // Image file not present, so skip the option
}
}
if case let .src(videoId) = option {
if let video = content.video(videoId) {
results.files.insert(video)
results.used(file: video)
let link = video.absoluteUrl
// TODO: Set correct video path?
return .src(link)
} else {
results.missing(file: videoId, markdown: markdown)
results.missing(file: videoId, source: "Video command source")
return nil // Video file not present, so skip the option
}
}
@@ -403,7 +348,7 @@ final class PageContentParser {
}
let fileId = arguments[0]
guard let file = content.file(fileId) else {
results.missing(file: fileId, markdown: markdown)
results.missing(file: fileId, source: "External HTML command")
return ""
}
let content = file.textContent()
@@ -435,7 +380,7 @@ final class PageContentParser {
let pageId = arguments[0]
guard let page = content.page(pageId) else {
results.missing(page: pageId, markdown: markdown)
results.missing(page: pageId, source: "Page link command")
return ""
}
guard !page.isDraft else {
@@ -443,6 +388,8 @@ final class PageContentParser {
return ""
}
results.linked(to: page)
let localized = page.localized(in: language)
let url = page.absoluteUrl(in: language)
let title = localized.linkPreviewTitle ?? localized.title
@@ -450,8 +397,8 @@ final class PageContentParser {
let image = localized.linkPreviewImage.map { image in
let size = content.settings.pages.pageLinkImageSize
results.files.insert(image)
results.imagesToGenerate.insert(.init(size: size, image: image))
results.used(file: image)
results.requireImageSet(for: image, size: size)
return RelatedPageLink.Image(
url: image.absoluteUrl,
@@ -478,7 +425,7 @@ final class PageContentParser {
let tagId = arguments[0]
guard let tag = content.tag(tagId) else {
results.missing(tag: tagId, markdown: markdown)
results.missing(tag: tagId, source: "Tag link command")
return ""
}
@@ -489,8 +436,7 @@ final class PageContentParser {
let image = localized.linkPreviewImage.map { image in
let size = content.settings.pages.pageLinkImageSize
results.files.insert(image)
results.imagesToGenerate.insert(.init(size: size, image: image))
results.requireImageSet(for: image, size: size)
return RelatedPageLink.Image(
url: image.absoluteUrl,
@@ -521,11 +467,11 @@ final class PageContentParser {
}
guard let file = content.file(fileId) else {
results.missing(file: fileId, markdown: markdown)
results.missing(file: fileId, source: "Model command")
return ""
}
results.files.insert(file)
results.requiredHeaders.insert(.modelViewer)
results.require(file: file)
results.require(header: .modelViewer)
let description = file.localized(in: language)
return ModelViewer(file: file.absoluteUrl, description: description).content
@@ -548,11 +494,10 @@ final class PageContentParser {
let imageId = arguments[0]
guard let image = content.image(imageId) else {
results.missing(file: imageId, markdown: markdown)
results.missing(file: imageId, source: "SVG command")
return ""
}
guard case .image(let imageType) = image.type,
imageType == .svg else {
guard image.type.isSvg else {
results.invalid(command: .svg, markdown)
return ""
}

View File

@@ -17,82 +17,219 @@ extension ImageToGenerate: Hashable {
final class PageGenerationResults: ObservableObject {
@Published
var linkedPages: Set<Page> = []
let itemId: ItemId
@Published
var linkedTags: Set<Tag> = []
private unowned let delegate: GenerationResults
@Published
var externalLinks: Set<String> = []
/// The files that could not be accessed
private(set) var inaccessibleFiles: Set<FileResource>
@Published
var files: Set<FileResource> = []
/// The files that could not be parsed, with the error message produced
private(set) var unparsableFiles: [FileResource : Set<String>]
@Published
var assets: Set<FileResource> = []
/// The missing files directly used by this page, and the source of the file
private(set) var missingFiles: [String: Set<String>]
@Published
var imagesToGenerate: Set<ImageToGenerate> = []
/// The missing files linked to from other files.
private(set) var missingLinkedFiles: [String : Set<FileResource>]
@Published
var missingPages: Set<String> = []
/// The missing tags linked to by this page, and the source of the link
private(set) var missingLinkedTags: [String : Set<String>]
@Published
var missingFiles: Set<String> = []
/// The missing pages linked to by this page, and the source of the link
private(set) var missingLinkedPages: [String : Set<String>]
@Published
var missingTags: Set<String> = []
/// The footer scripts or html to add to the end of the body
private(set) var requiredFooters: Set<String>
@Published
var invalidCommands: [(command: ShorthandMarkdownKey?, markdown: String)] = []
/// The known header elements to include in the page
private(set) var requiredHeaders: Set<KnownHeaderElement>
@Published
var requiredHeaders: Set<KnownHeaderElement> = []
/// The known icons that need to be included as hidden SVGs
private(set) var requiredIcons: Set<PageIcon>
@Published
var requiredFooters: Set<String> = []
/// The pages linked to by the page
private(set) var linkedPages: Set<Page>
@Published
var requiredIcons: Set<PageIcon> = []
/// The tags linked to by this page
private(set) var linkedTags: Set<Tag>
@Published
var issues: Set<PageContentAnomaly> = []
/// The links to external content in this page
private(set) var externalLinks: Set<String>
func reset() {
linkedPages = []
linkedTags = []
externalLinks = []
files = []
assets = []
imagesToGenerate = []
missingPages = []
missingFiles = []
missingTags = []
invalidCommands = []
/// The files used by this page, but not necessarily required in the output folder
private(set) var usedFiles: Set<FileResource>
/// The files that need to be copied
private(set) var requiredFiles: Set<FileResource>
/// The image versions required for this page
private(set) var imagesToGenerate: Set<ImageGenerationJob>
private(set) var invalidCommands: [(command: ShorthandMarkdownKey?, markdown: String)] = []
private(set) var warnings: Set<String>
/// The files that could not be saved to the output folder
private(set) var unsavedOutputFiles: [String: Set<ItemType>] = [:]
init(itemId: ItemId, delegate: GenerationResults) {
self.itemId = itemId
self.delegate = delegate
inaccessibleFiles = []
unparsableFiles = [:]
missingFiles = [:]
missingLinkedFiles = [:]
missingLinkedTags = [:]
missingLinkedPages = [:]
requiredHeaders = []
requiredFooters = []
requiredIcons = []
issues = []
linkedPages = []
linkedTags = []
externalLinks = []
usedFiles = []
requiredFiles = []
imagesToGenerate = []
invalidCommands = []
warnings = []
unsavedOutputFiles = [:]
}
private init(other: PageGenerationResults) {
self.itemId = other.itemId
self.delegate = other.delegate
inaccessibleFiles = other.inaccessibleFiles
unparsableFiles = other.unparsableFiles
missingFiles = other.missingFiles
missingLinkedFiles = other.missingLinkedFiles
missingLinkedTags = other.missingLinkedTags
missingLinkedPages = other.missingLinkedPages
requiredHeaders = other.requiredHeaders
requiredFooters = other.requiredFooters
requiredIcons = other.requiredIcons
linkedPages = other.linkedPages
linkedTags = other.linkedTags
externalLinks = other.externalLinks
usedFiles = other.usedFiles
requiredFiles = other.requiredFiles
imagesToGenerate = other.imagesToGenerate
invalidCommands = other.invalidCommands
warnings = other.warnings
unsavedOutputFiles = other.unsavedOutputFiles
}
func copy() -> PageGenerationResults {
.init(other: self)
}
// MARK: Adding entries
func inaccessibleContent(file: FileResource) {
inaccessibleFiles.insert(file)
delegate.inaccessibleContent(file: file)
}
func invalid(command: ShorthandMarkdownKey?, _ markdown: Substring) {
invalidCommands.append((command, String(markdown)))
issues.insert(.invalidCommand(command: command, markdown: String(markdown)))
let markdown = String(markdown)
invalidCommands.append((command, markdown))
delegate.invalidCommand(markdown)
}
func missing(page: String, markdown: Substring) {
missingPages.insert(page)
issues.insert(.missingPage(page: page, markdown: String(markdown)))
func missing(page: String, source: String) {
missingLinkedPages[page, default: []].insert(source)
delegate.missing(page: page)
}
func missing(tag: String, markdown: Substring) {
missingTags.insert(tag)
issues.insert(.missingTag(tag: tag, markdown: String(markdown)))
func missing(tag: String, source: String) {
missingLinkedTags[tag, default: []].insert(source)
delegate.missing(tag: tag)
}
func missing(file: String, markdown: Substring) {
missingFiles.insert(file)
issues.insert(.missingFile(file: file, markdown: String(markdown)))
func missing(file: String, source: String) {
missingFiles[file, default: []].insert(source)
delegate.missing(file: file)
}
func requireImageSet(for image: FileResource, size: Int) {
let jobs = ImageGenerationJob.imageSet(for: image.id, maxWidth: size, maxHeight: size)
imagesToGenerate.formUnion(jobs)
used(file: image)
delegate.generate(jobs)
}
func invalidFormat(file: FileResource, error: String) {
unparsableFiles[file, default: []].insert(error)
delegate.unparsable(file: file)
}
func missing(file: String, containedIn sourceFile: FileResource) {
missingLinkedFiles[file, default: []].insert(sourceFile)
delegate.missing(file: file)
}
func used(file: FileResource) {
usedFiles.insert(file)
// TODO: Notify delegate
}
func require(file: FileResource) {
requiredFiles.insert(file)
usedFiles.insert(file)
delegate.require(file: file)
}
func require(files: [FileResource]) {
requiredFiles.formUnion(files)
usedFiles.formUnion(files)
delegate.require(files: files)
}
func require(footer: String) {
requiredFooters.insert(footer)
}
func require(header: KnownHeaderElement) {
requiredHeaders.insert(header)
}
func require(headers: KnownHeaderElement...) {
requiredHeaders.formUnion(headers)
}
func require(icon: PageIcon) {
requiredIcons.insert(icon)
}
func require(icons: PageIcon...) {
requiredIcons.formUnion(icons)
}
func require(icons: [PageIcon]) {
requiredIcons.formUnion(icons)
}
func linked(to page: Page) {
linkedPages.insert(page)
}
func linked(to tag: Tag) {
linkedTags.insert(tag)
}
func externalLink(to url: String) {
externalLinks.insert(url)
delegate.externalLink(url)
}
func warning(_ warning: String) {
warnings.insert(warning)
delegate.warning(warning)
}
func unsavedOutput(_ path: String, source: ItemType) {
unsavedOutputFiles[path, default: []].insert(source)
delegate.unsaved(path)
}
}

View File

@@ -2,11 +2,8 @@ final class PageGenerator {
private let content: Content
private let imageGenerator: ImageGenerator
init(content: Content, imageGenerator: ImageGenerator) {
init(content: Content) {
self.content = content
self.imageGenerator = imageGenerator
}
private func makeHeaders(requiredItems: Set<KnownHeaderElement>) -> Set<HeaderElement> {
@@ -22,10 +19,10 @@ final class PageGenerator {
return result
}
func generate(page: Page, language: ContentLanguage) -> (page: String, results: PageGenerationResults)? {
func generate(page: Page, language: ContentLanguage, results: PageGenerationResults) -> String? {
let contentGenerator = PageContentParser(
content: content,
language: language)
language: language, results: results)
guard let rawPageContent = content.storage.pageContent(for: page.id, language: language) else {
return nil
@@ -33,8 +30,6 @@ final class PageGenerator {
let pageContent = contentGenerator.generatePage(from: rawPageContent)
contentGenerator.requestImages(imageGenerator)
let localized = page.localized(in: language)
let tags: [FeedEntryData.Tag] = page.tags.map { tag in
@@ -42,8 +37,8 @@ final class PageGenerator {
url: content.absoluteUrlToTag(tag, language: language))
}
let headers = makeHeaders(requiredItems: contentGenerator.results.requiredHeaders)
contentGenerator.results.assets.formUnion(headers.compactMap { $0.file })
let headers = makeHeaders(requiredItems: results.requiredHeaders)
results.require(files: headers.compactMap { $0.file })
let fullPage = ContentPage(
language: language,
@@ -55,10 +50,10 @@ final class PageGenerator {
navigationBarLinks: content.navigationBar(in: language),
pageContent: pageContent,
headers: headers,
footers: contentGenerator.results.requiredFooters.sorted(),
icons: contentGenerator.results.requiredIcons)
footers: results.requiredFooters.sorted(),
icons: results.requiredIcons)
.content
return (fullPage, contentGenerator.results)
return fullPage
}
}

View File

@@ -6,7 +6,7 @@ final class PostListPageGenerator {
private let content: Content
private let imageGenerator: ImageGenerator
private let results: PageGenerationResults
private let showTitle: Bool
@@ -17,28 +17,33 @@ final class PostListPageGenerator {
/// The url of the page, excluding the extension
private let pageUrlPrefix: String
init(language: ContentLanguage, content: Content, imageGenerator: ImageGenerator, showTitle: Bool, pageTitle: String, pageDescription: String, pageUrlPrefix: String) {
init(language: ContentLanguage,
content: Content,
results: PageGenerationResults,
showTitle: Bool, pageTitle: String,
pageDescription: String,
pageUrlPrefix: String) {
self.language = language
self.content = content
self.imageGenerator = imageGenerator
self.results = results
self.showTitle = showTitle
self.pageTitle = pageTitle
self.pageDescription = pageDescription
self.pageUrlPrefix = pageUrlPrefix
}
private var mainContentMaximumWidth: CGFloat {
CGFloat(content.settings.posts.contentWidth)
private var mainContentMaximumWidth: Int {
content.settings.posts.contentWidth
}
private var postsPerPage: Int {
content.settings.posts.postsPerPage
}
func createPages(for posts: [Post]) -> Bool {
func createPages(for posts: [Post]) {
let totalCount = posts.count
guard totalCount > 0 else {
return true
return
}
let numberOfPages = (totalCount + postsPerPage - 1) / postsPerPage // Round up
@@ -46,14 +51,11 @@ final class PostListPageGenerator {
let startIndex = (pageIndex - 1) * postsPerPage
let endIndex = min(pageIndex * postsPerPage, totalCount)
let postsOnPage = posts[startIndex..<endIndex]
guard createPostFeedPage(pageIndex, pageCount: numberOfPages, posts: postsOnPage) else {
return false
}
createPostFeedPage(pageIndex, pageCount: numberOfPages, posts: postsOnPage)
}
return true
}
private func createPostFeedPage(_ pageIndex: Int, pageCount: Int, posts: ArraySlice<Post>) -> Bool {
private func createPostFeedPage(_ pageIndex: Int, pageCount: Int, posts: ArraySlice<Post>) {
let posts: [FeedEntryData] = posts.map { post in
let localized: LocalizedPost = post.localized(in: language)
@@ -68,6 +70,8 @@ final class PostListPageGenerator {
url: content.absoluteUrlToTag(tag, language: language))
}
let images = localized.images.map(createFeedImage)
return FeedEntryData(
entryId: post.id,
title: localized.title,
@@ -75,7 +79,7 @@ final class PostListPageGenerator {
link: linkUrl,
tags: tags,
text: localized.text.components(separatedBy: "\n"),
images: localized.images.map(createImageSet))
images: images)
}
let feedPageGenerator = FeedPageGenerator(content: content)
@@ -88,23 +92,19 @@ final class PostListPageGenerator {
showTitle: showTitle,
pageNumber: pageIndex,
totalPages: pageCount)
if pageIndex == 1 {
return save(fileContent, to: "\(pageUrlPrefix).html")
} else {
return save(fileContent, to: "\(pageUrlPrefix)-\(pageIndex).html")
let filePath = "\(pageUrlPrefix)/\(pageIndex).html"
guard save(fileContent, to: filePath) else {
results.unsavedOutput(filePath, source: .feed)
return
}
}
private func createImageSet(for image: FileResource) -> FeedEntryData.Image {
imageGenerator.generateImageSet(
for: image.id,
maxWidth: mainContentMaximumWidth,
maxHeight: mainContentMaximumWidth)
private func createFeedImage(for image: FileResource) -> FeedEntryData.Image {
results.requireImageSet(for: image, size: mainContentMaximumWidth)
return .init(
rawImagePath: image.absoluteUrl,
width: Int(mainContentMaximumWidth),
height: Int(mainContentMaximumWidth),
width: mainContentMaximumWidth,
height: mainContentMaximumWidth,
altText: image.localized(in: language))
}