ch/ChWebsiteApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve storage

Browse Source
This commit is contained in:
Christoph Hagen
2024-12-19 16:25:05 +01:00
parent 9c828ff80a
commit 41887a1401
30 changed files with 926 additions and 831 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
CHDataManagement/Generator/HeaderElement.swift
View File

@ -64,6 +64,21 @@ enum HeaderElement {
return 102
}
}
var file: FileResource? {
switch self {
case .icon(let file, _, _):
return file
case .css(let file, _):
return file
case .js(let file, _):
return file
case .jsModule(let file):
return file
default:
return nil
}
}
}
extension HeaderElement: Hashable {

135
CHDataManagement/Generator/ImageGenerator.swift
View File

@ -9,35 +9,18 @@ final class ImageGenerator {
private let settings: Settings
private var relativeImageOutputPath: String {
settings.paths.imagesOutputFolderPath
}
private var generatedImages: [String : [String]] = [:]
private var generatedImages: [String : Set<String>] = [:]
private var jobs: [ImageGenerationJob] = []
init(storage: Storage, settings: Settings) {
self.storage = storage
self.settings = settings
do {
self.generatedImages = try storage.loadListOfGeneratedImages()
} catch {
print("Failed to load list of previously generated images: \(error)")
self.generatedImages = [:]
}
self.generatedImages = storage.loadListOfGeneratedImages() ?? [:]
}
func prepareForGeneration() -> Bool {
inOutputImagesFolder { imagesFolder in
do {
try imagesFolder.createIfNeeded()
return true
} catch {
print("Failed to create output images folder: \(error)")
return false
}
}
private var outputFolder: String {
settings.paths.imagesOutputFolderPath
}
func runJobs(callback: (String) -> Void) -> Bool {
@ -45,7 +28,7 @@ final class ImageGenerator {
return true
}
print("Generating \(jobs.count) images...")
for job in jobs {
while let job = jobs.popLast() {
callback("Generating image \(job.version)")
guard generate(job: job) else {
return false
@ -55,13 +38,11 @@ final class ImageGenerator {
}
func save() -> Bool {
do {
try storage.save(listOfGeneratedImages: generatedImages)
return true
} catch {
print("Failed to save list of generated images: \(error)")
guard storage.save(listOfGeneratedImages: generatedImages) else {
print("Failed to save list of generated images")
return false
}
return true
}
private func versionFileName(image: String, type: ImageFileType, width: CGFloat, height: CGFloat) -> String {
@ -88,12 +69,12 @@ final class ImageGenerator {
func generateVersion(for image: String, type: ImageFileType, maximumWidth: CGFloat, maximumHeight: CGFloat) {
let version = versionFileName(image: image, type: type, width: maximumWidth, height: maximumHeight)
if exists(version) {
hasNowGenerated(version: version, for: image)
guard needsToGenerate(version: version, for: image) else {
// Image already present
return
}
if hasPreviouslyGenerated(version: version, for: image), exists(version) {
// Don't add job again
guard !jobs.contains(where: { $0.version == version }) else {
// Job already in queue
return
}
@ -108,15 +89,29 @@ final class ImageGenerator {
jobs.append(job)
}
private func hasPreviouslyGenerated(version: String, for image: String) -> Bool {
guard let versions = generatedImages[image] else {
return false
}
return versions.contains(version)
/**
Remove all versions of an image, so that they will be recreated on the next run.
This function does not remove the images from the output folder.
*/
func removeVersions(of image: String) {
generatedImages[image] = nil
}
private func exists(imageVersion version: String) -> Bool {
inOutputImagesFolder { $0.appendingPathComponent(version).exists }
func recalculateGeneratedImages(by images: Set<String>) {
self.generatedImages = storage.calculateImages(generatedBy: images, in: outputFolder)
let versionCount = generatedImages.values.reduce(0) { $0 + $1.count }
print("Image generator: \(generatedImages.count)/\(images.count) images (\(versionCount) versions)")
}
private func needsToGenerate(version: String, for image: String) -> Bool {
guard let versions = generatedImages[image] else {
return true
}
guard versions.contains(version) else {
return true
}
return !exists(version)
}
private func hasNowGenerated(version: String, for image: String) {
@ -124,7 +119,7 @@ final class ImageGenerator {
generatedImages[image] = [version]
return
}
versions.append(version)
versions.insert(version)
generatedImages[image] = versions
}
@ -132,19 +127,29 @@ final class ImageGenerator {
generatedImages[image] = nil
}
// MARK: Files
private func exists(_ image: String) -> Bool {
storage.hasFileInOutputFolder(relativePath(for: image))
}
private func relativePath(for image: String) -> String {
outputFolder + "/" + image
}
private func write(imageData data: Data, version: String) -> Bool {
return storage.write(data, to: relativePath(for: version))
}
// MARK: Image operations
private func generate(job: ImageGenerationJob) -> Bool {
if hasPreviouslyGenerated(version: job.version, for: job.image), exists(job.version),
exists(imageVersion: job.version) {
guard needsToGenerate(version: job.version, for: job.image) else {
return true
}
let data: Data
do {
data = try storage.fileData(for: job.image)
} catch {
print("Failed to load image \(job.image): \(error)")
guard let data = storage.fileData(for: job.image) else {
print("Failed to load image \(job.image)")
return false
}
@ -161,14 +166,10 @@ final class ImageGenerator {
}
if job.type == .avif {
return inOutputImagesFolder { folder in
let url = folder.appendingPathComponent(job.version)
let out = url.path()
let input = url.deletingPathExtension().appendingPathExtension(job.image.fileExtension!).path()
print("avifenc -q 70 \(input) \(out)")
hasNowGenerated(version: job.version, for: job.image)
return true
}
let input = job.version.fileNameAndExtension.fileName + "." + job.image.fileExtension!
print("avifenc -q 70 \(input) \(job.version)")
hasNowGenerated(version: job.version, for: job.image)
return true
}
guard write(imageData: data, version: job.version) else {
@ -206,32 +207,6 @@ final class ImageGenerator {
return representation
}
private func write(imageData data: Data, version: String) -> Bool {
inOutputImagesFolder { folder in
let url = folder.appendingPathComponent(version)
do {
try data.write(to: url)
return true
} catch {
print("Failed to write image \(version): \(error)")
return false
}
}
}
private func exists(_ relativePath: String) -> Bool {
inOutputImagesFolder { folder in
folder.appendingPathComponent(relativePath).exists
}
}
private func inOutputImagesFolder(perform operation: (URL) -> Bool) -> Bool {
storage.write(in: .outputPath) { outputFolder in
let imagesFolder = outputFolder.appendingPathComponent(relativeImageOutputPath)
return operation(imagesFolder)
}
}
// MARK: Avif images
private func create(image: NSBitmapImageRep, type: ImageFileType, quality: CGFloat) -> Data? {

20
CHDataManagement/Generator/LocalizedWebsiteGenerator.swift
View File

@ -17,11 +17,8 @@ final class LocalizedWebsiteGenerator {
self.imageGenerator = ImageGenerator(
storage: content.storage,
settings: content.settings)
self.outputDirectory = content.storage.outputPath!
}
private let outputDirectory: URL
private var postsPerPage: Int {
content.settings.posts.postsPerPage
}
@ -31,9 +28,6 @@ final class LocalizedWebsiteGenerator {
}
func generateWebsite(callback: (String) -> Void) -> Bool {
guard imageGenerator.prepareForGeneration() else {
return false
}
guard createMainPostFeedPages() else {
return false
}
@ -90,11 +84,7 @@ final class LocalizedWebsiteGenerator {
guard !file.isExternallyStored else {
continue
}
do {
try content.storage.copy(file: file.id, to: file.absoluteUrl)
} catch {
print("Failed to copy file \(file.id): \(error)")
guard content.storage.copy(file: file.id, to: file.absoluteUrl) else {
return false
}
}
@ -102,12 +92,6 @@ final class LocalizedWebsiteGenerator {
}
private func save(_ content: String, to relativePath: String) -> Bool {
do {
try self.content.storage.write(content: content, to: relativePath)
return true
} catch {
print("Failed to write page \(relativePath)")
return false
}
self.content.storage.write(content, to: relativePath)
}
}

7
CHDataManagement/Generator/Page Content/AudioPlayerCommand.swift
View File

@ -30,12 +30,15 @@ struct AudioPlayerCommandProcessor: CommandProcessor {
results.missingFiles.insert(fileId)
return ""
}
guard let data = file.dataContent() else {
results.issues.insert(.failedToLoadContent)
return ""
}
let songs: [Song]
do {
let data = try file.dataContent()
songs = try JSONDecoder().decode([Song].self, from: data)
} catch {
results.issues.insert(.failedToLoadContent(error))
results.issues.insert(.failedToParseContent)
return ""
}

13
CHDataManagement/Generator/PageContentAnomaly.swift
View File

@ -1,6 +1,7 @@
enum PageContentAnomaly {
case failedToLoadContent(Error)
case failedToLoadContent
case failedToParseContent
case missingFile(file: String, markdown: String)
case missingPage(page: String, markdown: String)
case missingTag(tag: String, markdown: String)
@ -14,6 +15,8 @@ extension PageContentAnomaly: Identifiable {
switch self {
case .failedToLoadContent:
return "load-failed"
case .failedToParseContent:
return "parse-failed"
case .missingFile(let string, _):
return "missing-file-\(string)"
case .missingPage(let string, _):
@ -51,7 +54,7 @@ extension PageContentAnomaly {
var severity: Severity {
switch self {
case .failedToLoadContent:
case .failedToLoadContent, .failedToParseContent:
return .error
case .missingFile, .missingPage, .missingTag, .invalidCommand, .warning:
return .warning
@ -63,8 +66,10 @@ extension PageContentAnomaly: CustomStringConvertible {
var description: String {
switch self {
case .failedToLoadContent(let error):
return "Failed to load content: \(error)"
case .failedToLoadContent:
return "Failed to load content"
case .failedToParseContent:
return "Failed to parse content"
case .missingFile(let string, _):
return "Missing file: \(string)"
case .missingPage(let string, _):

4
CHDataManagement/Generator/PageGenerationResults.swift
View File

@ -29,6 +29,9 @@ final class PageGenerationResults: ObservableObject {
@Published
var files: Set<FileResource> = []
@Published
var assets: Set<FileResource> = []
@Published
var imagesToGenerate: Set<ImageToGenerate> = []
@ -61,6 +64,7 @@ final class PageGenerationResults: ObservableObject {
linkedTags = []
externalLinks = []
files = []
assets = []
imagesToGenerate = []
missingPages = []
missingFiles = []

8
CHDataManagement/Generator/PageGenerator.swift
View File

@ -22,12 +22,14 @@ final class PageGenerator {
return result
}
func generate(page: Page, language: ContentLanguage) throws -> (page: String, results: PageGenerationResults) {
func generate(page: Page, language: ContentLanguage) -> (page: String, results: PageGenerationResults)? {
let contentGenerator = PageContentParser(
content: content,
language: language)
let rawPageContent = try content.storage.pageContent(for: page.id, language: language)
guard let rawPageContent = content.storage.pageContent(for: page.id, language: language) else {
return nil
}
let pageContent = contentGenerator.generatePage(from: rawPageContent)
@ -41,7 +43,7 @@ final class PageGenerator {
}
let headers = makeHeaders(requiredItems: contentGenerator.results.requiredHeaders)
print("Headers for page: \(headers)")
contentGenerator.results.assets.formUnion(headers.compactMap { $0.file })
let fullPage = ContentPage(
language: language,

8
CHDataManagement/Generator/PostListPageGenerator.swift
View File

@ -109,12 +109,6 @@ final class PostListPageGenerator {
}
private func save(_ content: String, to relativePath: String) -> Bool {
do {
try self.content.storage.write(content: content, to: relativePath)
return true
} catch {
print("Failed to write page \(relativePath)")
return false
}
self.content.storage.write(content, to: relativePath)
}
}