ch/ChWebsiteApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve storage

Browse Source
This commit is contained in:
Christoph Hagen 2024-12-19 16:25:05 +01:00
parent 9c828ff80a
commit 41887a1401
30 changed files with 926 additions and 831 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHDataManagement.xcodeproj/project.pbxproj
View File

@ -49,6 +49,7 @@
E22990482D10B7B7009F8D77 /* StorageAccessError.swift in Sources */ = {isa = PBXBuildFile; fileRef = E22990472D10B7B7009F8D77 /* StorageAccessError.swift */; };
E229904A2D10BB90009F8D77 /* SecurityScopeBookmark.swift in Sources */ = {isa = PBXBuildFile; fileRef = E22990492D10BB90009F8D77 /* SecurityScopeBookmark.swift */; };
E229904C2D10BE5D009F8D77 /* InitialSetupView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E229904B2D10BE59009F8D77 /* InitialSetupView.swift */; };
E229904E2D13535C009F8D77 /* SecurityBookmark.swift in Sources */ = {isa = PBXBuildFile; fileRef = E229904D2D135349009F8D77 /* SecurityBookmark.swift */; };
E24252012C50E0A40029FF16 /* HighlightedTextEditor in Frameworks */ = {isa = PBXBuildFile; productRef = E24252002C50E0A40029FF16 /* HighlightedTextEditor */; };
E242520A2C52C9260029FF16 /* ContentLanguage.swift in Sources */ = {isa = PBXBuildFile; fileRef = E24252092C52C9260029FF16 /* ContentLanguage.swift */; };
E2581DED2C75202400F1F079 /* Tag.swift in Sources */ = {isa = PBXBuildFile; fileRef = E2581DEC2C75202400F1F079 /* Tag.swift */; };
@ -242,6 +243,7 @@
E22990472D10B7B7009F8D77 /* StorageAccessError.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StorageAccessError.swift; sourceTree = "<group>"; };
E22990492D10BB90009F8D77 /* SecurityScopeBookmark.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecurityScopeBookmark.swift; sourceTree = "<group>"; };
E229904B2D10BE59009F8D77 /* InitialSetupView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InitialSetupView.swift; sourceTree = "<group>"; };
E229904D2D135349009F8D77 /* SecurityBookmark.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecurityBookmark.swift; sourceTree = "<group>"; };
E24252092C52C9260029FF16 /* ContentLanguage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentLanguage.swift; sourceTree = "<group>"; };
E2581DEC2C75202400F1F079 /* Tag.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Tag.swift; sourceTree = "<group>"; };
E25A0B882CE4021400F33674 /* LocalizedPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalizedPage.swift; sourceTree = "<group>"; };
@ -650,6 +652,7 @@
E2A37D0F2CE5375E0000979F /* Storage */ = {
isa = PBXGroup;
children = (
E229904D2D135349009F8D77 /* SecurityBookmark.swift */,
E22990492D10BB90009F8D77 /* SecurityScopeBookmark.swift */,
E22990472D10B7B7009F8D77 /* StorageAccessError.swift */,
E22990452D10B7A6009F8D77 /* SecurityScopeStatus.swift */,
@ -941,6 +944,7 @@
E25DA51B2CFF08BB00AEF16D /* PostFeedPageNavigation.swift in Sources */,
E22990422D107A95009F8D77 /* ImageJob.swift in Sources */,
E29D317F2D086F4C0051B7F4 /* StatisticsIcons.swift in Sources */,
E229904E2D13535C009F8D77 /* SecurityBookmark.swift in Sources */,
E2A21C082CB17B870060935B /* TagView.swift in Sources */,
E29D313D2D047C1B0051B7F4 /* LocalizedPageContentView.swift in Sources */,
E2A37D2D2CED2EF10000979F /* OptionalTextField.swift in Sources */,

7
CHDataManagement/Extensions/Int+Random.swift
View File

@ -4,4 +4,11 @@ extension Int {
static func random() -> Int {
random(in: Int.min...Int.max)
}
mutating func increment(_ increment: Bool) {
guard increment else {
return
}
self += 1
}
}

4
CHDataManagement/Extensions/Optional+Extensions.swift
View File

@ -2,6 +2,10 @@ import Foundation
extension Optional {
func `default`(_ defaultValue: Wrapped) -> Wrapped {
self ?? defaultValue
}
func map<T>(_ transform: (Wrapped) throws -> T?) rethrows -> T? {
guard let self else { return nil }
return try transform(self)

18
CHDataManagement/Extensions/Sequence+Sorted.swift
View File

@ -8,4 +8,22 @@ extension Collection {
}
return sorted { conversion($0) < conversion($1) }
}
func count(where predicate: (Element) throws -> Bool) rethrows -> Int {
try reduce(0) { count, element in
try predicate(element) ? count + 1 : count
}
}
func countThrows(where predicate: (Element) throws -> Void) -> Int {
reduce(0) { count, element in
do {
try predicate(element)
return count
} catch {
return count + 1
}
}
}
}

4
CHDataManagement/Extensions/String+Extensions.swift
View File

@ -89,6 +89,10 @@ extension String {
extension String {
var fileNameWithoutExtension: String {
dropAfterLast(".")
}
var fileExtension: String? {
let parts = components(separatedBy: ".")
guard parts.count > 1 else { return nil }

15
CHDataManagement/Generator/HeaderElement.swift
View File

@ -64,6 +64,21 @@ enum HeaderElement {
return 102
}
}
var file: FileResource? {
switch self {
case .icon(let file, _, _):
return file
case .css(let file, _):
return file
case .js(let file, _):
return file
case .jsModule(let file):
return file
default:
return nil
}
}
}
extension HeaderElement: Hashable {

135
CHDataManagement/Generator/ImageGenerator.swift
View File

@ -9,35 +9,18 @@ final class ImageGenerator {
private let settings: Settings
private var relativeImageOutputPath: String {
settings.paths.imagesOutputFolderPath
}
private var generatedImages: [String : [String]] = [:]
private var generatedImages: [String : Set<String>] = [:]
private var jobs: [ImageGenerationJob] = []
init(storage: Storage, settings: Settings) {
self.storage = storage
self.settings = settings
do {
self.generatedImages = try storage.loadListOfGeneratedImages()
} catch {
print("Failed to load list of previously generated images: \(error)")
self.generatedImages = [:]
}
self.generatedImages = storage.loadListOfGeneratedImages() ?? [:]
}
func prepareForGeneration() -> Bool {
inOutputImagesFolder { imagesFolder in
do {
try imagesFolder.createIfNeeded()
return true
} catch {
print("Failed to create output images folder: \(error)")
return false
}
}
private var outputFolder: String {
settings.paths.imagesOutputFolderPath
}
func runJobs(callback: (String) -> Void) -> Bool {
@ -45,7 +28,7 @@ final class ImageGenerator {
return true
}
print("Generating \(jobs.count) images...")
for job in jobs {
while let job = jobs.popLast() {
callback("Generating image \(job.version)")
guard generate(job: job) else {
return false
@ -55,13 +38,11 @@ final class ImageGenerator {
}
func save() -> Bool {
do {
try storage.save(listOfGeneratedImages: generatedImages)
return true
} catch {
print("Failed to save list of generated images: \(error)")
guard storage.save(listOfGeneratedImages: generatedImages) else {
print("Failed to save list of generated images")
return false
}
return true
}
private func versionFileName(image: String, type: ImageFileType, width: CGFloat, height: CGFloat) -> String {
@ -88,12 +69,12 @@ final class ImageGenerator {
func generateVersion(for image: String, type: ImageFileType, maximumWidth: CGFloat, maximumHeight: CGFloat) {
let version = versionFileName(image: image, type: type, width: maximumWidth, height: maximumHeight)
if exists(version) {
hasNowGenerated(version: version, for: image)
guard needsToGenerate(version: version, for: image) else {
// Image already present
return
}
if hasPreviouslyGenerated(version: version, for: image), exists(version) {
// Don't add job again
guard !jobs.contains(where: { $0.version == version }) else {
// Job already in queue
return
}
@ -108,15 +89,29 @@ final class ImageGenerator {
jobs.append(job)
}
private func hasPreviouslyGenerated(version: String, for image: String) -> Bool {
guard let versions = generatedImages[image] else {
return false
}
return versions.contains(version)
/**
Remove all versions of an image, so that they will be recreated on the next run.
This function does not remove the images from the output folder.
*/
func removeVersions(of image: String) {
generatedImages[image] = nil
}
private func exists(imageVersion version: String) -> Bool {
inOutputImagesFolder { $0.appendingPathComponent(version).exists }
func recalculateGeneratedImages(by images: Set<String>) {
self.generatedImages = storage.calculateImages(generatedBy: images, in: outputFolder)
let versionCount = generatedImages.values.reduce(0) { $0 + $1.count }
print("Image generator: \(generatedImages.count)/\(images.count) images (\(versionCount) versions)")
}
private func needsToGenerate(version: String, for image: String) -> Bool {
guard let versions = generatedImages[image] else {
return true
}
guard versions.contains(version) else {
return true
}
return !exists(version)
}
private func hasNowGenerated(version: String, for image: String) {
@ -124,7 +119,7 @@ final class ImageGenerator {
generatedImages[image] = [version]
return
}
versions.append(version)
versions.insert(version)
generatedImages[image] = versions
}
@ -132,19 +127,29 @@ final class ImageGenerator {
generatedImages[image] = nil
}
// MARK: Files
private func exists(_ image: String) -> Bool {
storage.hasFileInOutputFolder(relativePath(for: image))
}
private func relativePath(for image: String) -> String {
outputFolder + "/" + image
}
private func write(imageData data: Data, version: String) -> Bool {
return storage.write(data, to: relativePath(for: version))
}
// MARK: Image operations
private func generate(job: ImageGenerationJob) -> Bool {
if hasPreviouslyGenerated(version: job.version, for: job.image), exists(job.version),
exists(imageVersion: job.version) {
guard needsToGenerate(version: job.version, for: job.image) else {
return true
}
let data: Data
do {
data = try storage.fileData(for: job.image)
} catch {
print("Failed to load image \(job.image): \(error)")
guard let data = storage.fileData(for: job.image) else {
print("Failed to load image \(job.image)")
return false
}
@ -161,14 +166,10 @@ final class ImageGenerator {
}
if job.type == .avif {
return inOutputImagesFolder { folder in
let url = folder.appendingPathComponent(job.version)
let out = url.path()
let input = url.deletingPathExtension().appendingPathExtension(job.image.fileExtension!).path()
print("avifenc -q 70 \(input) \(out)")
hasNowGenerated(version: job.version, for: job.image)
return true
}
let input = job.version.fileNameAndExtension.fileName + "." + job.image.fileExtension!
print("avifenc -q 70 \(input) \(job.version)")
hasNowGenerated(version: job.version, for: job.image)
return true
}
guard write(imageData: data, version: job.version) else {
@ -206,32 +207,6 @@ final class ImageGenerator {
return representation
}
private func write(imageData data: Data, version: String) -> Bool {
inOutputImagesFolder { folder in
let url = folder.appendingPathComponent(version)
do {
try data.write(to: url)
return true
} catch {
print("Failed to write image \(version): \(error)")
return false
}
}
}
private func exists(_ relativePath: String) -> Bool {
inOutputImagesFolder { folder in
folder.appendingPathComponent(relativePath).exists
}
}
private func inOutputImagesFolder(perform operation: (URL) -> Bool) -> Bool {
storage.write(in: .outputPath) { outputFolder in
let imagesFolder = outputFolder.appendingPathComponent(relativeImageOutputPath)
return operation(imagesFolder)
}
}
// MARK: Avif images
private func create(image: NSBitmapImageRep, type: ImageFileType, quality: CGFloat) -> Data? {

20
CHDataManagement/Generator/LocalizedWebsiteGenerator.swift
View File

@ -17,11 +17,8 @@ final class LocalizedWebsiteGenerator {
self.imageGenerator = ImageGenerator(
storage: content.storage,
settings: content.settings)
self.outputDirectory = content.storage.outputPath!
}
private let outputDirectory: URL
private var postsPerPage: Int {
content.settings.posts.postsPerPage
}
@ -31,9 +28,6 @@ final class LocalizedWebsiteGenerator {
}
func generateWebsite(callback: (String) -> Void) -> Bool {
guard imageGenerator.prepareForGeneration() else {
return false
}
guard createMainPostFeedPages() else {
return false
}
@ -90,11 +84,7 @@ final class LocalizedWebsiteGenerator {
guard !file.isExternallyStored else {
continue
}
do {
try content.storage.copy(file: file.id, to: file.absoluteUrl)
} catch {
print("Failed to copy file \(file.id): \(error)")
guard content.storage.copy(file: file.id, to: file.absoluteUrl) else {
return false
}
}
@ -102,12 +92,6 @@ final class LocalizedWebsiteGenerator {
}
private func save(_ content: String, to relativePath: String) -> Bool {
do {
try self.content.storage.write(content: content, to: relativePath)
return true
} catch {
print("Failed to write page \(relativePath)")
return false
}
self.content.storage.write(content, to: relativePath)
}
}

7
CHDataManagement/Generator/Page Content/AudioPlayerCommand.swift
View File

@ -30,12 +30,15 @@ struct AudioPlayerCommandProcessor: CommandProcessor {
results.missingFiles.insert(fileId)
return ""
}
guard let data = file.dataContent() else {
results.issues.insert(.failedToLoadContent)
return ""
}
let songs: [Song]
do {
let data = try file.dataContent()
songs = try JSONDecoder().decode([Song].self, from: data)
} catch {
results.issues.insert(.failedToLoadContent(error))
results.issues.insert(.failedToParseContent)
return ""
}

13
CHDataManagement/Generator/PageContentAnomaly.swift
View File

@ -1,6 +1,7 @@
enum PageContentAnomaly {
case failedToLoadContent(Error)
case failedToLoadContent
case failedToParseContent
case missingFile(file: String, markdown: String)
case missingPage(page: String, markdown: String)
case missingTag(tag: String, markdown: String)
@ -14,6 +15,8 @@ extension PageContentAnomaly: Identifiable {
switch self {
case .failedToLoadContent:
return "load-failed"
case .failedToParseContent:
return "parse-failed"
case .missingFile(let string, _):
return "missing-file-\(string)"
case .missingPage(let string, _):
@ -51,7 +54,7 @@ extension PageContentAnomaly {
var severity: Severity {
switch self {
case .failedToLoadContent:
case .failedToLoadContent, .failedToParseContent:
return .error
case .missingFile, .missingPage, .missingTag, .invalidCommand, .warning:
return .warning
@ -63,8 +66,10 @@ extension PageContentAnomaly: CustomStringConvertible {
var description: String {
switch self {
case .failedToLoadContent(let error):
return "Failed to load content: \(error)"
case .failedToLoadContent:
return "Failed to load content"
case .failedToParseContent:
return "Failed to parse content"
case .missingFile(let string, _):
return "Missing file: \(string)"
case .missingPage(let string, _):

4
CHDataManagement/Generator/PageGenerationResults.swift
View File

@ -29,6 +29,9 @@ final class PageGenerationResults: ObservableObject {
@Published
var files: Set<FileResource> = []
@Published
var assets: Set<FileResource> = []
@Published
var imagesToGenerate: Set<ImageToGenerate> = []
@ -61,6 +64,7 @@ final class PageGenerationResults: ObservableObject {
linkedTags = []
externalLinks = []
files = []
assets = []
imagesToGenerate = []
missingPages = []
missingFiles = []

8
CHDataManagement/Generator/PageGenerator.swift
View File

@ -22,12 +22,14 @@ final class PageGenerator {
return result
}
func generate(page: Page, language: ContentLanguage) throws -> (page: String, results: PageGenerationResults) {
func generate(page: Page, language: ContentLanguage) -> (page: String, results: PageGenerationResults)? {
let contentGenerator = PageContentParser(
content: content,
language: language)
let rawPageContent = try content.storage.pageContent(for: page.id, language: language)
guard let rawPageContent = content.storage.pageContent(for: page.id, language: language) else {
return nil
}
let pageContent = contentGenerator.generatePage(from: rawPageContent)
@ -41,7 +43,7 @@ final class PageGenerator {
}
let headers = makeHeaders(requiredItems: contentGenerator.results.requiredHeaders)
print("Headers for page: \(headers)")
contentGenerator.results.assets.formUnion(headers.compactMap { $0.file })
let fullPage = ContentPage(
language: language,

8
CHDataManagement/Generator/PostListPageGenerator.swift
View File

@ -109,12 +109,6 @@ final class PostListPageGenerator {
}
private func save(_ content: String, to relativePath: String) -> Bool {
do {
try self.content.storage.write(content: content, to: relativePath)
return true
} catch {
print("Failed to write page \(relativePath)")
return false
}
self.content.storage.write(content, to: relativePath)
}
}

4
CHDataManagement/Main/MainView.swift
View File

@ -162,7 +162,7 @@ struct MainView: App {
}.pickerStyle(.segmented)
}
ToolbarItem(placement: .primaryAction) {
if content.storage.hasContentFolders {
if content.storage.contentScope != nil {
Button(action: save) {
Text("Save")
}
@ -203,7 +203,7 @@ struct MainView: App {
}
private func loadContent() {
guard content.storage.hasContentFolders else {
guard content.storage.contentScope != nil else {
showInitialSheet()
return
}

121
CHDataManagement/Model/Content+Generation.swift
View File

@ -2,6 +2,40 @@ import Foundation
extension Content {
func generateFeed() -> Bool {
#warning("Implement feed generation")
return false
}
func generateAllPages() -> Bool {
guard startGenerating() else { return false }
defer { endGenerating() }
for page in pages {
for language in ContentLanguage.allCases {
guard generateInternal(page, in: language) else {
return false
}
}
}
let failedAssetCopies = results.values
.reduce(Set()) { $0.union($1.assets) }
.filter { !$0.isExternallyStored }
.filter { !storage.copy(file: $0.id, to: $0.assetUrl) }
let failedFileCopies = results.values
.reduce(Set()) { $0.union($1.files) }
.filter { !$0.isExternallyStored }
.filter { !storage.copy(file: $0.id, to: $0.absoluteUrl) }
guard imageGenerator.runJobs(callback: { _ in }) else {
return false
}
return true
}
func generatePage(_ page: Page) -> Bool {
guard startGenerating() else { return false }
defer { endGenerating() }
@ -11,6 +45,20 @@ extension Content {
return false
}
}
guard imageGenerator.runJobs(callback: { _ in }) else {
return false
}
let failedAssetCopies = results.values
.reduce(Set()) { $0.union($1.assets) }
.filter { !$0.isExternallyStored }
.filter { !storage.copy(file: $0.id, to: $0.assetUrl) }
let failedFileCopies = results.values
.reduce(Set()) { $0.union($1.files) }
.filter { !$0.isExternallyStored }
.filter { !storage.copy(file: $0.id, to: $0.absoluteUrl) }
return true
}
@ -73,6 +121,13 @@ extension Content {
return result
}
// MARK: Images
func recalculateGeneratedImages() {
let images = Set(self.images.map { $0.id })
imageGenerator.recalculateGeneratedImages(by: images)
}
// MARK: Generation
private func startGenerating() -> Bool {
@ -90,64 +145,36 @@ extension Content {
}
private func generateInternal(_ page: Page, in language: ContentLanguage) -> Bool {
let pagesFolder = settings.paths.pagesOutputFolderPath
guard storage.create(folder: pagesFolder, in: .outputPath) else {
print("Failed to generate output folder")
return false
}
let imageGenerator = ImageGenerator(
storage: storage,
settings: settings)
let pageGenerator = PageGenerator(
content: self,
imageGenerator: imageGenerator)
let content: String
let results: PageGenerationResults
do {
(content, results) = try pageGenerator.generate(page: page, language: language)
} catch {
print("Failed to generate page \(page.id) in language \(language): \(error)")
guard let (content, results) = pageGenerator.generate(page: page, language: language) else {
print("Failed to generate page \(page.id) in language \(language)")
return false
}
guard !content.trimmed.isEmpty else {
#warning("Generate page with placeholder content")
return true
DispatchQueue.main.async {
let id = ItemId(itemId: page.id, language: language, itemType: .page)
self.results[id] = results
}
let path = page.absoluteUrl(in: language) + ".html"
do {
try storage.write(content: content, to: path)
} catch {
print("Failed to save page \(page.id): \(error)")
guard storage.write(content, to: path) else {
print("Failed to save page \(page.id)")
return false
}
guard imageGenerator.runJobs(callback: { _ in }) else {
return false
}
guard copy(requiredFiles: results.files) else {
return false
}
return true
return true
}
private func copy(requiredFiles: Set<FileResource>) -> Bool {
//print("Copying \(requiredVideoFiles.count) files...")
for file in requiredFiles {
guard !file.isExternallyStored else {
continue
}
do {
try storage.copy(file: file.id, to: file.absoluteUrl)
} catch {
print("Failed to copy file \(file.id): \(error)")
return false
}
}
return true
}
}
prefix operator ~>
prefix func ~> (operation: () throws -> Void) -> Bool {
do {
try operation()
return true
} catch {
return false
}
}

40
CHDataManagement/Model/Content+Load.swift
View File

@ -41,28 +41,44 @@ extension Content {
}
func loadFromDisk() throws {
guard storage.hasContentFolders else {
guard storage.contentScope != nil else {
print("Storage not initialized, not loading content")
throw StorageAccessError.noBookmarkData
}
let settings = try storage.loadSettings() // Uses defaults if missing
let imageDescriptions = try storage.loadFileDescriptions().reduce(into: [:]) { descriptions, description in
descriptions[description.fileId] = description
let settings = storage.loadSettings() ?? .default
let imageDescriptions = storage.loadFileDescriptions()
.default([])
.reduce(into: [:]) { $0[$1.fileId] = $1 }
guard let tagData = storage.loadAllTags() else {
print("Failed to load file tags")
return
}
let tagData = try storage.loadAllTags()
let pagesData = try storage.loadAllPages()
let postsData = try storage.loadAllPosts()
let fileList = try storage.loadAllFiles()
let externalFiles = try storage.loadExternalFileList()
let tagOverviewData = try storage.loadTagOverview()
if tagData.isEmpty { print("No tags loaded") }
guard let pagesData = storage.loadAllPages() else {
print("Failed to load file pages")
return
}
if pagesData.isEmpty { print("No pages loaded") }
guard let postsData = storage.loadAllPosts() else {
print("Failed to load file posts")
return
}
if postsData.isEmpty { print("No posts loaded") }
guard let fileList = storage.loadAllFiles() else {
print("Failed to load file list")
return
}
if fileList.isEmpty { print("No files loaded") }
let externalFiles = storage.loadExternalFileList() ?? []
if externalFiles.isEmpty { print("No external files loaded") }
let tagOverviewData = storage.loadTagOverview()
if tagOverviewData == nil { print("No tag overview loaded") }
print("Loaded data from disk, processing...")

51
CHDataManagement/Model/Content+Save.swift
View File

@ -3,23 +3,16 @@ import Foundation
extension Content {
func saveToDisk() throws {
guard storage.hasContentFolders else {
guard storage.contentScope != nil else {
print("Storage not initialized, not saving content")
return
}
//print("Starting save")
for page in pages {
try storage.save(pageMetadata: page.pageFile, for: page.id)
}
for post in posts {
try storage.save(post: post.postFile, for: post.id)
}
for tag in tags {
try storage.save(tagMetadata: tag.tagFile, for: tag.id)
}
try storage.save(settings: settings.file)
var failedSaves = 0
failedSaves += pages.count { !storage.save(pageMetadata: $0.pageFile, for: $0.id) }
failedSaves += posts.count { !storage.save(post: $0.postFile, for: $0.id) }
failedSaves += tags.count { !storage.save(tagMetadata: $0.tagFile, for: $0.id) }
failedSaves.increment(!storage.save(settings: settings.file))
let fileDescriptions: [FileDescriptions] = files.sorted().compactMap { file in
guard !file.english.isEmpty || !file.german.isEmpty else {
@ -31,21 +24,33 @@ extension Content {
english: file.english.nonEmpty)
}
try storage.save(fileDescriptions: fileDescriptions)
try storage.save(tagOverview: tagOverview?.file)
failedSaves.increment(!storage.save(fileDescriptions: fileDescriptions))
failedSaves.increment(!storage.save(tagOverview: tagOverview?.file))
let externalFileList = files.filter { $0.isExternallyStored }.map { $0.id }
try storage.save(externalFileList: externalFileList)
failedSaves.increment(!storage.save(externalFileList: externalFileList))
do {
try storage.deletePostFiles(notIn: posts.map { $0.id })
try storage.deletePageFiles(notIn: pages.map { $0.id })
try storage.deleteTagFiles(notIn: tags.map { $0.id })
try storage.deleteFileResources(notIn: files.map { $0.id })
} catch {
print("Failed to remove unused files: \(error)")
if failedSaves > 0 {
print("Save partially failed with \(failedSaves) errors")
}
}
func removeUnlinkedFiles() -> Bool {
var success = true
if !storage.deletePostFiles(notIn: posts.map { $0.id }) {
success = false
}
if !storage.deletePageFiles(notIn: pages.map { $0.id }) {
success = false
}
if !storage.deleteTagFiles(notIn: tags.map { $0.id }) {
success = false
}
if !storage.deleteFileResources(notIn: files.map { $0.id }) {
success = false
}
return success
}
}
private extension Page {

21
CHDataManagement/Model/Content.swift
View File

@ -5,7 +5,7 @@ import Combine
final class Content: ObservableObject {
@ObservedObject
var storage = Storage()
var storage: Storage
@Published
var settings: Settings
@ -26,11 +26,13 @@ final class Content: ObservableObject {
var tagOverview: TagOverviewPage?
@Published
private(set) var results: [ItemId : PageGenerationResults]
var results: [ItemId : PageGenerationResults]
@Published
private(set) var isGeneratingWebsite = false
let imageGenerator: ImageGenerator
init(settings: Settings,
posts: [Post],
pages: [Page],
@ -44,16 +46,29 @@ final class Content: ObservableObject {
self.files = files
self.tagOverview = tagOverview
self.results = [:]
let storage = Storage()
self.storage = storage
self.imageGenerator = ImageGenerator(
storage: storage,
settings: settings)
}
init() {
self.settings = .default
let settings = Settings.default
self.settings = settings
self.posts = []
self.pages = []
self.tags = []
self.files = []
self.tagOverview = nil
self.results = [:]
let storage = Storage()
self.storage = storage
self.imageGenerator = ImageGenerator(
storage: storage,
settings: settings)
}
private func clear() {

26
CHDataManagement/Model/FileResource.swift
View File

@ -39,16 +39,11 @@ final class FileResource: Item {
// MARK: Text
func textContent() -> String {
do {
return try content.storage.fileContent(for: id)
} catch {
print("Failed to load text of file \(id): \(error)")
return ""
}
content.storage.fileContent(for: id) ?? ""
}
func dataContent() throws -> Data {
try content.storage.fileData(for: id)
func dataContent() -> Data? {
content.storage.fileData(for: id)
}
// MARK: Images
@ -61,11 +56,8 @@ final class FileResource: Item {
}
var imageToDisplay: Image {
let imageData: Data
do {
imageData = try content.storage.fileData(for: id)
} catch {
print("Failed to load data for image \(id): \(error)")
guard let imageData = content.storage.fileData(for: id) else {
print("Failed to load data for image \(id)")
return failureImage
}
guard let loadedImage = NSImage(data: imageData) else {
@ -123,14 +115,12 @@ final class FileResource: Item {
id = newId
return true
}
do {
try content.storage.move(file: id, to: newId)
id = newId
return true
} catch {
guard content.storage.move(file: id, to: newId) else {
print("Failed to move file \(id) to \(newId)")
return false
}
id = newId
return true
}
}

4
CHDataManagement/Model/Post.swift
View File

@ -73,9 +73,7 @@ final class Post: ObservableObject {
@discardableResult
func update(id newId: String) -> Bool {
do {
try content.storage.move(post: id, to: newId)
} catch {
guard content.storage.move(post: id, to: newId) else {
print("Failed to move file of post \(id)")
return false
}

393
CHDataManagement/Storage/SecurityBookmark.swift Normal file
View File

@ -0,0 +1,393 @@
import Foundation
struct SecurityBookmark {
enum OverwriteBehaviour {
case skip
case write
case writeIfChanged
case fail
}
let url: URL
let isStale: Bool
private let encoder = JSONEncoder()
private let decoder = JSONDecoder()
private let fm = FileManager.default
init(url: URL, isStale: Bool) {
self.url = url
self.isStale = isStale
self.encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
}
// MARK: Write
/**
Write the data of an encodable value to a relative path in the content folder,
or delete the file if nil is passed.
*/
func encode<T>(_ value: T?, to relativePath: String) -> Bool where T: Encodable {
guard let value else {
return deleteFile(at: relativePath)
}
return encode(value, to: relativePath)
}
/**
Write the data of an encodable value to a relative path in the content folder
*/
func encode<T>(_ value: T, to relativePath: String) -> Bool where T: Encodable {
let data: Data
do {
data = try encoder.encode(value)
} catch {
print("Failed to encode \(value): \(error)")
return false
}
return write(data, to: relativePath)
}
func write(_ string: String,
to relativePath: String,
createParentFolder: Bool = true,
ifFileExists overwrite: OverwriteBehaviour = .writeIfChanged) -> Bool {
guard let data = string.data(using: .utf8) else {
return false
}
return write(data, to: relativePath, createParentFolder: createParentFolder, ifFileExists: overwrite)
}
func write(_ data: Data,
to relativePath: String,
createParentFolder: Bool = true,
ifFileExists overwrite: OverwriteBehaviour = .writeIfChanged) -> Bool {
perform { url in
let file = url.appending(path: relativePath, directoryHint: .notDirectory)
if exists(file) {
switch overwrite {
case .fail: return false
case .skip: return true
case .write: break
case .writeIfChanged:
if let existingData = try? Data(contentsOf: file),
existingData == data {
return true
}
}
}
do {
try createParentIfNeeded(of: file)
try data.write(to: file)
} catch {
print("Failed to write to file \(url.path()): \(error)")
return false
}
return true
}
}
func create(folder: String) -> Bool {
with(relativePath: folder, perform: create)
}
// MARK: Read
func hasFile(at relativePath: String) -> Bool {
with(relativePath: relativePath, perform: exists)
}
func readString(at relativePath: String) -> String? {
guard let data = readData(at: relativePath) else {
return nil
}
return String(data: data, encoding: .utf8)
}
func readData(at relativePath: String) -> Data? {
with(relativePath: relativePath) { file in
guard exists(file) else {
return nil
}
do {
return try Data(contentsOf: file)
} catch {
print("Storage: Failed to read file \(relativePath): \(error)")
return nil
}
}
}
func decode<T>(at relativePath: String) -> T? where T: Decodable {
guard let data = readData(at: relativePath) else {
return nil
}
do {
return try decoder.decode(T.self, from: data)
} catch {
print("Failed to decode file \(relativePath): \(error)")
return nil
}
}
// MARK: Modify
func move(_ relativeSource: String,
to relativeDestination: String,
failIfMissing: Bool = true,
createParentFolder: Bool = true,
ifFileExists overwrite: OverwriteBehaviour = .fail) -> Bool {
with(relativePath: relativeSource) { source in
if !exists(source) {
return failIfMissing
}
let destination = url.appending(path: relativeDestination)
if exists(destination) {
switch overwrite {
case .fail: return false
case .skip: return true
case .write: break
case .writeIfChanged:
if let existingData = try? Data(contentsOf: destination),
let newData = try? Data(contentsOf: source),
existingData == newData {
return true
}
}
}
do {
if createParentFolder {
try createParentIfNeeded(of: destination)
}
try fm.moveItem(at: source, to: destination)
return true
} catch {
print("Failed to move \(source.path()) to \(destination.path())")
return false
}
}
}
func copy(externalFile: URL,
to relativePath: String,
createParentFolder: Bool = true,
ifFileExists overwrite: OverwriteBehaviour = .writeIfChanged) -> Bool {
with(relativePath: relativePath) { destination in
do {
if destination.exists {
switch overwrite {
case .fail: return false
case .skip: return true
case .write: break
case .writeIfChanged:
if let existingData = try? Data(contentsOf: destination),
let newData = try? Data(contentsOf: externalFile),
existingData == newData {
return true
}
}
try fm.removeItem(at: destination)
}
try createParentIfNeeded(of: destination)
try fm.copyItem(at: externalFile, to: destination)
return true
} catch {
print("Failed to copy \(externalFile.path()) to \(destination.path())")
return false
}
}
}
func deleteFile(at relativePath: String) -> Bool {
with(relativePath: relativePath) { file in
guard exists(file) else {
return true
}
do {
try fm.removeItem(at: file)
return true
} catch {
print("Failed to delete file \(file.path()): \(error)")
return false
}
}
}
// MARK: Writing files
/**
Delete files in a subPath of the content folder which are not in the given set of files
- Note: This function requires a security scope for the content path
*/
func deleteFiles(in relativePath: String, notIn fileSet: Set<String>) -> [String]? {
with(relativePath: relativePath) { folder in
if !exists(folder) {
return []
}
guard let files = files(in: folder) else {
return []
}
return files.compactMap { file in
guard !fileSet.contains(file.lastPathComponent) else {
return nil
}
guard remove(file) else {
return nil
}
return file.lastPathComponent
}
}
}
// MARK: Transfer
func transfer(file sourcePath: String,
to relativePath: String,
of scope: SecurityBookmark,
createParentFolder: Bool = true,
ifFileExists: OverwriteBehaviour = .writeIfChanged) -> Bool {
with(relativePath: sourcePath) { source in
scope.copy(
externalFile: source,
to: relativePath,
createParentFolder: createParentFolder,
ifFileExists: ifFileExists)
}
}
// MARK: Batch operations
func fileNames(inRelativeFolder relativePath: String) -> [String]? {
files(inRelativeFolder: relativePath)?.map { $0.lastPathComponent }
}
func files(inRelativeFolder relativePath: String) -> [URL]? {
with(relativePath: relativePath) { folder in
files(in: folder)
}
}
/**
- Note: This function requires a security scope for the content path
*/
func decodeJsonFiles<T>(in relativeFolder: String) -> [String : T]? where T: Decodable {
with(relativePath: relativeFolder) { folder in
guard let files = files(in: folder) else {
return nil
}
return files.filter { $0.pathExtension.lowercased() == "json" }
.reduce(into: [:]) { items, url in
let id = url.deletingPathExtension().lastPathComponent
let data: Data
do {
data = try Data(contentsOf: url)
} catch {
print("Storage: Failed to read file \(url.path()): \(error)")
return
}
do {
items[id] = try decoder.decode(T.self, from: data)
} catch {
print("Storage: Failed to decode file \(url.path()): \(error)")
return
}
}
}
}
// MARK: Generic operations
func with(relativePath: String, perform operation: (URL) -> Bool) -> Bool {
perform { operation($0.appending(path: relativePath)) }
}
func with<T>(relativePath: String, perform operation: (URL) -> T?) -> T? {
perform { operation($0.appending(path: relativePath)) }
}
/**
Run an operation in the security scope of a url.
*/
func perform(_ operation: (URL) -> Bool) -> Bool {
guard url.startAccessingSecurityScopedResource() else {
print("Failed to start security scope")
return false
}
defer { url.stopAccessingSecurityScopedResource() }
return operation(url)
}
/**
Run an operation in the content folder
*/
func perform<T>(_ operation: (URL) -> T?) -> T? {
guard url.startAccessingSecurityScopedResource() else {
print("Failed to start security scope")
return nil
}
defer { url.stopAccessingSecurityScopedResource() }
return operation(url)
}
// MARK: Unscoped helpers
private func create(folder: URL) -> Bool {
do {
try createIfNeeded(folder)
return true
} catch {
print("Failed to create folder \(folder.path())")
return false
}
}
private func exists(_ url: URL) -> Bool {
fm.fileExists(atPath: url.path())
}
private func remove(_ file: URL) -> Bool {
guard exists(url) else {
return true
}
do {
try fm.removeItem(at: file)
} catch {
print("Failed to remove \(file.path()): \(error)")
return false
}
return true
}
private func createParentIfNeeded(of file: URL) throws {
try createIfNeeded(file.deletingLastPathComponent())
}
private func createIfNeeded(_ folder: URL) throws {
if !exists(folder) {
try fm.createDirectory(at: folder, withIntermediateDirectories: true)
}
}
private func files(in folder: URL) throws -> [URL] {
try FileManager.default.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)
.filter { !$0.hasDirectoryPath }
}
private func files(in folder: URL) -> [URL]? {
do {
return try files(in: folder).filter { !$0.lastPathComponent.hasPrefix(".") }
} catch {
print("Failed to read list of files in \(folder.path())")
return nil
}
}
}

734
CHDataManagement/Storage/Storage.swift
View File

@ -24,88 +24,38 @@ final class Storage: ObservableObject {
private let tagsFolderName = "tags"
private let externalFileListName = "external-files.json"
private let fileDescriptionFilename = "file-descriptions.json"
private let generatedImagesListName = "generated-images.json"
private let outputPathFileName = "outputPath.bin"
private let settingsDataFileName = "settings.json"
private let tagOverviewFileName = "tag-overview.json"
private let contentPathBookmarkKey = "contentPathBookmark"
// MARK: Properties
private let encoder = JSONEncoder()
private let decoder = JSONDecoder()
private let fm = FileManager.default
@Published
var contentScope: SecurityBookmark?
@Published
var hasContentFolders = false
@Published
var contentPath: URL?
@Published
var outputPath: URL?
@Published
var contentPathUrlIsStale = false
@Published
var outputPathUrlIsStale = false
var outputScope: SecurityBookmark?
/**
Create the storage.
*/
init() {
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
loadContentPath()
createFolderStructure()
}
// MARK: Helper
private func files(in folder: URL) throws -> [URL] {
do {
return try fm.contentsOfDirectory(at: folder, includingPropertiesForKeys: [.isDirectoryKey])
.filter { !$0.hasDirectoryPath }
} catch {
print("Failed to get files in folder \(folder.path): \(error)")
throw error
}
}
// MARK: Folders
@discardableResult
func createFolderStructure() -> Bool {
do {
try inContentFolder { contentPath in
try pagesFolder(in: contentPath).createIfNeeded()
try filesFolder(in: contentPath).createIfNeeded()
try postsFolder(in: contentPath).createIfNeeded()
try tagsFolder(in: contentPath).createIfNeeded()
}
hasContentFolders = true
return true
} catch StorageAccessError.noBookmarkData {
hasContentFolders = false
} catch {
print("Failed to create storage folders: \(error)")
hasContentFolders = false
}
return false
loadContentScope()
loadOutputScope()
}
// MARK: Pages
/// The folder path where the markdown and metadata files of the pages are stored (by their id/url component)
private func pagesFolder(in folder: URL) -> URL {
folder.appending(path: pagesFolderName, directoryHint: .isDirectory)
}
private func pageContentFileName(_ id: String, _ language: ContentLanguage) -> String {
"\(id)-\(language.rawValue).md"
@ -123,68 +73,64 @@ final class Storage: ObservableObject {
id + ".json"
}
private func pageContentUrl(page pageId: String, language: ContentLanguage, in folder: URL) -> URL {
let fileName = pageContentFileName(pageId, language)
return pagesFolder(in: folder).appending(path: fileName, directoryHint: .notDirectory)
}
private func pageMetadataUrl(page pageId: String, in folder: URL) -> URL {
let fileName = pageFileName(pageId)
return pagesFolder(in: folder).appending(path: fileName, directoryHint: .notDirectory)
}
func save(pageContent: String, for pageId: String, language: ContentLanguage) throws {
func save(pageContent: String, for pageId: String, language: ContentLanguage) -> Bool {
guard let contentScope else { return false }
let path = pageContentPath(page: pageId, language: language)
try writeIfChanged(content: pageContent, to: path)
return contentScope.write(pageContent, to: path)
}
func save(pageMetadata: PageFile, for pageId: String) throws {
func save(pageMetadata: PageFile, for pageId: String) -> Bool {
guard let contentScope else { return false }
let path = pageMetadataPath(page: pageId)
try writeIfChanged(pageMetadata, to: path)
return contentScope.encode(pageMetadata, to: path)
}
func loadAllPages() throws -> [String : PageFile] {
try decodeAllFromJson(in: pagesFolderName)
func loadAllPages() -> [String : PageFile]? {
contentScope?.decodeJsonFiles(in: pagesFolderName)
}
func pageContent(for pageId: String, language: ContentLanguage) throws -> String {
func pageContent(for pageId: String, language: ContentLanguage) -> String? {
guard let contentScope else { return nil }
let path = pageContentPath(page: pageId, language: language)
return try readString(at: path, defaultValue: "")
return contentScope.readString(at: path)
}
/**
Delete all files associated with pages that are not in the given set
- Note: This function requires a security scope for the content path
*/
func deletePageFiles(notIn pages: [String]) throws {
func deletePageFiles(notIn pages: [String]) -> Bool {
guard let contentScope else { return false }
var files = Set(pages.map(pageFileName))
for language in ContentLanguage.allCases {
files.formUnion(pages.map { pageContentFileName($0, language) })
}
try deleteFiles(in: pagesFolderName, notIn: files)
}
func move(page pageId: String, to newFile: String) -> Bool {
do {
try operate(in: .contentPath) { contentPath in
// Move the metadata file
let source = pageMetadataUrl(page: pageId, in: contentPath)
let destination = pageMetadataUrl(page: newFile, in: contentPath)
try fm.moveItem(at: source, to: destination)
// Move the existing content files
for language in ContentLanguage.allCases {
let source = pageContentUrl(page: pageId, language: language, in: contentPath)
guard source.exists else { continue }
let destination = pageContentUrl(page: newFile, language: language, in: contentPath)
try fm.moveItem(at: source, to: destination)
}
}
return true
} catch {
print("Failed to move page file \(pageId) to \(newFile): \(error)")
guard let deleted = contentScope.deleteFiles(in: pagesFolderName, notIn: files) else {
return false
}
deleted.forEach { print("Deleted unused page file \($0)") }
return true
}
func move(page pageId: String, to newId: String) -> Bool {
guard let contentScope else { return false }
guard contentScope.move(pageFileName(pageId), to: pageFileName(newId)) else {
return false
}
// Move the existing content files
var result = true
for language in ContentLanguage.allCases {
// Copy as many files as possible, since metadata was already moved
// Don't fail early
if !contentScope.move(
pageContentFileName(pageId, language),
to: pageContentFileName(newId, language),
failIfMissing: false) {
result = false
}
}
return result
}
// MARK: Posts
@ -193,44 +139,37 @@ final class Storage: ObservableObject {
postId + ".json"
}
/// The folder path where the markdown files of the posts are stored (by their unique id/url component)
private func postsFolder(in folder: URL) -> URL {
folder.appending(path: postsFolderName, directoryHint: .isDirectory)
}
private func postFileUrl(post postId: String, in folder: URL) -> URL {
let path = postFilePath(post: postId)
return folder.appending(path: path, directoryHint: .notDirectory)
}
private func postFilePath(post postId: String) -> String {
postsFolderName + "/" + postFileName(postId)
}
func save(post: PostFile, for postId: String) throws {
func save(post: PostFile, for postId: String) -> Bool {
guard let contentScope else { return false }
let path = postFilePath(post: postId)
try writeIfChanged(post, to: path)
return contentScope.encode(post, to: path)
}
func loadAllPosts() throws -> [String : PostFile] {
try decodeAllFromJson(in: postsFolderName)
func loadAllPosts() -> [String : PostFile]? {
contentScope?.decodeJsonFiles(in: postsFolderName)
}
/**
Delete all files associated with posts that are not in the given set
- Note: This function requires a security scope for the content path
*/
func deletePostFiles(notIn posts: [String]) throws {
func deletePostFiles(notIn posts: [String]) -> Bool {
guard let contentScope else { return false }
let files = Set(posts.map(postFileName))
try deleteFiles(in: postsFolderName, notIn: files)
guard let deleted = contentScope.deleteFiles(in: postsFolderName, notIn: files) else {
return false
}
deleted.forEach { print("Deleted unused post file \($0)") }
return true
}
func move(post postId: String, to newFile: String) throws {
try operate(in: .contentPath) { contentPath in
let source = postFileUrl(post: postId, in: contentPath)
let destination = postFileUrl(post: newFile, in: contentPath)
try fm.moveItem(at: source, to: destination)
}
func move(post postId: String, to newId: String) -> Bool {
guard let contentScope else { return false }
return contentScope.move(postFilePath(post: postId), to: postFilePath(post: newId))
}
// MARK: Tags
@ -239,55 +178,54 @@ final class Storage: ObservableObject {
tagId + ".json"
}
/// The folder path where the source images are stored (by their unique name)
private func tagsFolder(in folder: URL) -> URL {
folder.appending(path: tagsFolderName)
}
private func relativeTagFilePath(tagId: String) -> String {
private func tagFilePath(tagId: String) -> String {
tagsFolderName + "/" + tagFileName(tagId: tagId)
}
func save(tagMetadata: TagFile, for tagId: String) throws {
let path = relativeTagFilePath(tagId: tagId)
try writeIfChanged(tagMetadata, to: path)
func save(tagMetadata: TagFile, for tagId: String) -> Bool {
guard let contentScope else { return false }
let path = tagFilePath(tagId: tagId)
return contentScope.encode(tagMetadata, to: path)
}
func loadAllTags() throws -> [String : TagFile] {
try decodeAllFromJson(in: tagsFolderName)
func loadAllTags() -> [String : TagFile]? {
contentScope?.decodeJsonFiles(in: tagsFolderName)
}
/**
Delete all files associated with tags that are not in the given set
- Note: This function requires a security scope for the content path
*/
func deleteTagFiles(notIn tags: [String]) throws {
func deleteTagFiles(notIn tags: [String]) -> Bool {
guard let contentScope else { return false }
let files = Set(tags.map { $0 + ".json" })
try deleteFiles(in: tagsFolderName, notIn: files)
guard let deleted = contentScope.deleteFiles(in: tagsFolderName, notIn: files) else {
return false
}
deleted.forEach { print("Deleted unused tag file \($0)") }
return true
}
// MARK: File descriptions
func loadFileDescriptions() throws -> [FileDescriptions] {
guard let descriptions: [FileDescriptions] = try read(at: fileDescriptionFilename) else {
print("Storage: No file descriptions loaded")
return []
}
return descriptions
func loadFileDescriptions() -> [FileDescriptions]? {
contentScope?.decode(at: fileDescriptionFilename)
}
func save(fileDescriptions: [FileDescriptions]) throws {
try writeIfChanged(fileDescriptions, to: fileDescriptionFilename)
func save(fileDescriptions: [FileDescriptions]) -> Bool {
guard let contentScope else { return false }
return contentScope.encode(fileDescriptions, to: fileDescriptionFilename)
}
// MARK: Tag overview
func loadTagOverview() throws -> TagOverviewFile? {
try read(at: tagOverviewFileName)
func loadTagOverview() -> TagOverviewFile? {
contentScope?.decode(at: tagOverviewFileName)
}
func save(tagOverview: TagOverviewFile?) throws {
try writeIfChanged(tagOverview, to: tagOverviewFileName)
func save(tagOverview: TagOverviewFile?) -> Bool {
guard let contentScope else { return false }
return contentScope.encode(tagOverview, to: tagOverviewFileName)
}
// MARK: Files
@ -296,185 +234,127 @@ final class Storage: ObservableObject {
filesFolderName + "/" + fileId
}
/// The folder path where other files are stored (by their unique name)
private func filesFolder(in folder: URL) -> URL {
folder.appending(path: filesFolderName, directoryHint: .isDirectory)
}
private func fileUrl(file: String, in folder: URL) -> URL {
filesFolder(in: folder).appending(path: file, directoryHint: .notDirectory)
}
/**
Copy an external file to the content folder
*/
func copyFile(at url: URL, fileId: String) throws {
try operate(in: .contentPath) { contentPath in
let destination = fileUrl(file: fileId, in: contentPath)
try fm.copyItem(at: url, to: destination)
}
func importExternalFile(at url: URL, fileId: String) -> Bool {
guard let contentScope else { return false }
return contentScope.copy(externalFile: url, to: filePath(file: fileId))
}
func move(file fileId: String, to newFile: String) throws {
try operate(in: .contentPath) { contentPath in
let source = fileUrl(file: fileId, in: contentPath)
let destination = fileUrl(file: newFile, in: contentPath)
try fm.moveItem(at: source, to: destination)
}
func move(file fileId: String, to newId: String) -> Bool {
guard let contentScope else { return false }
return contentScope.move(filePath(file: fileId), to: filePath(file: newId))
}
func copy(file fileId: String, to relativeOutputPath: String) throws {
let path = filePath(file: fileId)
try withScopedContent(file: path) { input in
try operate(in: .outputPath) { outputPath in
let output = outputPath.appending(path: relativeOutputPath, directoryHint: .notDirectory)
if output.exists {
return
}
try output.createParentFolderIfNeeded()
try FileManager.default.copyItem(at: input, to: output)
}
}
func copy(file fileId: String, to relativeOutputPath: String) -> Bool {
guard let contentScope, let outputScope else { return false }
return contentScope.transfer(
file: filePath(file: fileId),
to: relativeOutputPath, of: outputScope)
}
func loadAllFiles() throws -> [String] {
try inContentFolder(relativePath: filesFolderName) { try $0.containedFileNames() }
func loadAllFiles() -> [String]? {
contentScope?.fileNames(inRelativeFolder: filesFolderName)
}
/**
Delete all file resources that are not in the given set
- Note: This function requires a security scope for the content path
*/
func deleteFileResources(notIn fileSet: [String]) throws {
try deleteFiles(in: filesFolderName, notIn: Set(fileSet))
func deleteFileResources(notIn fileSet: [String]) -> Bool {
guard let contentScope else { return false }
guard let deleted = contentScope.deleteFiles(in: filesFolderName, notIn: Set(fileSet)) else {
return false
}
deleted.forEach { print("Deleted unused file \($0)") }
return true
}
func fileContent(for fileId: String) throws -> String {
func fileContent(for fileId: String) -> String? {
guard let contentScope else { return nil }
let path = filePath(file: fileId)
return try readString(at: path)
return contentScope.readString(at: path)
}
func fileData(for fileId: String) throws -> Data {
func fileData(for fileId: String) -> Data? {
guard let contentScope else { return nil }
let path = filePath(file: fileId)
return try readExistingFile(at: path)
return contentScope.readData(at: path)
}
// MARK: External file list
private let externalFileListName = "external-files.json"
func loadExternalFileList() throws -> [String] {
guard let files: [String] = try read(at: externalFileListName) else {
print("Storage: No external file list found")
return []
}
return files
func loadExternalFileList() -> [String]? {
guard let contentScope else { return nil }
return contentScope.decode(at: externalFileListName)
}
func save(externalFileList: [String]) throws {
try writeIfChanged(externalFileList.sorted(), to: externalFileListName)
func save(externalFileList: [String]) -> Bool {
guard let contentScope else { return false }
return contentScope.encode(externalFileList.sorted(), to: externalFileListName)
}
// MARK: Settings
private let settingsDataFileName: String = "settings.json"
func loadSettings() throws -> SettingsFile {
guard let settings: SettingsFile = try read(at: settingsDataFileName) else {
print("Storage: Loaded default settings")
return .default
}
return settings
func loadSettings() -> SettingsFile? {
guard let contentScope else { return nil }
return contentScope.decode(at: settingsDataFileName)
}
func save(settings: SettingsFile) throws {
try writeIfChanged(settings, to: settingsDataFileName)
func save(settings: SettingsFile) -> Bool {
guard let contentScope else { return false }
return contentScope.encode(settings, to: settingsDataFileName)
}
// MARK: Image generation data
func loadListOfGeneratedImages() throws -> [String : [String]] {
guard let images: [String : [String]] = try read(at: generatedImagesListName) else {
print("Storage: No generated images found")
func loadListOfGeneratedImages() -> [String : Set<String>]? {
guard let contentScope else { return nil }
return contentScope.decode(at: generatedImagesListName)
}
func save(listOfGeneratedImages: [String : Set<String>]) -> Bool {
guard let contentScope else { return false }
return contentScope.encode(listOfGeneratedImages, to: generatedImagesListName)
}
func calculateImages(generatedBy imageSet: Set<String>, in folder: String) -> [String : Set<String>] {
guard let outputScope else { return [:] }
guard let allImages = outputScope.fileNames(inRelativeFolder: folder) else {
print("Failed to get list of generated images in output folder")
return [:]
}
return images
}
func save(listOfGeneratedImages: [String : [String]]) throws {
try writeIfChanged(listOfGeneratedImages, to: generatedImagesListName)
}
// MARK: Output files
func write(content: String, to relativeOutputPath: String) throws {
try writeIfChanged(content: content, to: relativeOutputPath, in: .outputPath)
}
// MARK: Folder access
func create(folder relativePath: String, in scope: SecurityScopeBookmark) -> Bool {
return write(in: scope) { folder in
let url = folder.appendingPathComponent(relativePath, isDirectory: true)
do {
try url.createIfNeeded()
return true
} catch {
print("Failed to create folder \(url.path()): \(error)")
return false
guard !allImages.isEmpty else {
print("No images found in output folder \(folder)")
return [:]
}
print("Found \(allImages.count) generated images")
let images = Set(allImages)
return imageSet.reduce(into: [:]) { result, imageName in
let prefix = imageName.fileNameWithoutExtension + "@"
let versions = images.filter { $0.hasPrefix(prefix) }
if !versions.isEmpty {
result[imageName] = Set(versions)
}
}
}
func write(in scope: SecurityScopeBookmark, operation: (URL) -> Bool) -> Bool {
do {
return try operate(in: scope, operation: operation)
} catch {
print(error)
return false
}
// MARK: Output files
func write(_ content: String, to relativeOutputPath: String) -> Bool {
guard let outputScope else { return false }
return outputScope.write(content, to: relativeOutputPath)
}
private func withScopedContent<T>(file relativePath: String, in scope: SecurityScopeBookmark = .contentPath, _ operation: (URL) throws -> T) throws -> T {
try withScopedContent(relativePath, in: scope, directoryHint: .notDirectory, operation)
func write(_ data: Data, to relativeOutputPath: String) -> Bool {
guard let outputScope else { return false }
return outputScope.write(data, to: relativeOutputPath)
}
private func withScopedContent<T>(folder relativePath: String, in scope: SecurityScopeBookmark = .contentPath, _ operation: (URL) throws -> T) throws -> T {
try withScopedContent(relativePath, in: scope, directoryHint: .isDirectory, operation)
}
private func withScopedContent<T>(_ relativePath: String, in scope: SecurityScopeBookmark, directoryHint: URL.DirectoryHint, _ operation: (URL) throws -> T) throws -> T {
try operate(in: scope) {
let url = $0.appending(path: relativePath, directoryHint: directoryHint)
return try operation(url)
}
}
func operate<T>(in scope: SecurityScopeBookmark, operation: (URL) throws -> T) throws -> T {
guard let bookmarkData = UserDefaults.standard.data(forKey: scope.rawValue) else {
throw StorageAccessError.noBookmarkData
}
var isStale = false
let folderUrl: URL
do {
// Resolve the bookmark to get the folder URL
folderUrl = try URL(resolvingBookmarkData: bookmarkData, options: .withSecurityScope, relativeTo: nil, bookmarkDataIsStale: &isStale)
} catch {
throw StorageAccessError.bookmarkDataCorrupted(error)
}
if isStale {
print("Bookmark is stale, consider saving a new bookmark.")
#warning("Show warning about stale bookmark")
}
// Start accessing the security-scoped resource
guard folderUrl.startAccessingSecurityScopedResource() else {
throw StorageAccessError.folderAccessFailed(folderUrl)
}
defer { folderUrl.stopAccessingSecurityScopedResource() }
return try operation(folderUrl)
func hasFileInOutputFolder(_ relativeOutputPath: String) -> Bool {
guard let outputScope else { return false }
return outputScope.hasFile(at: relativeOutputPath)
}
// MARK: Security bookmarks
@ -493,10 +373,7 @@ final class Storage: ObservableObject {
return false
}
UserDefaults.standard.set(bookmarkData, forKey: contentPathBookmarkKey)
guard loadContentPath() else {
return false
}
return createFolderStructure()
return loadContentScope()
}
/**
@ -508,32 +385,36 @@ final class Storage: ObservableObject {
- Returns: `true`, if the url was loaded.
*/
@discardableResult
private func loadContentPath() -> Bool {
private func loadContentScope() -> Bool {
guard let bookmarkData = UserDefaults.standard.data(forKey: contentPathBookmarkKey) else {
print("No content path bookmark found")
contentPath = nil
contentPathUrlIsStale = false
contentScope = nil
return false
}
let (url, isStale) = decode(bookmark: bookmarkData)
contentPath = url
contentPathUrlIsStale = isStale
return url != nil
contentScope = decode(bookmark: bookmarkData)
return contentScope != nil
}
@discardableResult
private func loadOutputScope() -> Bool {
guard let contentScope else { return false }
guard let data = contentScope.readData(at: outputPathFileName) else {
return false
}
outputScope = decode(bookmark: data)
return outputScope != nil
}
func clearContentPath() {
UserDefaults.standard.removeObject(forKey: contentPathBookmarkKey)
contentPath = nil
contentPathUrlIsStale = false
hasContentFolders = false
outputPath = nil
outputPathUrlIsStale = false
contentScope = nil
outputScope = nil
}
/**
Decode the security scope data to get a url.
*/
private func decode(bookmark: Data) -> (url: URL?, isStale: Bool) {
private func decode(bookmark: Data) -> SecurityBookmark? {
do {
var isStale = false
let url = try URL(
@ -541,10 +422,10 @@ final class Storage: ObservableObject {
options: .withSecurityScope,
relativeTo: nil,
bookmarkDataIsStale: &isStale)
return (url, isStale)
return SecurityBookmark(url: url, isStale: isStale)
} catch {
print("Failed to resolve bookmark: \(error)")
return (nil, false)
return nil
}
}
@ -562,257 +443,14 @@ final class Storage: ObservableObject {
@discardableResult
func save(outputPath: URL) -> Bool {
guard let contentPath else { return false }
guard let bookmarkData = encode(url: outputPath) else { return false }
return write(bookmarkData, to: outputPathFileName, in: contentPath, onlyIfChanged: false)
}
/**
Run an operation in the content folder
*/
func inContentFolder<T>(perform operation: (URL) throws -> T) throws -> T {
try inSecurityScope(of: contentPath, perform: operation)
}
/**
Run an operation in the output folder
*/
func inOutputFolder<T>(perform operation: (URL) throws -> T) throws -> T {
try inSecurityScope(of: outputPath, perform: operation)
}
func inContentFolder<T>(relativePath: String, perform operation: (URL) throws -> T) throws -> T {
try inContentFolder { url in
try operation(url.appendingPathComponent(relativePath))
}
}
/**
Run an operation in the security scope of a url.
*/
private func inSecurityScope<T>(of url: URL?, perform: (URL) throws -> T) throws -> T {
guard let url else {
throw StorageAccessError.noBookmarkData
}
guard url.startAccessingSecurityScopedResource() else {
throw StorageAccessError.folderAccessFailed(url)
}
defer { url.stopAccessingSecurityScopedResource() }
return try perform(url)
}
private func writeContent(_ data: Data, to relativePath: String, onlyIfChanged: Bool = true) -> Bool {
guard let contentPath else { return false }
return write(data, to: relativePath, in: contentPath, onlyIfChanged: onlyIfChanged)
}
private func write(_ data: Data, to relativePath: String, in folder: URL, onlyIfChanged: Bool = true) -> Bool {
do {
try inSecurityScope(of: folder) { url in
let file = url.appending(path: relativePath, directoryHint: .notDirectory)
// Load previous file and compare
if onlyIfChanged,
fm.fileExists(atPath: file.path()),
let oldData = try? Data(contentsOf: file), // Write file again in case of read error
oldData == data {
return
}
try data.write(to: file)
}
return true
} catch {
print("Failed to write to file: \(error)")
#warning("Report error")
guard let contentScope,
let bookmarkData = encode(url: outputPath),
contentScope.write(bookmarkData, to: outputPathFileName) else {
outputScope = nil
return false
}
}
// MARK: Writing files
/**
Delete files in a subPath of the content folder which are not in the given set of files
- Note: This function requires a security scope for the content path
*/
private func deleteFiles(in folder: String, notIn fileSet: Set<String>) throws {
try withScopedContent(folder: folder) { folderUrl in
let filesToDelete = try files(in: folderUrl)
.filter { !fileSet.contains($0.lastPathComponent) }
for file in filesToDelete {
try fm.removeItem(at: file)
print("Deleted \(file.path())")
}
}
}
/**
Write the data of an encodable value to a relative path in the content folder,
or delete the file if nil is passed.
- Note: This function requires a security scope for the content path
*/
private func writeIfChanged<T>(_ value: T?, to relativePath: String) throws where T: Encodable {
guard let value else {
try deleteFile(at: relativePath)
return
}
return try writeIfChanged(value, to: relativePath)
}
/**
Write the data of an encodable value to a relative path in the content folder
- Note: This function requires a security scope for the content path
*/
private func writeIfChanged<T>(_ value: T, to relativePath: String) throws where T: Encodable {
let data = try encoder.encode(value)
try writeIfChanged(data: data, to: relativePath)
}
/**
Write the data of a string to a relative path in the content folder
- Note: This function requires a security scope for the content path
*/
private func writeIfChanged(content: String, to relativePath: String, in scope: SecurityScopeBookmark = .contentPath) throws {
guard let data = content.data(using: .utf8) else {
print("Failed to convert string to data for file at \(relativePath)")
throw StorageAccessError.stringConversionFailed
}
try writeIfChanged(data: data, to: relativePath, in: scope)
}
/**
Write the data to a relative path in the content folder
- Note: This function requires a security scope for the content path
*/
private func writeIfChanged(data: Data, to relativePath: String, in scope: SecurityScopeBookmark = .contentPath) throws {
try withScopedContent(file: relativePath, in: scope) { url in
if fm.fileExists(atPath: url.path()) {
// Check if content is the same, to prevent unnecessary writes
do {
let oldData = try Data(contentsOf: url)
if data == oldData {
// File is the same, don't write
return
}
} catch {
print("Failed to read file \(url.path()) for equality check: \(error)")
// No check possible, write file
}
} else {
print("Writing new file \(url.path())")
try url.createParentFolderIfNeeded()
}
try data.write(to: url)
print("Saved file \(url.path())")
}
}
/**
Read an object from a file, if the file exists
*/
private func read<T>(at relativePath: String) throws -> T? where T: Decodable {
guard let data = try readData(at: relativePath) else {
return nil
}
do {
return try decoder.decode(T.self, from: data)
} catch {
print("Failed to decode file \(relativePath): \(error)")
throw error
}
}
/**
- Note: This function requires a security scope for the content path
*/
private func readString(at relativePath: String, defaultValue: String? = nil) throws -> String {
try withScopedContent(file: relativePath) { url in
guard url.exists else {
guard let defaultValue else {
throw StorageAccessError.fileNotFound(relativePath)
}
return defaultValue
}
return try String(contentsOf: url, encoding: .utf8)
}
}
private func readExistingFile(at relativePath: String) throws -> Data {
guard let data = try readData(at: relativePath) else {
throw StorageAccessError.fileNotFound(relativePath)
}
return data
}
/**
- Note: This function requires a security scope for the content path
*/
private func readData(at relativePath: String) throws -> Data? {
try withScopedContent(file: relativePath) { url in
guard url.exists else {
return nil
}
do {
return try Data(contentsOf: url)
} catch {
print("Storage: Failed to read file \(relativePath): \(error)")
throw error
}
}
}
/**
- Note: This function requires a security scope for the content path
*/
private func decodeAllFromJson<T>(in folder: String) throws -> [String : T] where T: Decodable {
try inContentFolder(relativePath: folder) { folderUrl in
do {
return try folderUrl
.containedFiles()
.filter { $0.pathExtension.lowercased() == "json" }
.reduce(into: [:]) { items, url in
let id = url.deletingPathExtension().lastPathComponent
let data: Data
do {
data = try Data(contentsOf: url)
} catch {
print("Storage: Failed to read file \(url.path()): \(error)")
throw error
}
do {
items[id] = try decoder.decode(T.self, from: data)
} catch {
print("Storage: Failed to decode file \(url.path()): \(error)")
throw error
}
}
} catch {
print("Storage: Failed to decode files in \(folder): \(error)")
throw error
}
}
}
/**
- Note: This function requires a security scope for the content path
*/
private func copy(file: URL, to relativePath: String) throws {
try withScopedContent(file: relativePath) { destination in
try destination.createParentFolderIfNeeded()
try fm.copyItem(at: file, to: destination)
}
}
private func deleteFile(at relativePath: String) throws {
try withScopedContent(file: relativePath) { destination in
guard fm.fileExists(atPath: destination.path()) else {
return
}
try fm.removeItem(at: destination)
}
// TODO: Check if stale
outputScope = SecurityBookmark(url: outputPath, isStale: false)
return true
}
}

6
CHDataManagement/Views/Files/AddFileView.swift
View File

@ -92,10 +92,8 @@ struct AddFileView: View {
continue
}
if let url = file.url {
do {
try content.storage.copyFile(at: url, fileId: file.uniqueId)
} catch {
print("Failed to import file '\(file.uniqueId)' at \(url.path()): \(error)")
guard content.storage.importExternalFile(at: url, fileId: file.uniqueId) else {
print("Failed to import file '\(file.uniqueId)' at \(url.path())")
return
}
}

12
CHDataManagement/Views/Generic/FolderOnDiskPropertyView.swift
View File

@ -6,19 +6,15 @@ struct FolderOnDiskPropertyView: View {
let title: LocalizedStringKey
@Binding
var folder: URL?
@Binding
var isStale: Bool
var folder: SecurityBookmark?
let footer: LocalizedStringKey
let update: (URL) -> Void
init(title: LocalizedStringKey, folder: Binding<URL?>, isStale: Binding<Bool>, footer: LocalizedStringKey, update: @escaping (URL) -> Void) {
init(title: LocalizedStringKey, folder: Binding<SecurityBookmark?>, footer: LocalizedStringKey, update: @escaping (URL) -> Void) {
self.title = title
self._folder = folder
self._isStale = isStale
self.footer = footer
self.update = update
}
@ -29,7 +25,7 @@ struct FolderOnDiskPropertyView: View {
HStack(alignment: .firstTextBaseline) {
Text(title)
.font(.headline)
if isStale {
if folder == nil || folder?.isStale == true {
Image(systemSymbol: .exclamationmarkTriangle)
.foregroundStyle(.yellow)
}
@ -43,7 +39,7 @@ struct FolderOnDiskPropertyView: View {
}
}
}
Text(folder?.path() ?? "No folder selected")
Text(folder?.url.path() ?? "No folder selected")
.padding(.bottom, 5)
Text(footer)
.foregroundStyle(.secondary)

36
CHDataManagement/Views/Pages/LocalizedPageContentView.swift
View File

@ -65,22 +65,21 @@ struct LocalizedPageContentView: View {
private func loadContent() {
let language = language
do {
let content = try page.content.storage.pageContent(for: pageId, language: language)
guard content != "" else {
pageContent = "New file"
DispatchQueue.main.async {
didChangeContent = false
}
return
}
pageContent = content
checkContent()
} catch {
print("Failed to load page content: \(error)")
guard let content = page.content.storage.pageContent(for: pageId, language: language) else {
print("Failed to load page content")
pageContent = "Failed to load"
return
}
guard content != "" else {
pageContent = "New file"
DispatchQueue.main.async {
didChangeContent = false
}
return
}
pageContent = content
checkContent()
DispatchQueue.main.async {
didChangeContent = false
}
@ -94,12 +93,11 @@ struct LocalizedPageContentView: View {
guard didChangeContent else {
return
}
do {
try page.content.storage.save(pageContent: pageContent, for: pageId, language: language)
didChangeContent = false
} catch {
print("Failed to save content: \(error)")
guard page.content.storage.save(pageContent: pageContent, for: pageId, language: language) else {
print("Failed to save content")
return
}
didChangeContent = false
}
private func checkContent() {

11
CHDataManagement/Views/Pages/PageDetailView.swift
View File

@ -82,17 +82,8 @@ struct PageDetailView: View {
}
private func generate() {
guard let url = content.storage.outputPath else {
print("Invalid output path")
return
}
guard FileManager.default.fileExists(atPath: url.path) else {
print("Missing output folder")
return
}
DispatchQueue.global(qos: .userInitiated).async {
let success = content.generatePage(page)
let success = content.generateFeed()
DispatchQueue.main.async {
didGenerateWebsite = success
}

7
CHDataManagement/Views/Settings/Content/Pages/PageIssueChecker.swift
View File

@ -54,14 +54,13 @@ final class PageIssueChecker: ObservableObject {
let hasPreviousIssues = issues.contains { $0.page == page && $0.language == language }
let pageIssues: [PageIssue]
do {
let rawPageContent = try page.content.storage.pageContent(for: page.id, language: language)
if let rawPageContent = page.content.storage.pageContent(for: page.id, language: language) {
_ = parser.generatePage(from: rawPageContent)
pageIssues = parser.results.issues.map {
PageIssue(page: page, language: language, message: $0)
}
} catch {
let message = PageContentAnomaly.failedToLoadContent(error)
} else {
let message = PageContentAnomaly.failedToLoadContent
let error = PageIssue(page: page, language: language, message: message)
pageIssues = [error]
}

25
CHDataManagement/Views/Settings/Content/Pages/PageIssueView.swift
View File

@ -90,6 +90,8 @@ struct PageIssueView: View {
return [.init(name: "Retry", action: retryPageCheck)]
case .failedToLoadContent:
return [.init(name: "Retry", action: retryPageCheck)]
case .failedToParseContent:
return [.init(name: "Retry", action: retryPageCheck)]
case .missingFile(let missing, _):
return [
.init(name: "Select file", action: { selectFile(missingFile: missing) }),
@ -284,23 +286,22 @@ struct PageIssueView: View {
}
private func replace(_ oldString: String, with newString: String, in page: Page, language: ContentLanguage) {
do {
let pageContent = try content.storage.pageContent(for: page.id, language: language)
.replacingOccurrences(of: oldString, with: newString)
try content.storage.save(pageContent: pageContent, for: page.id, language: language)
guard let pageContent = content.storage.pageContent(for: page.id, language: language) else {
print("Failed to replace in page \(page.id) (\(language)), no content")
return
}
let modified = pageContent.replacingOccurrences(of: oldString, with: newString)
guard content.storage.save(pageContent: modified, for: page.id, language: language) else {
print("Replaced \(oldString) with \(newString) in page \(page.id) (\(language))")
} catch {
print("Failed to replace in page \(page.id) (\(language)): \(error)")
return
}
}
private func findOccurrences(of searchString: String, in page: Page, language: ContentLanguage) -> [String] {
let parts: [String]
do {
parts = try content.storage.pageContent(for: page.id, language: language)
.components(separatedBy: searchString)
} catch {
print("Failed to get page content to find occurrences: \(error.localizedDescription)")
guard let parts = content.storage.pageContent(for: page.id, language: language)?
.components(separatedBy: searchString) else {
print("Failed to get page content to find occurrences, no content")
return []
}

13
CHDataManagement/Views/Settings/GenerationContentView.swift
View File

@ -51,6 +51,9 @@ struct GenerationContentView: View {
.progressViewStyle(.circular)
.frame(height: 25)
}
Button(action: updateGeneratedImages) {
Text("Update images")
}
Text(generatorText)
Spacer()
}
@ -58,7 +61,16 @@ struct GenerationContentView: View {
}
}
private func updateGeneratedImages() {
content.recalculateGeneratedImages()
}
private func generateFeed() {
DispatchQueue.main.async {
_ = content.generateFeed()
}
#warning("Update feed generation")
/*
guard let url = content.storage.outputPath else {
print("Invalid output path")
return
@ -83,6 +95,7 @@ struct GenerationContentView: View {
self.generatorText = "Generation complete"
}
}
*/
}
}

6
CHDataManagement/Views/Settings/PathSettingsView.swift
View File

@ -17,16 +17,14 @@ struct PathSettingsView: View {
FolderOnDiskPropertyView(
title: "Content Folder",
folder: $content.storage.contentPath,
isStale: $content.storage.contentPathUrlIsStale,
folder: $content.storage.contentScope,
footer: "The folder where the raw content of the website is stored") { url in
content.update(contentPath: url)
}
FolderOnDiskPropertyView(
title: "Output Folder",
folder: $content.storage.outputPath,
isStale: $content.storage.outputPathUrlIsStale,
folder: $content.storage.outputScope,
footer: "The folder where the generated website is stored") { url in
content.storage.save(outputPath: url)
}