From 06b4c1ed76cb1a373062482116eebaf583f588e5 Mon Sep 17 00:00:00 2001 From: Christoph Hagen Date: Sat, 25 Jan 2025 22:14:31 +0100 Subject: [PATCH] Generate video thumbnails --- .../Generator/ImageGenerator.swift | 67 ++++++++++++++- CHDataManagement/Model/Content.swift | 15 ++++ CHDataManagement/Model/FileResource.swift | 50 +++++++---- CHDataManagement/Model/Item/Item.swift | 4 +- .../Storage/SecurityBookmark.swift | 10 +++ CHDataManagement/Storage/Storage.swift | 46 +++++++++- .../Views/Files/AddFileView.swift | 1 + .../Views/Files/FileContentView.swift | 27 ++++-- .../Generic/OptionalImagePropertyView.swift | 2 +- .../Views/Posts/PostContentView.swift | 86 ++++++++++++++----- 10 files changed, 254 insertions(+), 54 deletions(-) diff --git a/CHDataManagement/Generator/ImageGenerator.swift b/CHDataManagement/Generator/ImageGenerator.swift index 2f5dedc..e5dd4f0 100644 --- a/CHDataManagement/Generator/ImageGenerator.swift +++ b/CHDataManagement/Generator/ImageGenerator.swift @@ -2,6 +2,7 @@ import Foundation import AppKit import SDWebImageAVIFCoder import SDWebImageWebPCoder +import AVFoundation final class ImageGenerator { @@ -162,10 +163,9 @@ final class ImageGenerator { } private func createAvif(image: NSBitmapImageRep, quality: CGFloat) -> Data? { - return Data() -// let newImage = NSImage(size: image.size) -// newImage.addRepresentation(image) -// return SDImageAVIFCoder.shared.encodedData(with: newImage, format: .AVIF, options: [.encodeCompressionQuality: quality]) + let newImage = NSImage(size: image.size) + newImage.addRepresentation(image) + return SDImageAVIFCoder.shared.encodedData(with: newImage, format: .AVIF, options: [.encodeCompressionQuality: quality]) } private func createWebp(image: NSBitmapImageRep, quality: CGFloat) -> Data? { @@ -173,4 +173,63 @@ final class ImageGenerator { newImage.addRepresentation(image) return SDImageWebPCoder.shared.encodedData(with: newImage, format: .webP, options: [.encodeCompressionQuality: quality]) } + + // MARK: Video thumbnails + + @discardableResult + func createVideoThumbnail(for videoId: String) async -> Bool { + guard let image = await storage.with(file: videoId, perform: generateThumbnail) else { + print("Failed to generate thumbnail image for video \(videoId)") + return false + } + let scaled = create(image: image, width: image.size.width, height: image.size.height) + guard let data = scaled.representation(using: .jpeg, properties: [.compressionFactor: NSNumber(value: 0.6)]) else { + print("Failed to get thumbnail jpg data of video \(videoId)") + return false + } + if !storage.save(thumbnail: data, for: videoId) { + print("Failed to save thumbnail of video \(videoId)") + } + print("Generated video thumbnail for \(videoId)") + return true + } + + private func generateThumbnail(for url: URL) async -> NSImage? { + let time = CMTime(seconds: 1, preferredTimescale: 600) + let asset = AVURLAsset(url: url) + let imageGenerator = AVAssetImageGenerator(asset: asset) + imageGenerator.appliesPreferredTrackTransform = true // Correct for orientation + return await withCheckedContinuation { continuation in + imageGenerator.generateCGImageAsynchronously(for: time) { cgImage, _, error in + if let error { + print("Error generating thumbnail for \(url.path()): \(error.localizedDescription)") + } + if let cgImage { + let image = NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height)) + continuation.resume(returning: image) + } else { + continuation.resume(returning: nil) + } + } + } + } + + func getVideoDuration(for videoId: String) async -> TimeInterval? { + guard let duration = await storage.with(file: videoId, perform: getVideoDuration) else { + print("Failed to determine duration for video \(videoId)") + return nil + } + return duration + } + + private func getVideoDuration(url: URL) async -> TimeInterval? { + let asset = AVURLAsset(url: url) + do { + let duration = try await asset.load(.duration) + return CMTimeGetSeconds(duration) + } catch { + print("ImageGenerator: Failed to determine video duration: \(error.localizedDescription)") + return nil + } + } } diff --git a/CHDataManagement/Model/Content.swift b/CHDataManagement/Model/Content.swift index e01805d..f03dbf1 100644 --- a/CHDataManagement/Model/Content.swift +++ b/CHDataManagement/Model/Content.swift @@ -165,6 +165,21 @@ final class Content: ObservableObject { self.tagOverview = result.tagOverview self.didLoadContent = true callback([]) + self.generateMissingVideoThumbnails() + } + } + } + + func generateMissingVideoThumbnails() { + Task { + for file in self.files { + guard file.type.isVideo else { continue } + guard !file.isExternallyStored else { continue } + guard !storage.hasVideoThumbnail(for: file.id) else { continue } + if await imageGenerator.createVideoThumbnail(for: file.id) { + print("Generated thumbnail for \(file.id)") + file.didChange() + } } } } diff --git a/CHDataManagement/Model/FileResource.swift b/CHDataManagement/Model/FileResource.swift index 526143f..b2b8423 100644 --- a/CHDataManagement/Model/FileResource.swift +++ b/CHDataManagement/Model/FileResource.swift @@ -136,15 +136,14 @@ final class FileResource: Item, LocalizedItem { } } - var imageToDisplay: Image { - guard let imageData = content.storage.fileData(for: id) else { - print("Failed to load data for image \(id)") - return failureImage + var imageToDisplay: Image? { + guard let displayImageData else { + return nil } - update(fileSize: imageData.count) - guard let loadedImage = NSImage(data: imageData) else { + update(fileSize: displayImageData.count) + guard let loadedImage = NSImage(data: displayImageData) else { print("Failed to create image \(id)") - return failureImage + return nil } update(imageDimensions: loadedImage.size) @@ -167,14 +166,25 @@ final class FileResource: Item, LocalizedItem { return size } + private var displayImageData: Foundation.Data? { + if type.isImage { + guard let data = content.storage.fileData(for: id) else { + print("Failed to load data for image \(id)") + return nil + } + return data + } + if type.isVideo { + return content.storage.getVideoThumbnail(for: id) + } + return nil + } + private func getCurrentImageDimensions() -> CGSize? { - guard type.isImage else { + guard let displayImageData else { return nil } - guard let imageData = content.storage.fileData(for: id) else { - return nil - } - guard let loadedImage = NSImage(data: imageData) else { + guard let loadedImage = NSImage(data: displayImageData) else { return nil } return loadedImage.size @@ -214,10 +224,6 @@ final class FileResource: Item, LocalizedItem { self.generatedImageVersions = [] } - private var failureImage: Image { - Image(systemSymbol: .exclamationmarkTriangle) - } - /// The path to the output folder where image versions are stored (no leading slash) var outputImageFolder: String { "\(content.settings.paths.imagesOutputFolderPath)/\(id.fileNameWithoutExtension)" @@ -260,6 +266,18 @@ final class FileResource: Item, LocalizedItem { return content.settings.general.url + version.outputPath } + // MARK: Video thumbnail + + func createVideoThumbnail() { + guard type.isVideo else { return } + guard !content.storage.hasVideoThumbnail(for: id) else { return } + Task { + if await content.imageGenerator.createVideoThumbnail(for: id) { + didChange() + } + } + } + // MARK: Paths func removeFileFromOutputFolder() { diff --git a/CHDataManagement/Model/Item/Item.swift b/CHDataManagement/Model/Item/Item.swift index c7f0369..9cc8755 100644 --- a/CHDataManagement/Model/Item/Item.swift +++ b/CHDataManagement/Model/Item/Item.swift @@ -17,7 +17,9 @@ class Item: ObservableObject, Identifiable { } func didChange() { - changeToggle.toggle() + DispatchQueue.main.async { + self.changeToggle.toggle() + } } func makeCleanAbsolutePath(_ path: String) -> String { diff --git a/CHDataManagement/Storage/SecurityBookmark.swift b/CHDataManagement/Storage/SecurityBookmark.swift index 7d68d94..71c276d 100644 --- a/CHDataManagement/Storage/SecurityBookmark.swift +++ b/CHDataManagement/Storage/SecurityBookmark.swift @@ -349,6 +349,16 @@ struct SecurityBookmark { perform { operation($0.appending(path: relativePath.withLeadingSlashRemoved)) } } + func with(relativePath: String, perform operation: (URL) async -> T?) async -> T? { + let path = url.appending(path: relativePath.withLeadingSlashRemoved) + guard url.startAccessingSecurityScopedResource() else { + delegate?.securityBookmark(error: "Failed to start security scope") + return nil + } + defer { url.stopAccessingSecurityScopedResource() } + return await operation(path) + } + /** Run an operation in the security scope of a url. */ diff --git a/CHDataManagement/Storage/Storage.swift b/CHDataManagement/Storage/Storage.swift index 59d7fe2..45ff067 100644 --- a/CHDataManagement/Storage/Storage.swift +++ b/CHDataManagement/Storage/Storage.swift @@ -26,6 +26,8 @@ final class Storage: ObservableObject { private let tagsFolderName = "tags" + private let videoThumbnailFolderName = "thumbnails" + private let outputPathFileName = "outputPath.bin" private let settingsDataFileName = "settings.json" @@ -238,7 +240,10 @@ final class Storage: ObservableObject { guard contentScope.deleteFile(at: filePath(file: fileId)) else { return false } - return contentScope.deleteFile(at: fileInfoPath(file: fileId)) + guard contentScope.deleteFile(at: fileInfoPath(file: fileId)) else { + return false + } + return contentScope.deleteFile(at: videoThumbnailPath(fileId)) } /** @@ -248,7 +253,11 @@ final class Storage: ObservableObject { guard let contentScope else { return false } - return contentScope.deleteFile(at: filePath(file: fileId)) + guard contentScope.deleteFile(at: filePath(file: fileId)) else { + return false + } + // Delete video thumbnail, which may not exist (not generated / not a video) + return contentScope.deleteFile(at: videoThumbnailPath(fileId)) } /** @@ -366,6 +375,39 @@ final class Storage: ObservableObject { return contentScope.write(fileContent, to: path) } + func with(file fileId: String, perform operation: (URL) async -> T?) async -> T? { + guard let contentScope else { return nil } + let path = filePath(file: fileId) + return await contentScope.with(relativePath: path, perform: operation) + } + + // MARK: Video thumbnails + + func hasVideoThumbnail(for videoId: String) -> Bool { + guard let contentScope else { return false } + let path = videoThumbnailPath(videoId) + return contentScope.hasFile(at: path) + } + + private func videoThumbnailPath(_ videoId: String) -> String { + guard !videoId.hasSuffix("jpg") else { + return videoThumbnailFolderName + "/" + videoId + } + return videoThumbnailFolderName + "/" + videoId + ".jpg" + } + + func save(thumbnail: Data, for videoId: String) -> Bool { + guard let contentScope else { return false } + let path = videoThumbnailPath(videoId) + return contentScope.write(thumbnail, to: path) + } + + func getVideoThumbnail(for videoId: String) -> Data? { + guard let contentScope else { return nil } + let path = videoThumbnailPath(videoId) + return contentScope.readData(at: path) + } + // MARK: Settings func loadSettings() -> Settings.Data? { diff --git a/CHDataManagement/Views/Files/AddFileView.swift b/CHDataManagement/Views/Files/AddFileView.swift index 9762a28..ada9cc9 100644 --- a/CHDataManagement/Views/Files/AddFileView.swift +++ b/CHDataManagement/Views/Files/AddFileView.swift @@ -104,6 +104,7 @@ struct AddFileView: View { content.add(resource) selectedFile = resource } + content.generateMissingVideoThumbnails() dismiss() } } diff --git a/CHDataManagement/Views/Files/FileContentView.swift b/CHDataManagement/Views/Files/FileContentView.swift index 31c0b9a..3dd2a04 100644 --- a/CHDataManagement/Views/Files/FileContentView.swift +++ b/CHDataManagement/Views/Files/FileContentView.swift @@ -26,7 +26,7 @@ struct FileContentView: View { } else { switch file.type.category { case .image: - file.imageToDisplay + (file.imageToDisplay ?? Image(systemSymbol: .exclamationmarkTriangle)) .resizable() .aspectRatio(contentMode: .fit) case .model: @@ -44,14 +44,19 @@ struct FileContentView: View { .id(file.id) case .video: VStack { - Image(systemSymbol: .film) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(width: iconSize) - Text("No preview available") - .font(.title) - } - .foregroundStyle(.secondary) + if let image = file.imageToDisplay { + image + .resizable() + .aspectRatio(contentMode: .fit) + } else { + Image(systemSymbol: .film) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(width: iconSize) + Button("Generate preview", action: generateVideoPreview) + .font(.body) + } + }.foregroundStyle(.secondary) case .resource: VStack { Image(systemSymbol: .docQuestionmark) @@ -76,6 +81,10 @@ struct FileContentView: View { } }.padding() } + + private func generateVideoPreview() { + file.createVideoThumbnail() + } } extension FileContentView: MainContentView { diff --git a/CHDataManagement/Views/Generic/OptionalImagePropertyView.swift b/CHDataManagement/Views/Generic/OptionalImagePropertyView.swift index 9404c91..c25f2e9 100644 --- a/CHDataManagement/Views/Generic/OptionalImagePropertyView.swift +++ b/CHDataManagement/Views/Generic/OptionalImagePropertyView.swift @@ -28,7 +28,7 @@ struct OptionalImagePropertyView: View { } if let image = selectedImage { - image.imageToDisplay + (image.imageToDisplay ?? Image(systemSymbol: .exclamationmarkTriangle)) .resizable() .aspectRatio(contentMode: .fit) .frame(maxHeight: 300) diff --git a/CHDataManagement/Views/Posts/PostContentView.swift b/CHDataManagement/Views/Posts/PostContentView.swift index 646dbfe..0355b02 100644 --- a/CHDataManagement/Views/Posts/PostContentView.swift +++ b/CHDataManagement/Views/Posts/PostContentView.swift @@ -42,6 +42,70 @@ private struct LinkedPageTagView: View { } } +private struct PostImageView: View { + + @ObservedObject + var image: FileResource + + var body: some View { + if let preview = image.imageToDisplay { + preview + .resizable() + .aspectRatio(contentMode: .fill) + .frame(maxWidth: 300, maxHeight: 200) + .cornerRadius(8) + } else if image.type.isImage { + VStack { + Image(systemSymbol: .exclamationmarkTriangle) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(height: 100) + Text(image.id) + .font(.title) + Text("Failed to load image") + .font(.body) + } + .frame(width: 300, height: 200) + .background(Color.gray) + .cornerRadius(8) + } else if image.type.isVideo { + VStack { + Image(systemSymbol: .film) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(height: 100) + Text(image.id) + .font(.title) + Button("Generate preview") { + generateVideoPreview(image) + } + .font(.body) + } + .frame(width: 300, height: 200) + .background(Color.gray) + .cornerRadius(8) + } else { + VStack { + Image(systemSymbol: .exclamationmarkTriangle) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(height: 100) + Text(image.id) + .font(.title) + Text("Invalid media type") + .font(.body) + } + .frame(width: 300, height: 200) + .background(Color.gray) + .cornerRadius(8) + } + } + + private func generateVideoPreview(_ image: FileResource) { + image.createVideoThumbnail() + } +} + struct LocalizedPostContentView: View { @Environment(\.language) @@ -98,27 +162,7 @@ struct LocalizedPostContentView: View { ScrollView(.horizontal) { HStack(alignment: .center, spacing: 8) { ForEach(post.images) { image in - if image.type.isImage { - image.imageToDisplay - .resizable() - .aspectRatio(contentMode: .fill) - .frame(maxWidth: 300, maxHeight: 200) - .cornerRadius(8) - - } else { - VStack { - Image(systemSymbol: .film) - .resizable() - .aspectRatio(contentMode: .fit) - .frame(height: 100) - Text(image.id) - .font(.title) - } - //.foregroundStyle(.secondary) - .frame(width: 300, height: 200) - .background(Color.gray) - .cornerRadius(8) - } + PostImageView(image: image) } } }