Generate video thumbnails

CHDataManagement/Generator/ImageGenerator.swift

@@ -2,6 +2,7 @@ import Foundation
 import AppKit
 import SDWebImageAVIFCoder
 import SDWebImageWebPCoder
+import AVFoundation
 
 final class ImageGenerator {
 
@@ -162,10 +163,9 @@ final class ImageGenerator {
     }
 
     private func createAvif(image: NSBitmapImageRep, quality: CGFloat) -> Data? {
-        return Data()
+        let newImage = NSImage(size: image.size)
-//        let newImage = NSImage(size: image.size)
+        newImage.addRepresentation(image)
-//        newImage.addRepresentation(image)
+        return SDImageAVIFCoder.shared.encodedData(with: newImage, format: .AVIF, options: [.encodeCompressionQuality: quality])
-//        return SDImageAVIFCoder.shared.encodedData(with: newImage, format: .AVIF, options: [.encodeCompressionQuality: quality])
     }
 
     private func createWebp(image: NSBitmapImageRep, quality: CGFloat) -> Data? {
@@ -173,4 +173,63 @@ final class ImageGenerator {
         newImage.addRepresentation(image)
         return SDImageWebPCoder.shared.encodedData(with: newImage, format: .webP, options: [.encodeCompressionQuality: quality])
     }
+
+    // MARK: Video thumbnails
+
+    @discardableResult
+    func createVideoThumbnail(for videoId: String) async -> Bool {
+        guard let image = await storage.with(file: videoId, perform: generateThumbnail) else {
+            print("Failed to generate thumbnail image for video \(videoId)")
+            return false
+        }
+        let scaled = create(image: image, width: image.size.width, height: image.size.height)
+        guard let data = scaled.representation(using: .jpeg, properties: [.compressionFactor: NSNumber(value: 0.6)]) else {
+            print("Failed to get thumbnail jpg data of video \(videoId)")
+            return false
+        }
+        if !storage.save(thumbnail: data, for: videoId) {
+            print("Failed to save thumbnail of video \(videoId)")
+        }
+        print("Generated video thumbnail for \(videoId)")
+        return true
+    }
+
+    private func generateThumbnail(for url: URL) async -> NSImage? {
+        let time = CMTime(seconds: 1, preferredTimescale: 600)
+        let asset = AVURLAsset(url: url)
+        let imageGenerator = AVAssetImageGenerator(asset: asset)
+        imageGenerator.appliesPreferredTrackTransform = true // Correct for orientation
+        return await withCheckedContinuation { continuation in
+            imageGenerator.generateCGImageAsynchronously(for: time) { cgImage, _, error in
+                if let error {
+                    print("Error generating thumbnail for \(url.path()): \(error.localizedDescription)")
+                }
+                if let cgImage {
+                    let image = NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height))
+                    continuation.resume(returning: image)
+                } else {
+                    continuation.resume(returning: nil)
+                }
+            }
+        }
+    }
+
+    func getVideoDuration(for videoId: String) async -> TimeInterval? {
+        guard let duration = await storage.with(file: videoId, perform: getVideoDuration) else {
+            print("Failed to determine duration for video \(videoId)")
+            return nil
+        }
+        return duration
+    }
+
+    private func getVideoDuration(url: URL) async -> TimeInterval? {
+        let asset = AVURLAsset(url: url)
+        do {
+            let duration = try await asset.load(.duration)
+            return CMTimeGetSeconds(duration)
+        } catch {
+            print("ImageGenerator: Failed to determine video duration: \(error.localizedDescription)")
+            return nil
+        }
+    }
 }

CHDataManagement/Model/Content.swift

@@ -165,6 +165,21 @@ final class Content: ObservableObject {
                 self.tagOverview = result.tagOverview
                 self.didLoadContent = true
                 callback([])
+                self.generateMissingVideoThumbnails()
+            }
+        }
+    }
+
+    func generateMissingVideoThumbnails() {
+        Task {
+            for file in self.files {
+                guard file.type.isVideo else { continue }
+                guard !file.isExternallyStored else { continue }
+                guard !storage.hasVideoThumbnail(for: file.id) else { continue }
+                if await imageGenerator.createVideoThumbnail(for: file.id) {
+                    print("Generated thumbnail for \(file.id)")
+                    file.didChange()
+                }
             }
         }
     }

CHDataManagement/Model/FileResource.swift

@@ -136,15 +136,14 @@ final class FileResource: Item, LocalizedItem {
         }
     }
 
-    var imageToDisplay: Image {
+    var imageToDisplay: Image? {
-        guard let imageData = content.storage.fileData(for: id) else {
+        guard let displayImageData else {
-            print("Failed to load data for image \(id)")
+            return nil
-            return failureImage
         }
-        update(fileSize: imageData.count)
+        update(fileSize: displayImageData.count)
-        guard let loadedImage = NSImage(data: imageData) else {
+        guard let loadedImage = NSImage(data: displayImageData) else {
             print("Failed to create image \(id)")
-            return failureImage
+            return nil
         }
         update(imageDimensions: loadedImage.size)
 
@@ -167,14 +166,25 @@ final class FileResource: Item, LocalizedItem {
         return size
     }
 
+    private var displayImageData: Foundation.Data? {
+        if type.isImage {
+            guard let data = content.storage.fileData(for: id) else {
+                print("Failed to load data for image \(id)")
+                return nil
+            }
+            return data
+        }
+        if type.isVideo {
+            return content.storage.getVideoThumbnail(for: id)
+        }
+        return nil
+    }
+
     private func getCurrentImageDimensions() -> CGSize? {
-        guard type.isImage else {
+        guard let displayImageData else {
             return nil
         }
-        guard let imageData = content.storage.fileData(for: id) else {
+        guard let loadedImage = NSImage(data: displayImageData) else {
-            return nil
-        }
-        guard let loadedImage = NSImage(data: imageData) else {
             return nil
         }
         return loadedImage.size
@@ -214,10 +224,6 @@ final class FileResource: Item, LocalizedItem {
         self.generatedImageVersions = []
     }
 
-    private var failureImage: Image {
-        Image(systemSymbol: .exclamationmarkTriangle)
-    }
-
     /// The path to the output folder where image versions are stored (no leading slash)
     var outputImageFolder: String {
         "\(content.settings.paths.imagesOutputFolderPath)/\(id.fileNameWithoutExtension)"
@@ -260,6 +266,18 @@ final class FileResource: Item, LocalizedItem {
         return content.settings.general.url + version.outputPath
     }
 
+    // MARK: Video thumbnail
+
+    func createVideoThumbnail() {
+        guard type.isVideo else { return }
+        guard !content.storage.hasVideoThumbnail(for: id) else { return }
+        Task {
+            if await content.imageGenerator.createVideoThumbnail(for: id) {
+                didChange()
+            }
+        }
+    }
+
     // MARK: Paths
 
     func removeFileFromOutputFolder() {

CHDataManagement/Model/Item/Item.swift

@@ -17,7 +17,9 @@ class Item: ObservableObject, Identifiable {
     }
 
     func didChange() {
-        changeToggle.toggle()
+        DispatchQueue.main.async {
+            self.changeToggle.toggle()
+        }
     }
 
     func makeCleanAbsolutePath(_ path: String) -> String {

CHDataManagement/Storage/SecurityBookmark.swift

@@ -349,6 +349,16 @@ struct SecurityBookmark {
         perform { operation($0.appending(path: relativePath.withLeadingSlashRemoved)) }
     }
 
+    func with<T>(relativePath: String, perform operation: (URL) async -> T?) async -> T? {
+        let path = url.appending(path: relativePath.withLeadingSlashRemoved)
+        guard url.startAccessingSecurityScopedResource() else {
+            delegate?.securityBookmark(error: "Failed to start security scope")
+            return nil
+        }
+        defer { url.stopAccessingSecurityScopedResource() }
+        return await operation(path)
+    }
+
     /**
      Run an operation in the security scope of a url.
      */

CHDataManagement/Storage/Storage.swift

@@ -26,6 +26,8 @@ final class Storage: ObservableObject {
 
     private let tagsFolderName = "tags"
 
+    private let videoThumbnailFolderName = "thumbnails"
+
     private let outputPathFileName = "outputPath.bin"
 
     private let settingsDataFileName = "settings.json"
@@ -238,7 +240,10 @@ final class Storage: ObservableObject {
         guard contentScope.deleteFile(at: filePath(file: fileId)) else {
             return false
         }
-        return contentScope.deleteFile(at: fileInfoPath(file: fileId))
+        guard contentScope.deleteFile(at: fileInfoPath(file: fileId)) else {
+            return false
+        }
+        return contentScope.deleteFile(at: videoThumbnailPath(fileId))
     }
 
     /**
@@ -248,7 +253,11 @@ final class Storage: ObservableObject {
         guard let contentScope else {
             return false
         }
-        return contentScope.deleteFile(at: filePath(file: fileId))
+        guard contentScope.deleteFile(at: filePath(file: fileId)) else {
+            return false
+        }
+        // Delete video thumbnail, which may not exist (not generated / not a video)
+        return contentScope.deleteFile(at: videoThumbnailPath(fileId))
     }
 
     /**
@@ -366,6 +375,39 @@ final class Storage: ObservableObject {
         return contentScope.write(fileContent, to: path)
     }
 
+    func with<T>(file fileId: String, perform operation: (URL) async -> T?) async -> T? {
+        guard let contentScope else { return nil }
+        let path = filePath(file: fileId)
+        return await contentScope.with(relativePath: path, perform: operation)
+    }
+
+    // MARK: Video thumbnails
+
+    func hasVideoThumbnail(for videoId: String) -> Bool {
+        guard let contentScope else { return false }
+        let path = videoThumbnailPath(videoId)
+        return contentScope.hasFile(at: path)
+    }
+
+    private func videoThumbnailPath(_ videoId: String) -> String {
+        guard !videoId.hasSuffix("jpg") else {
+            return videoThumbnailFolderName + "/" + videoId
+        }
+        return videoThumbnailFolderName + "/" + videoId + ".jpg"
+    }
+
+    func save(thumbnail: Data, for videoId: String) -> Bool {
+        guard let contentScope else { return false }
+        let path = videoThumbnailPath(videoId)
+        return contentScope.write(thumbnail, to: path)
+    }
+
+    func getVideoThumbnail(for videoId: String) -> Data? {
+        guard let contentScope else { return nil }
+        let path = videoThumbnailPath(videoId)
+        return contentScope.readData(at: path)
+    }
+
     // MARK: Settings
 
     func loadSettings() -> Settings.Data? {

CHDataManagement/Views/Files/AddFileView.swift

@@ -104,6 +104,7 @@ struct AddFileView: View {
             content.add(resource)
             selectedFile = resource
         }
+        content.generateMissingVideoThumbnails()
         dismiss()
     }
 }

CHDataManagement/Views/Files/FileContentView.swift

@@ -26,7 +26,7 @@ struct FileContentView: View {
             } else {
                 switch file.type.category {
                 case .image:
-                    file.imageToDisplay
+                    (file.imageToDisplay ?? Image(systemSymbol: .exclamationmarkTriangle))
                         .resizable()
                         .aspectRatio(contentMode: .fit)
                 case .model:
@@ -44,14 +44,19 @@ struct FileContentView: View {
                         .id(file.id)
                 case .video:
                     VStack {
-                        Image(systemSymbol: .film)
+                        if let image = file.imageToDisplay {
-                            .resizable()
+                            image
-                            .aspectRatio(contentMode: .fit)
+                                .resizable()
-                            .frame(width: iconSize)
+                                .aspectRatio(contentMode: .fit)
-                        Text("No preview available")
+                        } else {
-                            .font(.title)
+                            Image(systemSymbol: .film)
-                    }
+                                .resizable()
-                    .foregroundStyle(.secondary)
+                                .aspectRatio(contentMode: .fit)
+                                .frame(width: iconSize)
+                            Button("Generate preview", action: generateVideoPreview)
+                                .font(.body)
+                        }
+                    }.foregroundStyle(.secondary)
                 case .resource:
                     VStack {
                         Image(systemSymbol: .docQuestionmark)
@@ -76,6 +81,10 @@ struct FileContentView: View {
             }
         }.padding()
     }
+
+    private func generateVideoPreview() {
+        file.createVideoThumbnail()
+    }
 }
 
 extension FileContentView: MainContentView {

CHDataManagement/Views/Generic/OptionalImagePropertyView.swift

@@ -28,7 +28,7 @@ struct OptionalImagePropertyView: View {
             }
 
             if let image = selectedImage {
-                image.imageToDisplay
+                (image.imageToDisplay ?? Image(systemSymbol: .exclamationmarkTriangle))
                     .resizable()
                     .aspectRatio(contentMode: .fit)
                     .frame(maxHeight: 300)

CHDataManagement/Views/Posts/PostContentView.swift

@@ -42,6 +42,70 @@ private struct LinkedPageTagView: View {
     }
 }
 
+private struct PostImageView: View {
+
+    @ObservedObject
+    var image: FileResource
+
+    var body: some View {
+        if let preview = image.imageToDisplay {
+            preview
+                .resizable()
+                .aspectRatio(contentMode: .fill)
+                .frame(maxWidth: 300, maxHeight: 200)
+                .cornerRadius(8)
+        } else if image.type.isImage {
+            VStack {
+                Image(systemSymbol: .exclamationmarkTriangle)
+                    .resizable()
+                    .aspectRatio(contentMode: .fit)
+                    .frame(height: 100)
+                Text(image.id)
+                    .font(.title)
+                Text("Failed to load image")
+                    .font(.body)
+            }
+            .frame(width: 300, height: 200)
+            .background(Color.gray)
+            .cornerRadius(8)
+        } else if image.type.isVideo {
+            VStack {
+                Image(systemSymbol: .film)
+                    .resizable()
+                    .aspectRatio(contentMode: .fit)
+                    .frame(height: 100)
+                Text(image.id)
+                    .font(.title)
+                Button("Generate preview") {
+                    generateVideoPreview(image)
+                }
+                .font(.body)
+            }
+            .frame(width: 300, height: 200)
+            .background(Color.gray)
+            .cornerRadius(8)
+        } else {
+            VStack {
+                Image(systemSymbol: .exclamationmarkTriangle)
+                    .resizable()
+                    .aspectRatio(contentMode: .fit)
+                    .frame(height: 100)
+                Text(image.id)
+                    .font(.title)
+                Text("Invalid media type")
+                    .font(.body)
+            }
+            .frame(width: 300, height: 200)
+            .background(Color.gray)
+            .cornerRadius(8)
+        }
+    }
+
+    private func generateVideoPreview(_ image: FileResource) {
+        image.createVideoThumbnail()
+    }
+}
+
 struct LocalizedPostContentView: View {
 
     @Environment(\.language)
@@ -98,27 +162,7 @@ struct LocalizedPostContentView: View {
             ScrollView(.horizontal) {
                 HStack(alignment: .center, spacing: 8) {
                     ForEach(post.images) { image in
-                        if image.type.isImage {
+                        PostImageView(image: image)
-                            image.imageToDisplay
-                                .resizable()
-                                .aspectRatio(contentMode: .fill)
-                                .frame(maxWidth: 300, maxHeight: 200)
-                                .cornerRadius(8)
-                            
-                        } else {
-                            VStack {
-                                Image(systemSymbol: .film)
-                                    .resizable()
-                                    .aspectRatio(contentMode: .fit)
-                                    .frame(height: 100)
-                                Text(image.id)
-                                    .font(.title)
-                            }
-                            //.foregroundStyle(.secondary)
-                            .frame(width: 300, height: 200)
-                            .background(Color.gray)
-                            .cornerRadius(8)
-                        }
                     }
                 }
             }