Optimizing Background Video/Image Renderer

CollapseLauncher.slnx

@@ -53,6 +53,10 @@
   <Project Path="Hi3Helper.LocaleSourceGen/Hi3Helper.LocaleSourceGen.csproj">
     <Platform Project="x64" />
   </Project>
+  <Project Path="Hi3Helper.MagicScaler.LibheifCL/Hi3Helper.MagicScaler.LibheifCL.csproj">
+    <BuildType Solution="Publish|*" Project="Release" />
+    <Platform Project="x64" />
+  </Project>
   <Project Path="Hi3Helper.Plugin.Core/Hi3Helper.Plugin.Core.csproj">
     <BuildType Solution="Publish|*" Project="Release" />
     <Platform Project="x64" />

CollapseLauncher/App.xaml.cs

@@ -137,12 +137,6 @@ protected override void OnLaunched(LaunchActivatedEventArgs args)
                 {
                     ImageLoaderHelper.InitWaifu2X();
                 }
-
-                // Initialize support for MagicScaler's WebP decoding
-                CodecManager.Configure(codecs =>
-                {
-                    codecs.UseLibwebp();
-                });
             }
             catch (Exception ex)
             {

CollapseLauncher/Classes/AnimatedVisuals/Lottie/NewLogoTitleIntro.cs

@@ -43,6 +43,9 @@
 // | Gradient stops           |     6 |
 // | CompositionVisualSurface |    42 |
 // ------------------------------------
+
+using CollapseLauncher.GameManagement.ImageBackground;
+using CollapseLauncher.Helper.Image;
 using Hi3Helper.Shared.Region;
 using Microsoft.Graphics.Canvas;
 using Microsoft.Graphics.Canvas.Effects;
@@ -55,6 +58,7 @@
 using System.ComponentModel;
 using System.IO;
 using System.Numerics;
+using System.Threading.Tasks;
 using Windows.Foundation;
 using Windows.UI;
 
@@ -156,9 +160,9 @@ void EnsureImageLoadingStarted()
             if (!_isImageLoadingStarted)
             {
                 var eventHandler = new TypedEventHandler<LoadedImageSurface, LoadedImageSourceLoadCompletedEventArgs>(HandleLoadCompleted);
-                _image_image_0 = LoadedImageSurface.StartLoadFromStream(File.OpenRead(Path.Combine(LauncherConfig.AppExecutableDir, @"Assets\CollapseLauncherLogo.png")).AsRandomAccessStream());
+                _image_image_0               =  LoadedImageSurface.StartLoadFromStream(File.OpenRead(Path.Combine(LauncherConfig.AppExecutableDir, @"Assets\CollapseLauncherLogo.png")).AsRandomAccessStream());
                 _image_image_0.LoadCompleted += eventHandler;
-                _isImageLoadingStarted = true;
+                _isImageLoadingStarted       =  true;
             }
         }
 

CollapseLauncher/Classes/Extension/UIElementExtensions.cs

@@ -148,14 +148,14 @@ internal static void SetAllControlsCursorRecursive(this UIElement? element, Inpu
             }
         }
 
-        internal static void BindProperty(this FrameworkElement element,
-                                          object?               objectToBind,
-                                          string                propertyName,
-                                          DependencyProperty    dependencyProperty,
-                                          BindingMode           bindingMode,
-                                          IValueConverter?      converter = null)
-        {
-            element.SetBinding(dependencyProperty, new Binding
+        internal static void BindProperty(this FrameworkElement? element,
+                                          object?                objectToBind,
+                                          string                 propertyName,
+                                          DependencyProperty     dependencyProperty,
+                                          BindingMode            bindingMode,
+                                          IValueConverter?       converter = null)
+        {
+            element?.SetBinding(dependencyProperty, new Binding
             {
                 Source    = objectToBind,
                 Mode      = bindingMode,

CollapseLauncher/Classes/GameManagement/ImageBackground/ClipboardUtility.cs

@@ -1,5 +1,4 @@
-using CollapseLauncher.Extension;
-using CollapseLauncher.XAMLs.Theme.CustomControls;
+using CollapseLauncher.XAMLs.Theme.CustomControls;
 using Hi3Helper;
 using Hi3Helper.Win32.ManagedTools;
 using Hi3Helper.Win32.Native.Enums;
@@ -8,13 +7,11 @@
 using Hi3Helper.Win32.WinRT.WindowsStream;
 using Microsoft.Extensions.Logging;
 using Microsoft.Graphics.Canvas;
-using Microsoft.UI.Dispatching;
 using System;
 using System.IO;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Runtime.InteropServices.WindowsRuntime;
-using System.Threading;
 using System.Threading.Tasks;
 using Windows.Graphics.DirectX;
 using Windows.Graphics.DirectX.Direct3D11;
@@ -30,10 +27,8 @@ public static class ClipboardUtility
 {
     public static async void CopyCurrentFrameToClipboard(
         ImageBackgroundManager? instance,
-        FrameToCopyType frameToCopyType)
+        FrameToCopyType         frameToCopyType)
     {
-        bool isBackgroundVideo = false;
-
         try
         {
             if (instance == null)
@@ -43,157 +38,21 @@ public static async void CopyCurrentFrameToClipboard(
 
             bool hasBackground = frameToCopyType.HasFlag(FrameToCopyType.Background);
             bool hasForeground = frameToCopyType.HasFlag(FrameToCopyType.Foreground);
-            bool hasBoth       = frameToCopyType == FrameToCopyType.Both;
 
             // Try to get video frame buffer first (if the background is video).
             LayeredBackgroundImage? currentElement = instance.CurrentBackgroundElement;
-            bool isStaticBackgroundShowing = currentElement is { BackgroundStaticSource: not null, IsVideoPlay: false };
-
-            string? usedBackgroundPath = isStaticBackgroundShowing
-                ? instance.CurrentSelectedBackgroundContext?
-                   .OriginBackgroundStaticImagePath
-                : instance.CurrentSelectedBackgroundContext?
-                   .OriginBackgroundImagePath;
-
-            CanvasRenderTarget? backgroundCanvas = hasBackground
-                ? isStaticBackgroundShowing ? null : currentElement?.LockCanvasRenderTarget()
-                : null;
-
-            ICanvasResourceCreator? canvasDevice = backgroundCanvas;
-            canvasDevice ??= CanvasDevice.GetSharedDevice();
-
-            if (backgroundCanvas != null)
-            {
-                // Copy bitmap to avoid arbitrary write to video frame buffer.
-                CanvasRenderTarget oldCanvas = backgroundCanvas;
-                backgroundCanvas = new CanvasRenderTarget(canvasDevice, backgroundCanvas.Size._width, backgroundCanvas.Size._height, 96f);
-                backgroundCanvas.CopyPixelsFromBitmap(oldCanvas);
-
-                isBackgroundVideo = true; // Mark as video so we can unlock the frame later.
-            }
-
-            // If it's still null (which most likely an image), get the canvas.
-            backgroundCanvas ??= hasBackground
-                ? await GetImageCanvasTargetBuffer(canvasDevice,
-                                                   usedBackgroundPath)
-                : await GetImageCanvasTargetBuffer(canvasDevice,
-                                                   instance.CurrentSelectedBackgroundContext?
-                                                      .OriginOverlayImagePath);
-
-            // If background canvas is still null, abort.
-            if (backgroundCanvas == null)
-            {
+            if (currentElement == null)
                 return;
-            }
-
-            // Get foreground canvas if available.
-            CanvasRenderTarget? foregroundCanvas = hasBoth && hasForeground
-                    ? await GetImageCanvasTargetBuffer(canvasDevice,
-                                                       instance.CurrentSelectedBackgroundContext?
-                                                          .OriginOverlayImagePath)
-                    : null;
-
-            int widthOfForeground = (int)(foregroundCanvas?.SizeInPixels.Width ?? 0u);
-            int heightOfForeground = (int)(foregroundCanvas?.SizeInPixels.Height ?? 0u);
-            if (foregroundCanvas != null &&
-                (backgroundCanvas.SizeInPixels.Width != widthOfForeground ||
-                 backgroundCanvas.SizeInPixels.Height != heightOfForeground))
-            {
-                backgroundCanvas = ResizeCanvasToSizeOf(backgroundCanvas, widthOfForeground, heightOfForeground);
-            }
 
-            MergeCanvasTargetWith(backgroundCanvas, foregroundCanvas); // Merge overlay (foreground) to background.
-            await CopyCanvasTargetBufferToClipboard(backgroundCanvas); // Copy to clipboard
+            using CanvasRenderTarget? sourceFrameCanvas = await currentElement.CopyCurrentFrame(hasBackground, hasForeground);
+            await CopyCanvasTargetBufferToClipboard(sourceFrameCanvas); // Copy to clipboard
         }
         catch (Exception ex)
         {
             Logger.LogWriteLine($"[ImageBackgroundManager::CopyCurrentFrameToClipboard] An error occurred while trying to copy current background frame to clipboard\r\n{ex}",
                                 LogType.Error,
                                 true);
         }
-        finally
-        {
-            if (isBackgroundVideo)
-            {
-                instance?.CurrentBackgroundElement?.UnlockCanvasRenderTarget();
-            }
-        }
-    }
-
-    private static CanvasRenderTarget ResizeCanvasToSizeOf(CanvasRenderTarget canvasToResize, int width, int height)
-    {
-        CanvasRenderTarget resizedCanvas = new(canvasToResize,
-                                               width,
-                                               height,
-                                               canvasToResize.Dpi);
-        using CanvasDrawingSession ds = resizedCanvas.CreateDrawingSession();
-        ds.DrawImage(canvasToResize,
-                     new Windows.Foundation.Rect(0, 0, width, height),
-                     new Windows.Foundation.Rect(0, 0,
-                                                 canvasToResize.SizeInPixels.Width,
-                                                 canvasToResize.SizeInPixels.Height),
-                     1f,
-                     CanvasImageInterpolation.HighQualityCubic);
-
-        canvasToResize.Dispose();
-        return resizedCanvas;
-    }
-
-    private static async Task<CanvasRenderTarget?> GetImageCanvasTargetBuffer(ICanvasResourceCreator canvasDevice, string? path)
-    {
-        if (string.IsNullOrEmpty(path))
-        {
-            return null; // Ignore if path is null or empty.
-        }
-
-        // Get source image as native or decoded image (if it requires external codec decoding).
-        Uri sourceUri = new(path);
-        (sourceUri, _) = await ImageBackgroundManager
-           .GetNativeOrDecodedImagePath(sourceUri, CancellationToken.None);
-
-        using CanvasBitmap bitmap = sourceUri.IsFile ?
-            await CanvasBitmap.LoadAsync(canvasDevice,
-                                         sourceUri.LocalPath,
-                                         96f,
-                                         CanvasAlphaMode.Premultiplied) :
-            await CanvasBitmap.LoadAsync(canvasDevice,
-                                         sourceUri,
-                                         96f,
-                                         CanvasAlphaMode.Premultiplied);
-
-        // Load to render target
-        return await Task.Run(() =>
-        {
-            CanvasRenderTarget renderTarget = new(canvasDevice,
-                                                  bitmap.SizeInPixels.Width,
-                                                  bitmap.SizeInPixels.Height, bitmap.Dpi);
-            using CanvasDrawingSession ds = renderTarget.CreateDrawingSession();
-            ds.DrawImage(bitmap);
-            return renderTarget;
-        });
-    }
-
-    private static void MergeCanvasTargetWith(CanvasRenderTarget sourceCanvasTarget,
-                                              CanvasRenderTarget? withCanvasTarget)
-    {
-        if (withCanvasTarget == null)
-        {
-            return; // Abort
-        }
-
-        CanvasDrawingSession? ds = sourceCanvasTarget.CreateDrawingSession();
-        ds.DrawImage(withCanvasTarget);
-
-        // Dispose the overlay canvas
-        withCanvasTarget.Dispose();
-
-        if (DispatcherQueueExtensions.CurrentDispatcherQueue.HasThreadAccess)
-        {
-            ds.Dispose();
-            return;
-        }
-
-        DispatcherQueueExtensions.CurrentDispatcherQueue.TryEnqueue(DispatcherQueuePriority.High, ds.Dispose);
     }
 
     private static double GetBitmapBitSize(DirectXPixelFormat pixelFormat)
@@ -300,9 +159,7 @@ private static async Task<int> WriteCanvasTargetBufferType(
     private static async Task CopyCanvasTargetBufferToClipboard(CanvasRenderTarget? sourceCanvasTarget)
     {
         if (sourceCanvasTarget == null)
-        {
             return;
-        }
 
         bool    isClipboardOpened = true;
         byte[]? buffer            = null;

CollapseLauncher/Classes/GameManagement/ImageBackground/ImageBackgroundManager.CodecDetect.cs

@@ -6,10 +6,11 @@
 using System;
 using System.IO;
 using System.Threading.Tasks;
-
+// ReSharper disable CommentTypo
 // ReSharper disable IdentifierTypo
 // ReSharper disable StringLiteralTypo
 // ReSharper disable CheckNamespace
+#pragma warning disable IDE0130
 
 #nullable enable
 namespace CollapseLauncher.GameManagement.ImageBackground;
@@ -18,72 +19,60 @@ public partial class ImageBackgroundManager
 {
     #region Codec Checks
 
-    private async ValueTask<(bool IsSupported, bool IsVideo, bool ForceFFmpeg)> CheckCodecOrSpawnDialog(Uri? fileUri)
+    private async ValueTask<(bool IsSupported, bool ForceFFmpeg)> CheckCodecOrSpawnDialog(string? localPath)
     {
-        // -- Cancel if null or URI is not a local file
-        if (fileUri == null || !fileUri.IsFile)
-        {
-            return (false, false, false);
-        }
-
-        string filePath = fileUri.LocalPath;
+        if (localPath == null)
+            return (false, false);
 
         // -- Check for supported extension first
-        if (!IsMediaFileExtensionSupported(filePath))
+        if (!IsMediaFileExtensionSupported(localPath))
         {
-            await SimpleDialogs.Dialog_SpawnMediaExtensionNotSupportedDialog(filePath);
-            return (false, false, false);
+            await SimpleDialogs.Dialog_SpawnMediaExtensionNotSupportedDialog(localPath);
+            return (false, false);
         }
 
         // -- Check for supported image codec
-        if (IsImageMediaFileExtensionSupported(filePath))
+        if (IsImageMediaFileExtensionSupported(localPath))
         {
-            if (WindowsCodecHelper.IsFileSupportedImage(filePath))
+            if (WindowsCodecHelper.IsFileSupportedImage(localPath))
             {
-                return (true, false, false);
+                return (true, false);
             }
 
-            await SimpleDialogs.Dialog_SpawnImageNotSupportedDialog(filePath);
-            return (false, false, false);
+            await SimpleDialogs.Dialog_SpawnImageNotSupportedDialog(localPath);
+            return (false, false);
         }
 
         // -- Check using FFmpeg if available
-        MediaSupport    codecInfo   = FFmpegPInvoke.GetMediaSupport(filePath);
+        MediaSupport    codecInfo   = FFmpegPInvoke.GetMediaSupport(localPath);
+        PixelColorModel pixelFormat = codecInfo.VideoPixelFormatInfo.ColorModel;
         if ((GlobalIsUseFFmpeg && GlobalIsFFmpegAvailable) ||
-            RequiresFFmpegDecoder(codecInfo))
+            (codecInfo.Video.DecoderAvailable && pixelFormat.HasFlag(PixelColorModel.Rgb)))
         {
-            return (true, true, true);
+            return (true, true);
         }
 
         // -- Check for supported video codec
-        if (WindowsCodecHelper.IsFileSupportedVideo(filePath,
+        if (WindowsCodecHelper.IsFileSupportedVideo(localPath,
                                                     out bool canPlayVideo,
                                                     out bool canPlayAudio,
                                                     out Guid videoCodecGuid,
                                                     out Guid audioCodecGuid) ||
             (GlobalIsUseFFmpeg && GlobalIsFFmpegAvailable))
         {
-            return (true, true, true);
+            return (true, false);
         }
 
         return (await SimpleDialogs
-                   .Dialog_SpawnVideoNotSupportedDialog(filePath,
+                   .Dialog_SpawnVideoNotSupportedDialog(localPath,
                                                         canPlayVideo,
                                                         canPlayAudio,
                                                         videoCodecGuid,
-                                                        audioCodecGuid), true, false);
+                                                        audioCodecGuid), false);
     }
 
     #endregion
 
-    private static bool RequiresFFmpegDecoder(MediaSupport codecInfo)
-    {
-        // Windows' H.264 decoder does not support the High 10 profile used by some backgrounds.
-        return codecInfo.Video.DecoderAvailable &&
-               (codecInfo.VideoPixelFormatInfo.ColorModel.HasFlag(PixelColorModel.Rgb) ||
-                codecInfo.VideoPixelFormatInfo.Format is AVPixelFormat.AV_PIX_FMT_YUV420P10LE or AVPixelFormat.AV_PIX_FMT_YUV420P10BE);
-    }
-
     private static bool IsMediaFileExtensionSupported(ReadOnlySpan<char> filePath)
     {
         return IsImageMediaFileExtensionSupported(filePath) ||
@@ -98,7 +87,7 @@ private static bool IsImageMediaFileExtensionSupported(ReadOnlySpan<char> filePa
                LayeredBackgroundImage.SupportedImageVectorExtensionsLookup.Contains(extension);
     }
 
-    private static bool IsVideoMediaFileExtensionSupported(ReadOnlySpan<char> filePath)
+    public static bool IsVideoMediaFileExtensionSupported(ReadOnlySpan<char> filePath)
     {
         ReadOnlySpan<char> extension = Path.GetExtension(filePath);
         return LayeredBackgroundImage.SupportedVideoExtensionsLookup.Contains(extension);