From 3c5b6c496c1826293f1eb32ea2ed00a140a71c3b Mon Sep 17 00:00:00 2001 From: Huy Nguyen Date: Thu, 15 Jan 2026 14:51:24 +1030 Subject: [PATCH] Change ImportGames logic to allow cancelling operation Retrieving orderKeys prior to processing records and ScrapeOrders now uses GetOrders with a cancellation token GetOrders now implements IEnumerable which may be cancelled Use Sidebar menu to convey progress of scraping data from Humble API Made HumbleKeysAccountClient a dependent lazy loaded instance --- HumbleKeysLibrary.cs | 280 ++++++++++++++++------------ Services/HumbleKeysAccountClient.cs | 19 +- changelog.md | 4 + 3 files changed, 182 insertions(+), 121 deletions(-) diff --git a/HumbleKeysLibrary.cs b/HumbleKeysLibrary.cs index 233e501..4d40261 100644 --- a/HumbleKeysLibrary.cs +++ b/HumbleKeysLibrary.cs @@ -4,7 +4,11 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.ComponentModel; +using System.IO; using System.Linq; +using System.Reflection; +using System.Threading; using System.Windows.Controls; using HumbleKeys.Models; using HumbleKeys.Services; @@ -35,6 +39,26 @@ public class HumbleKeysLibrary : LibraryPlugin private Platform switchPlatform; private readonly KeyInfo humbleKeysSource = new KeyInfo { Name = "Unknown" }; public override string Name => "Humble Keys"; + private SidebarItem importProgress; + private HumbleKeysAccountClient humbleApi; + + private HumbleKeysAccountClient HumbleApi + { + get + { + if (humbleApi != null) return humbleApi; + + var view = PlayniteApi.WebViews.CreateOffscreenView(new WebViewSettings { JavaScriptEnabled = false }); + humbleApi = new HumbleKeysAccountClient(view, + new HumbleKeysAccountClientSettings + { + CacheEnabled = Settings.CacheEnabled, + CachePath = $"{PlayniteApi.Paths.ExtensionsDataPath}\\{Id}" + }); + return humbleApi; + } + } + #endregion #region === Accessors ================ @@ -50,6 +74,13 @@ public HumbleKeysLibrary(IPlayniteAPI api) : base(api) { Properties = new LibraryPluginProperties { CanShutdownClient = false, HasCustomizedGameImport = true }; Settings = new HumbleKeysLibrarySettings(this); + Settings.PropertyChanged += OnSettingsOnPropertyChanged; + } + + private void OnSettingsOnPropertyChanged(object sender, PropertyChangedEventArgs e) + { + humbleApi.Dispose(); + humbleApi = null; } public override ISettings GetSettings(bool firstRunSettings) @@ -72,10 +103,12 @@ public override IEnumerable ImportGames(LibraryImportGamesArgs args) try { - var orders = ScrapeOrders(); - var selectedTpkds = SelectTpkds(orders); - logger.Trace("ImportGames: Selected Tpkds Count = " + selectedTpkds.Count()); - ProcessOrders(orders, selectedTpkds, ref importedGames, ref removedGames); + var orderKeys = HumbleApi.GetLibraryKeys(); + var orders = ScrapeOrders(orderKeys, args.CancelToken); + //var orderDictionary = orders.ToDictionary(order => order.gamekey, order => order); + //var selectedTpkds = SelectTpkds(orderDictionary); + //logger.Trace("ImportGames: Selected Tpkds Count = " + selectedTpkds.Count()); + ProcessOrders(orders, orderKeys, ref importedGames, ref removedGames); } catch (Exception e) { @@ -99,38 +132,29 @@ public override IEnumerable ImportGames(LibraryImportGamesArgs args) } logger.Trace($"ImportGames: Imported {importedGames.Count} games, Removed {removedGames.Count} games"); + humbleApi.Dispose(); + humbleApi = null; return importedGames; } - public Dictionary ScrapeOrders() + public override IEnumerable GetSidebarItems() { - Dictionary orders; - using (var view = PlayniteApi.WebViews.CreateOffscreenView( - new WebViewSettings { JavaScriptEnabled = false })) + importProgress = new SidebarItem { - var api = new Services.HumbleKeysAccountClient(view, - new HumbleKeysAccountClientSettings - { - CacheEnabled = Settings.CacheEnabled, - CachePath = $"{PlayniteApi.Paths.ExtensionsDataPath}\\{Id}" - }); - var keys = api.GetLibraryKeys(); - logger.Trace("ScrapeOrders: Keys Count = " + keys.Count); - orders = api.GetOrders(keys, Settings.ImportChoiceKeys); - logger.Trace("ScrapeOrders: Orders Count = " + orders.Count); - } - - return orders; + ProgressMaximum = 100f, + ProgressValue = 0f, + Type = SiderbarItemType.Button, + Icon = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? string.Empty, "icon.png"), + Title = "Humble Remote Progress", + Visible = false + }; + yield return importProgress; } - public IEnumerable> SelectTpkds(Dictionary orders) + private IEnumerable ScrapeOrders(List orderKeys, CancellationToken cancellationToken = default) { - return orders.Select(kv => kv.Value) - .SelectMany(a => a.tpkd_dict?.all_tpks) - .Where(t => t != null - && Settings.keyTypeWhitelist.ContainsKey(t.key_type) - && !string.IsNullOrWhiteSpace(t.gamekey) - ).GroupBy(tpk => tpk.gamekey); + logger.Trace($"ScrapeOrders: Keys Count = {orderKeys.Count}" ); + return HumbleApi.GetOrders(orderKeys, Settings.ImportChoiceKeys, cancellationToken); } /// @@ -138,10 +162,10 @@ public Dictionary ScrapeOrders() /// an Order may be a single purchase, a bundle purchase or a monthly subscription /// /// - /// + /// /// List of Games added from orders /// List of Games removed from orders due to settings - protected void ProcessOrders(Dictionary orders, IEnumerable> tpkds, ref List importedGames, ref List removedGames) + private void ProcessOrders(IEnumerable orders, List orderKeys, ref List importedGames, ref List removedGames) { var redeemedTag = PlayniteApi.Database.Tags.Add(REDEEMED_STR); var unredeemedTag = PlayniteApi.Database.Tags.Add(UNREDEEMED_STR); @@ -157,44 +181,57 @@ protected void ProcessOrders(Dictionary orders, IEnumerable 1) + var tpkdGroupEnumerable = order.tpkd_dict.all_tpks + .Where(t => t != null + && Settings.keyTypeWhitelist.ContainsKey(t.key_type) + && !string.IsNullOrWhiteSpace(t.gamekey)) + .GroupBy(tpk => tpk.gamekey); + foreach (var tpkdGroup in tpkdGroupEnumerable) { - var isHumbleMonthly = orders[tpkdGroup.Key].product.human_name.Contains("Humble Monthly"); - if (tagMethod == TagMethodology.All || tagMethod == TagMethodology.Monthly && isHumbleMonthly) + var tpkdGroupEntries = tpkdGroup.AsEnumerable(); + Tag humbleChoiceTag = null; + var groupEntries = tpkdGroupEntries.ToList(); + if (Settings.ImportChoiceKeys && tagMethod != TagMethodology.None && groupEntries.Count() > 1) { - humbleChoiceTag = PlayniteApi.Database.Tags.Add($"Bundle: {orders[tpkdGroup.Key].product.human_name}"); + var isHumbleMonthly = order.product.human_name.Contains("Humble Monthly"); + if (tagMethod == TagMethodology.All || tagMethod == TagMethodology.Monthly && isHumbleMonthly) + { + humbleChoiceTag = PlayniteApi.Database.Tags.Add($"Bundle: {order.product.human_name}"); + } } - } - var bundleContainsUnredeemableKeys = false; - var sourceOrder = orders[tpkdGroup.Key]; - if (sourceOrder != null && sourceOrder.product.category != "storefront" && sourceOrder.total_choices > 0 && sourceOrder.product.is_subs_v2_product) - { - bundleContainsUnredeemableKeys = sourceOrder.choices_remaining == 0; - } + var bundleContainsUnredeemableKeys = false; + var sourceOrder = order; + if (sourceOrder != null && sourceOrder.product.category != "storefront" && sourceOrder.total_choices > 0 && sourceOrder.product.is_subs_v2_product) + { + bundleContainsUnredeemableKeys = sourceOrder.choices_remaining == 0; + } - // Monthly bundle has all choices made - if (bundleContainsUnredeemableKeys && humbleChoiceTag != null) - { - // search Playnite db for all games that are not included in groupEntries, these can be removed - var virtualOrders = groupEntries.Where(tpk => tpk.is_virtual).Select(GetGameId) ?? - new List(); - var gameKeys = virtualOrders.ToList(); - // for this bundle, get all games from the database that are not in the keys collection for this order - var libraryKeysNotInOrder = PlayniteApi.Database.Games - .Where(game => - game.TagIds != null && game.TagIds.Contains(humbleChoiceTag.Id) && gameKeys.Contains(game.GameId)) - .ToList(); - foreach (var game in libraryKeysNotInOrder) + // Monthly bundle has all choices made + if (bundleContainsUnredeemableKeys && humbleChoiceTag != null) { - switch (unredeemableMethod) + // search Playnite db for all games that are not included in groupEntries, these can be removed + var virtualOrders = order.tpkd_dict.all_tpks.Where(tpk => tpk.is_virtual).Select(GetGameId) ?? + new List(); + var gameKeys = virtualOrders.ToList(); + // for this bundle, get all games from the database that are not in the keys collection for this order + var libraryKeysNotInOrder = PlayniteApi.Database.Games + .Where(game => + game.TagIds != null && game.TagIds.Contains(humbleChoiceTag.Id) && gameKeys.Contains(game.GameId)) + .ToList(); + foreach (var game in libraryKeysNotInOrder) { - case UnredeemableMethodology.Tag: + switch (unredeemableMethod) + { + case UnredeemableMethodology.Tag: { game.TagIds.Remove(unredeemedTag.Id); if (game.TagIds.Contains(unredeemableTag.Id)) continue; @@ -212,7 +249,7 @@ protected void ProcessOrders(Dictionary orders, IEnumerable orders, IEnumerable game.GameId == gameId && game.PluginId == Id); + var alreadyImported = PlayniteApi.Database.Games.FirstOrDefault(game => game.GameId == gameId && game.PluginId == Id); - if (alreadyImported == null) - { - if (!Settings.IgnoreRedeemedKeys || (Settings.IgnoreRedeemedKeys && !IsKeyPresent(tpkd))) + if (alreadyImported == null) { - importedGames.Add(ImportNewGame(tpkd, humbleChoiceTag)); + if (!Settings.IgnoreRedeemedKeys || (Settings.IgnoreRedeemedKeys && !IsKeyPresent(tpkd))) + { + importedGames.Add(ImportNewGame(tpkd, humbleChoiceTag)); + } } - } - else - { - if (!Settings.IgnoreRedeemedKeys || (Settings.IgnoreRedeemedKeys && !IsKeyPresent(tpkd))) + else { - var tagsUpdated = UpdateRedemptionStatus(alreadyImported, tpkd, humbleChoiceTag); - var otherUpdated = UpdatePlatform(alreadyImported, tpkd); - if (UpdateRedemptionStore(alreadyImported, tpkd)) otherUpdated = true; - - if (Settings.AddLinks) + if (!Settings.IgnoreRedeemedKeys || (Settings.IgnoreRedeemedKeys && !IsKeyPresent(tpkd))) { - if (alreadyImported.Links == null) + var tagsUpdated = UpdateRedemptionStatus(alreadyImported, tpkd, humbleChoiceTag); + var otherUpdated = UpdatePlatform(alreadyImported, tpkd); + if (UpdateRedemptionStore(alreadyImported, tpkd)) otherUpdated = true; + + if (Settings.AddLinks) { - alreadyImported.Links = new ObservableCollection(); - } + if (alreadyImported.Links == null) + { + alreadyImported.Links = new ObservableCollection(); + } - if (UpdateStoreLinks(alreadyImported.Links, tpkd, true)) otherUpdated = true; - } + if (UpdateStoreLinks(alreadyImported.Links, tpkd, true)) otherUpdated = true; + } - if (!tagsUpdated && !otherUpdated) - { - logger.Trace($"ProcessOrders: No update needed for '{alreadyImported.Name}' with GameId = {alreadyImported.GameId}"); - continue; - } + if (!tagsUpdated && !otherUpdated) + { + logger.Trace($"ProcessOrders: No update needed for '{alreadyImported.Name}' with GameId = {alreadyImported.GameId}"); + continue; + } - if (alreadyImported.TagIds != null && alreadyImported.TagIds.Contains(unredeemableTag.Id)) - { - switch (unredeemableMethod) + if (alreadyImported.TagIds != null && alreadyImported.TagIds.Contains(unredeemableTag.Id)) { - case UnredeemableMethodology.Tag: + switch (unredeemableMethod) + { + case UnredeemableMethodology.Tag: { PlayniteApi.Database.Games.Update(alreadyImported); PlayniteApi.Notifications.Add( @@ -281,44 +318,57 @@ protected void ProcessOrders(Dictionary orders, IEnumerable + { + if (PlayniteApi.ApplicationInfo.Mode == ApplicationMode.Fullscreen) return; + PlayniteApi.MainView.SelectGame(alreadyImported.Id); + }) + ); + } } } else { - PlayniteApi.Database.Games.Update(alreadyImported); - logger.Trace($"ProcessOrders: Updated '{alreadyImported.Name}' with GameId = {alreadyImported.GameId}"); - if (tagsUpdated) - { - PlayniteApi.Notifications.Add( - new NotificationMessage("HumbleKeysLibraryUpdate_" + alreadyImported.Id, - $"Tags Updated for {alreadyImported.Name}: " + GetOrderRedemptionTagState(tpkd), NotificationType.Info, - () => - { - if (PlayniteApi.ApplicationInfo.Mode == ApplicationMode.Fullscreen) return; - PlayniteApi.MainView.SelectGame(alreadyImported.Id); - }) - ); - } + // Remove Existing Game? + PlayniteApi.Database.Games.Remove(alreadyImported); + logger.Trace( + $"Removing game '{alreadyImported.Name}' with GameId = {alreadyImported.GameId} since Settings.IgnoreRedeemedKeys is: [{Settings.IgnoreRedeemedKeys}] and IsKeyPresent() is [{IsKeyPresent(tpkd)}]"); } } - else - { - // Remove Existing Game? - PlayniteApi.Database.Games.Remove(alreadyImported); - logger.Trace( - $"Removing game '{alreadyImported.Name}' with GameId = {alreadyImported.GameId} since Settings.IgnoreRedeemedKeys is: [{Settings.IgnoreRedeemedKeys}] and IsKeyPresent() is [{IsKeyPresent(tpkd)}]"); - } } } + + currentOrderIndex++; + if (importProgress == null) continue; + + importProgress.ProgressValue = ((float)currentOrderIndex / orderKeys.Count) * 100; } + + if (importProgress == null) return; + + importProgress.ProgressValue = 100f; + importProgress.Visible = false; + } finally { diff --git a/Services/HumbleKeysAccountClient.cs b/Services/HumbleKeysAccountClient.cs index e1442e4..99f33a0 100644 --- a/Services/HumbleKeysAccountClient.cs +++ b/Services/HumbleKeysAccountClient.cs @@ -6,11 +6,12 @@ using System.Linq; using System.Reflection; using System.Text.RegularExpressions; +using System.Threading; using Playnite.SDK.Data; namespace HumbleKeys.Services { - public class HumbleKeysAccountClient + public class HumbleKeysAccountClient: IDisposable { private static readonly ILogger logger = LogManager.GetLogger(); private readonly IWebView webView; @@ -136,13 +137,15 @@ void CreateCacheContent(string cacheFilename, string strCacheEntry) streamWriter.Close(); } - internal Dictionary GetOrders(List gameKeys, bool includeChoiceMonths = false) + internal IEnumerable GetOrders(List gameKeys, bool includeChoiceMonths = false, CancellationToken cancellationToken = default) { - var orders = new Dictionary(); logger.Trace($"GetOrders: Processing {gameKeys.Count} game keys"); + var processedOrderCount = gameKeys.Count; foreach (var key in gameKeys) { + if (cancellationToken.IsCancellationRequested) break; + var orderUri = string.Format(orderUrlMask, key); var cacheFileName = $"{localCachePath}/order/{key}.json"; Order order = null; @@ -173,12 +176,11 @@ internal Dictionary GetOrders(List gameKeys, bool include { AddChoiceMonthlyGames(order); } - orders.Add(order.gamekey, order); logger.Trace($"GetOrders: Added order {order.gamekey} with {order.tpkd_dict.all_tpks.Count} total tpks"); + yield return order; } - logger.Trace($"GetOrders: Completed processing {orders.Count} orders"); - return orders; + logger.Trace($"GetOrders: Completed processing {processedOrderCount} orders"); } void AddChoiceMonthlyGames(Order order) @@ -303,5 +305,10 @@ internal List GetOrders(string cachePath) return orders; } + + public void Dispose() + { + webView?.Dispose(); + } } } diff --git a/changelog.md b/changelog.md index 83e6d9a..39d9092 100644 --- a/changelog.md +++ b/changelog.md @@ -1,4 +1,8 @@ ## What's Changed +# 0.3.10 +* Display a progress bar in the sidebar to convey the status of scraping orders from Humble API +* Cancelling during scraping is now possible + # 0.3.9 * IMPORTANT - Platform field is no longer used by default for the Redemption Store (i.e. Steam, GOG, etc.), but now Source is (helps some metadata plugins properly match games) * Added dropdown setting to add Redemption Store (e.g. Steam) to either Source (now default), Tag, Category, or Platform (no longer default) field, or None (disabled)