GoldbergGUI.Core/GoldbergGUI.Core.csproj

@@ -9,6 +9,8 @@
 
   <ItemGroup>
     <PackageReference Include="AngleSharp" Version="0.14.0" />
+    <PackageReference Include="LiteDB" Version="5.0.10" />
+    <PackageReference Include="LiteDB.Async" Version="0.0.8" />
     <PackageReference Include="MvvmCross" Version="7.1.2" />
     <PackageReference Include="NinjaNye.SearchExtensions" Version="3.0.1" />
     <PackageReference Include="SharpCompress" Version="0.26.0" />

GoldbergGUI.Core/Models/SteamAppModel.cs

@@ -79,7 +79,7 @@ namespace GoldbergGUI.Core.Models
 
     public class AppType
     {
-        private AppType(string value) => Value = value;
+        public AppType(string value) => Value = value;
 
         public string Value { get; }
 

GoldbergGUI.Core/Services/SteamService.cs

@@ -1,19 +1,17 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Net.Http;
-using System.Text;
-using System.Text.Json;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using AngleSharp.Dom;
 using AngleSharp.Html.Parser;
 using GoldbergGUI.Core.Models;
 using GoldbergGUI.Core.Utils;
+using LiteDB;
+using LiteDB.Async;
 using MvvmCross.Logging;
-using NinjaNye.SearchExtensions;
 using SteamStorefrontAPI;
+using JsonSerializer = System.Text.Json.JsonSerializer;
 
 namespace GoldbergGUI.Core.Services
 {
@@ -29,24 +27,250 @@ namespace GoldbergGUI.Core.Services
 
     class SteamCache
     {
-        public string Filename { get; }
         public string SteamUri { get; }
         public Type ApiVersion { get; }
         public AppType SteamAppType { get; }
-        public HashSet<SteamApp> Cache { get; set; } = new HashSet<SteamApp>();
 
-        public SteamCache(string filename, string uri, Type apiVersion, AppType steamAppType)
+        public SteamCache(string uri, Type apiVersion, AppType steamAppType)
         {
-            Filename = filename;
             SteamUri = uri;
             ApiVersion = apiVersion;
             SteamAppType = steamAppType;
         }
     }
 
+    public class SteamService : ISteamService
+    {
+        private const string UserAgent =
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
+            "Chrome/87.0.4280.88 Safari/537.36";
+
+        private readonly Dictionary<AppType, SteamCache> _caches =
+            new Dictionary<AppType, SteamCache>
+            {
+                {
+                    AppType.Game,
+                    new SteamCache(
+                        "https://api.steampowered.com/IStoreService/GetAppList/v1/" +
+                        "?max_results=50000" +
+                        "&include_games=1" +
+                        "&key=" + Secrets.SteamWebApiKey(),
+                        typeof(SteamAppsV1),
+                        AppType.Game
+                    )
+                },
+                {
+                    AppType.DLC,
+                    new SteamCache(
+                        "https://api.steampowered.com/IStoreService/GetAppList/v1/" +
+                        "?max_results=50000" +
+                        "&include_games=0" +
+                        "&include_dlc=1" +
+                        "&key=" + Secrets.SteamWebApiKey(),
+                        typeof(SteamAppsV1),
+                        AppType.DLC
+                    )
+                }
+            };
+
+        private static readonly Secrets Secrets = new Secrets();
+        private IMvxLog _log;
+
+        public async Task Initialize(IMvxLog log)
+        {
+            _log = log;
+
+            static SteamApps DeserializeSteamApps(Type type, string cacheString)
+            {
+                if (type == typeof(SteamAppsV1))
+                    return JsonSerializer.Deserialize<SteamAppsV1>(cacheString);
+                else if (type == typeof(SteamAppsV2))
+                    return JsonSerializer.Deserialize<SteamAppsV2>(cacheString);
+                return null;
+            }
+
+            if (DateTime.Now.Subtract(File.GetLastWriteTimeUtc("steamapps.db")).TotalDays >= 1)
+            {
+                using var db = new LiteDatabaseAsync("steamapps.db");
+                var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+                var deleteAllResult = await steamAppCollection.DeleteAllAsync().ConfigureAwait(false);
+                _log.Debug($"deleteAllResult: {deleteAllResult}");
+                foreach (var (type, steamCache) in _caches)
+                {
+                    bool haveMoreResults;
+                    long lastAppId = 0;
+                    var client = new HttpClient();
+                    var cacheRaw = new HashSet<SteamApp>();
+                    do
+                    {
+                        var steamCacheSteamUri = steamCache.SteamUri;
+                        if (lastAppId > 0)
+                        {
+                            steamCacheSteamUri += "&last_appid=" + lastAppId;
+                        }
+                        var response = await client.GetAsync(steamCacheSteamUri).ConfigureAwait(false);
+                        var responseBody = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+                        var steamApps = DeserializeSteamApps(steamCache.ApiVersion, responseBody);
+                        foreach (var appListApp in steamApps.AppList.Apps) cacheRaw.Add(appListApp);
+                        haveMoreResults = steamApps.AppList.HaveMoreResults;
+                        lastAppId = steamApps.AppList.LastAppid;
+                    } while (haveMoreResults);
+
+                    var cache = new HashSet<SteamApp>();
+                    foreach (var steamApp in cacheRaw)
+                    {
+                        steamApp.type = steamCache.SteamAppType;
+                        cache.Add(steamApp);
+                    }
+
+                    var bulkInsertResult = await steamAppCollection.InsertBulkAsync(cache).ConfigureAwait(false);
+                    _log.Debug($"bulkInsertResult: {bulkInsertResult}");
+                    if (cache.Count.Equals(bulkInsertResult))
+                    {
+                        _log.Info($"Successfully added cache to DB (type: {type.Value})");
+                    }
+                    else
+                    {
+                        _log.Error($"Error: could not add all items to DB (type: {type.Value})");
+                    }
+                }
+            }
+        }
+
+        public IEnumerable<SteamApp> GetListOfAppsByName(string name)
+        {
+            using var db = new LiteDatabase("steamapps.db");
+            var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+            steamAppCollection.EnsureIndex(x => x.Name);
+            return steamAppCollection.Query()
+                .Where(x => x.Name.Contains(name))
+                .OrderBy(x => x.AppId)
+                .ToList();
+        }
+
+        public SteamApp GetAppByName(string name)
+        {
+            using var db = new LiteDatabase("steamapps.db");
+            var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+            steamAppCollection.EnsureIndex(x => x.Name);
+            return steamAppCollection.FindOne(x => x.Name.Contains(name));
+        }
+
+        public SteamApp GetAppById(int appid)
+        {
+            using var db = new LiteDatabase("steamapps.db");
+            var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+            steamAppCollection.EnsureIndex(x => x.AppId);
+            return steamAppCollection.FindOne(x => x.AppId.Equals(appid));
+        }
+
+        public async Task<List<SteamApp>> GetListOfDlc(SteamApp steamApp, bool useSteamDb)
+        {
+            /*using var db = new LiteDatabaseAsync("steamapps.db");
+            var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+            var findOneAsync = 
+                await steamAppCollection.FindOneAsync(x => x.Equals(steamApp)).ConfigureAwait(false);
+            return new List<SteamApp>();*/
+            _log.Info("Get DLC");
+            var dlcList = new List<SteamApp>();
+            if (steamApp != null)
+            {
+                var task = AppDetails.GetAsync(steamApp.AppId);
+                var steamAppDetails = await task.ConfigureAwait(true);
+                if (steamAppDetails.Type == AppType.Game.Value)
+                {
+                    var db = new LiteDatabase("steamapps.db");
+                    steamAppDetails.DLC.ForEach(x =>
+                    {
+                        /*var result = _caches[AppType.DLC].Cache.FirstOrDefault(y => y.AppId.Equals(x))
+                                     ?? new SteamApp {AppId = x, Name = $"Unknown DLC {x}"};*/
+                        var steamAppCollection = db.GetCollection<SteamApp>("steamapps");
+                        steamAppCollection.EnsureIndex(y => y.AppId);
+                        var result = steamAppCollection.FindOne(y => y.AppId.Equals(x))
+                                     ?? new SteamApp {AppId = x, Name = $"Unknown DLC {x}"};
+                        dlcList.Add(result);
+                    });
+                    db.Dispose();
+
+                    dlcList.ForEach(x => _log.Debug($"{x.AppId}={x.Name}"));
+                    _log.Info("Got DLC successfully...");
+
+                    // Get DLC from SteamDB
+                    // Get Cloudflare cookie
+                    // Scrape and parse HTML page
+                    // Add missing to DLC list
+
+                    // ReSharper disable once InvertIf
+                    if (useSteamDb)
+                    {
+                        var steamDbUri = new Uri($"https://steamdb.info/app/{steamApp.AppId}/dlc/");
+
+                        var client = new HttpClient();
+                        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent);
+
+                        _log.Info("Get SteamDB App");
+                        var httpCall = client.GetAsync(steamDbUri);
+                        var response = await httpCall.ConfigureAwait(false);
+                        _log.Debug(httpCall.Status.ToString());
+                        _log.Debug(response.EnsureSuccessStatusCode().ToString());
+
+                        var readAsStringAsync = response.Content.ReadAsStringAsync();
+                        var responseBody = await readAsStringAsync.ConfigureAwait(false);
+                        _log.Debug(readAsStringAsync.Status.ToString());
+
+                        var parser = new HtmlParser();
+                        var doc = parser.ParseDocument(responseBody);
+
+                        var query1 = doc.QuerySelector("#dlc");
+                        if (query1 != null)
+                        {
+                            var query2 = query1.QuerySelectorAll(".app");
+                            foreach (var element in query2)
+                            {
+                                var dlcId = element.GetAttribute("data-appid");
+                                var query3 = element.QuerySelectorAll("td");
+                                var dlcName = query3 != null
+                                    ? query3[1].Text().Replace("\n", "").Trim()
+                                    : $"Unknown DLC {dlcId}";
+                                var dlcApp = new SteamApp {AppId = Convert.ToInt32(dlcId), Name = dlcName};
+                                var i = dlcList.FindIndex(x => x.AppId.Equals(dlcApp.AppId));
+                                if (i > -1)
+                                {
+                                    if (dlcList[i].Name.Contains("Unknown DLC")) dlcList[i] = dlcApp;
+                                }
+                                else
+                                {
+                                    dlcList.Add(dlcApp);
+                                }
+                            }
+
+                            dlcList.ForEach(x => _log.Debug($"{x.AppId}={x.Name}"));
+                            _log.Info("Got DLC from SteamDB successfully...");
+                        }
+                        else
+                        {
+                            _log.Error("Could not get DLC from SteamDB!");
+                        }
+                    }
+                }
+                else
+                {
+                    _log.Error("Could not get DLC: Steam App is not of type \"game\"");
+                }
+            }
+            else
+            {
+                _log.Error("Could not get DLC: Invalid Steam App");
+            }
+
+            return dlcList;
+        }
+    }
+
+    /*
     // ReSharper disable once UnusedType.Global
     // ReSharper disable once ClassNeverInstantiated.Global
-    public class SteamService : ISteamService
+    public class OldSteamService : ISteamService
     {
         // ReSharper disable StringLiteralTypo
         private readonly Dictionary<AppType, SteamCache> _caches =
@@ -282,4 +506,5 @@ namespace GoldbergGUI.Core.Services
             return dlcList;
         }
     }
+    */
 }

GoldbergGUI.Core/ViewModels/MainViewModel.cs

@@ -292,7 +292,7 @@ namespace GoldbergGUI.Core.ViewModels
 
             DllPath = dialog.FileName;
             await ReadConfig().ConfigureAwait(false);
-            if (!GoldbergApplied) await GetListOfDlc().ConfigureAwait(false);
+            // if (!GoldbergApplied) await GetListOfDlc().ConfigureAwait(false);
             MainWindowEnabled = true;
             StatusText = "Ready.";
         }
@@ -349,7 +349,7 @@ namespace GoldbergGUI.Core.ViewModels
                     await FindIdInList(steamApps).ConfigureAwait(false);
                 }
             }
-            await GetListOfDlc().ConfigureAwait(false);
+            //await GetListOfDlc().ConfigureAwait(false);
             MainWindowEnabled = true;
             StatusText = "Ready.";
         }