From bc19e79d6973eaa28c390d72cb41f3133eaf5203 Mon Sep 17 00:00:00 2001 From: Jeddunk Date: Sun, 21 Mar 2021 15:20:46 +0100 Subject: [PATCH] Added SQLite support --- GoldbergGUI.Core/GoldbergGUI.Core.csproj | 1 + GoldbergGUI.Core/Models/SteamAppModel.cs | 57 ++++++--- GoldbergGUI.Core/Services/SteamService.cs | 134 +++++++++------------- 3 files changed, 93 insertions(+), 99 deletions(-) diff --git a/GoldbergGUI.Core/GoldbergGUI.Core.csproj b/GoldbergGUI.Core/GoldbergGUI.Core.csproj index 79fab83..5eab489 100644 --- a/GoldbergGUI.Core/GoldbergGUI.Core.csproj +++ b/GoldbergGUI.Core/GoldbergGUI.Core.csproj @@ -12,6 +12,7 @@ + diff --git a/GoldbergGUI.Core/Models/SteamAppModel.cs b/GoldbergGUI.Core/Models/SteamAppModel.cs index 6fbf277..b5f245f 100644 --- a/GoldbergGUI.Core/Models/SteamAppModel.cs +++ b/GoldbergGUI.Core/Models/SteamAppModel.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; using System.Text.RegularExpressions; using GoldbergGUI.Core.Utils; +using SQLite; // ReSharper disable UnusedMember.Global // ReSharper disable ClassNeverInstantiated.Global @@ -11,44 +12,51 @@ using GoldbergGUI.Core.Utils; // ReSharper disable InconsistentNaming namespace GoldbergGUI.Core.Models { + [Table("steamapp")] public class SteamApp { private string _name; - private string _comparableName; - [JsonPropertyName("appid")] public int AppId { get; set; } + + [JsonPropertyName("appid")] + [Column("appid")] + [PrimaryKey] + public int AppId { get; set; } /// /// Name of Steam app /// [JsonPropertyName("name")] + [Column("name")] public string Name { get => _name; set { _name = value; - _comparableName = Regex.Replace(value, Misc.AlphaNumOnlyRegex, "").ToLower(); + ComparableName = Regex.Replace(value, Misc.AlphaNumOnlyRegex, "").ToLower(); } } - - /// - /// Trimmed and cleaned name of Steam app, used for comparisons. - /// - public bool CompareName(string value) => _comparableName.Equals(value); + + [Column("comparable_name")] + public string ComparableName { get; private set; } /// /// App type (Game, DLC, ...) /// - public AppType type { get; set; } + [Column("type")] + public string type { get; set; } public override string ToString() { return $"{AppId}={Name}"; } - [JsonPropertyName("last_modified")] public long LastModified { get; set; } + [JsonPropertyName("last_modified")] + [Ignore] + public long LastModified { get; set; } [JsonPropertyName("price_change_number")] + [Ignore] public long PriceChangeNumber { get; set; } } @@ -77,18 +85,29 @@ namespace GoldbergGUI.Core.Models [JsonPropertyName("response")] public override AppList AppList { get; set; } } + /*[Table("apptype")] public class AppType { - private AppType(string value) => Value = value; + private AppType(string value) + { + var db = new SQLiteConnection("steamapps.db"); + db.CreateTable(); + Value = value; + } + [PrimaryKey, AutoIncrement] + [Column("id")] + public int Id { get; } + + [Column("value")] public string Value { get; } - public static AppType Game { get; } = new AppType("game"); - public static AppType DLC { get; } = new AppType("dlc"); - public static AppType Music { get; } = new AppType("music"); - public static AppType Demo { get; } = new AppType("demo"); - public static AppType Ad { get; } = new AppType("advertising"); - public static AppType Mod { get; } = new AppType("mod"); - public static AppType Video { get; } = new AppType("video"); - } + [Ignore] public static AppType Game { get; } = new AppType("game"); + [Ignore] public static AppType DLC { get; } = new AppType("dlc"); + [Ignore] public static AppType Music { get; } = new AppType("music"); + [Ignore] public static AppType Demo { get; } = new AppType("demo"); + [Ignore] public static AppType Ad { get; } = new AppType("advertising"); + [Ignore] public static AppType Mod { get; } = new AppType("mod"); + [Ignore] public static AppType Video { get; } = new AppType("video"); + }*/ } \ No newline at end of file diff --git a/GoldbergGUI.Core/Services/SteamService.cs b/GoldbergGUI.Core/Services/SteamService.cs index 300961e..eab75cf 100644 --- a/GoldbergGUI.Core/Services/SteamService.cs +++ b/GoldbergGUI.Core/Services/SteamService.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; -using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -13,6 +12,7 @@ using GoldbergGUI.Core.Models; using GoldbergGUI.Core.Utils; using MvvmCross.Logging; using NinjaNye.SearchExtensions; +using SQLite; using SteamStorefrontAPI; namespace GoldbergGUI.Core.Services @@ -29,15 +29,12 @@ namespace GoldbergGUI.Core.Services class SteamCache { - public string Filename { get; } public string SteamUri { get; } public Type ApiVersion { get; } - public AppType SteamAppType { get; } - public HashSet Cache { get; set; } = new HashSet(); + public string SteamAppType { get; } - public SteamCache(string filename, string uri, Type apiVersion, AppType steamAppType) + public SteamCache(string uri, Type apiVersion, string steamAppType) { - Filename = filename; SteamUri = uri; ApiVersion = apiVersion; SteamAppType = steamAppType; @@ -49,32 +46,30 @@ namespace GoldbergGUI.Core.Services public class SteamService : ISteamService { // ReSharper disable StringLiteralTypo - private readonly Dictionary _caches = - new Dictionary + private readonly Dictionary _caches = + new Dictionary { { - AppType.Game, + AppTypeGame, new SteamCache( - "steamapps_games.json", "https://api.steampowered.com/IStoreService/GetAppList/v1/" + "?max_results=50000" + "&include_games=1" + "&key=" + Secrets.SteamWebApiKey(), typeof(SteamAppsV1), - AppType.Game + AppTypeGame ) }, { - AppType.DLC, + AppTypeDlc, new SteamCache( - "steamapps_dlc.json", "https://api.steampowered.com/IStoreService/GetAppList/v1/" + "?max_results=50000" + "&include_games=0" + "&include_dlc=1" + "&key=" + Secrets.SteamWebApiKey(), typeof(SteamAppsV1), - AppType.DLC + AppTypeDlc ) } }; @@ -85,90 +80,66 @@ namespace GoldbergGUI.Core.Services "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/87.0.4280.88 Safari/537.36"; + private const string AppTypeGame = "game"; + private const string AppTypeDlc = "dlc"; + private const string Database = "steamapps.db"; + private IMvxLog _log; + private SQLiteConnection _db; + public async Task Initialize(IMvxLog log) { //var (path, uri, jsonType, appType) = _caches[0]; static SteamApps DeserializeSteamApps(Type type, string cacheString) { - if (type == typeof(SteamAppsV1)) - return JsonSerializer.Deserialize(cacheString); - else if (type == typeof(SteamAppsV2)) - return JsonSerializer.Deserialize(cacheString); - return null; + return type == typeof(SteamAppsV2) + ? (SteamApps) JsonSerializer.Deserialize(cacheString) + : JsonSerializer.Deserialize(cacheString); } - foreach (var (k, c) in _caches) - { - _log = log; - _log.Info($"Updating cache ({k.Value})..."); - var updateNeeded = - DateTime.Now.Subtract(File.GetLastWriteTimeUtc(c.Filename)).TotalDays >= 1; - SteamApps steamApps; - try - { - var temp = await GetCache(updateNeeded, c.SteamUri, c.Filename) - .ConfigureAwait(false); - steamApps = DeserializeSteamApps(c.ApiVersion, temp); - } - catch (JsonException) - { - _log.Error("Local cache broken, forcing update..."); - var temp = await GetCache(true, c.SteamUri, c.Filename).ConfigureAwait(false); - steamApps = DeserializeSteamApps(c.ApiVersion, temp); - } + _log = log; + _db = new SQLiteConnection(Database); + _db.CreateTable(); - try + if (DateTime.Now.Subtract(File.GetLastWriteTimeUtc(Database)).TotalDays >= 1 || !_db.Table().Any()) + { + foreach (var (appType, steamCache) in _caches) { - var cacheRaw = new HashSet(steamApps.AppList.Apps); + _log.Info($"Updating cache ({appType})..."); + bool haveMoreResults; + long lastAppId = 0; + var client = new HttpClient(); + var cacheRaw = new HashSet(); + do + { + var response = lastAppId > 0 + ? await client.GetAsync($"{steamCache.SteamUri}&last_appid={lastAppId}") + .ConfigureAwait(false) + : await client.GetAsync(steamCache.SteamUri).ConfigureAwait(false); + var responseBody = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var steamApps = DeserializeSteamApps(steamCache.ApiVersion, responseBody); + foreach (var appListApp in steamApps.AppList.Apps) cacheRaw.Add(appListApp); + haveMoreResults = steamApps.AppList.HaveMoreResults; + lastAppId = steamApps.AppList.LastAppid; + } while (haveMoreResults); + var cache = new HashSet(); foreach (var steamApp in cacheRaw) { - steamApp.type = c.SteamAppType; + steamApp.type = steamCache.SteamAppType; cache.Add(steamApp); } - c.Cache = cache; - - _log.Info("Loaded cache into memory!"); - } - catch (NullReferenceException e) - { - Console.WriteLine(e); - throw; + _db.InsertAll(cache); } } } - private async Task GetCache(bool updateNeeded, string steamUri, string cachePath) - { - string cacheString; - if (updateNeeded) - { - _log.Info("Getting content from API..."); - var client = new HttpClient(); - var response = await client.GetAsync(steamUri).ConfigureAwait(false); - var responseBody = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - _log.Info("Got content from API successfully. Writing to file..."); - await File.WriteAllTextAsync(cachePath, responseBody, Encoding.UTF8).ConfigureAwait(false); - - _log.Info("Cache written to file successfully."); - cacheString = responseBody; - } - else - { - _log.Info("Cache already up to date!"); - cacheString = await File.ReadAllTextAsync(cachePath).ConfigureAwait(false); - } - - return cacheString; - } - public IEnumerable GetListOfAppsByName(string name) { - var listOfAppsByName = _caches[AppType.Game].Cache.Search(x => x.Name) + var listOfAppsByName = _db.Table() + .Where(x => x.type == AppTypeGame).Search(x => x.Name) .SetCulture(StringComparison.OrdinalIgnoreCase) .ContainingAll(name.Split(' ')); return listOfAppsByName; @@ -178,7 +149,8 @@ namespace GoldbergGUI.Core.Services { _log.Info($"Trying to get app {name}"); var comparableName = Regex.Replace(name, Misc.AlphaNumOnlyRegex, "").ToLower(); - var app = _caches[AppType.Game].Cache.FirstOrDefault(x => x.CompareName(comparableName)); + var app = _db.Table().Where(x => x.type == AppTypeGame) + .FirstOrDefault(x => x.ComparableName.Equals(comparableName)); if (app != null) _log.Info($"Successfully got app {app}"); return app; } @@ -186,7 +158,8 @@ namespace GoldbergGUI.Core.Services public SteamApp GetAppById(int appid) { _log.Info($"Trying to get app with ID {appid}"); - var app = _caches[AppType.Game].Cache.FirstOrDefault(x => x.AppId.Equals(appid)); + var app = _db.Table().Where(x => x.type == AppTypeGame) + .FirstOrDefault(x => x.AppId.Equals(appid)); if (app != null) _log.Info($"Successfully got app {app}"); return app; } @@ -199,11 +172,12 @@ namespace GoldbergGUI.Core.Services { var task = AppDetails.GetAsync(steamApp.AppId); var steamAppDetails = await task.ConfigureAwait(true); - if (steamAppDetails.Type == AppType.Game.Value) + if (steamAppDetails.Type == AppTypeGame) { steamAppDetails.DLC.ForEach(x => { - var result = _caches[AppType.DLC].Cache.FirstOrDefault(y => y.AppId.Equals(x)) + var result = _db.Table().Where(z => z.type == AppTypeDlc) + .FirstOrDefault(y => y.AppId.Equals(x)) ?? new SteamApp {AppId = x, Name = $"Unknown DLC {x}"}; dlcList.Add(result); }); @@ -215,7 +189,7 @@ namespace GoldbergGUI.Core.Services // Get Cloudflare cookie // Scrape and parse HTML page // Add missing to DLC list - + // ReSharper disable once InvertIf if (useSteamDb) {