From fdc2e0277fed8e0a3cafccc344976560611d8dde Mon Sep 17 00:00:00 2001 From: Jeddunk Date: Thu, 11 Mar 2021 14:04:07 +0100 Subject: [PATCH] Replaced old cache system with DB based one (using LiteDB) Fixed crashes related to global settings --- GoldbergGUI.Core/GoldbergGUI.Core.csproj | 2 + GoldbergGUI.Core/Services/GoldbergService.cs | 49 ++-- GoldbergGUI.Core/Services/SteamService.cs | 245 ++++++++++++++++++- 3 files changed, 270 insertions(+), 26 deletions(-) diff --git a/GoldbergGUI.Core/GoldbergGUI.Core.csproj b/GoldbergGUI.Core/GoldbergGUI.Core.csproj index 79fab83..84a806c 100644 --- a/GoldbergGUI.Core/GoldbergGUI.Core.csproj +++ b/GoldbergGUI.Core/GoldbergGUI.Core.csproj @@ -9,6 +9,8 @@ + + diff --git a/GoldbergGUI.Core/Services/GoldbergService.cs b/GoldbergGUI.Core/Services/GoldbergService.cs index 2bd56b2..299ba96 100644 --- a/GoldbergGUI.Core/Services/GoldbergService.cs +++ b/GoldbergGUI.Core/Services/GoldbergService.cs @@ -5,7 +5,6 @@ using System.IO.Compression; using System.Linq; using System.Net.Http; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; using GoldbergGUI.Core.Models; using GoldbergGUI.Core.Utils; @@ -24,8 +23,8 @@ namespace GoldbergGUI.Core.Services public Task GetGlobalSettings(); public Task SetGlobalSettings(GoldbergGlobalConfiguration configuration); public bool GoldbergApplied(string path); - public Task Download(); - public Task Extract(string archivePath); + // public Task Download(); + // public Task Extract(string archivePath); public Task GenerateInterfacesFile(string filePath); public List Languages(); } @@ -34,8 +33,10 @@ namespace GoldbergGUI.Core.Services public class GoldbergService : IGoldbergService { private IMvxLog _log; - private const string GoldbergUrl = "https://mr_goldberg.gitlab.io/goldberg_emulator/"; + private const string DefaultAccountName = "Mr_Goldberg"; + private const long DefaultSteamId = 76561197960287930; private const string DefaultLanguage = "english"; + private const string GoldbergUrl = "https://mr_goldberg.gitlab.io/goldberg_emulator/"; private readonly string _goldbergZipPath = Path.Combine(Directory.GetCurrentDirectory(), "goldberg.zip"); private readonly string _goldbergPath = Path.Combine(Directory.GetCurrentDirectory(), "goldberg"); @@ -48,6 +49,7 @@ namespace GoldbergGUI.Core.Services private readonly string _languagePath = Path.Combine(GlobalSettingsPath, "settings/language.txt"); private readonly string _customBroadcastIpsPath = Path.Combine(GlobalSettingsPath, "settings/custom_broadcasts.txt"); + // ReSharper disable StringLiteralTypo private readonly List _interfaceNames = new List { "SteamClient", @@ -90,10 +92,11 @@ namespace GoldbergGUI.Core.Services public async Task GetGlobalSettings() { _log.Info("Getting global settings..."); - var accountName = "Account name..."; - long steamId = -1; + var accountName = DefaultAccountName; + var steamId = DefaultSteamId; var language = DefaultLanguage; var customBroadcastIps = new List(); + if (!File.Exists(GlobalSettingsPath)) Directory.CreateDirectory(Path.Join(GlobalSettingsPath, "settings")); await Task.Run(() => { if (File.Exists(_accountNamePath)) accountName = File.ReadLines(_accountNamePath).First().Trim(); @@ -125,44 +128,58 @@ namespace GoldbergGUI.Core.Services var customBroadcastIps = c.CustomBroadcastIps; _log.Info("Setting global settings..."); // Account Name - if (accountName != null && accountName != "Account name...") + if (!string.IsNullOrEmpty(accountName)) { _log.Info("Setting account name..."); + if (!File.Exists(_accountNamePath)) + await File.Create(_accountNamePath).DisposeAsync().ConfigureAwait(false); await File.WriteAllTextAsync(_accountNamePath, accountName).ConfigureAwait(false); } else { _log.Info("Invalid account name! Skipping..."); - await File.WriteAllTextAsync(_accountNamePath, "Goldberg").ConfigureAwait(false); + if (!File.Exists(_accountNamePath)) + await File.Create(_accountNamePath).DisposeAsync().ConfigureAwait(false); + await File.WriteAllTextAsync(_accountNamePath, DefaultAccountName).ConfigureAwait(false); } // User SteamID if (userSteamId >= 76561197960265729 && userSteamId <= 76561202255233023) { _log.Info("Setting user Steam ID..."); + if (!File.Exists(_userSteamIdPath)) + await File.Create(_userSteamIdPath).DisposeAsync().ConfigureAwait(false); await File.WriteAllTextAsync(_userSteamIdPath, userSteamId.ToString()).ConfigureAwait(false); } else { _log.Info("Invalid user Steam ID! Skipping..."); - await Task.Run(() => File.Delete(_userSteamIdPath)).ConfigureAwait(false); + if (!File.Exists(_userSteamIdPath)) + await File.Create(_userSteamIdPath).DisposeAsync().ConfigureAwait(false); + await File.WriteAllTextAsync(_userSteamIdPath, DefaultSteamId.ToString()).ConfigureAwait(false); } // Language - if (language != null) + if (!string.IsNullOrEmpty(language)) { _log.Info("Setting language..."); + if (!File.Exists(_languagePath)) + await File.Create(_languagePath).DisposeAsync().ConfigureAwait(false); await File.WriteAllTextAsync(_languagePath, language).ConfigureAwait(false); } else { _log.Info("Invalid language! Skipping..."); + if (!File.Exists(_languagePath)) + await File.Create(_languagePath).DisposeAsync().ConfigureAwait(false); await File.WriteAllTextAsync(_languagePath, DefaultLanguage).ConfigureAwait(false); } // Custom Broadcast IPs - if (customBroadcastIps.Count > 0) + if (customBroadcastIps != null && customBroadcastIps.Count > 0) { _log.Info("Setting custom broadcast IPs..."); var result = customBroadcastIps.Aggregate("", (current, address) => $"{current}{address}\n"); + if (!File.Exists(_customBroadcastIpsPath)) + await File.Create(_customBroadcastIpsPath).DisposeAsync().ConfigureAwait(false); await File.WriteAllTextAsync(_customBroadcastIpsPath, result).ConfigureAwait(false); } else @@ -329,11 +346,11 @@ namespace GoldbergGUI.Core.Services return steamSettingsDirExists && steamAppIdTxtExists; } - // Get webpage - // Get job id, compare with local if exists, save it if false or missing - // Get latest archive if mismatch, call Extract - public async Task Download() + private async Task Download() { + // Get webpage + // Get job id, compare with local if exists, save it if false or missing + // Get latest archive if mismatch, call Extract _log.Info("Initializing download..."); if (!Directory.Exists(_goldbergPath)) Directory.CreateDirectory(_goldbergPath); var client = new HttpClient(); @@ -376,7 +393,7 @@ namespace GoldbergGUI.Core.Services // Empty subfolder ./goldberg/ // Extract all from archive to subfolder ./goldberg/ - public async Task Extract(string archivePath) + private async Task Extract(string archivePath) { _log.Debug("Start extraction..."); await Task.Run(() => diff --git a/GoldbergGUI.Core/Services/SteamService.cs b/GoldbergGUI.Core/Services/SteamService.cs index 300961e..b197912 100644 --- a/GoldbergGUI.Core/Services/SteamService.cs +++ b/GoldbergGUI.Core/Services/SteamService.cs @@ -1,19 +1,17 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; using System.Threading.Tasks; using AngleSharp.Dom; using AngleSharp.Html.Parser; using GoldbergGUI.Core.Models; using GoldbergGUI.Core.Utils; +using LiteDB; +using LiteDB.Async; using MvvmCross.Logging; -using NinjaNye.SearchExtensions; using SteamStorefrontAPI; +using JsonSerializer = System.Text.Json.JsonSerializer; namespace GoldbergGUI.Core.Services { @@ -29,24 +27,250 @@ namespace GoldbergGUI.Core.Services class SteamCache { - public string Filename { get; } public string SteamUri { get; } public Type ApiVersion { get; } public AppType SteamAppType { get; } - public HashSet Cache { get; set; } = new HashSet(); - public SteamCache(string filename, string uri, Type apiVersion, AppType steamAppType) + public SteamCache(string uri, Type apiVersion, AppType steamAppType) { - Filename = filename; SteamUri = uri; ApiVersion = apiVersion; SteamAppType = steamAppType; } } + public class SteamService : ISteamService + { + private const string UserAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/87.0.4280.88 Safari/537.36"; + + private readonly Dictionary _caches = + new Dictionary + { + { + AppType.Game, + new SteamCache( + "https://api.steampowered.com/IStoreService/GetAppList/v1/" + + "?max_results=50000" + + "&include_games=1" + + "&key=" + Secrets.SteamWebApiKey(), + typeof(SteamAppsV1), + AppType.Game + ) + }, + { + AppType.DLC, + new SteamCache( + "https://api.steampowered.com/IStoreService/GetAppList/v1/" + + "?max_results=50000" + + "&include_games=0" + + "&include_dlc=1" + + "&key=" + Secrets.SteamWebApiKey(), + typeof(SteamAppsV1), + AppType.DLC + ) + } + }; + + private static readonly Secrets Secrets = new Secrets(); + private IMvxLog _log; + + public async Task Initialize(IMvxLog log) + { + _log = log; + + static SteamApps DeserializeSteamApps(Type type, string cacheString) + { + if (type == typeof(SteamAppsV1)) + return JsonSerializer.Deserialize(cacheString); + else if (type == typeof(SteamAppsV2)) + return JsonSerializer.Deserialize(cacheString); + return null; + } + + if (DateTime.Now.Subtract(File.GetLastWriteTimeUtc("steamapps.db")).TotalDays >= 1) + { + using var db = new LiteDatabaseAsync("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + var deleteAllResult = await steamAppCollection.DeleteAllAsync().ConfigureAwait(false); + _log.Debug($"deleteAllResult: {deleteAllResult}"); + foreach (var (type, steamCache) in _caches) + { + bool haveMoreResults; + long lastAppId = 0; + var client = new HttpClient(); + var cacheRaw = new HashSet(); + do + { + var steamCacheSteamUri = steamCache.SteamUri; + if (lastAppId > 0) + { + steamCacheSteamUri += "&last_appid=" + lastAppId; + } + var response = await client.GetAsync(steamCacheSteamUri).ConfigureAwait(false); + var responseBody = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + var steamApps = DeserializeSteamApps(steamCache.ApiVersion, responseBody); + foreach (var appListApp in steamApps.AppList.Apps) cacheRaw.Add(appListApp); + haveMoreResults = steamApps.AppList.HaveMoreResults; + lastAppId = steamApps.AppList.LastAppid; + } while (haveMoreResults); + + var cache = new HashSet(); + foreach (var steamApp in cacheRaw) + { + steamApp.type = steamCache.SteamAppType; + cache.Add(steamApp); + } + + var bulkInsertResult = await steamAppCollection.InsertBulkAsync(cache).ConfigureAwait(false); + _log.Debug($"bulkInsertResult: {bulkInsertResult}"); + if (cache.Count.Equals(bulkInsertResult)) + { + _log.Info($"Successfully added cache to DB (type: {type.Value})"); + } + else + { + _log.Error($"Error: could not add all items to DB (type: {type.Value})"); + } + } + } + } + + public IEnumerable GetListOfAppsByName(string name) + { + using var db = new LiteDatabase("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + steamAppCollection.EnsureIndex(x => x.Name); + return steamAppCollection.Query() + .Where(x => x.Name.Contains(name)) + .OrderBy(x => x.AppId) + .ToList(); + } + + public SteamApp GetAppByName(string name) + { + using var db = new LiteDatabase("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + steamAppCollection.EnsureIndex(x => x.Name); + return steamAppCollection.FindOne(x => x.Name.Contains(name)); + } + + public SteamApp GetAppById(int appid) + { + using var db = new LiteDatabase("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + steamAppCollection.EnsureIndex(x => x.AppId); + return steamAppCollection.FindOne(x => x.AppId.Equals(appid)); + } + + public async Task> GetListOfDlc(SteamApp steamApp, bool useSteamDb) + { + /*using var db = new LiteDatabaseAsync("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + var findOneAsync = + await steamAppCollection.FindOneAsync(x => x.Equals(steamApp)).ConfigureAwait(false); + return new List();*/ + _log.Info("Get DLC"); + var dlcList = new List(); + if (steamApp != null) + { + var task = AppDetails.GetAsync(steamApp.AppId); + var steamAppDetails = await task.ConfigureAwait(true); + if (steamAppDetails.Type == AppType.Game.Value) + { + steamAppDetails.DLC.ForEach(x => + { + /*var result = _caches[AppType.DLC].Cache.FirstOrDefault(y => y.AppId.Equals(x)) + ?? new SteamApp {AppId = x, Name = $"Unknown DLC {x}"};*/ + + using var db = new LiteDatabase("steamapps.db"); + var steamAppCollection = db.GetCollection("steamapps"); + steamAppCollection.EnsureIndex(y => y.AppId); + var result = steamAppCollection.FindOne(y => y.AppId.Equals(x)) + ?? new SteamApp {AppId = x, Name = $"Unknown DLC {x}"}; + dlcList.Add(result); + }); + + dlcList.ForEach(x => _log.Debug($"{x.AppId}={x.Name}")); + _log.Info("Got DLC successfully..."); + + // Get DLC from SteamDB + // Get Cloudflare cookie + // Scrape and parse HTML page + // Add missing to DLC list + + // ReSharper disable once InvertIf + if (useSteamDb) + { + var steamDbUri = new Uri($"https://steamdb.info/app/{steamApp.AppId}/dlc/"); + + var client = new HttpClient(); + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + + _log.Info("Get SteamDB App"); + var httpCall = client.GetAsync(steamDbUri); + var response = await httpCall.ConfigureAwait(false); + _log.Debug(httpCall.Status.ToString()); + _log.Debug(response.EnsureSuccessStatusCode().ToString()); + + var readAsStringAsync = response.Content.ReadAsStringAsync(); + var responseBody = await readAsStringAsync.ConfigureAwait(false); + _log.Debug(readAsStringAsync.Status.ToString()); + + var parser = new HtmlParser(); + var doc = parser.ParseDocument(responseBody); + + var query1 = doc.QuerySelector("#dlc"); + if (query1 != null) + { + var query2 = query1.QuerySelectorAll(".app"); + foreach (var element in query2) + { + var dlcId = element.GetAttribute("data-appid"); + var query3 = element.QuerySelectorAll("td"); + var dlcName = query3 != null + ? query3[1].Text().Replace("\n", "").Trim() + : $"Unknown DLC {dlcId}"; + var dlcApp = new SteamApp {AppId = Convert.ToInt32(dlcId), Name = dlcName}; + var i = dlcList.FindIndex(x => x.AppId.Equals(dlcApp.AppId)); + if (i > -1) + { + if (dlcList[i].Name.Contains("Unknown DLC")) dlcList[i] = dlcApp; + } + else + { + dlcList.Add(dlcApp); + } + } + + dlcList.ForEach(x => _log.Debug($"{x.AppId}={x.Name}")); + _log.Info("Got DLC from SteamDB successfully..."); + } + else + { + _log.Error("Could not get DLC from SteamDB!"); + } + } + } + else + { + _log.Error("Could not get DLC: Steam App is not of type \"game\""); + } + } + else + { + _log.Error("Could not get DLC: Invalid Steam App"); + } + + return dlcList; + } + } + + /* // ReSharper disable once UnusedType.Global // ReSharper disable once ClassNeverInstantiated.Global - public class SteamService : ISteamService + public class OldSteamService : ISteamService { // ReSharper disable StringLiteralTypo private readonly Dictionary _caches = @@ -282,4 +506,5 @@ namespace GoldbergGUI.Core.Services return dlcList; } } + */ } \ No newline at end of file