From 992f2fb20c0b7a1c047e190c15cbd916c009ec77 Mon Sep 17 00:00:00 2001 From: Jeddunk Date: Wed, 21 Oct 2020 15:47:58 +0200 Subject: [PATCH] Fixed and enabled SteamDB DLC functionality --- auto-cream-api.iml | 21 +++- pom.xml | 11 +- .../xyz/jeddunk/autocreamapi/Controller.java | 11 +- .../autocreamapi/util/CreamApiDllHandler.java | 6 +- .../autocreamapi/util/SteamAppsListCache.java | 110 +++++++++++++----- 5 files changed, 114 insertions(+), 45 deletions(-) diff --git a/auto-cream-api.iml b/auto-cream-api.iml index 9654196..3730131 100644 --- a/auto-cream-api.iml +++ b/auto-cream-api.iml @@ -56,7 +56,7 @@ - + @@ -79,5 +79,24 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index bc682b0..8aefdfd 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ xyz.jeddunk auto-cream-api - 1.2.1 + 1.2.2 UTF-8 @@ -150,7 +150,7 @@ false bintray-jerady-maven - bintray + bintray-jerady https://dl.bintray.com/jerady/maven @@ -160,7 +160,7 @@ false bintray-jerady-maven - bintray-plugins + bintray-jerady-plugins https://dl.bintray.com/jerady/maven @@ -229,5 +229,10 @@ jansi 1.18 + + net.sourceforge.htmlunit + htmlunit + 2.44.0 + \ No newline at end of file diff --git a/src/main/java/xyz/jeddunk/autocreamapi/Controller.java b/src/main/java/xyz/jeddunk/autocreamapi/Controller.java index 4257bc1..baa5cb4 100644 --- a/src/main/java/xyz/jeddunk/autocreamapi/Controller.java +++ b/src/main/java/xyz/jeddunk/autocreamapi/Controller.java @@ -132,10 +132,7 @@ public class Controller implements Initializable { @FXML @Override public void initialize(URL location, ResourceBundle resources) { - // steamdb_dlc_checkbox.setSelected(true); - // Disable SteamDB DLC checking for now until it's fix - steamdb_dlc_checkbox.setSelected(false); - steamdb_dlc_checkbox.setDisable(true); + steamdb_dlc_checkbox.setSelected(true); appId_textfield.textProperty().addListener(appIdChangesGameName()); game_name_textfield.setOnKeyReleased(event -> { KeyCode code = event.getCode(); @@ -147,7 +144,6 @@ public class Controller implements Initializable { language_combobox.setTooltip(new Tooltip()); language_combobox.setEditable(false); new ComboBoxAutoComplete<>(language_combobox); - //retrieveDlcList_button.setDisable(true); // WIP generate_tooltips(); fix_dlc_textarea_prompt_text(); reset(true); @@ -186,10 +182,9 @@ public class Controller implements Initializable { extra_protection_checkbox.setTooltip(extra_protection_tooltip); Tooltip.install(extra_protection_checkbox, extra_protection_tooltip); - /*Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" + + Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" + "Especially useful for DLC that is not listed on the Steam Store.\n" + - "This sometimes doesn't work because SteamDB may block access for scrapers.");*/ - Tooltip steamdb_dlc_tooltip = new Tooltip("Currently broken."); + "This sometimes doesn't work because SteamDB may block access for scrapers."); steamdb_dlc_checkbox.setTooltip(steamdb_dlc_tooltip); Tooltip.install(steamdb_dlc_checkbox, steamdb_dlc_tooltip); } diff --git a/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java b/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java index 940a545..fc6f441 100644 --- a/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java +++ b/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java @@ -31,10 +31,8 @@ public class CreamApiDllHandler { private final String steamApi64DllMd5; private CreamApiDllHandler() throws IOException { - String steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath)); - String steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath)); - this.steamApiDllMd5 = steamApiDllMd5; - this.steamApi64DllMd5 = steamApi64DllMd5; + this.steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath)); + this.steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath)); } public static synchronized CreamApiDllHandler getInstance() throws IOException { diff --git a/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java b/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java index b1f7655..03f205d 100644 --- a/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java +++ b/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java @@ -15,6 +15,11 @@ package xyz.jeddunk.autocreamapi.util; +import com.gargoylesoftware.htmlunit.BrowserVersion; +import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; +import com.gargoylesoftware.htmlunit.WebClient; +import com.gargoylesoftware.htmlunit.html.HtmlPage; +import com.gargoylesoftware.htmlunit.util.Cookie; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; @@ -35,6 +40,7 @@ import xyz.jeddunk.autocreamapi.util.env.MainEnv; import java.io.*; import java.lang.reflect.Type; +import java.net.URL; import java.time.Duration; import java.time.Instant; import java.util.*; @@ -47,7 +53,7 @@ public class SteamAppsListCache { private SteamAppsList list = new SteamAppsList(); private final File cacheFile = new File("steamapps.json"); private MainEnv env; - //private String key; + private String cfduid = ""; public SteamAppsListCache() { try { @@ -151,12 +157,13 @@ public class SteamAppsListCache { } public Map getDlcMap(String appId, boolean use_steamdb) { + logger.info("Getting list of DLC for AppID: " + appId + "..."); Map steamStoreDLCs = new HashMap<>(); Map steamDbDLCs = new HashMap<>(); - //StringBuilder sb = new StringBuilder(); // Steam Store try { + logger.info("Trying to get DLC from Steam store page..."); Document steamDoc = Jsoup .connect("https://store.steampowered.com/app/" + appId + "/") .cookie("birthtime", "470703601") @@ -170,13 +177,12 @@ public class SteamAppsListCache { .text().replace("\n", "").trim(); steamStoreDLCs.put(Integer.parseInt(dlc_id), dlc_name); } + logger.info("Got DLC from Steam store page successfully."); } catch (HttpStatusException e) { logger.error(e.getUrl()); if (e.getStatusCode() == 404) { logger.error("App ID empty or not found! (HTTP Status Code: 404)"); } else { - /*System.err.printf("Error occurred while trying to get list of DLCs " + - "(HTTP Status Code: %d)%n", e.getStatusCode());*/ logger.error("Error occurred while trying to get list of DLCs " + "(HTTP Status Code: " + e.getStatusCode() + ")"); } @@ -186,39 +192,85 @@ public class SteamAppsListCache { // SteamDB if (use_steamdb) { - try { - Document steamDbDoc = Jsoup - .connect("https://steamdb.info/app/" + appId + "/dlc/") - .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0") - .get(); - Element steamDbDlcSection = steamDbDoc.getElementById("dlc"); - Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app"); - for (Element dlc : steamDbDLCElements) { - String dlc_id = dlc.attr("data-appid"); - String dlc_name = "Unknown DLC " + dlc_id; - Elements td = dlc.getElementsByTag("td"); - if (!td.isEmpty()) { - dlc_name = td.get(1).text().replace("\n", "").trim(); + + logger.info("Trying to get DLC from SteamDB..."); + if (cfduid.equals("")) { + logger.info("Missing CF cookie, trying to get it..."); + WebClient client = new WebClient(BrowserVersion.CHROME); + + client.getOptions().setCssEnabled(false); + client.getOptions().setJavaScriptEnabled(false); + client.getOptions().setThrowExceptionOnFailingStatusCode(false); + client.getOptions().setRedirectEnabled(true); + client.getCache().setMaxSize(0); + /*client.waitForBackgroundJavaScript(10000); + client.setJavaScriptTimeout(10000); + client.waitForBackgroundJavaScriptStartingBefore(10000);*/ + + try { + + String urlString = "https://steamdb.info/"; + + HtmlPage page = client.getPage(urlString); + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (page) { + page.wait(8000); } - steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name); + //Print cookies for test purposes. Comment out in production. + URL url = new URL(urlString); + for (Cookie c : client.getCookies(url)) { + //System.out.println(c.getName() +"="+c.getValue()); + if (c.getName().equals("__cfduid")) { + cfduid = c.getValue(); + logger.info("Got CF cookie successfully."); + } + } + + //This prints the content after bypassing Cloudflare. + // System.out.println(client.getPage(url).getWebResponse().getContentAsString()); + } catch (FailingHttpStatusCodeException | IOException | InterruptedException e) { + // e.printStackTrace(); + logger.error("Could not get CF cookie, skipping SteamDB DLC list..."); } - } catch (HttpStatusException e) { - logger.error(e.getUrl()); - if (e.getStatusCode() == 404) { - logger.error("App ID empty or not found! (HTTP Status Code: 404)"); - } else { - /*System.err.printf("Error occurred while trying to get list of DLCs " + - "(HTTP Status Code: %d)%n", e.getStatusCode());*/ - logger.error("Error occurred while trying to get list of DLCs " + - "(HTTP Status Code: " + e.getStatusCode() + ")"); + } + + if ((!(cfduid.equals(""))) && (!(cfduid.equals("N/A")))) { + try { + Document steamDbDoc = Jsoup + .connect("https://steamdb.info/app/" + appId + "/dlc/") + .cookie("__cfduid", cfduid) + .userAgent(BrowserVersion.CHROME.getUserAgent()) + .get(); + Element steamDbDlcSection = steamDbDoc.getElementById("dlc"); + Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app"); + for (Element dlc : steamDbDLCElements) { + String dlc_id = dlc.attr("data-appid"); + String dlc_name = "Unknown DLC " + dlc_id; + Elements td = dlc.getElementsByTag("td"); + if (!td.isEmpty()) { + dlc_name = td.get(1).text().replace("\n", "").trim(); + } + steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name); + } + logger.info("Got DLC from SteamDB successfully."); + } catch (HttpStatusException e) { + logger.error(e.getUrl()); + if (e.getStatusCode() == 404) { + logger.error("App ID empty or not found! (HTTP Status Code: 404)"); + } else { + logger.error("Error occurred while trying to get list of DLCs " + + "(HTTP Status Code: " + e.getStatusCode() + ")"); + } + } catch (NullPointerException | IOException e) { + // ignore } - } catch (NullPointerException | IOException e) { - // ignore } } + logger.info("Merging both DLC lists..."); Map allDLCs = new HashMap<>(steamStoreDLCs); steamDbDLCs.forEach(allDLCs::putIfAbsent); + logger.info("DLC list is done!"); return allDLCs.entrySet().stream() .sorted(Map.Entry.comparingByKey()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,