diff --git a/auto-cream-api.iml b/auto-cream-api.iml
index 9654196..3730131 100644
--- a/auto-cream-api.iml
+++ b/auto-cream-api.iml
@@ -56,7 +56,7 @@
-
+
@@ -79,5 +79,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index bc682b0..8aefdfd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,7 +21,7 @@
xyz.jeddunk
auto-cream-api
- 1.2.1
+ 1.2.2
UTF-8
@@ -150,7 +150,7 @@
false
bintray-jerady-maven
- bintray
+ bintray-jerady
https://dl.bintray.com/jerady/maven
@@ -160,7 +160,7 @@
false
bintray-jerady-maven
- bintray-plugins
+ bintray-jerady-plugins
https://dl.bintray.com/jerady/maven
@@ -229,5 +229,10 @@
jansi
1.18
+
+ net.sourceforge.htmlunit
+ htmlunit
+ 2.44.0
+
\ No newline at end of file
diff --git a/src/main/java/xyz/jeddunk/autocreamapi/Controller.java b/src/main/java/xyz/jeddunk/autocreamapi/Controller.java
index 4257bc1..baa5cb4 100644
--- a/src/main/java/xyz/jeddunk/autocreamapi/Controller.java
+++ b/src/main/java/xyz/jeddunk/autocreamapi/Controller.java
@@ -132,10 +132,7 @@ public class Controller implements Initializable {
@FXML
@Override
public void initialize(URL location, ResourceBundle resources) {
- // steamdb_dlc_checkbox.setSelected(true);
- // Disable SteamDB DLC checking for now until it's fix
- steamdb_dlc_checkbox.setSelected(false);
- steamdb_dlc_checkbox.setDisable(true);
+ steamdb_dlc_checkbox.setSelected(true);
appId_textfield.textProperty().addListener(appIdChangesGameName());
game_name_textfield.setOnKeyReleased(event -> {
KeyCode code = event.getCode();
@@ -147,7 +144,6 @@ public class Controller implements Initializable {
language_combobox.setTooltip(new Tooltip());
language_combobox.setEditable(false);
new ComboBoxAutoComplete<>(language_combobox);
- //retrieveDlcList_button.setDisable(true); // WIP
generate_tooltips();
fix_dlc_textarea_prompt_text();
reset(true);
@@ -186,10 +182,9 @@ public class Controller implements Initializable {
extra_protection_checkbox.setTooltip(extra_protection_tooltip);
Tooltip.install(extra_protection_checkbox, extra_protection_tooltip);
- /*Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
+ Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
"Especially useful for DLC that is not listed on the Steam Store.\n" +
- "This sometimes doesn't work because SteamDB may block access for scrapers.");*/
- Tooltip steamdb_dlc_tooltip = new Tooltip("Currently broken.");
+ "This sometimes doesn't work because SteamDB may block access for scrapers.");
steamdb_dlc_checkbox.setTooltip(steamdb_dlc_tooltip);
Tooltip.install(steamdb_dlc_checkbox, steamdb_dlc_tooltip);
}
diff --git a/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java b/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java
index 940a545..fc6f441 100644
--- a/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java
+++ b/src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java
@@ -31,10 +31,8 @@ public class CreamApiDllHandler {
private final String steamApi64DllMd5;
private CreamApiDllHandler() throws IOException {
- String steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
- String steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
- this.steamApiDllMd5 = steamApiDllMd5;
- this.steamApi64DllMd5 = steamApi64DllMd5;
+ this.steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
+ this.steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
}
public static synchronized CreamApiDllHandler getInstance() throws IOException {
diff --git a/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java b/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java
index b1f7655..03f205d 100644
--- a/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java
+++ b/src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java
@@ -15,6 +15,11 @@
package xyz.jeddunk.autocreamapi.util;
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import com.gargoylesoftware.htmlunit.util.Cookie;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
@@ -35,6 +40,7 @@ import xyz.jeddunk.autocreamapi.util.env.MainEnv;
import java.io.*;
import java.lang.reflect.Type;
+import java.net.URL;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
@@ -47,7 +53,7 @@ public class SteamAppsListCache {
private SteamAppsList list = new SteamAppsList();
private final File cacheFile = new File("steamapps.json");
private MainEnv env;
- //private String key;
+ private String cfduid = "";
public SteamAppsListCache() {
try {
@@ -151,12 +157,13 @@ public class SteamAppsListCache {
}
public Map getDlcMap(String appId, boolean use_steamdb) {
+ logger.info("Getting list of DLC for AppID: " + appId + "...");
Map steamStoreDLCs = new HashMap<>();
Map steamDbDLCs = new HashMap<>();
- //StringBuilder sb = new StringBuilder();
// Steam Store
try {
+ logger.info("Trying to get DLC from Steam store page...");
Document steamDoc = Jsoup
.connect("https://store.steampowered.com/app/" + appId + "/")
.cookie("birthtime", "470703601")
@@ -170,13 +177,12 @@ public class SteamAppsListCache {
.text().replace("\n", "").trim();
steamStoreDLCs.put(Integer.parseInt(dlc_id), dlc_name);
}
+ logger.info("Got DLC from Steam store page successfully.");
} catch (HttpStatusException e) {
logger.error(e.getUrl());
if (e.getStatusCode() == 404) {
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
} else {
- /*System.err.printf("Error occurred while trying to get list of DLCs " +
- "(HTTP Status Code: %d)%n", e.getStatusCode());*/
logger.error("Error occurred while trying to get list of DLCs " +
"(HTTP Status Code: " + e.getStatusCode() + ")");
}
@@ -186,39 +192,85 @@ public class SteamAppsListCache {
// SteamDB
if (use_steamdb) {
- try {
- Document steamDbDoc = Jsoup
- .connect("https://steamdb.info/app/" + appId + "/dlc/")
- .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0")
- .get();
- Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
- Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
- for (Element dlc : steamDbDLCElements) {
- String dlc_id = dlc.attr("data-appid");
- String dlc_name = "Unknown DLC " + dlc_id;
- Elements td = dlc.getElementsByTag("td");
- if (!td.isEmpty()) {
- dlc_name = td.get(1).text().replace("\n", "").trim();
+
+ logger.info("Trying to get DLC from SteamDB...");
+ if (cfduid.equals("")) {
+ logger.info("Missing CF cookie, trying to get it...");
+ WebClient client = new WebClient(BrowserVersion.CHROME);
+
+ client.getOptions().setCssEnabled(false);
+ client.getOptions().setJavaScriptEnabled(false);
+ client.getOptions().setThrowExceptionOnFailingStatusCode(false);
+ client.getOptions().setRedirectEnabled(true);
+ client.getCache().setMaxSize(0);
+ /*client.waitForBackgroundJavaScript(10000);
+ client.setJavaScriptTimeout(10000);
+ client.waitForBackgroundJavaScriptStartingBefore(10000);*/
+
+ try {
+
+ String urlString = "https://steamdb.info/";
+
+ HtmlPage page = client.getPage(urlString);
+ //noinspection SynchronizationOnLocalVariableOrMethodParameter
+ synchronized (page) {
+ page.wait(8000);
}
- steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
+ //Print cookies for test purposes. Comment out in production.
+ URL url = new URL(urlString);
+ for (Cookie c : client.getCookies(url)) {
+ //System.out.println(c.getName() +"="+c.getValue());
+ if (c.getName().equals("__cfduid")) {
+ cfduid = c.getValue();
+ logger.info("Got CF cookie successfully.");
+ }
+ }
+
+ //This prints the content after bypassing Cloudflare.
+ // System.out.println(client.getPage(url).getWebResponse().getContentAsString());
+ } catch (FailingHttpStatusCodeException | IOException | InterruptedException e) {
+ // e.printStackTrace();
+ logger.error("Could not get CF cookie, skipping SteamDB DLC list...");
}
- } catch (HttpStatusException e) {
- logger.error(e.getUrl());
- if (e.getStatusCode() == 404) {
- logger.error("App ID empty or not found! (HTTP Status Code: 404)");
- } else {
- /*System.err.printf("Error occurred while trying to get list of DLCs " +
- "(HTTP Status Code: %d)%n", e.getStatusCode());*/
- logger.error("Error occurred while trying to get list of DLCs " +
- "(HTTP Status Code: " + e.getStatusCode() + ")");
+ }
+
+ if ((!(cfduid.equals(""))) && (!(cfduid.equals("N/A")))) {
+ try {
+ Document steamDbDoc = Jsoup
+ .connect("https://steamdb.info/app/" + appId + "/dlc/")
+ .cookie("__cfduid", cfduid)
+ .userAgent(BrowserVersion.CHROME.getUserAgent())
+ .get();
+ Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
+ Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
+ for (Element dlc : steamDbDLCElements) {
+ String dlc_id = dlc.attr("data-appid");
+ String dlc_name = "Unknown DLC " + dlc_id;
+ Elements td = dlc.getElementsByTag("td");
+ if (!td.isEmpty()) {
+ dlc_name = td.get(1).text().replace("\n", "").trim();
+ }
+ steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
+ }
+ logger.info("Got DLC from SteamDB successfully.");
+ } catch (HttpStatusException e) {
+ logger.error(e.getUrl());
+ if (e.getStatusCode() == 404) {
+ logger.error("App ID empty or not found! (HTTP Status Code: 404)");
+ } else {
+ logger.error("Error occurred while trying to get list of DLCs " +
+ "(HTTP Status Code: " + e.getStatusCode() + ")");
+ }
+ } catch (NullPointerException | IOException e) {
+ // ignore
}
- } catch (NullPointerException | IOException e) {
- // ignore
}
}
+ logger.info("Merging both DLC lists...");
Map allDLCs = new HashMap<>(steamStoreDLCs);
steamDbDLCs.forEach(allDLCs::putIfAbsent);
+ logger.info("DLC list is done!");
return allDLCs.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,