Fixed and enabled SteamDB DLC functionality

auto-cream-api.iml

@@ -56,7 +56,7 @@
     <orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-lightweight:1.0-beta-2" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-shared:1.0-beta-2" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="Maven: jtidy:jtidy:4aug2000r7-dev" level="project" />
-    <orderEntry type="library" scope="RUNTIME" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
+    <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
     <orderEntry type="library" name="Maven: org.apache.maven.reporting:maven-reporting-api:2.0.9" level="project" />
     <orderEntry type="library" name="Maven: org.apache.maven.doxia:doxia-sink-api:1.0-alpha-10" level="project" />
     <orderEntry type="library" name="Maven: org.apache.maven.wagon:wagon-provider-api:1.0-beta-2" level="project" />
@@ -79,5 +79,24 @@
     <orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
     <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
     <orderEntry type="library" name="Maven: org.fusesource.jansi:jansi:1.18" level="project" />
+    <orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit:2.44.0" level="project" />
+    <orderEntry type="library" name="Maven: xalan:xalan:2.7.2" level="project" />
+    <orderEntry type="library" name="Maven: xalan:serializer:2.7.2" level="project" />
+    <orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-core-js:2.44.0" level="project" />
+    <orderEntry type="library" name="Maven: net.sourceforge.htmlunit:neko-htmlunit:2.44.0" level="project" />
+    <orderEntry type="library" name="Maven: xerces:xercesImpl:2.12.0" level="project" />
+    <orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-cssparser:1.6.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:2.8.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-net:commons-net:3.7.1" level="project" />
+    <orderEntry type="library" name="Maven: org.brotli:dec:0.1.2" level="project" />
+    <orderEntry type="library" name="Maven: com.shapesecurity:salvation2:3.0.0" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-client:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-client:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-http:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-xml:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-util:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-io:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-common:9.4.32.v20200930" level="project" />
+    <orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-api:9.4.32.v20200930" level="project" />
   </component>
 </module>

pom.xml

@@ -21,7 +21,7 @@
 
     <groupId>xyz.jeddunk</groupId>
     <artifactId>auto-cream-api</artifactId>
-    <version>1.2.1</version>
+    <version>1.2.2</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -150,7 +150,7 @@
                 <enabled>false</enabled>
             </snapshots>
             <id>bintray-jerady-maven</id>
-            <name>bintray</name>
+            <name>bintray-jerady</name>
             <url>https://dl.bintray.com/jerady/maven</url>
         </repository>
     </repositories>
@@ -160,7 +160,7 @@
                 <enabled>false</enabled>
             </snapshots>
             <id>bintray-jerady-maven</id>
-            <name>bintray-plugins</name>
+            <name>bintray-jerady-plugins</name>
             <url>https://dl.bintray.com/jerady/maven</url>
         </pluginRepository>
     </pluginRepositories>
@@ -229,5 +229,10 @@
             <artifactId>jansi</artifactId>
             <version>1.18</version>
         </dependency>
+        <dependency>
+            <groupId>net.sourceforge.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>2.44.0</version>
+        </dependency>
     </dependencies>
 </project>

src/main/java/xyz/jeddunk/autocreamapi/Controller.java

@@ -132,10 +132,7 @@ public class Controller implements Initializable {
     @FXML
     @Override
     public void initialize(URL location, ResourceBundle resources) {
-        // steamdb_dlc_checkbox.setSelected(true);
+        steamdb_dlc_checkbox.setSelected(true);
-        // Disable SteamDB DLC checking for now until it's fix
-        steamdb_dlc_checkbox.setSelected(false);
-        steamdb_dlc_checkbox.setDisable(true);
         appId_textfield.textProperty().addListener(appIdChangesGameName());
         game_name_textfield.setOnKeyReleased(event -> {
             KeyCode code = event.getCode();
@@ -147,7 +144,6 @@ public class Controller implements Initializable {
         language_combobox.setTooltip(new Tooltip());
         language_combobox.setEditable(false);
         new ComboBoxAutoComplete<>(language_combobox);
-        //retrieveDlcList_button.setDisable(true); // WIP
         generate_tooltips();
         fix_dlc_textarea_prompt_text();
         reset(true);
@@ -186,10 +182,9 @@ public class Controller implements Initializable {
         extra_protection_checkbox.setTooltip(extra_protection_tooltip);
         Tooltip.install(extra_protection_checkbox, extra_protection_tooltip);
 
-        /*Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
+        Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
                 "Especially useful for DLC that is not listed on the Steam Store.\n" +
-                "This sometimes doesn't work because SteamDB may block access for scrapers.");*/
+                "This sometimes doesn't work because SteamDB may block access for scrapers.");
-        Tooltip steamdb_dlc_tooltip = new Tooltip("Currently broken.");
         steamdb_dlc_checkbox.setTooltip(steamdb_dlc_tooltip);
         Tooltip.install(steamdb_dlc_checkbox, steamdb_dlc_tooltip);
     }

src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java

@@ -31,10 +31,8 @@ public class CreamApiDllHandler {
     private final String steamApi64DllMd5;
 
     private CreamApiDllHandler() throws IOException {
-        String steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
+        this.steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
-        String steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
+        this.steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
-        this.steamApiDllMd5 = steamApiDllMd5;
-        this.steamApi64DllMd5 = steamApi64DllMd5;
     }
 
     public static synchronized CreamApiDllHandler getInstance() throws IOException {

src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java

@@ -15,6 +15,11 @@
 
 package xyz.jeddunk.autocreamapi.util;
 
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import com.gargoylesoftware.htmlunit.util.Cookie;
 import com.google.gson.Gson;
 import com.google.gson.JsonSyntaxException;
 import com.google.gson.reflect.TypeToken;
@@ -35,6 +40,7 @@ import xyz.jeddunk.autocreamapi.util.env.MainEnv;
 
 import java.io.*;
 import java.lang.reflect.Type;
+import java.net.URL;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
@@ -47,7 +53,7 @@ public class SteamAppsListCache {
     private SteamAppsList list = new SteamAppsList();
     private final File cacheFile = new File("steamapps.json");
     private MainEnv env;
-    //private String key;
+    private String cfduid = "";
 
     public SteamAppsListCache() {
         try {
@@ -151,12 +157,13 @@ public class SteamAppsListCache {
     }
 
     public Map<Integer, String> getDlcMap(String appId, boolean use_steamdb) {
+        logger.info("Getting list of DLC for AppID: " + appId + "...");
         Map<Integer, String> steamStoreDLCs = new HashMap<>();
         Map<Integer, String> steamDbDLCs = new HashMap<>();
-        //StringBuilder sb = new StringBuilder();
 
         // Steam Store
         try {
+            logger.info("Trying to get DLC from Steam store page...");
             Document steamDoc = Jsoup
                     .connect("https://store.steampowered.com/app/" + appId + "/")
                     .cookie("birthtime", "470703601")
@@ -170,13 +177,12 @@ public class SteamAppsListCache {
                         .text().replace("\n", "").trim();
                 steamStoreDLCs.put(Integer.parseInt(dlc_id), dlc_name);
             }
+            logger.info("Got DLC from Steam store page successfully.");
         } catch (HttpStatusException e) {
             logger.error(e.getUrl());
             if (e.getStatusCode() == 404) {
                 logger.error("App ID empty or not found! (HTTP Status Code: 404)");
             } else {
-                /*System.err.printf("Error occurred while trying to get list of DLCs " +
-                        "(HTTP Status Code: %d)%n", e.getStatusCode());*/
                 logger.error("Error occurred while trying to get list of DLCs " +
                         "(HTTP Status Code: " + e.getStatusCode() + ")");
             }
@@ -186,39 +192,85 @@ public class SteamAppsListCache {
 
         // SteamDB
         if (use_steamdb) {
-            try {
+
-                Document steamDbDoc = Jsoup
+            logger.info("Trying to get DLC from SteamDB...");
-                        .connect("https://steamdb.info/app/" + appId + "/dlc/")
+            if (cfduid.equals("")) {
-                        .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0")
+                logger.info("Missing CF cookie, trying to get it...");
-                        .get();
+                WebClient client = new WebClient(BrowserVersion.CHROME);
-                Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
+
-                Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
+                client.getOptions().setCssEnabled(false);
-                for (Element dlc : steamDbDLCElements) {
+                client.getOptions().setJavaScriptEnabled(false);
-                    String dlc_id = dlc.attr("data-appid");
+                client.getOptions().setThrowExceptionOnFailingStatusCode(false);
-                    String dlc_name = "Unknown DLC " + dlc_id;
+                client.getOptions().setRedirectEnabled(true);
-                    Elements td = dlc.getElementsByTag("td");
+                client.getCache().setMaxSize(0);
-                    if (!td.isEmpty()) {
+                /*client.waitForBackgroundJavaScript(10000);
-                        dlc_name = td.get(1).text().replace("\n", "").trim();
+                client.setJavaScriptTimeout(10000);
+                client.waitForBackgroundJavaScriptStartingBefore(10000);*/
+
+                try {
+
+                    String urlString = "https://steamdb.info/";
+
+                    HtmlPage page = client.getPage(urlString);
+                    //noinspection SynchronizationOnLocalVariableOrMethodParameter
+                    synchronized (page) {
+                        page.wait(8000);
                     }
-                    steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
+                    //Print cookies for test purposes. Comment out in production.
+                    URL url = new URL(urlString);
+                    for (Cookie c : client.getCookies(url)) {
+                        //System.out.println(c.getName() +"="+c.getValue());
+                        if (c.getName().equals("__cfduid")) {
+                            cfduid = c.getValue();
+                            logger.info("Got CF cookie successfully.");
+                        }
+                    }
+
+                    //This prints the content after bypassing Cloudflare.
+                    // System.out.println(client.getPage(url).getWebResponse().getContentAsString());
+                } catch (FailingHttpStatusCodeException | IOException | InterruptedException e) {
+                    // e.printStackTrace();
+                    logger.error("Could not get CF cookie, skipping SteamDB DLC list...");
                 }
-            } catch (HttpStatusException e) {
+            }
-                logger.error(e.getUrl());
+
-                if (e.getStatusCode() == 404) {
+            if ((!(cfduid.equals(""))) && (!(cfduid.equals("N/A")))) {
-                    logger.error("App ID empty or not found! (HTTP Status Code: 404)");
+                try {
-                } else {
+                    Document steamDbDoc = Jsoup
-                    /*System.err.printf("Error occurred while trying to get list of DLCs " +
+                            .connect("https://steamdb.info/app/" + appId + "/dlc/")
-                            "(HTTP Status Code: %d)%n", e.getStatusCode());*/
+                            .cookie("__cfduid", cfduid)
-                    logger.error("Error occurred while trying to get list of DLCs " +
+                            .userAgent(BrowserVersion.CHROME.getUserAgent())
-                            "(HTTP Status Code: " + e.getStatusCode() + ")");
+                            .get();
+                    Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
+                    Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
+                    for (Element dlc : steamDbDLCElements) {
+                        String dlc_id = dlc.attr("data-appid");
+                        String dlc_name = "Unknown DLC " + dlc_id;
+                        Elements td = dlc.getElementsByTag("td");
+                        if (!td.isEmpty()) {
+                            dlc_name = td.get(1).text().replace("\n", "").trim();
+                        }
+                        steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
+                    }
+                    logger.info("Got DLC from SteamDB successfully.");
+                } catch (HttpStatusException e) {
+                    logger.error(e.getUrl());
+                    if (e.getStatusCode() == 404) {
+                        logger.error("App ID empty or not found! (HTTP Status Code: 404)");
+                    } else {
+                        logger.error("Error occurred while trying to get list of DLCs " +
+                                "(HTTP Status Code: " + e.getStatusCode() + ")");
+                    }
+                } catch (NullPointerException | IOException e) {
+                    // ignore
                 }
-            } catch (NullPointerException | IOException e) {
-                // ignore
             }
         }
 
+        logger.info("Merging both DLC lists...");
         Map<Integer, String> allDLCs = new HashMap<>(steamStoreDLCs);
         steamDbDLCs.forEach(allDLCs::putIfAbsent);
+        logger.info("DLC list is done!");
         return allDLCs.entrySet().stream()
                 .sorted(Map.Entry.comparingByKey())
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,