jeddunk/auto-creamapi
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Fixed and enabled SteamDB DLC functionality

Browse Source
This commit is contained in:
Jeddunk 2020-10-21 15:47:58 +02:00
parent 26872ec8fb
commit 992f2fb20c
5 changed files with 114 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
auto-cream-api.iml
View File

@ -56,7 +56,7 @@
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-lightweight:1.0-beta-2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-shared:1.0-beta-2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: jtidy:jtidy:4aug2000r7-dev" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
<orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
<orderEntry type="library" name="Maven: org.apache.maven.reporting:maven-reporting-api:2.0.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.maven.doxia:doxia-sink-api:1.0-alpha-10" level="project" />
<orderEntry type="library" name="Maven: org.apache.maven.wagon:wagon-provider-api:1.0-beta-2" level="project" />
@ -79,5 +79,24 @@
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
<orderEntry type="library" name="Maven: org.fusesource.jansi:jansi:1.18" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit:2.44.0" level="project" />
<orderEntry type="library" name="Maven: xalan:xalan:2.7.2" level="project" />
<orderEntry type="library" name="Maven: xalan:serializer:2.7.2" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-core-js:2.44.0" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:neko-htmlunit:2.44.0" level="project" />
<orderEntry type="library" name="Maven: xerces:xercesImpl:2.12.0" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-cssparser:1.6.0" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.8.0" level="project" />
<orderEntry type="library" name="Maven: commons-net:commons-net:3.7.1" level="project" />
<orderEntry type="library" name="Maven: org.brotli:dec:0.1.2" level="project" />
<orderEntry type="library" name="Maven: com.shapesecurity:salvation2:3.0.0" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-client:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-client:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-http:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-xml:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-util:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-io:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-common:9.4.32.v20200930" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-api:9.4.32.v20200930" level="project" />
</component>
</module>

11
pom.xml
View File

@ -21,7 +21,7 @@
<groupId>xyz.jeddunk</groupId>
<artifactId>auto-cream-api</artifactId>
<version>1.2.1</version>
<version>1.2.2</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@ -150,7 +150,7 @@
<enabled>false</enabled>
</snapshots>
<id>bintray-jerady-maven</id>
<name>bintray</name>
<name>bintray-jerady</name>
<url>https://dl.bintray.com/jerady/maven</url>
</repository>
</repositories>
@ -160,7 +160,7 @@
<enabled>false</enabled>
</snapshots>
<id>bintray-jerady-maven</id>
<name>bintray-plugins</name>
<name>bintray-jerady-plugins</name>
<url>https://dl.bintray.com/jerady/maven</url>
</pluginRepository>
</pluginRepositories>
@ -229,5 +229,10 @@
<artifactId>jansi</artifactId>
<version>1.18</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>2.44.0</version>
</dependency>
</dependencies>
</project>

11
src/main/java/xyz/jeddunk/autocreamapi/Controller.java
View File

@ -132,10 +132,7 @@ public class Controller implements Initializable {
@FXML
@Override
public void initialize(URL location, ResourceBundle resources) {
// steamdb_dlc_checkbox.setSelected(true);
// Disable SteamDB DLC checking for now until it's fix
steamdb_dlc_checkbox.setSelected(false);
steamdb_dlc_checkbox.setDisable(true);
steamdb_dlc_checkbox.setSelected(true);
appId_textfield.textProperty().addListener(appIdChangesGameName());
game_name_textfield.setOnKeyReleased(event -> {
KeyCode code = event.getCode();
@ -147,7 +144,6 @@ public class Controller implements Initializable {
language_combobox.setTooltip(new Tooltip());
language_combobox.setEditable(false);
new ComboBoxAutoComplete<>(language_combobox);
//retrieveDlcList_button.setDisable(true); // WIP
generate_tooltips();
fix_dlc_textarea_prompt_text();
reset(true);
@ -186,10 +182,9 @@ public class Controller implements Initializable {
extra_protection_checkbox.setTooltip(extra_protection_tooltip);
Tooltip.install(extra_protection_checkbox, extra_protection_tooltip);
/*Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
"Especially useful for DLC that is not listed on the Steam Store.\n" +
"This sometimes doesn't work because SteamDB may block access for scrapers.");*/
Tooltip steamdb_dlc_tooltip = new Tooltip("Currently broken.");
"This sometimes doesn't work because SteamDB may block access for scrapers.");
steamdb_dlc_checkbox.setTooltip(steamdb_dlc_tooltip);
Tooltip.install(steamdb_dlc_checkbox, steamdb_dlc_tooltip);
}

6
src/main/java/xyz/jeddunk/autocreamapi/util/CreamApiDllHandler.java
View File

@ -31,10 +31,8 @@ public class CreamApiDllHandler {
private final String steamApi64DllMd5;
private CreamApiDllHandler() throws IOException {
String steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
String steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
this.steamApiDllMd5 = steamApiDllMd5;
this.steamApi64DllMd5 = steamApi64DllMd5;
this.steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
this.steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
}
public static synchronized CreamApiDllHandler getInstance() throws IOException {

66
src/main/java/xyz/jeddunk/autocreamapi/util/SteamAppsListCache.java
View File

@ -15,6 +15,11 @@
package xyz.jeddunk.autocreamapi.util;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.Cookie;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
@ -35,6 +40,7 @@ import xyz.jeddunk.autocreamapi.util.env.MainEnv;
import java.io.*;
import java.lang.reflect.Type;
import java.net.URL;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
@ -47,7 +53,7 @@ public class SteamAppsListCache {
private SteamAppsList list = new SteamAppsList();
private final File cacheFile = new File("steamapps.json");
private MainEnv env;
//private String key;
private String cfduid = "";
public SteamAppsListCache() {
try {
@ -151,12 +157,13 @@ public class SteamAppsListCache {
}
public Map<Integer, String> getDlcMap(String appId, boolean use_steamdb) {
logger.info("Getting list of DLC for AppID: " + appId + "...");
Map<Integer, String> steamStoreDLCs = new HashMap<>();
Map<Integer, String> steamDbDLCs = new HashMap<>();
//StringBuilder sb = new StringBuilder();
// Steam Store
try {
logger.info("Trying to get DLC from Steam store page...");
Document steamDoc = Jsoup
.connect("https://store.steampowered.com/app/" + appId + "/")
.cookie("birthtime", "470703601")
@ -170,13 +177,12 @@ public class SteamAppsListCache {
.text().replace("\n", "").trim();
steamStoreDLCs.put(Integer.parseInt(dlc_id), dlc_name);
}
logger.info("Got DLC from Steam store page successfully.");
} catch (HttpStatusException e) {
logger.error(e.getUrl());
if (e.getStatusCode() == 404) {
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
} else {
/*System.err.printf("Error occurred while trying to get list of DLCs " +
"(HTTP Status Code: %d)%n", e.getStatusCode());*/
logger.error("Error occurred while trying to get list of DLCs " +
"(HTTP Status Code: " + e.getStatusCode() + ")");
}
@ -186,10 +192,54 @@ public class SteamAppsListCache {
// SteamDB
if (use_steamdb) {
logger.info("Trying to get DLC from SteamDB...");
if (cfduid.equals("")) {
logger.info("Missing CF cookie, trying to get it...");
WebClient client = new WebClient(BrowserVersion.CHROME);
client.getOptions().setCssEnabled(false);
client.getOptions().setJavaScriptEnabled(false);
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
client.getOptions().setRedirectEnabled(true);
client.getCache().setMaxSize(0);
/*client.waitForBackgroundJavaScript(10000);
client.setJavaScriptTimeout(10000);
client.waitForBackgroundJavaScriptStartingBefore(10000);*/
try {
String urlString = "https://steamdb.info/";
HtmlPage page = client.getPage(urlString);
//noinspection SynchronizationOnLocalVariableOrMethodParameter
synchronized (page) {
page.wait(8000);
}
//Print cookies for test purposes. Comment out in production.
URL url = new URL(urlString);
for (Cookie c : client.getCookies(url)) {
//System.out.println(c.getName() +"="+c.getValue());
if (c.getName().equals("__cfduid")) {
cfduid = c.getValue();
logger.info("Got CF cookie successfully.");
}
}
//This prints the content after bypassing Cloudflare.
// System.out.println(client.getPage(url).getWebResponse().getContentAsString());
} catch (FailingHttpStatusCodeException | IOException | InterruptedException e) {
// e.printStackTrace();
logger.error("Could not get CF cookie, skipping SteamDB DLC list...");
}
}
if ((!(cfduid.equals(""))) && (!(cfduid.equals("N/A")))) {
try {
Document steamDbDoc = Jsoup
.connect("https://steamdb.info/app/" + appId + "/dlc/")
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0")
.cookie("__cfduid", cfduid)
.userAgent(BrowserVersion.CHROME.getUserAgent())
.get();
Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
@ -202,13 +252,12 @@ public class SteamAppsListCache {
}
steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
}
logger.info("Got DLC from SteamDB successfully.");
} catch (HttpStatusException e) {
logger.error(e.getUrl());
if (e.getStatusCode() == 404) {
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
} else {
/*System.err.printf("Error occurred while trying to get list of DLCs " +
"(HTTP Status Code: %d)%n", e.getStatusCode());*/
logger.error("Error occurred while trying to get list of DLCs " +
"(HTTP Status Code: " + e.getStatusCode() + ")");
}
@ -216,9 +265,12 @@ public class SteamAppsListCache {
// ignore
}
}
}
logger.info("Merging both DLC lists...");
Map<Integer, String> allDLCs = new HashMap<>(steamStoreDLCs);
steamDbDLCs.forEach(allDLCs::putIfAbsent);
logger.info("DLC list is done!");
return allDLCs.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,