Fixed and enabled SteamDB DLC functionality
This commit is contained in:
parent
26872ec8fb
commit
992f2fb20c
@ -56,7 +56,7 @@
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-lightweight:1.0-beta-2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.maven.wagon:wagon-http-shared:1.0-beta-2" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: jtidy:jtidy:4aug2000r7-dev" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
|
||||
<orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.maven.reporting:maven-reporting-api:2.0.9" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.maven.doxia:doxia-sink-api:1.0-alpha-10" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.maven.wagon:wagon-provider-api:1.0-beta-2" level="project" />
|
||||
@ -79,5 +79,24 @@
|
||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.fusesource.jansi:jansi:1.18" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit:2.44.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: xalan:xalan:2.7.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: xalan:serializer:2.7.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-core-js:2.44.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:neko-htmlunit:2.44.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: xerces:xercesImpl:2.12.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sourceforge.htmlunit:htmlunit-cssparser:1.6.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-io:commons-io:2.8.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-net:commons-net:3.7.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.brotli:dec:0.1.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.shapesecurity:salvation2:3.0.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-client:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-client:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-http:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-xml:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-util:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-io:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-common:9.4.32.v20200930" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty.websocket:websocket-api:9.4.32.v20200930" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
11
pom.xml
11
pom.xml
@ -21,7 +21,7 @@
|
||||
|
||||
<groupId>xyz.jeddunk</groupId>
|
||||
<artifactId>auto-cream-api</artifactId>
|
||||
<version>1.2.1</version>
|
||||
<version>1.2.2</version>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
@ -150,7 +150,7 @@
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<id>bintray-jerady-maven</id>
|
||||
<name>bintray</name>
|
||||
<name>bintray-jerady</name>
|
||||
<url>https://dl.bintray.com/jerady/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
@ -160,7 +160,7 @@
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
<id>bintray-jerady-maven</id>
|
||||
<name>bintray-plugins</name>
|
||||
<name>bintray-jerady-plugins</name>
|
||||
<url>https://dl.bintray.com/jerady/maven</url>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
@ -229,5 +229,10 @@
|
||||
<artifactId>jansi</artifactId>
|
||||
<version>1.18</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.htmlunit</groupId>
|
||||
<artifactId>htmlunit</artifactId>
|
||||
<version>2.44.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@ -132,10 +132,7 @@ public class Controller implements Initializable {
|
||||
@FXML
|
||||
@Override
|
||||
public void initialize(URL location, ResourceBundle resources) {
|
||||
// steamdb_dlc_checkbox.setSelected(true);
|
||||
// Disable SteamDB DLC checking for now until it's fix
|
||||
steamdb_dlc_checkbox.setSelected(false);
|
||||
steamdb_dlc_checkbox.setDisable(true);
|
||||
steamdb_dlc_checkbox.setSelected(true);
|
||||
appId_textfield.textProperty().addListener(appIdChangesGameName());
|
||||
game_name_textfield.setOnKeyReleased(event -> {
|
||||
KeyCode code = event.getCode();
|
||||
@ -147,7 +144,6 @@ public class Controller implements Initializable {
|
||||
language_combobox.setTooltip(new Tooltip());
|
||||
language_combobox.setEditable(false);
|
||||
new ComboBoxAutoComplete<>(language_combobox);
|
||||
//retrieveDlcList_button.setDisable(true); // WIP
|
||||
generate_tooltips();
|
||||
fix_dlc_textarea_prompt_text();
|
||||
reset(true);
|
||||
@ -186,10 +182,9 @@ public class Controller implements Initializable {
|
||||
extra_protection_checkbox.setTooltip(extra_protection_tooltip);
|
||||
Tooltip.install(extra_protection_checkbox, extra_protection_tooltip);
|
||||
|
||||
/*Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
|
||||
Tooltip steamdb_dlc_tooltip = new Tooltip("Additionally get DLC data from SteamDB.\n" +
|
||||
"Especially useful for DLC that is not listed on the Steam Store.\n" +
|
||||
"This sometimes doesn't work because SteamDB may block access for scrapers.");*/
|
||||
Tooltip steamdb_dlc_tooltip = new Tooltip("Currently broken.");
|
||||
"This sometimes doesn't work because SteamDB may block access for scrapers.");
|
||||
steamdb_dlc_checkbox.setTooltip(steamdb_dlc_tooltip);
|
||||
Tooltip.install(steamdb_dlc_checkbox, steamdb_dlc_tooltip);
|
||||
}
|
||||
|
||||
@ -31,10 +31,8 @@ public class CreamApiDllHandler {
|
||||
private final String steamApi64DllMd5;
|
||||
|
||||
private CreamApiDllHandler() throws IOException {
|
||||
String steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
|
||||
String steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
|
||||
this.steamApiDllMd5 = steamApiDllMd5;
|
||||
this.steamApi64DllMd5 = steamApi64DllMd5;
|
||||
this.steamApiDllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApiDllPath));
|
||||
this.steamApi64DllMd5 = DigestUtils.md5Hex(Files.newInputStream(steamApi64DllPath));
|
||||
}
|
||||
|
||||
public static synchronized CreamApiDllHandler getInstance() throws IOException {
|
||||
|
||||
@ -15,6 +15,11 @@
|
||||
|
||||
package xyz.jeddunk.autocreamapi.util;
|
||||
|
||||
import com.gargoylesoftware.htmlunit.BrowserVersion;
|
||||
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
|
||||
import com.gargoylesoftware.htmlunit.WebClient;
|
||||
import com.gargoylesoftware.htmlunit.html.HtmlPage;
|
||||
import com.gargoylesoftware.htmlunit.util.Cookie;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonSyntaxException;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
@ -35,6 +40,7 @@ import xyz.jeddunk.autocreamapi.util.env.MainEnv;
|
||||
|
||||
import java.io.*;
|
||||
import java.lang.reflect.Type;
|
||||
import java.net.URL;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
@ -47,7 +53,7 @@ public class SteamAppsListCache {
|
||||
private SteamAppsList list = new SteamAppsList();
|
||||
private final File cacheFile = new File("steamapps.json");
|
||||
private MainEnv env;
|
||||
//private String key;
|
||||
private String cfduid = "";
|
||||
|
||||
public SteamAppsListCache() {
|
||||
try {
|
||||
@ -151,12 +157,13 @@ public class SteamAppsListCache {
|
||||
}
|
||||
|
||||
public Map<Integer, String> getDlcMap(String appId, boolean use_steamdb) {
|
||||
logger.info("Getting list of DLC for AppID: " + appId + "...");
|
||||
Map<Integer, String> steamStoreDLCs = new HashMap<>();
|
||||
Map<Integer, String> steamDbDLCs = new HashMap<>();
|
||||
//StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Steam Store
|
||||
try {
|
||||
logger.info("Trying to get DLC from Steam store page...");
|
||||
Document steamDoc = Jsoup
|
||||
.connect("https://store.steampowered.com/app/" + appId + "/")
|
||||
.cookie("birthtime", "470703601")
|
||||
@ -170,13 +177,12 @@ public class SteamAppsListCache {
|
||||
.text().replace("\n", "").trim();
|
||||
steamStoreDLCs.put(Integer.parseInt(dlc_id), dlc_name);
|
||||
}
|
||||
logger.info("Got DLC from Steam store page successfully.");
|
||||
} catch (HttpStatusException e) {
|
||||
logger.error(e.getUrl());
|
||||
if (e.getStatusCode() == 404) {
|
||||
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
|
||||
} else {
|
||||
/*System.err.printf("Error occurred while trying to get list of DLCs " +
|
||||
"(HTTP Status Code: %d)%n", e.getStatusCode());*/
|
||||
logger.error("Error occurred while trying to get list of DLCs " +
|
||||
"(HTTP Status Code: " + e.getStatusCode() + ")");
|
||||
}
|
||||
@ -186,39 +192,85 @@ public class SteamAppsListCache {
|
||||
|
||||
// SteamDB
|
||||
if (use_steamdb) {
|
||||
try {
|
||||
Document steamDbDoc = Jsoup
|
||||
.connect("https://steamdb.info/app/" + appId + "/dlc/")
|
||||
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0")
|
||||
.get();
|
||||
Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
|
||||
Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
|
||||
for (Element dlc : steamDbDLCElements) {
|
||||
String dlc_id = dlc.attr("data-appid");
|
||||
String dlc_name = "Unknown DLC " + dlc_id;
|
||||
Elements td = dlc.getElementsByTag("td");
|
||||
if (!td.isEmpty()) {
|
||||
dlc_name = td.get(1).text().replace("\n", "").trim();
|
||||
|
||||
logger.info("Trying to get DLC from SteamDB...");
|
||||
if (cfduid.equals("")) {
|
||||
logger.info("Missing CF cookie, trying to get it...");
|
||||
WebClient client = new WebClient(BrowserVersion.CHROME);
|
||||
|
||||
client.getOptions().setCssEnabled(false);
|
||||
client.getOptions().setJavaScriptEnabled(false);
|
||||
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
|
||||
client.getOptions().setRedirectEnabled(true);
|
||||
client.getCache().setMaxSize(0);
|
||||
/*client.waitForBackgroundJavaScript(10000);
|
||||
client.setJavaScriptTimeout(10000);
|
||||
client.waitForBackgroundJavaScriptStartingBefore(10000);*/
|
||||
|
||||
try {
|
||||
|
||||
String urlString = "https://steamdb.info/";
|
||||
|
||||
HtmlPage page = client.getPage(urlString);
|
||||
//noinspection SynchronizationOnLocalVariableOrMethodParameter
|
||||
synchronized (page) {
|
||||
page.wait(8000);
|
||||
}
|
||||
steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
|
||||
//Print cookies for test purposes. Comment out in production.
|
||||
URL url = new URL(urlString);
|
||||
for (Cookie c : client.getCookies(url)) {
|
||||
//System.out.println(c.getName() +"="+c.getValue());
|
||||
if (c.getName().equals("__cfduid")) {
|
||||
cfduid = c.getValue();
|
||||
logger.info("Got CF cookie successfully.");
|
||||
}
|
||||
}
|
||||
|
||||
//This prints the content after bypassing Cloudflare.
|
||||
// System.out.println(client.getPage(url).getWebResponse().getContentAsString());
|
||||
} catch (FailingHttpStatusCodeException | IOException | InterruptedException e) {
|
||||
// e.printStackTrace();
|
||||
logger.error("Could not get CF cookie, skipping SteamDB DLC list...");
|
||||
}
|
||||
} catch (HttpStatusException e) {
|
||||
logger.error(e.getUrl());
|
||||
if (e.getStatusCode() == 404) {
|
||||
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
|
||||
} else {
|
||||
/*System.err.printf("Error occurred while trying to get list of DLCs " +
|
||||
"(HTTP Status Code: %d)%n", e.getStatusCode());*/
|
||||
logger.error("Error occurred while trying to get list of DLCs " +
|
||||
"(HTTP Status Code: " + e.getStatusCode() + ")");
|
||||
}
|
||||
|
||||
if ((!(cfduid.equals(""))) && (!(cfduid.equals("N/A")))) {
|
||||
try {
|
||||
Document steamDbDoc = Jsoup
|
||||
.connect("https://steamdb.info/app/" + appId + "/dlc/")
|
||||
.cookie("__cfduid", cfduid)
|
||||
.userAgent(BrowserVersion.CHROME.getUserAgent())
|
||||
.get();
|
||||
Element steamDbDlcSection = steamDbDoc.getElementById("dlc");
|
||||
Elements steamDbDLCElements = steamDbDlcSection.getElementsByClass("app");
|
||||
for (Element dlc : steamDbDLCElements) {
|
||||
String dlc_id = dlc.attr("data-appid");
|
||||
String dlc_name = "Unknown DLC " + dlc_id;
|
||||
Elements td = dlc.getElementsByTag("td");
|
||||
if (!td.isEmpty()) {
|
||||
dlc_name = td.get(1).text().replace("\n", "").trim();
|
||||
}
|
||||
steamDbDLCs.put(Integer.parseInt(dlc_id), dlc_name);
|
||||
}
|
||||
logger.info("Got DLC from SteamDB successfully.");
|
||||
} catch (HttpStatusException e) {
|
||||
logger.error(e.getUrl());
|
||||
if (e.getStatusCode() == 404) {
|
||||
logger.error("App ID empty or not found! (HTTP Status Code: 404)");
|
||||
} else {
|
||||
logger.error("Error occurred while trying to get list of DLCs " +
|
||||
"(HTTP Status Code: " + e.getStatusCode() + ")");
|
||||
}
|
||||
} catch (NullPointerException | IOException e) {
|
||||
// ignore
|
||||
}
|
||||
} catch (NullPointerException | IOException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("Merging both DLC lists...");
|
||||
Map<Integer, String> allDLCs = new HashMap<>(steamStoreDLCs);
|
||||
steamDbDLCs.forEach(allDLCs::putIfAbsent);
|
||||
logger.info("DLC list is done!");
|
||||
return allDLCs.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByKey())
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user