mirrors/PCUGameManager
mirrors
/
PCUGameManager
mirror of https://github.com/lantzelot-swe/PCUGameManager.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: more fixes for c64.com scraper

This commit is contained in:
lantzelot-swe 2021-01-13 23:29:31 +01:00
parent 22682e68ac
commit 900b4d4ab6
5 changed files with 68 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java
View File

@ -290,7 +290,6 @@ public class GameDetailsBackgroundPanel extends JPanel
MainWindow.getInstance().setWaitCursor(false);
if (scraperFields.isScreenshots())
{
List<BufferedImage> screenshots = scraperManager.getScreenshots();
if (screenshots.size() > 2)
{

32
src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java
View File

@ -38,7 +38,7 @@ public class C64comOptionsPanel extends JPanel
private JCheckBox titleCheckBox;
private JCheckBox authorCheckBox;
private JCheckBox yearCheckBox;
private JCheckBox descriptionCheckBox;
private JCheckBox gameCheckBox;
private JCheckBox coverCheckBox;
private JCheckBox screensCheckBox;
private ScraperManager scraper;
@ -194,12 +194,12 @@ public class C64comOptionsPanel extends JPanel
gbc_yearCheckBox.gridx = 0;
gbc_yearCheckBox.gridy = 3;
fieldsPanel.add(getYearCheckBox(), gbc_yearCheckBox);
GridBagConstraints gbc_descriptionCheckBox = new GridBagConstraints();
gbc_descriptionCheckBox.insets = new Insets(0, 0, 5, 5);
gbc_descriptionCheckBox.anchor = GridBagConstraints.WEST;
gbc_descriptionCheckBox.gridx = 1;
gbc_descriptionCheckBox.gridy = 1;
fieldsPanel.add(getDescriptionCheckBox(), gbc_descriptionCheckBox);
GridBagConstraints gbc_gameCheckBox = new GridBagConstraints();
gbc_gameCheckBox.insets = new Insets(0, 0, 5, 5);
gbc_gameCheckBox.anchor = GridBagConstraints.WEST;
gbc_gameCheckBox.gridx = 1;
gbc_gameCheckBox.gridy = 1;
fieldsPanel.add(getGameCheckBox(), gbc_gameCheckBox);
GridBagConstraints gbc_genreCheckBox = new GridBagConstraints();
gbc_genreCheckBox.insets = new Insets(0, 0, 0, 5);
gbc_genreCheckBox.anchor = GridBagConstraints.WEST;
@ -258,15 +258,15 @@ public class C64comOptionsPanel extends JPanel
return yearCheckBox;
}
private JCheckBox getDescriptionCheckBox()
private JCheckBox getGameCheckBox()
{
if (descriptionCheckBox == null)
if (gameCheckBox == null)
{
descriptionCheckBox = new JCheckBox("Description");
descriptionCheckBox.setSelected(false);
descriptionCheckBox.setEnabled(false);
gameCheckBox = new JCheckBox("Game file");
gameCheckBox.setSelected(true);
gameCheckBox.setEnabled(false);
}
return descriptionCheckBox;
return gameCheckBox;
}
private JCheckBox getCoverCheckBox()
@ -296,7 +296,7 @@ public class C64comOptionsPanel extends JPanel
titleCheckBox.setEnabled(enable);
authorCheckBox.setEnabled(enable);
yearCheckBox.setEnabled(enable);
// descriptionCheckBox.setEnabled(enable);
gameCheckBox.setEnabled(enable);
coverCheckBox.setEnabled(enable);
screensCheckBox.setEnabled(enable);
genreCheckBox.setEnabled(enable);
@ -310,12 +310,12 @@ public class C64comOptionsPanel extends JPanel
returnValue.setAuthor(authorCheckBox.isSelected());
returnValue.setYear(yearCheckBox.isSelected());
returnValue.setGenre(genreCheckBox.isSelected());
//No description available
returnValue.setDescription(false);
returnValue.setCover(coverCheckBox.isSelected());
returnValue.setScreenshots(screensCheckBox.isSelected());
returnValue.setComposer(composerCheckBox.isSelected());
//TODO:add option
returnValue.setGame(true);
returnValue.setGame(gameCheckBox.isSelected());
return returnValue;
}

22
src/main/java/se/lantz/gui/scraper/ScraperDialog.java
View File

@ -20,6 +20,10 @@ import se.lantz.model.data.ScraperFields;
public class ScraperDialog extends BaseDialog
{
private static final String MOBY = "moby";
private static final String C64COM = "c64com";
private static final String WWW_C64_COM = "www.c64.com";
private static final String WWW_MOBYGAMES_COM = "www.mobygames.com";
private MobyGamesOptionsPanel mobyGamesPanel;
private C64comOptionsPanel c64comPanel;
private final ScraperManager scraper;
@ -38,8 +42,8 @@ public class ScraperDialog extends BaseDialog
content.setLayout(new BorderLayout());
cardPanel = new JPanel();
cardPanel.setLayout(cardLayout);
cardPanel.add(getMobyGamesPanel(), "moby");
cardPanel.add(getC64comPanel(), "c64com");
cardPanel.add(getMobyGamesPanel(), MOBY);
cardPanel.add(getC64comPanel(), C64COM);
content.add(cardPanel, BorderLayout.CENTER);
addContent(content);
content.add(getScraperSelectionPanel(), BorderLayout.NORTH);
@ -66,7 +70,7 @@ public class ScraperDialog extends BaseDialog
public ScraperFields getScraperFields()
{
if (getScraperComboBox().getSelectedItem().equals("www.mobygames.com"))
if (getScraperComboBox().getSelectedItem().equals(WWW_MOBYGAMES_COM))
{
return getMobyGamesPanel().getScraperFields();
}
@ -98,6 +102,8 @@ public class ScraperDialog extends BaseDialog
gbc_scraperInfoLabel.gridy = 0;
scraperSelectionPanel.add(getScraperInfoLabel(), gbc_scraperInfoLabel);
GridBagConstraints gbc_scraperComboBox = new GridBagConstraints();
gbc_scraperComboBox.anchor = GridBagConstraints.WEST;
gbc_scraperComboBox.weightx = 1.0;
gbc_scraperComboBox.insets = new Insets(10, 0, 5, 5);
gbc_scraperComboBox.gridx = 1;
gbc_scraperComboBox.gridy = 0;
@ -124,22 +130,22 @@ public class ScraperDialog extends BaseDialog
{
public void actionPerformed(ActionEvent arg0)
{
if (scraperComboBox.getSelectedItem().equals("www.mobygames.com"))
if (scraperComboBox.getSelectedItem().equals(WWW_MOBYGAMES_COM))
{
cardLayout.show(cardPanel, "moby");
cardLayout.show(cardPanel, MOBY);
scraper.setScrapertoUse(SCRAPER.moby);
getRootPane().setDefaultButton(getMobyGamesPanel().getConnectButton());
}
else
{
cardLayout.show(cardPanel, "c64com");
cardLayout.show(cardPanel, C64COM);
scraper.setScrapertoUse(SCRAPER.c64com);
getRootPane().setDefaultButton(getC64comPanel().getConnectButton());
}
}
});
scraperComboBox.addItem("www.mobygames.com");
scraperComboBox.addItem("www.c64.com");
scraperComboBox.addItem(WWW_MOBYGAMES_COM);
scraperComboBox.addItem(WWW_C64_COM);
}
return scraperComboBox;
}

37
src/main/java/se/lantz/scraper/C64comScraper.java
View File

@ -64,7 +64,22 @@ public class C64comScraper implements Scraper
public void connect(String url) throws IOException
{
this.c64comGameUrl = "";
Jsoup.connect(url).method(Connection.Method.GET).execute();
//c64.com gives no errors for invalid urls. Check if there is an title to make sure it's valid
Connection.Response result = Jsoup.connect(url).method(Connection.Method.GET).execute();
Document doc = result.parse();
//Fetch right frame
Document mainFrameDocument = Jsoup.connect(doc.select("frame[name=text]").first().absUrl("src")).get();
//Fetch title
Elements queryElements = mainFrameDocument.select(titleCssQuery);
Element first = queryElements.first();
if (first != null)
{
if (first.text().isEmpty())
{
throw new IllegalArgumentException();
}
}
this.c64comGameUrl = url;
resetFields();
}
@ -172,32 +187,18 @@ public class C64comScraper implements Scraper
Response response = Jsoup.connect(url)
.header("Accept-Encoding", "gzip, deflate")
.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0")
// .referrer(URL_TO_PARSE)
.ignoreContentType(true)
.maxBodySize(0)
.timeout(600000)
.execute();
// .bodyAsBytes();
//TODO: add to temp folder instead
File file = new File("scrapedFile.zip");
BufferedInputStream inputStream = response.bodyStream();
FileOutputStream fos = new FileOutputStream(file);
byte[] buffer = new byte[1024];
int len;
while ((len = inputStream.read(buffer)) != -1) {
fos.write(buffer, 0, len);
}
inputStream.close();
fos.close();
//Unzip
scrapedFile = FileManager.unzipAndPickFirstEntry(file.getAbsolutePath(), ".");
//create a temp file and fetch the content
scrapedFile = FileManager.createTempFileForScraper(response.bodyStream());
logger.debug("File to include as game: {}", scrapedFile.getAbsolutePath());
}
catch (IOException e)
{
logger.error("", e);
logger.error("Could not scrape game file for " + scrapedTitle , e);
}
}
}

25
src/main/java/se/lantz/util/FileManager.java
View File

@ -3,6 +3,7 @@ package se.lantz.util;
import java.awt.Graphics;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -828,13 +829,25 @@ public class FileManager
return newImage;
}
public static File unzipAndPickFirstEntry(String zipFilePath, String destDir)
public static File createTempFileForScraper(BufferedInputStream inputStream) throws IOException
{
Files.createDirectories(TEMP_PATH);
//TODO: add to temp folder instead
File file = new File(TEMP_PATH + File.separator +"scrapedFile.zip");
FileOutputStream fos = new FileOutputStream(file, false);
byte[] buffer = new byte[1024];
int len;
while ((len = inputStream.read(buffer)) != -1) {
fos.write(buffer, 0, len);
}
inputStream.close();
fos.close();
return unzipAndPickFirstEntry(file.getAbsolutePath());
}
public static File unzipAndPickFirstEntry(String zipFilePath)
{
Path filePath = null;
File dir = new File(destDir);
// create output directory if it doesn't exist
if (!dir.exists())
dir.mkdirs();
FileInputStream fis;
//buffer for read and write data to file
byte[] buffer = new byte[1024];
@ -846,7 +859,7 @@ public class FileManager
if (ze != null)
{
String fileName = ze.getName();
File newFile = new File(destDir + File.separator + fileName);
File newFile = new File(TEMP_PATH + File.separator + fileName);
System.out.println("Unzipping to " + newFile.getAbsolutePath());
//create directories for sub directories in zip
new File(newFile.getParent()).mkdirs();