diff --git a/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java b/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java index 69cef6b..5927f65 100644 --- a/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java +++ b/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java @@ -290,7 +290,6 @@ public class GameDetailsBackgroundPanel extends JPanel MainWindow.getInstance().setWaitCursor(false); if (scraperFields.isScreenshots()) { - List screenshots = scraperManager.getScreenshots(); if (screenshots.size() > 2) { diff --git a/src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java b/src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java index 86e549e..b570582 100644 --- a/src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java +++ b/src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java @@ -38,7 +38,7 @@ public class C64comOptionsPanel extends JPanel private JCheckBox titleCheckBox; private JCheckBox authorCheckBox; private JCheckBox yearCheckBox; - private JCheckBox descriptionCheckBox; + private JCheckBox gameCheckBox; private JCheckBox coverCheckBox; private JCheckBox screensCheckBox; private ScraperManager scraper; @@ -194,12 +194,12 @@ public class C64comOptionsPanel extends JPanel gbc_yearCheckBox.gridx = 0; gbc_yearCheckBox.gridy = 3; fieldsPanel.add(getYearCheckBox(), gbc_yearCheckBox); - GridBagConstraints gbc_descriptionCheckBox = new GridBagConstraints(); - gbc_descriptionCheckBox.insets = new Insets(0, 0, 5, 5); - gbc_descriptionCheckBox.anchor = GridBagConstraints.WEST; - gbc_descriptionCheckBox.gridx = 1; - gbc_descriptionCheckBox.gridy = 1; - fieldsPanel.add(getDescriptionCheckBox(), gbc_descriptionCheckBox); + GridBagConstraints gbc_gameCheckBox = new GridBagConstraints(); + gbc_gameCheckBox.insets = new Insets(0, 0, 5, 5); + gbc_gameCheckBox.anchor = GridBagConstraints.WEST; + gbc_gameCheckBox.gridx = 1; + gbc_gameCheckBox.gridy = 1; + fieldsPanel.add(getGameCheckBox(), gbc_gameCheckBox); GridBagConstraints gbc_genreCheckBox = new GridBagConstraints(); gbc_genreCheckBox.insets = new Insets(0, 0, 0, 5); gbc_genreCheckBox.anchor = GridBagConstraints.WEST; @@ -258,15 +258,15 @@ public class C64comOptionsPanel extends JPanel return yearCheckBox; } - private JCheckBox getDescriptionCheckBox() + private JCheckBox getGameCheckBox() { - if (descriptionCheckBox == null) + if (gameCheckBox == null) { - descriptionCheckBox = new JCheckBox("Description"); - descriptionCheckBox.setSelected(false); - descriptionCheckBox.setEnabled(false); + gameCheckBox = new JCheckBox("Game file"); + gameCheckBox.setSelected(true); + gameCheckBox.setEnabled(false); } - return descriptionCheckBox; + return gameCheckBox; } private JCheckBox getCoverCheckBox() @@ -296,7 +296,7 @@ public class C64comOptionsPanel extends JPanel titleCheckBox.setEnabled(enable); authorCheckBox.setEnabled(enable); yearCheckBox.setEnabled(enable); -// descriptionCheckBox.setEnabled(enable); + gameCheckBox.setEnabled(enable); coverCheckBox.setEnabled(enable); screensCheckBox.setEnabled(enable); genreCheckBox.setEnabled(enable); @@ -310,12 +310,12 @@ public class C64comOptionsPanel extends JPanel returnValue.setAuthor(authorCheckBox.isSelected()); returnValue.setYear(yearCheckBox.isSelected()); returnValue.setGenre(genreCheckBox.isSelected()); + //No description available returnValue.setDescription(false); returnValue.setCover(coverCheckBox.isSelected()); returnValue.setScreenshots(screensCheckBox.isSelected()); returnValue.setComposer(composerCheckBox.isSelected()); - //TODO:add option - returnValue.setGame(true); + returnValue.setGame(gameCheckBox.isSelected()); return returnValue; } diff --git a/src/main/java/se/lantz/gui/scraper/ScraperDialog.java b/src/main/java/se/lantz/gui/scraper/ScraperDialog.java index 08dd5a6..2dedd7a 100644 --- a/src/main/java/se/lantz/gui/scraper/ScraperDialog.java +++ b/src/main/java/se/lantz/gui/scraper/ScraperDialog.java @@ -20,6 +20,10 @@ import se.lantz.model.data.ScraperFields; public class ScraperDialog extends BaseDialog { + private static final String MOBY = "moby"; + private static final String C64COM = "c64com"; + private static final String WWW_C64_COM = "www.c64.com"; + private static final String WWW_MOBYGAMES_COM = "www.mobygames.com"; private MobyGamesOptionsPanel mobyGamesPanel; private C64comOptionsPanel c64comPanel; private final ScraperManager scraper; @@ -38,8 +42,8 @@ public class ScraperDialog extends BaseDialog content.setLayout(new BorderLayout()); cardPanel = new JPanel(); cardPanel.setLayout(cardLayout); - cardPanel.add(getMobyGamesPanel(), "moby"); - cardPanel.add(getC64comPanel(), "c64com"); + cardPanel.add(getMobyGamesPanel(), MOBY); + cardPanel.add(getC64comPanel(), C64COM); content.add(cardPanel, BorderLayout.CENTER); addContent(content); content.add(getScraperSelectionPanel(), BorderLayout.NORTH); @@ -66,7 +70,7 @@ public class ScraperDialog extends BaseDialog public ScraperFields getScraperFields() { - if (getScraperComboBox().getSelectedItem().equals("www.mobygames.com")) + if (getScraperComboBox().getSelectedItem().equals(WWW_MOBYGAMES_COM)) { return getMobyGamesPanel().getScraperFields(); } @@ -98,6 +102,8 @@ public class ScraperDialog extends BaseDialog gbc_scraperInfoLabel.gridy = 0; scraperSelectionPanel.add(getScraperInfoLabel(), gbc_scraperInfoLabel); GridBagConstraints gbc_scraperComboBox = new GridBagConstraints(); + gbc_scraperComboBox.anchor = GridBagConstraints.WEST; + gbc_scraperComboBox.weightx = 1.0; gbc_scraperComboBox.insets = new Insets(10, 0, 5, 5); gbc_scraperComboBox.gridx = 1; gbc_scraperComboBox.gridy = 0; @@ -124,22 +130,22 @@ public class ScraperDialog extends BaseDialog { public void actionPerformed(ActionEvent arg0) { - if (scraperComboBox.getSelectedItem().equals("www.mobygames.com")) + if (scraperComboBox.getSelectedItem().equals(WWW_MOBYGAMES_COM)) { - cardLayout.show(cardPanel, "moby"); + cardLayout.show(cardPanel, MOBY); scraper.setScrapertoUse(SCRAPER.moby); getRootPane().setDefaultButton(getMobyGamesPanel().getConnectButton()); } else { - cardLayout.show(cardPanel, "c64com"); + cardLayout.show(cardPanel, C64COM); scraper.setScrapertoUse(SCRAPER.c64com); getRootPane().setDefaultButton(getC64comPanel().getConnectButton()); } } }); - scraperComboBox.addItem("www.mobygames.com"); - scraperComboBox.addItem("www.c64.com"); + scraperComboBox.addItem(WWW_MOBYGAMES_COM); + scraperComboBox.addItem(WWW_C64_COM); } return scraperComboBox; } diff --git a/src/main/java/se/lantz/scraper/C64comScraper.java b/src/main/java/se/lantz/scraper/C64comScraper.java index 01a57c4..19efe1e 100644 --- a/src/main/java/se/lantz/scraper/C64comScraper.java +++ b/src/main/java/se/lantz/scraper/C64comScraper.java @@ -64,7 +64,22 @@ public class C64comScraper implements Scraper public void connect(String url) throws IOException { this.c64comGameUrl = ""; - Jsoup.connect(url).method(Connection.Method.GET).execute(); + //c64.com gives no errors for invalid urls. Check if there is an title to make sure it's valid + Connection.Response result = Jsoup.connect(url).method(Connection.Method.GET).execute(); + Document doc = result.parse(); + //Fetch right frame + Document mainFrameDocument = Jsoup.connect(doc.select("frame[name=text]").first().absUrl("src")).get(); + //Fetch title + Elements queryElements = mainFrameDocument.select(titleCssQuery); + Element first = queryElements.first(); + if (first != null) + { + if (first.text().isEmpty()) + { + throw new IllegalArgumentException(); + } + } + this.c64comGameUrl = url; resetFields(); } @@ -172,32 +187,18 @@ public class C64comScraper implements Scraper Response response = Jsoup.connect(url) .header("Accept-Encoding", "gzip, deflate") .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0") - // .referrer(URL_TO_PARSE) .ignoreContentType(true) .maxBodySize(0) .timeout(600000) .execute(); - // .bodyAsBytes(); - //TODO: add to temp folder instead - File file = new File("scrapedFile.zip"); - BufferedInputStream inputStream = response.bodyStream(); - FileOutputStream fos = new FileOutputStream(file); - byte[] buffer = new byte[1024]; - int len; - while ((len = inputStream.read(buffer)) != -1) { - fos.write(buffer, 0, len); - } - inputStream.close(); - fos.close(); - - //Unzip - scrapedFile = FileManager.unzipAndPickFirstEntry(file.getAbsolutePath(), "."); + //create a temp file and fetch the content + scrapedFile = FileManager.createTempFileForScraper(response.bodyStream()); logger.debug("File to include as game: {}", scrapedFile.getAbsolutePath()); } catch (IOException e) { - logger.error("", e); + logger.error("Could not scrape game file for " + scrapedTitle , e); } } } diff --git a/src/main/java/se/lantz/util/FileManager.java b/src/main/java/se/lantz/util/FileManager.java index e8059bb..1f2cd6d 100644 --- a/src/main/java/se/lantz/util/FileManager.java +++ b/src/main/java/se/lantz/util/FileManager.java @@ -3,6 +3,7 @@ package se.lantz.util; import java.awt.Graphics; import java.awt.Image; import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -827,14 +828,26 @@ public class FileManager g.drawImage(newImage, 0, 0, null); return newImage; } + + public static File createTempFileForScraper(BufferedInputStream inputStream) throws IOException + { + Files.createDirectories(TEMP_PATH); + //TODO: add to temp folder instead + File file = new File(TEMP_PATH + File.separator +"scrapedFile.zip"); + FileOutputStream fos = new FileOutputStream(file, false); + byte[] buffer = new byte[1024]; + int len; + while ((len = inputStream.read(buffer)) != -1) { + fos.write(buffer, 0, len); + } + inputStream.close(); + fos.close(); + return unzipAndPickFirstEntry(file.getAbsolutePath()); + } - public static File unzipAndPickFirstEntry(String zipFilePath, String destDir) + public static File unzipAndPickFirstEntry(String zipFilePath) { Path filePath = null; - File dir = new File(destDir); - // create output directory if it doesn't exist - if (!dir.exists()) - dir.mkdirs(); FileInputStream fis; //buffer for read and write data to file byte[] buffer = new byte[1024]; @@ -846,7 +859,7 @@ public class FileManager if (ze != null) { String fileName = ze.getName(); - File newFile = new File(destDir + File.separator + fileName); + File newFile = new File(TEMP_PATH + File.separator + fileName); System.out.println("Unzipping to " + newFile.getAbsolutePath()); //create directories for sub directories in zip new File(newFile.getParent()).mkdirs();