fix: more fixes for c64.com scraper

src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java

@@ -290,7 +290,6 @@ public class GameDetailsBackgroundPanel extends JPanel
       MainWindow.getInstance().setWaitCursor(false);
       if (scraperFields.isScreenshots())
       {
-
         List<BufferedImage> screenshots = scraperManager.getScreenshots();
         if (screenshots.size() > 2)
         {

src/main/java/se/lantz/gui/scraper/C64comOptionsPanel.java

@@ -38,7 +38,7 @@ public class C64comOptionsPanel extends JPanel
   private JCheckBox titleCheckBox;
   private JCheckBox authorCheckBox;
   private JCheckBox yearCheckBox;
-  private JCheckBox descriptionCheckBox;
+  private JCheckBox gameCheckBox;
   private JCheckBox coverCheckBox;
   private JCheckBox screensCheckBox;
   private ScraperManager scraper;
@@ -194,12 +194,12 @@ public class C64comOptionsPanel extends JPanel
       gbc_yearCheckBox.gridx = 0;
       gbc_yearCheckBox.gridy = 3;
       fieldsPanel.add(getYearCheckBox(), gbc_yearCheckBox);
-      GridBagConstraints gbc_descriptionCheckBox = new GridBagConstraints();
+      GridBagConstraints gbc_gameCheckBox = new GridBagConstraints();
-      gbc_descriptionCheckBox.insets = new Insets(0, 0, 5, 5);
+      gbc_gameCheckBox.insets = new Insets(0, 0, 5, 5);
-      gbc_descriptionCheckBox.anchor = GridBagConstraints.WEST;
+      gbc_gameCheckBox.anchor = GridBagConstraints.WEST;
-      gbc_descriptionCheckBox.gridx = 1;
+      gbc_gameCheckBox.gridx = 1;
-      gbc_descriptionCheckBox.gridy = 1;
+      gbc_gameCheckBox.gridy = 1;
-      fieldsPanel.add(getDescriptionCheckBox(), gbc_descriptionCheckBox);
+      fieldsPanel.add(getGameCheckBox(), gbc_gameCheckBox);
       GridBagConstraints gbc_genreCheckBox = new GridBagConstraints();
       gbc_genreCheckBox.insets = new Insets(0, 0, 0, 5);
       gbc_genreCheckBox.anchor = GridBagConstraints.WEST;
@@ -258,15 +258,15 @@ public class C64comOptionsPanel extends JPanel
     return yearCheckBox;
   }
 
-  private JCheckBox getDescriptionCheckBox()
+  private JCheckBox getGameCheckBox()
   {
-    if (descriptionCheckBox == null)
+    if (gameCheckBox == null)
     {
-      descriptionCheckBox = new JCheckBox("Description");
+      gameCheckBox = new JCheckBox("Game file");
-      descriptionCheckBox.setSelected(false);
+      gameCheckBox.setSelected(true);
-      descriptionCheckBox.setEnabled(false);
+      gameCheckBox.setEnabled(false);
     }
-    return descriptionCheckBox;
+    return gameCheckBox;
   }
 
   private JCheckBox getCoverCheckBox()
@@ -296,7 +296,7 @@ public class C64comOptionsPanel extends JPanel
     titleCheckBox.setEnabled(enable);
     authorCheckBox.setEnabled(enable);
     yearCheckBox.setEnabled(enable);
-//    descriptionCheckBox.setEnabled(enable);
+    gameCheckBox.setEnabled(enable);
     coverCheckBox.setEnabled(enable);
     screensCheckBox.setEnabled(enable);
     genreCheckBox.setEnabled(enable);
@@ -310,12 +310,12 @@ public class C64comOptionsPanel extends JPanel
     returnValue.setAuthor(authorCheckBox.isSelected());
     returnValue.setYear(yearCheckBox.isSelected());
     returnValue.setGenre(genreCheckBox.isSelected());
+    //No description available
     returnValue.setDescription(false);
     returnValue.setCover(coverCheckBox.isSelected());
     returnValue.setScreenshots(screensCheckBox.isSelected());
     returnValue.setComposer(composerCheckBox.isSelected());
-    //TODO:add option
+    returnValue.setGame(gameCheckBox.isSelected());
-    returnValue.setGame(true);
     return returnValue;
   }
 

src/main/java/se/lantz/gui/scraper/ScraperDialog.java

@@ -20,6 +20,10 @@ import se.lantz.model.data.ScraperFields;
 
 public class ScraperDialog extends BaseDialog
 {
+  private static final String MOBY = "moby";
+  private static final String C64COM = "c64com";
+  private static final String WWW_C64_COM = "www.c64.com";
+  private static final String WWW_MOBYGAMES_COM = "www.mobygames.com";
   private MobyGamesOptionsPanel mobyGamesPanel;
   private C64comOptionsPanel c64comPanel;
   private final ScraperManager scraper;
@@ -38,8 +42,8 @@ public class ScraperDialog extends BaseDialog
     content.setLayout(new BorderLayout());
     cardPanel = new JPanel();
     cardPanel.setLayout(cardLayout);
-    cardPanel.add(getMobyGamesPanel(), "moby");
+    cardPanel.add(getMobyGamesPanel(), MOBY);
-    cardPanel.add(getC64comPanel(), "c64com");
+    cardPanel.add(getC64comPanel(), C64COM);
     content.add(cardPanel, BorderLayout.CENTER);
     addContent(content);
     content.add(getScraperSelectionPanel(), BorderLayout.NORTH);
@@ -66,7 +70,7 @@ public class ScraperDialog extends BaseDialog
 
   public ScraperFields getScraperFields()
   {
-    if (getScraperComboBox().getSelectedItem().equals("www.mobygames.com"))
+    if (getScraperComboBox().getSelectedItem().equals(WWW_MOBYGAMES_COM))
     {
       return getMobyGamesPanel().getScraperFields();
     }
@@ -98,6 +102,8 @@ public class ScraperDialog extends BaseDialog
       gbc_scraperInfoLabel.gridy = 0;
       scraperSelectionPanel.add(getScraperInfoLabel(), gbc_scraperInfoLabel);
       GridBagConstraints gbc_scraperComboBox = new GridBagConstraints();
+      gbc_scraperComboBox.anchor = GridBagConstraints.WEST;
+      gbc_scraperComboBox.weightx = 1.0;
       gbc_scraperComboBox.insets = new Insets(10, 0, 5, 5);
       gbc_scraperComboBox.gridx = 1;
       gbc_scraperComboBox.gridy = 0;
@@ -124,22 +130,22 @@ public class ScraperDialog extends BaseDialog
         {
           public void actionPerformed(ActionEvent arg0)
           {
-            if (scraperComboBox.getSelectedItem().equals("www.mobygames.com"))
+            if (scraperComboBox.getSelectedItem().equals(WWW_MOBYGAMES_COM))
             {
-              cardLayout.show(cardPanel, "moby");
+              cardLayout.show(cardPanel, MOBY);
               scraper.setScrapertoUse(SCRAPER.moby);
               getRootPane().setDefaultButton(getMobyGamesPanel().getConnectButton());
             }
             else
             {
-              cardLayout.show(cardPanel, "c64com");
+              cardLayout.show(cardPanel, C64COM);
               scraper.setScrapertoUse(SCRAPER.c64com);
               getRootPane().setDefaultButton(getC64comPanel().getConnectButton());
             }
           }
         });
-      scraperComboBox.addItem("www.mobygames.com");
+      scraperComboBox.addItem(WWW_MOBYGAMES_COM);
-      scraperComboBox.addItem("www.c64.com");
+      scraperComboBox.addItem(WWW_C64_COM);
     }
     return scraperComboBox;
   }

src/main/java/se/lantz/scraper/C64comScraper.java

@@ -64,7 +64,22 @@ public class C64comScraper implements Scraper
   public void connect(String url) throws IOException
   {
     this.c64comGameUrl = "";
-    Jsoup.connect(url).method(Connection.Method.GET).execute();
+    //c64.com gives no errors for invalid urls. Check if there is an title to make sure it's valid
+    Connection.Response result = Jsoup.connect(url).method(Connection.Method.GET).execute();
+    Document doc = result.parse();     
+    //Fetch right frame 
+    Document mainFrameDocument = Jsoup.connect(doc.select("frame[name=text]").first().absUrl("src")).get();
+    //Fetch title
+    Elements queryElements = mainFrameDocument.select(titleCssQuery);
+    Element first = queryElements.first();
+    if (first != null)
+    {
+      if (first.text().isEmpty())
+      {
+        throw new IllegalArgumentException();
+      }     
+    }
+    
     this.c64comGameUrl = url;
     resetFields();
   }
@@ -172,32 +187,18 @@ public class C64comScraper implements Scraper
                 Response response = Jsoup.connect(url)
                   .header("Accept-Encoding", "gzip, deflate")
                   .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0")
-  //                .referrer(URL_TO_PARSE)
                   .ignoreContentType(true)
                   .maxBodySize(0)
                   .timeout(600000)
                   .execute();
-  //                .bodyAsBytes();
                 
-                //TODO: add to temp folder instead
+                //create a temp file and fetch the content
-                File file = new File("scrapedFile.zip");
+                scrapedFile = FileManager.createTempFileForScraper(response.bodyStream());
-                BufferedInputStream inputStream = response.bodyStream();
-                FileOutputStream fos = new FileOutputStream(file);
-                byte[] buffer = new byte[1024];
-                int len;
-                while ((len = inputStream.read(buffer)) != -1) {
-                    fos.write(buffer, 0, len);
-                }
-                inputStream.close();
-                fos.close();
-                
-                //Unzip
-                scrapedFile = FileManager.unzipAndPickFirstEntry(file.getAbsolutePath(), ".");
                 logger.debug("File to include as game: {}", scrapedFile.getAbsolutePath());
               }
               catch (IOException e)
               {
-                logger.error("", e);
+                logger.error("Could not scrape game file for " + scrapedTitle , e);
               }
             }
           }

src/main/java/se/lantz/util/FileManager.java

@@ -3,6 +3,7 @@ package se.lantz.util;
 import java.awt.Graphics;
 import java.awt.Image;
 import java.awt.image.BufferedImage;
+import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -827,14 +828,26 @@ public class FileManager
     g.drawImage(newImage, 0, 0, null);
     return newImage;
   }
+  
+  public static File createTempFileForScraper(BufferedInputStream inputStream) throws IOException
+  {
+    Files.createDirectories(TEMP_PATH);
+    //TODO: add to temp folder instead
+    File file = new File(TEMP_PATH +  File.separator +"scrapedFile.zip");
+    FileOutputStream fos = new FileOutputStream(file, false);
+    byte[] buffer = new byte[1024];
+    int len;
+    while ((len = inputStream.read(buffer)) != -1) {
+        fos.write(buffer, 0, len);
+    }
+    inputStream.close();
+    fos.close();
+    return unzipAndPickFirstEntry(file.getAbsolutePath());
+  }
 
-  public static File unzipAndPickFirstEntry(String zipFilePath, String destDir)
+  public static File unzipAndPickFirstEntry(String zipFilePath)
   {
     Path filePath = null;
-    File dir = new File(destDir);
-    // create output directory if it doesn't exist
-    if (!dir.exists())
-      dir.mkdirs();
     FileInputStream fis;
     //buffer for read and write data to file
     byte[] buffer = new byte[1024];
@@ -846,7 +859,7 @@ public class FileManager
       if (ze != null)
       {
         String fileName = ze.getName();
-        File newFile = new File(destDir + File.separator + fileName);
+        File newFile = new File(TEMP_PATH + File.separator + fileName);
         System.out.println("Unzipping to " + newFile.getAbsolutePath());
         //create directories for sub directories in zip
         new File(newFile.getParent()).mkdirs();