Moby games scraper finished

src/main/java/se/lantz/gui/BaseDialog.java

@@ -124,6 +124,7 @@ public class BaseDialog extends JDialog
 
   public boolean showDialog()
   {
+    okPressed = false;
     this.setVisible(true);
     return okPressed;
   }

src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java

@@ -19,7 +19,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import se.lantz.gui.scraper.ScraperDialog;
+import se.lantz.gui.scraper.ScraperProgressDialog;
+import se.lantz.gui.scraper.ScraperWorker;
 import se.lantz.gui.scraper.ScreenshotsSelectionDialog;
+import se.lantz.manager.ScraperManager;
 import se.lantz.model.MainViewModel;
 import se.lantz.model.data.GameListData;
 import se.lantz.model.data.ScraperFields;
@@ -29,7 +32,7 @@ public class GameDetailsBackgroundPanel extends JPanel
   private static final String EMPTY = "empty";
   private static final String DETAILS = "details";
   private static final Logger logger = LoggerFactory.getLogger(GameDetailsBackgroundPanel.class);
-  private MainViewModel model;
+  private final MainViewModel model;
   private JPanel settingsPanel;
   private InfoPanel infoPanel;
   private CombinedJoystickPanel joystickPanel;
@@ -42,9 +45,12 @@ public class GameDetailsBackgroundPanel extends JPanel
   private CardLayout cardLayout;
   private JButton scrapeButton;
 
+  private final ScraperManager scraperManager;
+
   public GameDetailsBackgroundPanel(MainViewModel model)
   {
     this.model = model;
+    this.scraperManager = new ScraperManager(model.getInfoModel());
     this.setMinimumSize(new Dimension(1250, 800));
     cardLayout = new CardLayout();
     setLayout(cardLayout);
@@ -198,57 +204,82 @@ public class GameDetailsBackgroundPanel extends JPanel
     }
     return saveButton;
   }
-  private JButton getScrapeButton() {
+
-    if (scrapeButton == null) {
+  private JButton getScrapeButton()
-    	scrapeButton = new JButton("Scrape...");
+  {
-    	scrapeButton.addActionListener(new ActionListener() {
+    if (scrapeButton == null)
-    	  public void actionPerformed(ActionEvent e) {
+    {
-    	    scrapeGamesInformation();
+      scrapeButton = new JButton("Scrape...");
-    	  }
+      scrapeButton.addActionListener(new ActionListener()
-    	});
+        {
+          public void actionPerformed(ActionEvent e)
+          {
+            scrapeGamesInformation();
+          }
+        });
     }
     return scrapeButton;
   }
-  
+
   private ScraperDialog scraperDialog = null;
-  
+
   private void scrapeGamesInformation()
   {
     if (scraperDialog == null)
     {
-      scraperDialog = new ScraperDialog(MainWindow.getInstance(), model);
+      scraperDialog = new ScraperDialog(MainWindow.getInstance(), scraperManager);
       scraperDialog.pack();
       scraperDialog.setLocationRelativeTo(MainWindow.getInstance());
     }
-    
+
     if (scraperDialog.showDialog())
     {
       MainWindow.getInstance().setWaitCursor(true);
+
       ScraperFields scraperFields = scraperDialog.getScraperFields();
-      model.scrapeGameInformation(scraperFields);
+      ScraperProgressDialog dialog = new ScraperProgressDialog(MainWindow.getInstance());
+      dialog.pack();
+      dialog.setLocationRelativeTo(MainWindow.getInstance());
+
+      ScraperWorker worker = new ScraperWorker(scraperManager, scraperFields, dialog);
+      worker.execute();
+      dialog.setVisible(true);
+
       MainWindow.getInstance().setWaitCursor(false);
       if (scraperFields.isScreenshots())
       {
-        //Scrape the screens and check how many there are.
+
-        List<BufferedImage> screenshots =  model.scrapeScreenshots();
+        List<BufferedImage> screenshots = scraperManager.getScreenshots();
         if (screenshots.size() > 2)
         {
           //Show dialog for selecting screenshots
-          ScreenshotsSelectionDialog screenDialog = new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots);
+          ScreenshotsSelectionDialog screenDialog =
+            new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots);
           screenDialog.pack();
           screenDialog.setLocationRelativeTo(MainWindow.getInstance());
           if (screenDialog.showDialog())
           {
-            List<BufferedImage> selectedScreenshots = screenDialog.getSelectedScreenshots();
+            screenshots = screenDialog.getSelectedScreenshots();
-            model.setScreenshotImages(selectedScreenshots.get(0), selectedScreenshots.get(1));
           }
+          else
+          {
+            return;
+          }
+        }
+        //Update with screenshots
+        if (screenshots.size() >= 2)
+        {
+          scraperManager.updateModelWithScreenshotImages(screenshots.get(0), screenshots.get(1));
+        }
+        else if (screenshots.size() == 1)
+        {
+          scraperManager.updateModelWithScreenshotImages(screenshots.get(0), null);
         }
         else
         {
-          //TODO
+          //Do nothing
         }
       }
     }
-    
   }
 }

src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java

@@ -20,6 +20,7 @@ import javax.swing.JTextField;
 import javax.swing.UIManager;
 import javax.swing.event.HyperlinkEvent;
 
+import se.lantz.manager.ScraperManager;
 import se.lantz.model.MainViewModel;
 import se.lantz.model.data.ScraperFields;
 import se.lantz.util.ExceptionHandler;
@@ -36,7 +37,7 @@ public class MobyGamesOptionsPanel extends JPanel
   private JCheckBox descriptionCheckBox;
   private JCheckBox coverCheckBox;
   private JCheckBox screensCheckBox;
-  private MainViewModel model;
+  private ScraperManager scraper;
   private JButton connectButton;
   private JLabel connectionStatusLabel;
   
@@ -46,9 +47,9 @@ public class MobyGamesOptionsPanel extends JPanel
   private JCheckBox genreCheckBox;
   
 
-  public MobyGamesOptionsPanel(MainViewModel model, JButton okButton)
+  public MobyGamesOptionsPanel(ScraperManager scraper, JButton okButton)
   {
-    this.model = model;
+    this.scraper = scraper;
     this.okButton = okButton;
     GridBagLayout gridBagLayout = new GridBagLayout();
     setLayout(gridBagLayout);
@@ -74,8 +75,8 @@ public class MobyGamesOptionsPanel extends JPanel
     gbc_connectButton.gridy = 1;
     add(getConnectButton(), gbc_connectButton);
     GridBagConstraints gbc_connectionStatusLabel = new GridBagConstraints();
-    gbc_connectionStatusLabel.anchor = GridBagConstraints.WEST;
+    gbc_connectionStatusLabel.gridwidth = 2;
-    gbc_connectionStatusLabel.insets = new Insets(0, 10, 0, 5);
+    gbc_connectionStatusLabel.insets = new Insets(0, 10, 10, 5);
     gbc_connectionStatusLabel.gridx = 0;
     gbc_connectionStatusLabel.gridy = 2;
     add(getConnectionStatusLabel(), gbc_connectionStatusLabel);
@@ -288,6 +289,8 @@ public class MobyGamesOptionsPanel extends JPanel
     returnValue.setYear(yearCheckBox.isSelected());
     returnValue.setGenre(genreCheckBox.isSelected());
     returnValue.setDescription(descriptionCheckBox.isSelected());
+    returnValue.setCover(coverCheckBox.isSelected());
+    returnValue.setScreenshots(screensCheckBox.isSelected());
     return returnValue;
   }
   private JButton getConnectButton() {
@@ -298,7 +301,7 @@ public class MobyGamesOptionsPanel extends JPanel
     	    try
           {
     	      MobyGamesOptionsPanel.this.setCursor(waitCursor);
-            model.connectScraper(urlTextField.getText());
+            scraper.connectScraper(urlTextField.getText());
             getConnectionStatusLabel().setText("Connection status: OK");
             enableCheckBoxes(true);
             okButton.setEnabled(true);

src/main/java/se/lantz/gui/scraper/ScraperDialog.java

@@ -6,17 +6,17 @@ import java.awt.Frame;
 import javax.swing.JPanel;
 
 import se.lantz.gui.BaseDialog;
-import se.lantz.model.MainViewModel;
+import se.lantz.manager.ScraperManager;
 import se.lantz.model.data.ScraperFields;
 
 public class ScraperDialog extends BaseDialog
 {
   private MobyGamesOptionsPanel mbyGamesPanel;
-  private final MainViewModel model;
+  private final ScraperManager scraper;
-  public ScraperDialog(Frame owner, MainViewModel model)
+  public ScraperDialog(Frame owner, ScraperManager scraper)
   {
     super(owner);
-    this.model = model;
+    this.scraper = scraper;
     JPanel content = new JPanel();
     content.setLayout(new BorderLayout());
     content.add(getMobyGamesPanel(), BorderLayout.CENTER);
@@ -29,7 +29,7 @@ public class ScraperDialog extends BaseDialog
   {
     if (mbyGamesPanel == null)
     {
-      mbyGamesPanel = new MobyGamesOptionsPanel(model, getOkButton());
+      mbyGamesPanel = new MobyGamesOptionsPanel(scraper, getOkButton());
     }
     return mbyGamesPanel;
   }

src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java

@@ -0,0 +1,72 @@
+package se.lantz.gui.scraper;
+
+import java.awt.Frame;
+
+import javax.swing.JDialog;
+import java.awt.GridBagLayout;
+import javax.swing.JProgressBar;
+import javax.swing.WindowConstants;
+
+import java.awt.GridBagConstraints;
+import javax.swing.JLabel;
+import java.awt.Insets;
+
+public class ScraperProgressDialog extends JDialog
+{
+  private static final long serialVersionUID = 1L;
+  private JProgressBar progressBar;
+  private JLabel textLabel;
+
+  public ScraperProgressDialog(Frame frame)
+  {
+    super(frame,"Scraping", true);
+    setAlwaysOnTop(true);
+    setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE);
+    setResizable(false);
+    GridBagLayout gridBagLayout = new GridBagLayout();
+    getContentPane().setLayout(gridBagLayout);
+    GridBagConstraints gbc_progressBar = new GridBagConstraints();
+    gbc_progressBar.weightx = 1.0;
+    gbc_progressBar.anchor = GridBagConstraints.NORTH;
+    gbc_progressBar.fill = GridBagConstraints.HORIZONTAL;
+    gbc_progressBar.insets = new Insets(5, 5, 5, 5);
+    gbc_progressBar.gridx = 0;
+    gbc_progressBar.gridy = 0;
+    getContentPane().add(getProgressBar(), gbc_progressBar);
+    GridBagConstraints gbc_textLabel = new GridBagConstraints();
+    gbc_textLabel.insets = new Insets(5, 5, 5, 5);
+    gbc_textLabel.anchor = GridBagConstraints.NORTH;
+    gbc_textLabel.weightx = 1.0;
+    gbc_textLabel.weighty = 1.0;
+    gbc_textLabel.gridx = 0;
+    gbc_textLabel.gridy = 1;
+    getContentPane().add(getTextLabel(), gbc_textLabel);
+  }
+
+  public void updateProgress()
+  {
+    getTextLabel().setText("Fetching Screenshots...");
+    this.repaint();
+  }
+  
+  public void finish()
+  {
+    progressBar.setIndeterminate(false);
+    getProgressBar().setValue( getProgressBar().getMaximum());
+    dispose();
+  }
+  
+  private JProgressBar getProgressBar() {
+    if (progressBar == null) {
+    	progressBar = new JProgressBar();
+    	progressBar.setIndeterminate(true);
+    }
+    return progressBar;
+  }
+  private JLabel getTextLabel() {
+    if (textLabel == null) {
+    	textLabel = new JLabel("Fetching games information...");
+    }
+    return textLabel;
+  }
+}

src/main/java/se/lantz/gui/scraper/ScraperWorker.java

@@ -0,0 +1,45 @@
+package se.lantz.gui.scraper;
+
+import java.util.List;
+
+import javax.swing.SwingWorker;
+
+import se.lantz.manager.ScraperManager;
+import se.lantz.model.data.ScraperFields;
+
+public class ScraperWorker extends SwingWorker<Void, String>
+{
+
+  private final ScraperManager scraperManager;
+  private final ScraperFields fields;
+  private ScraperProgressDialog dialog;
+
+  public ScraperWorker(ScraperManager scraperManager, ScraperFields fields, ScraperProgressDialog dialog)
+  {
+    this.scraperManager = scraperManager;
+    this.fields = fields;
+    this.dialog = dialog;
+  }
+
+  @Override
+  protected Void doInBackground() throws Exception
+  {
+    scraperManager.scrapeGameInformation(fields);
+    publish("");
+    scraperManager.scrapeScreenshots();
+    return null;
+  }
+
+  @Override
+  protected void process(List<String> chunks)
+  {
+    scraperManager.updateModelWithGamesInfo();  
+    dialog.updateProgress();
+  }
+
+  @Override
+  protected void done()
+  {
+    dialog.finish();
+  }
+}

src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java

@@ -28,11 +28,12 @@ public class ScreenshotsSelectionPanel extends JPanel
     GridBagConstraints gbc_infoLabel = new GridBagConstraints();
     gbc_infoLabel.weightx = 1.0;
     gbc_infoLabel.anchor = GridBagConstraints.WEST;
-    gbc_infoLabel.insets = new Insets(10, 5, 5, 0);
+    gbc_infoLabel.insets = new Insets(10, 10, 5, 0);
     gbc_infoLabel.gridx = 0;
     gbc_infoLabel.gridy = 0;
     add(getInfoLabel(), gbc_infoLabel);
     GridBagConstraints gbc_screenPanel = new GridBagConstraints();
+    gbc_screenPanel.insets = new Insets(5, 10, 10, 10);
     gbc_screenPanel.weighty = 1.0;
     gbc_screenPanel.weightx = 1.0;
     gbc_screenPanel.fill = GridBagConstraints.BOTH;
@@ -60,6 +61,7 @@ public class ScreenshotsSelectionPanel extends JPanel
       {
         ScreenshotCheckBoxPanel checkBox = new ScreenshotCheckBoxPanel();
         checkBox.getImageLabel().setIcon(new ImageIcon(screenshots.get(i)));
+        checkBox.getCheckBox().setText("Screenshot " + (i+1));
         screenshotCheckBoxList.add(checkBox);
         screenPanel.add(checkBox);
       }

src/main/java/se/lantz/manager/ScraperManager.java

@@ -0,0 +1,82 @@
+package se.lantz.manager;
+
+import java.awt.image.BufferedImage;
+import java.util.ArrayList;
+import java.util.List;
+
+import se.lantz.model.InfoModel;
+import se.lantz.model.data.ScraperFields;
+import se.lantz.scraper.MobyGamesScraper;
+
+public class ScraperManager
+{
+  MobyGamesScraper scraper = new MobyGamesScraper();
+  List<BufferedImage> screenshotsList = new ArrayList<>();
+  private InfoModel infoModel;
+  private ScraperFields fields;
+  
+  public ScraperManager(InfoModel model)
+  {
+    this.infoModel = model;
+  }
+  
+  public void connectScraper(String url) throws Exception
+  {
+    scraper.connectToMobyGames(url);
+  }
+  
+  public void scrapeGameInformation(ScraperFields fields)
+  {
+    this.fields = fields;
+    scraper.scrapeInformation(fields);
+  }
+  
+  public void scrapeScreenshots()
+  {
+    screenshotsList = scraper.scrapeScreenshots();
+  }
+  
+  
+  public List<BufferedImage> getScreenshots()
+  {
+    return screenshotsList;
+  }
+  
+  public void updateModelWithGamesInfo()
+  {
+    if (fields.isTitle())
+    {
+      infoModel.setTitle(scraper.getTitle());
+    }
+    if (fields.isAuthor())
+    {
+      infoModel.setAuthor(scraper.getAuthor());
+    }
+    if (fields.isYear())
+    {    
+      infoModel.setYear(scraper.getYear());
+    }
+    if (fields.isDescription())
+    {
+      infoModel.setDescription(scraper.getDescription());
+    }
+    if (fields.isGenre())
+    {
+      String genre = scraper.getGenre();
+      if (!genre.isEmpty())
+      {
+        infoModel.setGenre(genre);
+      }
+    }
+    if (fields.isCover())
+    {
+      infoModel.setCoverImage(scraper.getCover());
+    }
+  }
+  
+  public void updateModelWithScreenshotImages(BufferedImage screen1, BufferedImage screen2)
+  {
+    infoModel.setScreen1Image(screen1);
+    infoModel.setScreen2Image(screen2);
+  }
+}

src/main/java/se/lantz/model/MainViewModel.java

@@ -20,8 +20,8 @@ import se.lantz.model.data.GameDetails;
 import se.lantz.model.data.GameListData;
 import se.lantz.model.data.GameView;
 import se.lantz.model.data.ScraperFields;
+import se.lantz.scraper.MobyGamesScraper;
 import se.lantz.util.FileManager;
-import se.lantz.util.MobyGamesScraper;
 
 public class MainViewModel extends AbstractModel
 {
@@ -48,7 +48,7 @@ public class MainViewModel extends AbstractModel
   private PropertyChangeListener requiredFieldsListener;
 
   private GameListData selectedData;
-  
+
   MobyGamesScraper scraper = new MobyGamesScraper();
 
   public void initialize()
@@ -63,10 +63,10 @@ public class MainViewModel extends AbstractModel
 
     joy2Model.setPrimaryChangeListener(e -> joy1Model
       .setPrimaryWithoutListenerNotification(!Boolean.valueOf(e.getActionCommand())));
-    
+
     resetDataChangedAfterInit();
   }
-  
+
   private void setupGameViews()
   {
     //Setup game views
@@ -81,7 +81,7 @@ public class MainViewModel extends AbstractModel
       gameViewModel.addElement(gameView);
     }
   }
-  
+
   public void reloadGameViews()
   {
     this.disableChangeNotification(true);
@@ -256,7 +256,7 @@ public class MainViewModel extends AbstractModel
     return infoModel.isDataChanged() || joy1Model.isDataChanged() || joy2Model.isDataChanged() ||
       systemModel.isDataChanged();
   }
-  
+
   @Override
   public void resetDataChanged()
   {
@@ -265,7 +265,7 @@ public class MainViewModel extends AbstractModel
     joy2Model.resetDataChanged();
     systemModel.resetDataChanged();
   }
-  
+
   @Override
   public void resetDataChangedAfterInit()
   {
@@ -371,7 +371,7 @@ public class MainViewModel extends AbstractModel
     //Reload the current view
     reloadCurrentGameView();
   }
-  
+
   public void deleteGameView(GameView view)
   {
     if (view.getGameViewId() != GameView.ALL_GAMES_ID)
@@ -420,53 +420,4 @@ public class MainViewModel extends AbstractModel
       resetDataChanged();
     }
   }
-  public void connectScraper(String url) throws Exception
-  {
-    scraper.connectToMobyGames(url);
-  }
-  
-  public void scrapeGameInformation(ScraperFields fields)
-  {
-    scraper.scrapeInformation(fields);
-      
-    if (fields.isTitle())
-    {
-      infoModel.setTitle(scraper.getTitle());
-    }
-    if (fields.isAuthor())
-    {
-      infoModel.setAuthor(scraper.getAuthor());
-    }
-    if (fields.isYear())
-    {    
-      infoModel.setYear(scraper.getYear());
-    }
-    if (fields.isDescription())
-    {
-      infoModel.setDescription(scraper.getDescription());
-    }
-    if (fields.isGenre())
-    {
-      String genre = scraper.getGenre();
-      if (!genre.isEmpty())
-      {
-        infoModel.setGenre(genre);
-      }
-    }
-    if (fields.isCover())
-    {
-      infoModel.setCoverImage(scraper.getCover());
-    }
-  }
-  
-  public List<BufferedImage> scrapeScreenshots()
-  {
-    return scraper.scrapeScreenshots();
-  }
-  
-  public void setScreenshotImages(BufferedImage screen1, BufferedImage screen2)
-  {
-    getInfoModel().setScreen1Image(screen1);
-    getInfoModel().setScreen2Image(screen2);
-  }
 }

src/main/java/se/lantz/scraper/MobyGamesScraper.java

@@ -1,4 +1,4 @@
-package se.lantz.util;
+package se.lantz.scraper;
 
 import java.awt.image.BufferedImage;
 import java.io.IOException;
@@ -22,17 +22,14 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import se.lantz.model.data.ScraperFields;
+import se.lantz.util.ExceptionHandler;
 
 public class MobyGamesScraper
 {
   private static final Logger logger = LoggerFactory.getLogger(MobyGamesScraper.class);
 
-  private String mobyGamesBaseUrl = "https://www.mobygames.com/game/c64/*";
-
   private String mobyGamesGameUrl = "";
 
-  private String game = "rushn-attack";
-
   private String descriptionCssQuery = "#main > div > div:eq(2) > div";
 
   private String titleCssQuery = ".niceHeaderTitle > a";
@@ -65,14 +62,6 @@ public class MobyGamesScraper
 
   public MobyGamesScraper()
   {
-    // TODO Auto-generated constructor stub
-
-    //*[@id="main"]/div/div[2]/h1/a
-
-    //*[@id="main"]/div/div[3]/div[1]/h2[1]
-
-    //*[@id="coreGameCover"]/a/img
-
     //Keys are Genres defined on MobyGames, values are supported genres in the tool
     genreMap.put("Adventure, Role-Playing (RPG)", "adventure");
     genreMap.put("Racing / driving", "driving");
@@ -135,25 +124,11 @@ public class MobyGamesScraper
       {
         scrapedCover = scrapeCover(doc);
       }
-      //TODO: Screens
     }
     catch (IOException e)
     {
       ExceptionHandler.handleException(e, "Could not scrape title");
     }
-    //    
-    //    if (fields.isScreenshots())
-    //    {  //TODO: Make it possible to select which screenshot to use
-    //      List<BufferedImage> images = scraper.scrapeScreenshots();
-    //      if (images.size() > 0)
-    //      {
-    //        infoModel.setScreen1Image(images.get(0));
-    //      }
-    //      if (images.size() > 1)
-    //      {
-    //        infoModel.setScreen2Image(images.get(1));
-    //      }
-    //    }
   }
 
   public String getTitle()
@@ -186,29 +161,6 @@ public class MobyGamesScraper
     return scrapedCover;
   }
 
-  //  public String scrapeTitle()
-  //  {
-  //    String value = "";
-  //    Document doc;
-  //    try
-  //    {
-  //      Connection.Response result = Jsoup.connect(mobyGamesGameUrl).method(Connection.Method.GET).execute();
-  //      doc = result.parse();
-  //      //Fetch the right element
-  //      Elements queryElements = doc.select(titleCssQuery);
-  //      Element first = queryElements.first();
-  //      if (first != null)
-  //      {
-  //        value = first.text();
-  //      }
-  //    }
-  //    catch (IOException e)
-  //    {
-  //      ExceptionHandler.handleException(e, "Could not scrape title");
-  //    }
-  //    return value;
-  //  }
-
   public String scrapeDescription(Document doc)
   {
     //Fetch the right element
@@ -316,9 +268,10 @@ public class MobyGamesScraper
       Elements coverElements = doc.select(screensCssQuery);
 
       logger.debug("Number of screenshots found: {}", coverElements.size());
-      for (Element element : coverElements)
+      //Scrape max 6 screens
+      for (int i = 0; i < Math.min(6, coverElements.size()); i++)
       {
-        String bigScreenUrl = element.attr("href");
+        String bigScreenUrl = coverElements.get(i).attr("href");
         logger.debug("Screen URL = " + bigScreenUrl);
         returnList.add(scrapeBigScreenshot(bigScreenUrl));
       }