From 19df5c4b8b8d0c11cd229c2002071d4c1d77c2af Mon Sep 17 00:00:00 2001 From: lantzelot-swe Date: Sat, 26 Dec 2020 23:48:02 +0100 Subject: [PATCH] Moby games scraper finished --- src/main/java/se/lantz/gui/BaseDialog.java | 1 + .../lantz/gui/GameDetailsBackgroundPanel.java | 73 ++++++++++++----- .../gui/scraper/MobyGamesOptionsPanel.java | 15 ++-- .../se/lantz/gui/scraper/ScraperDialog.java | 10 +-- .../gui/scraper/ScraperProgressDialog.java | 72 ++++++++++++++++ .../se/lantz/gui/scraper/ScraperWorker.java | 45 ++++++++++ .../scraper/ScreenshotsSelectionPanel.java | 4 +- .../java/se/lantz/manager/ScraperManager.java | 82 +++++++++++++++++++ .../java/se/lantz/model/MainViewModel.java | 65 ++------------- .../{util => scraper}/MobyGamesScraper.java | 57 ++----------- 10 files changed, 282 insertions(+), 142 deletions(-) create mode 100644 src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java create mode 100644 src/main/java/se/lantz/gui/scraper/ScraperWorker.java create mode 100644 src/main/java/se/lantz/manager/ScraperManager.java rename src/main/java/se/lantz/{util => scraper}/MobyGamesScraper.java (84%) diff --git a/src/main/java/se/lantz/gui/BaseDialog.java b/src/main/java/se/lantz/gui/BaseDialog.java index 2761362..f9e0a1f 100644 --- a/src/main/java/se/lantz/gui/BaseDialog.java +++ b/src/main/java/se/lantz/gui/BaseDialog.java @@ -124,6 +124,7 @@ public class BaseDialog extends JDialog public boolean showDialog() { + okPressed = false; this.setVisible(true); return okPressed; } diff --git a/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java b/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java index bd6c00c..9e662fb 100644 --- a/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java +++ b/src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java @@ -19,7 +19,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import se.lantz.gui.scraper.ScraperDialog; +import se.lantz.gui.scraper.ScraperProgressDialog; +import se.lantz.gui.scraper.ScraperWorker; import se.lantz.gui.scraper.ScreenshotsSelectionDialog; +import se.lantz.manager.ScraperManager; import se.lantz.model.MainViewModel; import se.lantz.model.data.GameListData; import se.lantz.model.data.ScraperFields; @@ -29,7 +32,7 @@ public class GameDetailsBackgroundPanel extends JPanel private static final String EMPTY = "empty"; private static final String DETAILS = "details"; private static final Logger logger = LoggerFactory.getLogger(GameDetailsBackgroundPanel.class); - private MainViewModel model; + private final MainViewModel model; private JPanel settingsPanel; private InfoPanel infoPanel; private CombinedJoystickPanel joystickPanel; @@ -42,9 +45,12 @@ public class GameDetailsBackgroundPanel extends JPanel private CardLayout cardLayout; private JButton scrapeButton; + private final ScraperManager scraperManager; + public GameDetailsBackgroundPanel(MainViewModel model) { this.model = model; + this.scraperManager = new ScraperManager(model.getInfoModel()); this.setMinimumSize(new Dimension(1250, 800)); cardLayout = new CardLayout(); setLayout(cardLayout); @@ -198,57 +204,82 @@ public class GameDetailsBackgroundPanel extends JPanel } return saveButton; } - private JButton getScrapeButton() { - if (scrapeButton == null) { - scrapeButton = new JButton("Scrape..."); - scrapeButton.addActionListener(new ActionListener() { - public void actionPerformed(ActionEvent e) { - scrapeGamesInformation(); - } - }); + + private JButton getScrapeButton() + { + if (scrapeButton == null) + { + scrapeButton = new JButton("Scrape..."); + scrapeButton.addActionListener(new ActionListener() + { + public void actionPerformed(ActionEvent e) + { + scrapeGamesInformation(); + } + }); } return scrapeButton; } - + private ScraperDialog scraperDialog = null; - + private void scrapeGamesInformation() { if (scraperDialog == null) { - scraperDialog = new ScraperDialog(MainWindow.getInstance(), model); + scraperDialog = new ScraperDialog(MainWindow.getInstance(), scraperManager); scraperDialog.pack(); scraperDialog.setLocationRelativeTo(MainWindow.getInstance()); } - + if (scraperDialog.showDialog()) { MainWindow.getInstance().setWaitCursor(true); + ScraperFields scraperFields = scraperDialog.getScraperFields(); - model.scrapeGameInformation(scraperFields); + ScraperProgressDialog dialog = new ScraperProgressDialog(MainWindow.getInstance()); + dialog.pack(); + dialog.setLocationRelativeTo(MainWindow.getInstance()); + + ScraperWorker worker = new ScraperWorker(scraperManager, scraperFields, dialog); + worker.execute(); + dialog.setVisible(true); + MainWindow.getInstance().setWaitCursor(false); if (scraperFields.isScreenshots()) { - //Scrape the screens and check how many there are. - List screenshots = model.scrapeScreenshots(); + + List screenshots = scraperManager.getScreenshots(); if (screenshots.size() > 2) { //Show dialog for selecting screenshots - ScreenshotsSelectionDialog screenDialog = new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots); + ScreenshotsSelectionDialog screenDialog = + new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots); screenDialog.pack(); screenDialog.setLocationRelativeTo(MainWindow.getInstance()); if (screenDialog.showDialog()) { - List selectedScreenshots = screenDialog.getSelectedScreenshots(); - model.setScreenshotImages(selectedScreenshots.get(0), selectedScreenshots.get(1)); + screenshots = screenDialog.getSelectedScreenshots(); } + else + { + return; + } + } + //Update with screenshots + if (screenshots.size() >= 2) + { + scraperManager.updateModelWithScreenshotImages(screenshots.get(0), screenshots.get(1)); + } + else if (screenshots.size() == 1) + { + scraperManager.updateModelWithScreenshotImages(screenshots.get(0), null); } else { - //TODO + //Do nothing } } } - } } diff --git a/src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java b/src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java index 848538a..a2a0d84 100644 --- a/src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java +++ b/src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java @@ -20,6 +20,7 @@ import javax.swing.JTextField; import javax.swing.UIManager; import javax.swing.event.HyperlinkEvent; +import se.lantz.manager.ScraperManager; import se.lantz.model.MainViewModel; import se.lantz.model.data.ScraperFields; import se.lantz.util.ExceptionHandler; @@ -36,7 +37,7 @@ public class MobyGamesOptionsPanel extends JPanel private JCheckBox descriptionCheckBox; private JCheckBox coverCheckBox; private JCheckBox screensCheckBox; - private MainViewModel model; + private ScraperManager scraper; private JButton connectButton; private JLabel connectionStatusLabel; @@ -46,9 +47,9 @@ public class MobyGamesOptionsPanel extends JPanel private JCheckBox genreCheckBox; - public MobyGamesOptionsPanel(MainViewModel model, JButton okButton) + public MobyGamesOptionsPanel(ScraperManager scraper, JButton okButton) { - this.model = model; + this.scraper = scraper; this.okButton = okButton; GridBagLayout gridBagLayout = new GridBagLayout(); setLayout(gridBagLayout); @@ -74,8 +75,8 @@ public class MobyGamesOptionsPanel extends JPanel gbc_connectButton.gridy = 1; add(getConnectButton(), gbc_connectButton); GridBagConstraints gbc_connectionStatusLabel = new GridBagConstraints(); - gbc_connectionStatusLabel.anchor = GridBagConstraints.WEST; - gbc_connectionStatusLabel.insets = new Insets(0, 10, 0, 5); + gbc_connectionStatusLabel.gridwidth = 2; + gbc_connectionStatusLabel.insets = new Insets(0, 10, 10, 5); gbc_connectionStatusLabel.gridx = 0; gbc_connectionStatusLabel.gridy = 2; add(getConnectionStatusLabel(), gbc_connectionStatusLabel); @@ -288,6 +289,8 @@ public class MobyGamesOptionsPanel extends JPanel returnValue.setYear(yearCheckBox.isSelected()); returnValue.setGenre(genreCheckBox.isSelected()); returnValue.setDescription(descriptionCheckBox.isSelected()); + returnValue.setCover(coverCheckBox.isSelected()); + returnValue.setScreenshots(screensCheckBox.isSelected()); return returnValue; } private JButton getConnectButton() { @@ -298,7 +301,7 @@ public class MobyGamesOptionsPanel extends JPanel try { MobyGamesOptionsPanel.this.setCursor(waitCursor); - model.connectScraper(urlTextField.getText()); + scraper.connectScraper(urlTextField.getText()); getConnectionStatusLabel().setText("Connection status: OK"); enableCheckBoxes(true); okButton.setEnabled(true); diff --git a/src/main/java/se/lantz/gui/scraper/ScraperDialog.java b/src/main/java/se/lantz/gui/scraper/ScraperDialog.java index 98d8a81..4899d55 100644 --- a/src/main/java/se/lantz/gui/scraper/ScraperDialog.java +++ b/src/main/java/se/lantz/gui/scraper/ScraperDialog.java @@ -6,17 +6,17 @@ import java.awt.Frame; import javax.swing.JPanel; import se.lantz.gui.BaseDialog; -import se.lantz.model.MainViewModel; +import se.lantz.manager.ScraperManager; import se.lantz.model.data.ScraperFields; public class ScraperDialog extends BaseDialog { private MobyGamesOptionsPanel mbyGamesPanel; - private final MainViewModel model; - public ScraperDialog(Frame owner, MainViewModel model) + private final ScraperManager scraper; + public ScraperDialog(Frame owner, ScraperManager scraper) { super(owner); - this.model = model; + this.scraper = scraper; JPanel content = new JPanel(); content.setLayout(new BorderLayout()); content.add(getMobyGamesPanel(), BorderLayout.CENTER); @@ -29,7 +29,7 @@ public class ScraperDialog extends BaseDialog { if (mbyGamesPanel == null) { - mbyGamesPanel = new MobyGamesOptionsPanel(model, getOkButton()); + mbyGamesPanel = new MobyGamesOptionsPanel(scraper, getOkButton()); } return mbyGamesPanel; } diff --git a/src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java b/src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java new file mode 100644 index 0000000..b0a728d --- /dev/null +++ b/src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java @@ -0,0 +1,72 @@ +package se.lantz.gui.scraper; + +import java.awt.Frame; + +import javax.swing.JDialog; +import java.awt.GridBagLayout; +import javax.swing.JProgressBar; +import javax.swing.WindowConstants; + +import java.awt.GridBagConstraints; +import javax.swing.JLabel; +import java.awt.Insets; + +public class ScraperProgressDialog extends JDialog +{ + private static final long serialVersionUID = 1L; + private JProgressBar progressBar; + private JLabel textLabel; + + public ScraperProgressDialog(Frame frame) + { + super(frame,"Scraping", true); + setAlwaysOnTop(true); + setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE); + setResizable(false); + GridBagLayout gridBagLayout = new GridBagLayout(); + getContentPane().setLayout(gridBagLayout); + GridBagConstraints gbc_progressBar = new GridBagConstraints(); + gbc_progressBar.weightx = 1.0; + gbc_progressBar.anchor = GridBagConstraints.NORTH; + gbc_progressBar.fill = GridBagConstraints.HORIZONTAL; + gbc_progressBar.insets = new Insets(5, 5, 5, 5); + gbc_progressBar.gridx = 0; + gbc_progressBar.gridy = 0; + getContentPane().add(getProgressBar(), gbc_progressBar); + GridBagConstraints gbc_textLabel = new GridBagConstraints(); + gbc_textLabel.insets = new Insets(5, 5, 5, 5); + gbc_textLabel.anchor = GridBagConstraints.NORTH; + gbc_textLabel.weightx = 1.0; + gbc_textLabel.weighty = 1.0; + gbc_textLabel.gridx = 0; + gbc_textLabel.gridy = 1; + getContentPane().add(getTextLabel(), gbc_textLabel); + } + + public void updateProgress() + { + getTextLabel().setText("Fetching Screenshots..."); + this.repaint(); + } + + public void finish() + { + progressBar.setIndeterminate(false); + getProgressBar().setValue( getProgressBar().getMaximum()); + dispose(); + } + + private JProgressBar getProgressBar() { + if (progressBar == null) { + progressBar = new JProgressBar(); + progressBar.setIndeterminate(true); + } + return progressBar; + } + private JLabel getTextLabel() { + if (textLabel == null) { + textLabel = new JLabel("Fetching games information..."); + } + return textLabel; + } +} diff --git a/src/main/java/se/lantz/gui/scraper/ScraperWorker.java b/src/main/java/se/lantz/gui/scraper/ScraperWorker.java new file mode 100644 index 0000000..6296753 --- /dev/null +++ b/src/main/java/se/lantz/gui/scraper/ScraperWorker.java @@ -0,0 +1,45 @@ +package se.lantz.gui.scraper; + +import java.util.List; + +import javax.swing.SwingWorker; + +import se.lantz.manager.ScraperManager; +import se.lantz.model.data.ScraperFields; + +public class ScraperWorker extends SwingWorker +{ + + private final ScraperManager scraperManager; + private final ScraperFields fields; + private ScraperProgressDialog dialog; + + public ScraperWorker(ScraperManager scraperManager, ScraperFields fields, ScraperProgressDialog dialog) + { + this.scraperManager = scraperManager; + this.fields = fields; + this.dialog = dialog; + } + + @Override + protected Void doInBackground() throws Exception + { + scraperManager.scrapeGameInformation(fields); + publish(""); + scraperManager.scrapeScreenshots(); + return null; + } + + @Override + protected void process(List chunks) + { + scraperManager.updateModelWithGamesInfo(); + dialog.updateProgress(); + } + + @Override + protected void done() + { + dialog.finish(); + } +} diff --git a/src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java b/src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java index 9187f63..3488155 100644 --- a/src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java +++ b/src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java @@ -28,11 +28,12 @@ public class ScreenshotsSelectionPanel extends JPanel GridBagConstraints gbc_infoLabel = new GridBagConstraints(); gbc_infoLabel.weightx = 1.0; gbc_infoLabel.anchor = GridBagConstraints.WEST; - gbc_infoLabel.insets = new Insets(10, 5, 5, 0); + gbc_infoLabel.insets = new Insets(10, 10, 5, 0); gbc_infoLabel.gridx = 0; gbc_infoLabel.gridy = 0; add(getInfoLabel(), gbc_infoLabel); GridBagConstraints gbc_screenPanel = new GridBagConstraints(); + gbc_screenPanel.insets = new Insets(5, 10, 10, 10); gbc_screenPanel.weighty = 1.0; gbc_screenPanel.weightx = 1.0; gbc_screenPanel.fill = GridBagConstraints.BOTH; @@ -60,6 +61,7 @@ public class ScreenshotsSelectionPanel extends JPanel { ScreenshotCheckBoxPanel checkBox = new ScreenshotCheckBoxPanel(); checkBox.getImageLabel().setIcon(new ImageIcon(screenshots.get(i))); + checkBox.getCheckBox().setText("Screenshot " + (i+1)); screenshotCheckBoxList.add(checkBox); screenPanel.add(checkBox); } diff --git a/src/main/java/se/lantz/manager/ScraperManager.java b/src/main/java/se/lantz/manager/ScraperManager.java new file mode 100644 index 0000000..f58bac4 --- /dev/null +++ b/src/main/java/se/lantz/manager/ScraperManager.java @@ -0,0 +1,82 @@ +package se.lantz.manager; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.List; + +import se.lantz.model.InfoModel; +import se.lantz.model.data.ScraperFields; +import se.lantz.scraper.MobyGamesScraper; + +public class ScraperManager +{ + MobyGamesScraper scraper = new MobyGamesScraper(); + List screenshotsList = new ArrayList<>(); + private InfoModel infoModel; + private ScraperFields fields; + + public ScraperManager(InfoModel model) + { + this.infoModel = model; + } + + public void connectScraper(String url) throws Exception + { + scraper.connectToMobyGames(url); + } + + public void scrapeGameInformation(ScraperFields fields) + { + this.fields = fields; + scraper.scrapeInformation(fields); + } + + public void scrapeScreenshots() + { + screenshotsList = scraper.scrapeScreenshots(); + } + + + public List getScreenshots() + { + return screenshotsList; + } + + public void updateModelWithGamesInfo() + { + if (fields.isTitle()) + { + infoModel.setTitle(scraper.getTitle()); + } + if (fields.isAuthor()) + { + infoModel.setAuthor(scraper.getAuthor()); + } + if (fields.isYear()) + { + infoModel.setYear(scraper.getYear()); + } + if (fields.isDescription()) + { + infoModel.setDescription(scraper.getDescription()); + } + if (fields.isGenre()) + { + String genre = scraper.getGenre(); + if (!genre.isEmpty()) + { + infoModel.setGenre(genre); + } + } + if (fields.isCover()) + { + infoModel.setCoverImage(scraper.getCover()); + } + } + + public void updateModelWithScreenshotImages(BufferedImage screen1, BufferedImage screen2) + { + infoModel.setScreen1Image(screen1); + infoModel.setScreen2Image(screen2); + } +} diff --git a/src/main/java/se/lantz/model/MainViewModel.java b/src/main/java/se/lantz/model/MainViewModel.java index da52d75..dfa9ee2 100644 --- a/src/main/java/se/lantz/model/MainViewModel.java +++ b/src/main/java/se/lantz/model/MainViewModel.java @@ -20,8 +20,8 @@ import se.lantz.model.data.GameDetails; import se.lantz.model.data.GameListData; import se.lantz.model.data.GameView; import se.lantz.model.data.ScraperFields; +import se.lantz.scraper.MobyGamesScraper; import se.lantz.util.FileManager; -import se.lantz.util.MobyGamesScraper; public class MainViewModel extends AbstractModel { @@ -48,7 +48,7 @@ public class MainViewModel extends AbstractModel private PropertyChangeListener requiredFieldsListener; private GameListData selectedData; - + MobyGamesScraper scraper = new MobyGamesScraper(); public void initialize() @@ -63,10 +63,10 @@ public class MainViewModel extends AbstractModel joy2Model.setPrimaryChangeListener(e -> joy1Model .setPrimaryWithoutListenerNotification(!Boolean.valueOf(e.getActionCommand()))); - + resetDataChangedAfterInit(); } - + private void setupGameViews() { //Setup game views @@ -81,7 +81,7 @@ public class MainViewModel extends AbstractModel gameViewModel.addElement(gameView); } } - + public void reloadGameViews() { this.disableChangeNotification(true); @@ -256,7 +256,7 @@ public class MainViewModel extends AbstractModel return infoModel.isDataChanged() || joy1Model.isDataChanged() || joy2Model.isDataChanged() || systemModel.isDataChanged(); } - + @Override public void resetDataChanged() { @@ -265,7 +265,7 @@ public class MainViewModel extends AbstractModel joy2Model.resetDataChanged(); systemModel.resetDataChanged(); } - + @Override public void resetDataChangedAfterInit() { @@ -371,7 +371,7 @@ public class MainViewModel extends AbstractModel //Reload the current view reloadCurrentGameView(); } - + public void deleteGameView(GameView view) { if (view.getGameViewId() != GameView.ALL_GAMES_ID) @@ -420,53 +420,4 @@ public class MainViewModel extends AbstractModel resetDataChanged(); } } - public void connectScraper(String url) throws Exception - { - scraper.connectToMobyGames(url); - } - - public void scrapeGameInformation(ScraperFields fields) - { - scraper.scrapeInformation(fields); - - if (fields.isTitle()) - { - infoModel.setTitle(scraper.getTitle()); - } - if (fields.isAuthor()) - { - infoModel.setAuthor(scraper.getAuthor()); - } - if (fields.isYear()) - { - infoModel.setYear(scraper.getYear()); - } - if (fields.isDescription()) - { - infoModel.setDescription(scraper.getDescription()); - } - if (fields.isGenre()) - { - String genre = scraper.getGenre(); - if (!genre.isEmpty()) - { - infoModel.setGenre(genre); - } - } - if (fields.isCover()) - { - infoModel.setCoverImage(scraper.getCover()); - } - } - - public List scrapeScreenshots() - { - return scraper.scrapeScreenshots(); - } - - public void setScreenshotImages(BufferedImage screen1, BufferedImage screen2) - { - getInfoModel().setScreen1Image(screen1); - getInfoModel().setScreen2Image(screen2); - } } diff --git a/src/main/java/se/lantz/util/MobyGamesScraper.java b/src/main/java/se/lantz/scraper/MobyGamesScraper.java similarity index 84% rename from src/main/java/se/lantz/util/MobyGamesScraper.java rename to src/main/java/se/lantz/scraper/MobyGamesScraper.java index 7c1a49b..a282a82 100644 --- a/src/main/java/se/lantz/util/MobyGamesScraper.java +++ b/src/main/java/se/lantz/scraper/MobyGamesScraper.java @@ -1,4 +1,4 @@ -package se.lantz.util; +package se.lantz.scraper; import java.awt.image.BufferedImage; import java.io.IOException; @@ -22,17 +22,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import se.lantz.model.data.ScraperFields; +import se.lantz.util.ExceptionHandler; public class MobyGamesScraper { private static final Logger logger = LoggerFactory.getLogger(MobyGamesScraper.class); - private String mobyGamesBaseUrl = "https://www.mobygames.com/game/c64/*"; - private String mobyGamesGameUrl = ""; - private String game = "rushn-attack"; - private String descriptionCssQuery = "#main > div > div:eq(2) > div"; private String titleCssQuery = ".niceHeaderTitle > a"; @@ -65,14 +62,6 @@ public class MobyGamesScraper public MobyGamesScraper() { - // TODO Auto-generated constructor stub - - //*[@id="main"]/div/div[2]/h1/a - - //*[@id="main"]/div/div[3]/div[1]/h2[1] - - //*[@id="coreGameCover"]/a/img - //Keys are Genres defined on MobyGames, values are supported genres in the tool genreMap.put("Adventure, Role-Playing (RPG)", "adventure"); genreMap.put("Racing / driving", "driving"); @@ -135,25 +124,11 @@ public class MobyGamesScraper { scrapedCover = scrapeCover(doc); } - //TODO: Screens } catch (IOException e) { ExceptionHandler.handleException(e, "Could not scrape title"); } - // - // if (fields.isScreenshots()) - // { //TODO: Make it possible to select which screenshot to use - // List images = scraper.scrapeScreenshots(); - // if (images.size() > 0) - // { - // infoModel.setScreen1Image(images.get(0)); - // } - // if (images.size() > 1) - // { - // infoModel.setScreen2Image(images.get(1)); - // } - // } } public String getTitle() @@ -186,29 +161,6 @@ public class MobyGamesScraper return scrapedCover; } - // public String scrapeTitle() - // { - // String value = ""; - // Document doc; - // try - // { - // Connection.Response result = Jsoup.connect(mobyGamesGameUrl).method(Connection.Method.GET).execute(); - // doc = result.parse(); - // //Fetch the right element - // Elements queryElements = doc.select(titleCssQuery); - // Element first = queryElements.first(); - // if (first != null) - // { - // value = first.text(); - // } - // } - // catch (IOException e) - // { - // ExceptionHandler.handleException(e, "Could not scrape title"); - // } - // return value; - // } - public String scrapeDescription(Document doc) { //Fetch the right element @@ -316,9 +268,10 @@ public class MobyGamesScraper Elements coverElements = doc.select(screensCssQuery); logger.debug("Number of screenshots found: {}", coverElements.size()); - for (Element element : coverElements) + //Scrape max 6 screens + for (int i = 0; i < Math.min(6, coverElements.size()); i++) { - String bigScreenUrl = element.attr("href"); + String bigScreenUrl = coverElements.get(i).attr("href"); logger.debug("Screen URL = " + bigScreenUrl); returnList.add(scrapeBigScreenshot(bigScreenUrl)); }