mirrors/PCUGameManager
mirrors
/
PCUGameManager
mirror of https://github.com/lantzelot-swe/PCUGameManager.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Moby games scraper finished

This commit is contained in:
lantzelot-swe 2020-12-26 23:48:02 +01:00
parent 38315712b0
commit 19df5c4b8b
10 changed files with 282 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/se/lantz/gui/BaseDialog.java
View File

@ -124,6 +124,7 @@ public class BaseDialog extends JDialog
public boolean showDialog()
{
okPressed = false;
this.setVisible(true);
return okPressed;
}

73
src/main/java/se/lantz/gui/GameDetailsBackgroundPanel.java
View File

@ -19,7 +19,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.lantz.gui.scraper.ScraperDialog;
import se.lantz.gui.scraper.ScraperProgressDialog;
import se.lantz.gui.scraper.ScraperWorker;
import se.lantz.gui.scraper.ScreenshotsSelectionDialog;
import se.lantz.manager.ScraperManager;
import se.lantz.model.MainViewModel;
import se.lantz.model.data.GameListData;
import se.lantz.model.data.ScraperFields;
@ -29,7 +32,7 @@ public class GameDetailsBackgroundPanel extends JPanel
private static final String EMPTY = "empty";
private static final String DETAILS = "details";
private static final Logger logger = LoggerFactory.getLogger(GameDetailsBackgroundPanel.class);
private MainViewModel model;
private final MainViewModel model;
private JPanel settingsPanel;
private InfoPanel infoPanel;
private CombinedJoystickPanel joystickPanel;
@ -42,9 +45,12 @@ public class GameDetailsBackgroundPanel extends JPanel
private CardLayout cardLayout;
private JButton scrapeButton;
private final ScraperManager scraperManager;
public GameDetailsBackgroundPanel(MainViewModel model)
{
this.model = model;
this.scraperManager = new ScraperManager(model.getInfoModel());
this.setMinimumSize(new Dimension(1250, 800));
cardLayout = new CardLayout();
setLayout(cardLayout);
@ -198,57 +204,82 @@ public class GameDetailsBackgroundPanel extends JPanel
}
return saveButton;
}
private JButton getScrapeButton() {
if (scrapeButton == null) {
scrapeButton = new JButton("Scrape...");
scrapeButton.addActionListener(new ActionListener() {
public void actionPerformed(ActionEvent e) {
scrapeGamesInformation();
}
});
private JButton getScrapeButton()
{
if (scrapeButton == null)
{
scrapeButton = new JButton("Scrape...");
scrapeButton.addActionListener(new ActionListener()
{
public void actionPerformed(ActionEvent e)
{
scrapeGamesInformation();
}
});
}
return scrapeButton;
}
private ScraperDialog scraperDialog = null;
private void scrapeGamesInformation()
{
if (scraperDialog == null)
{
scraperDialog = new ScraperDialog(MainWindow.getInstance(), model);
scraperDialog = new ScraperDialog(MainWindow.getInstance(), scraperManager);
scraperDialog.pack();
scraperDialog.setLocationRelativeTo(MainWindow.getInstance());
}
if (scraperDialog.showDialog())
{
MainWindow.getInstance().setWaitCursor(true);
ScraperFields scraperFields = scraperDialog.getScraperFields();
model.scrapeGameInformation(scraperFields);
ScraperProgressDialog dialog = new ScraperProgressDialog(MainWindow.getInstance());
dialog.pack();
dialog.setLocationRelativeTo(MainWindow.getInstance());
ScraperWorker worker = new ScraperWorker(scraperManager, scraperFields, dialog);
worker.execute();
dialog.setVisible(true);
MainWindow.getInstance().setWaitCursor(false);
if (scraperFields.isScreenshots())
{
//Scrape the screens and check how many there are.
List<BufferedImage> screenshots = model.scrapeScreenshots();
List<BufferedImage> screenshots = scraperManager.getScreenshots();
if (screenshots.size() > 2)
{
//Show dialog for selecting screenshots
ScreenshotsSelectionDialog screenDialog = new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots);
ScreenshotsSelectionDialog screenDialog =
new ScreenshotsSelectionDialog(MainWindow.getInstance(), screenshots);
screenDialog.pack();
screenDialog.setLocationRelativeTo(MainWindow.getInstance());
if (screenDialog.showDialog())
{
List<BufferedImage> selectedScreenshots = screenDialog.getSelectedScreenshots();
model.setScreenshotImages(selectedScreenshots.get(0), selectedScreenshots.get(1));
screenshots = screenDialog.getSelectedScreenshots();
}
else
{
return;
}
}
//Update with screenshots
if (screenshots.size() >= 2)
{
scraperManager.updateModelWithScreenshotImages(screenshots.get(0), screenshots.get(1));
}
else if (screenshots.size() == 1)
{
scraperManager.updateModelWithScreenshotImages(screenshots.get(0), null);
}
else
{
//TODO
//Do nothing
}
}
}
}
}

15
src/main/java/se/lantz/gui/scraper/MobyGamesOptionsPanel.java
View File

@ -20,6 +20,7 @@ import javax.swing.JTextField;
import javax.swing.UIManager;
import javax.swing.event.HyperlinkEvent;
import se.lantz.manager.ScraperManager;
import se.lantz.model.MainViewModel;
import se.lantz.model.data.ScraperFields;
import se.lantz.util.ExceptionHandler;
@ -36,7 +37,7 @@ public class MobyGamesOptionsPanel extends JPanel
private JCheckBox descriptionCheckBox;
private JCheckBox coverCheckBox;
private JCheckBox screensCheckBox;
private MainViewModel model;
private ScraperManager scraper;
private JButton connectButton;
private JLabel connectionStatusLabel;
@ -46,9 +47,9 @@ public class MobyGamesOptionsPanel extends JPanel
private JCheckBox genreCheckBox;
public MobyGamesOptionsPanel(MainViewModel model, JButton okButton)
public MobyGamesOptionsPanel(ScraperManager scraper, JButton okButton)
{
this.model = model;
this.scraper = scraper;
this.okButton = okButton;
GridBagLayout gridBagLayout = new GridBagLayout();
setLayout(gridBagLayout);
@ -74,8 +75,8 @@ public class MobyGamesOptionsPanel extends JPanel
gbc_connectButton.gridy = 1;
add(getConnectButton(), gbc_connectButton);
GridBagConstraints gbc_connectionStatusLabel = new GridBagConstraints();
gbc_connectionStatusLabel.anchor = GridBagConstraints.WEST;
gbc_connectionStatusLabel.insets = new Insets(0, 10, 0, 5);
gbc_connectionStatusLabel.gridwidth = 2;
gbc_connectionStatusLabel.insets = new Insets(0, 10, 10, 5);
gbc_connectionStatusLabel.gridx = 0;
gbc_connectionStatusLabel.gridy = 2;
add(getConnectionStatusLabel(), gbc_connectionStatusLabel);
@ -288,6 +289,8 @@ public class MobyGamesOptionsPanel extends JPanel
returnValue.setYear(yearCheckBox.isSelected());
returnValue.setGenre(genreCheckBox.isSelected());
returnValue.setDescription(descriptionCheckBox.isSelected());
returnValue.setCover(coverCheckBox.isSelected());
returnValue.setScreenshots(screensCheckBox.isSelected());
return returnValue;
}
private JButton getConnectButton() {
@ -298,7 +301,7 @@ public class MobyGamesOptionsPanel extends JPanel
try
{
MobyGamesOptionsPanel.this.setCursor(waitCursor);
model.connectScraper(urlTextField.getText());
scraper.connectScraper(urlTextField.getText());
getConnectionStatusLabel().setText("Connection status: OK");
enableCheckBoxes(true);
okButton.setEnabled(true);

10
src/main/java/se/lantz/gui/scraper/ScraperDialog.java
View File

@ -6,17 +6,17 @@ import java.awt.Frame;
import javax.swing.JPanel;
import se.lantz.gui.BaseDialog;
import se.lantz.model.MainViewModel;
import se.lantz.manager.ScraperManager;
import se.lantz.model.data.ScraperFields;
public class ScraperDialog extends BaseDialog
{
private MobyGamesOptionsPanel mbyGamesPanel;
private final MainViewModel model;
public ScraperDialog(Frame owner, MainViewModel model)
private final ScraperManager scraper;
public ScraperDialog(Frame owner, ScraperManager scraper)
{
super(owner);
this.model = model;
this.scraper = scraper;
JPanel content = new JPanel();
content.setLayout(new BorderLayout());
content.add(getMobyGamesPanel(), BorderLayout.CENTER);
@ -29,7 +29,7 @@ public class ScraperDialog extends BaseDialog
{
if (mbyGamesPanel == null)
{
mbyGamesPanel = new MobyGamesOptionsPanel(model, getOkButton());
mbyGamesPanel = new MobyGamesOptionsPanel(scraper, getOkButton());
}
return mbyGamesPanel;
}

72
src/main/java/se/lantz/gui/scraper/ScraperProgressDialog.java Normal file
View File

@ -0,0 +1,72 @@
package se.lantz.gui.scraper;
import java.awt.Frame;
import javax.swing.JDialog;
import java.awt.GridBagLayout;
import javax.swing.JProgressBar;
import javax.swing.WindowConstants;
import java.awt.GridBagConstraints;
import javax.swing.JLabel;
import java.awt.Insets;
public class ScraperProgressDialog extends JDialog
{
private static final long serialVersionUID = 1L;
private JProgressBar progressBar;
private JLabel textLabel;
public ScraperProgressDialog(Frame frame)
{
super(frame,"Scraping", true);
setAlwaysOnTop(true);
setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE);
setResizable(false);
GridBagLayout gridBagLayout = new GridBagLayout();
getContentPane().setLayout(gridBagLayout);
GridBagConstraints gbc_progressBar = new GridBagConstraints();
gbc_progressBar.weightx = 1.0;
gbc_progressBar.anchor = GridBagConstraints.NORTH;
gbc_progressBar.fill = GridBagConstraints.HORIZONTAL;
gbc_progressBar.insets = new Insets(5, 5, 5, 5);
gbc_progressBar.gridx = 0;
gbc_progressBar.gridy = 0;
getContentPane().add(getProgressBar(), gbc_progressBar);
GridBagConstraints gbc_textLabel = new GridBagConstraints();
gbc_textLabel.insets = new Insets(5, 5, 5, 5);
gbc_textLabel.anchor = GridBagConstraints.NORTH;
gbc_textLabel.weightx = 1.0;
gbc_textLabel.weighty = 1.0;
gbc_textLabel.gridx = 0;
gbc_textLabel.gridy = 1;
getContentPane().add(getTextLabel(), gbc_textLabel);
}
public void updateProgress()
{
getTextLabel().setText("Fetching Screenshots...");
this.repaint();
}
public void finish()
{
progressBar.setIndeterminate(false);
getProgressBar().setValue( getProgressBar().getMaximum());
dispose();
}
private JProgressBar getProgressBar() {
if (progressBar == null) {
progressBar = new JProgressBar();
progressBar.setIndeterminate(true);
}
return progressBar;
}
private JLabel getTextLabel() {
if (textLabel == null) {
textLabel = new JLabel("Fetching games information...");
}
return textLabel;
}
}

45
src/main/java/se/lantz/gui/scraper/ScraperWorker.java Normal file
View File

@ -0,0 +1,45 @@
package se.lantz.gui.scraper;
import java.util.List;
import javax.swing.SwingWorker;
import se.lantz.manager.ScraperManager;
import se.lantz.model.data.ScraperFields;
public class ScraperWorker extends SwingWorker<Void, String>
{
private final ScraperManager scraperManager;
private final ScraperFields fields;
private ScraperProgressDialog dialog;
public ScraperWorker(ScraperManager scraperManager, ScraperFields fields, ScraperProgressDialog dialog)
{
this.scraperManager = scraperManager;
this.fields = fields;
this.dialog = dialog;
}
@Override
protected Void doInBackground() throws Exception
{
scraperManager.scrapeGameInformation(fields);
publish("");
scraperManager.scrapeScreenshots();
return null;
}
@Override
protected void process(List<String> chunks)
{
scraperManager.updateModelWithGamesInfo();
dialog.updateProgress();
}
@Override
protected void done()
{
dialog.finish();
}
}

4
src/main/java/se/lantz/gui/scraper/ScreenshotsSelectionPanel.java
View File

@ -28,11 +28,12 @@ public class ScreenshotsSelectionPanel extends JPanel
GridBagConstraints gbc_infoLabel = new GridBagConstraints();
gbc_infoLabel.weightx = 1.0;
gbc_infoLabel.anchor = GridBagConstraints.WEST;
gbc_infoLabel.insets = new Insets(10, 5, 5, 0);
gbc_infoLabel.insets = new Insets(10, 10, 5, 0);
gbc_infoLabel.gridx = 0;
gbc_infoLabel.gridy = 0;
add(getInfoLabel(), gbc_infoLabel);
GridBagConstraints gbc_screenPanel = new GridBagConstraints();
gbc_screenPanel.insets = new Insets(5, 10, 10, 10);
gbc_screenPanel.weighty = 1.0;
gbc_screenPanel.weightx = 1.0;
gbc_screenPanel.fill = GridBagConstraints.BOTH;
@ -60,6 +61,7 @@ public class ScreenshotsSelectionPanel extends JPanel
{
ScreenshotCheckBoxPanel checkBox = new ScreenshotCheckBoxPanel();
checkBox.getImageLabel().setIcon(new ImageIcon(screenshots.get(i)));
checkBox.getCheckBox().setText("Screenshot " + (i+1));
screenshotCheckBoxList.add(checkBox);
screenPanel.add(checkBox);
}

82
src/main/java/se/lantz/manager/ScraperManager.java Normal file
View File

@ -0,0 +1,82 @@
package se.lantz.manager;
import java.awt.image.BufferedImage;
import java.util.ArrayList;
import java.util.List;
import se.lantz.model.InfoModel;
import se.lantz.model.data.ScraperFields;
import se.lantz.scraper.MobyGamesScraper;
public class ScraperManager
{
MobyGamesScraper scraper = new MobyGamesScraper();
List<BufferedImage> screenshotsList = new ArrayList<>();
private InfoModel infoModel;
private ScraperFields fields;
public ScraperManager(InfoModel model)
{
this.infoModel = model;
}
public void connectScraper(String url) throws Exception
{
scraper.connectToMobyGames(url);
}
public void scrapeGameInformation(ScraperFields fields)
{
this.fields = fields;
scraper.scrapeInformation(fields);
}
public void scrapeScreenshots()
{
screenshotsList = scraper.scrapeScreenshots();
}
public List<BufferedImage> getScreenshots()
{
return screenshotsList;
}
public void updateModelWithGamesInfo()
{
if (fields.isTitle())
{
infoModel.setTitle(scraper.getTitle());
}
if (fields.isAuthor())
{
infoModel.setAuthor(scraper.getAuthor());
}
if (fields.isYear())
{
infoModel.setYear(scraper.getYear());
}
if (fields.isDescription())
{
infoModel.setDescription(scraper.getDescription());
}
if (fields.isGenre())
{
String genre = scraper.getGenre();
if (!genre.isEmpty())
{
infoModel.setGenre(genre);
}
}
if (fields.isCover())
{
infoModel.setCoverImage(scraper.getCover());
}
}
public void updateModelWithScreenshotImages(BufferedImage screen1, BufferedImage screen2)
{
infoModel.setScreen1Image(screen1);
infoModel.setScreen2Image(screen2);
}
}

65
src/main/java/se/lantz/model/MainViewModel.java
View File

@ -20,8 +20,8 @@ import se.lantz.model.data.GameDetails;
import se.lantz.model.data.GameListData;
import se.lantz.model.data.GameView;
import se.lantz.model.data.ScraperFields;
import se.lantz.scraper.MobyGamesScraper;
import se.lantz.util.FileManager;
import se.lantz.util.MobyGamesScraper;
public class MainViewModel extends AbstractModel
{
@ -48,7 +48,7 @@ public class MainViewModel extends AbstractModel
private PropertyChangeListener requiredFieldsListener;
private GameListData selectedData;
MobyGamesScraper scraper = new MobyGamesScraper();
public void initialize()
@ -63,10 +63,10 @@ public class MainViewModel extends AbstractModel
joy2Model.setPrimaryChangeListener(e -> joy1Model
.setPrimaryWithoutListenerNotification(!Boolean.valueOf(e.getActionCommand())));
resetDataChangedAfterInit();
}
private void setupGameViews()
{
//Setup game views
@ -81,7 +81,7 @@ public class MainViewModel extends AbstractModel
gameViewModel.addElement(gameView);
}
}
public void reloadGameViews()
{
this.disableChangeNotification(true);
@ -256,7 +256,7 @@ public class MainViewModel extends AbstractModel
return infoModel.isDataChanged() || joy1Model.isDataChanged() || joy2Model.isDataChanged() ||
systemModel.isDataChanged();
}
@Override
public void resetDataChanged()
{
@ -265,7 +265,7 @@ public class MainViewModel extends AbstractModel
joy2Model.resetDataChanged();
systemModel.resetDataChanged();
}
@Override
public void resetDataChangedAfterInit()
{
@ -371,7 +371,7 @@ public class MainViewModel extends AbstractModel
//Reload the current view
reloadCurrentGameView();
}
public void deleteGameView(GameView view)
{
if (view.getGameViewId() != GameView.ALL_GAMES_ID)
@ -420,53 +420,4 @@ public class MainViewModel extends AbstractModel
resetDataChanged();
}
}
public void connectScraper(String url) throws Exception
{
scraper.connectToMobyGames(url);
}
public void scrapeGameInformation(ScraperFields fields)
{
scraper.scrapeInformation(fields);
if (fields.isTitle())
{
infoModel.setTitle(scraper.getTitle());
}
if (fields.isAuthor())
{
infoModel.setAuthor(scraper.getAuthor());
}
if (fields.isYear())
{
infoModel.setYear(scraper.getYear());
}
if (fields.isDescription())
{
infoModel.setDescription(scraper.getDescription());
}
if (fields.isGenre())
{
String genre = scraper.getGenre();
if (!genre.isEmpty())
{
infoModel.setGenre(genre);
}
}
if (fields.isCover())
{
infoModel.setCoverImage(scraper.getCover());
}
}
public List<BufferedImage> scrapeScreenshots()
{
return scraper.scrapeScreenshots();
}
public void setScreenshotImages(BufferedImage screen1, BufferedImage screen2)
{
getInfoModel().setScreen1Image(screen1);
getInfoModel().setScreen2Image(screen2);
}
}

57
src/main/java/se/lantz/util/MobyGamesScraper.java → src/main/java/se/lantz/scraper/MobyGamesScraper.java
View File

@ -1,4 +1,4 @@
package se.lantz.util;
package se.lantz.scraper;
import java.awt.image.BufferedImage;
import java.io.IOException;
@ -22,17 +22,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.lantz.model.data.ScraperFields;
import se.lantz.util.ExceptionHandler;
public class MobyGamesScraper
{
private static final Logger logger = LoggerFactory.getLogger(MobyGamesScraper.class);
private String mobyGamesBaseUrl = "https://www.mobygames.com/game/c64/*";
private String mobyGamesGameUrl = "";
private String game = "rushn-attack";
private String descriptionCssQuery = "#main > div > div:eq(2) > div";
private String titleCssQuery = ".niceHeaderTitle > a";
@ -65,14 +62,6 @@ public class MobyGamesScraper
public MobyGamesScraper()
{
// TODO Auto-generated constructor stub
//*[@id="main"]/div/div[2]/h1/a
//*[@id="main"]/div/div[3]/div[1]/h2[1]
//*[@id="coreGameCover"]/a/img
//Keys are Genres defined on MobyGames, values are supported genres in the tool
genreMap.put("Adventure, Role-Playing (RPG)", "adventure");
genreMap.put("Racing / driving", "driving");
@ -135,25 +124,11 @@ public class MobyGamesScraper
{
scrapedCover = scrapeCover(doc);
}
//TODO: Screens
}
catch (IOException e)
{
ExceptionHandler.handleException(e, "Could not scrape title");
}
//
// if (fields.isScreenshots())
// { //TODO: Make it possible to select which screenshot to use
// List<BufferedImage> images = scraper.scrapeScreenshots();
// if (images.size() > 0)
// {
// infoModel.setScreen1Image(images.get(0));
// }
// if (images.size() > 1)
// {
// infoModel.setScreen2Image(images.get(1));
// }
// }
}
public String getTitle()
@ -186,29 +161,6 @@ public class MobyGamesScraper
return scrapedCover;
}
// public String scrapeTitle()
// {
// String value = "";
// Document doc;
// try
// {
// Connection.Response result = Jsoup.connect(mobyGamesGameUrl).method(Connection.Method.GET).execute();
// doc = result.parse();
// //Fetch the right element
// Elements queryElements = doc.select(titleCssQuery);
// Element first = queryElements.first();
// if (first != null)
// {
// value = first.text();
// }
// }
// catch (IOException e)
// {
// ExceptionHandler.handleException(e, "Could not scrape title");
// }
// return value;
// }
public String scrapeDescription(Document doc)
{
//Fetch the right element
@ -316,9 +268,10 @@ public class MobyGamesScraper
Elements coverElements = doc.select(screensCssQuery);
logger.debug("Number of screenshots found: {}", coverElements.size());
for (Element element : coverElements)
//Scrape max 6 screens
for (int i = 0; i < Math.min(6, coverElements.size()); i++)
{
String bigScreenUrl = element.attr("href");
String bigScreenUrl = coverElements.get(i).attr("href");
logger.debug("Screen URL = " + bigScreenUrl);
returnList.add(scrapeBigScreenshot(bigScreenUrl));
}