feat: add scraping of multiple covers from MobyGames

A selection dialog is shown if there is more than one front cover image.
This commit is contained in:
mikael.lantz 2021-06-29 14:29:56 +02:00
parent 678c636b38
commit cec5b25e4a
12 changed files with 402 additions and 87 deletions

View File

@ -10,6 +10,8 @@ import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.KeyEvent;
import java.awt.image.BufferedImage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
@ -29,6 +31,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.lantz.gui.ScrollablePanel.ScrollableSizeHint;
import se.lantz.gui.scraper.CoverSelectionDialog;
import se.lantz.gui.scraper.ScraperDialog;
import se.lantz.gui.scraper.ScraperProgressDialog;
import se.lantz.gui.scraper.ScraperWorker;
@ -315,6 +318,34 @@ public class GameDetailsBackgroundPanel extends JPanel
dialog.setVisible(true);
MainWindow.getInstance().setWaitCursor(false);
if (scraperFields.isCover())
{
List<BufferedImage> covers = scraperManager.getCovers();
if (covers.size() > 1)
{
//Show dialog for selecting cover image
CoverSelectionDialog coverDialog = new CoverSelectionDialog(MainWindow.getInstance(), covers);
coverDialog.pack();
coverDialog.setLocationRelativeTo(MainWindow.getInstance());
if (coverDialog.showDialog())
{
covers = Arrays.asList(coverDialog.getSelectedCover());
}
else
{
covers = new ArrayList<>();
}
}
//Update with cover
if (covers.size() == 1)
{
scraperManager.updateModelWithCoverImage(covers.get(0));
}
else
{
//Do nothing
}
}
if (scraperFields.isScreenshots())
{
List<BufferedImage> screenshots = scraperManager.getScreenshots();
@ -331,7 +362,7 @@ public class GameDetailsBackgroundPanel extends JPanel
}
else
{
return;
screenshots = new ArrayList<>();
}
}
//Update with screenshots

View File

@ -0,0 +1,49 @@
package se.lantz.gui.scraper;
import java.awt.LayoutManager;
import javax.swing.JPanel;
import javax.swing.JRadioButton;
import java.awt.GridBagLayout;
import javax.swing.JLabel;
import java.awt.GridBagConstraints;
import javax.swing.JCheckBox;
import java.awt.Insets;
public class CoverRadioButtonPanel extends JPanel
{
private JLabel imageLabel;
private JRadioButton radioButton;
public CoverRadioButtonPanel()
{
GridBagLayout gridBagLayout = new GridBagLayout();
setLayout(gridBagLayout);
GridBagConstraints gbc_imageLabel = new GridBagConstraints();
gbc_imageLabel.weightx = 1.0;
gbc_imageLabel.insets = new Insets(0, 0, 5, 0);
gbc_imageLabel.gridx = 0;
gbc_imageLabel.gridy = 0;
add(getImageLabel(), gbc_imageLabel);
GridBagConstraints gbc_button = new GridBagConstraints();
gbc_button.anchor = GridBagConstraints.NORTH;
gbc_button.weighty = 1.0;
gbc_button.weightx = 1.0;
gbc_button.gridx = 0;
gbc_button.gridy = 1;
add(getRadioButton(), gbc_button);
}
public JLabel getImageLabel() {
if (imageLabel == null) {
imageLabel = new JLabel("");
}
return imageLabel;
}
public JRadioButton getRadioButton() {
if (radioButton == null) {
radioButton = new JRadioButton("");
}
return radioButton;
}
}

View File

@ -0,0 +1,42 @@
package se.lantz.gui.scraper;
import java.awt.BorderLayout;
import java.awt.Frame;
import java.awt.image.BufferedImage;
import java.util.List;
import javax.swing.JPanel;
import se.lantz.gui.BaseDialog;
public class CoverSelectionDialog extends BaseDialog
{
private CoverSelectionPanel mbyGamesPanel;
private List<BufferedImage> coverInfoList;
public CoverSelectionDialog(Frame owner, List<BufferedImage> coverInfoList)
{
super(owner);
this.coverInfoList = coverInfoList;
JPanel content = new JPanel();
content.setLayout(new BorderLayout());
content.add(getCoverSelectionPanel(), BorderLayout.CENTER);
addContent(content);
setTitle("Scrape game information");
this.setResizable(false);
}
private CoverSelectionPanel getCoverSelectionPanel()
{
if (mbyGamesPanel == null)
{
mbyGamesPanel = new CoverSelectionPanel(coverInfoList);
}
return mbyGamesPanel;
}
public BufferedImage getSelectedCover()
{
return getCoverSelectionPanel().getSelectedCover();
}
}

View File

@ -0,0 +1,146 @@
package se.lantz.gui.scraper;
import java.awt.Dimension;
import java.awt.GridBagConstraints;
import java.awt.GridBagLayout;
import java.awt.GridLayout;
import java.awt.Insets;
import java.awt.Rectangle;
import java.awt.image.BufferedImage;
import java.util.ArrayList;
import java.util.List;
import javax.swing.ButtonGroup;
import javax.swing.ImageIcon;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.Scrollable;
public class CoverSelectionPanel extends JPanel
{
private List<BufferedImage> covers;
private List<CoverRadioButtonPanel> coverRadioButtonList = new ArrayList<>();
private JLabel infoLabel;
private CoversPanel coverPanel;
private JScrollPane scrollPane;
private ButtonGroup buttonGroup = new ButtonGroup();
public CoverSelectionPanel(List<BufferedImage> covers)
{
this.covers = covers;
GridBagLayout gridBagLayout = new GridBagLayout();
setLayout(gridBagLayout);
GridBagConstraints gbc_infoLabel = new GridBagConstraints();
gbc_infoLabel.weightx = 1.0;
gbc_infoLabel.anchor = GridBagConstraints.WEST;
gbc_infoLabel.insets = new Insets(10, 10, 5, 0);
gbc_infoLabel.gridx = 0;
gbc_infoLabel.gridy = 0;
add(getInfoLabel(), gbc_infoLabel);
GridBagConstraints gbc_scrollPane = new GridBagConstraints();
gbc_scrollPane.insets = new Insets(0, 5, 5, 5);
gbc_scrollPane.weighty = 1.0;
gbc_scrollPane.weightx = 1.0;
gbc_scrollPane.fill = GridBagConstraints.BOTH;
gbc_scrollPane.gridx = 0;
gbc_scrollPane.gridy = 1;
add(getScrollPane(), gbc_scrollPane);
}
private JLabel getInfoLabel()
{
if (infoLabel == null)
{
infoLabel = new JLabel("Select one cover below:");
}
return infoLabel;
}
private CoversPanel getCoverPanel()
{
if (coverPanel == null)
{
coverPanel = new CoversPanel();
for (int i = 0; i < covers.size(); i++)
{
CoverRadioButtonPanel radioButton = new CoverRadioButtonPanel();
radioButton.getImageLabel().setIcon(new ImageIcon(covers.get(i)));
radioButton.getRadioButton().setText("Cover " + (i + 1));
coverRadioButtonList.add(radioButton);
coverPanel.add(radioButton);
buttonGroup.add(radioButton.getRadioButton());
if (i == 0)
{
radioButton.getRadioButton().setSelected(true);
}
}
}
return coverPanel;
}
public BufferedImage getSelectedCover()
{
for (int i = 0; i < coverRadioButtonList.size(); i++)
{
if (coverRadioButtonList.get(i).getRadioButton().isSelected())
{
return covers.get(i);
}
}
return null;
}
private JScrollPane getScrollPane()
{
if (scrollPane == null)
{
scrollPane = new JScrollPane();
scrollPane.setViewportView(getCoverPanel());
scrollPane.setPreferredSize(new Dimension(Math.min(1400, getCoverPanel().getPreferredSize().width + 2),
Math.min(800, getCoverPanel().getPreferredSize().height + 2)));
}
return scrollPane;
}
}
class CoversPanel extends JPanel implements Scrollable
{
public CoversPanel()
{
this.setLayout(new GridLayout(1, 0, 5, 5));
}
@Override
public Dimension getPreferredScrollableViewportSize()
{
// TODO Auto-generated method stub
return null;
}
@Override
public int getScrollableUnitIncrement(Rectangle visibleRect, int orientation, int direction)
{
return 10;
}
@Override
public int getScrollableBlockIncrement(Rectangle visibleRect, int orientation, int direction)
{
return 40;
}
@Override
public boolean getScrollableTracksViewportWidth()
{
return false;
}
@Override
public boolean getScrollableTracksViewportHeight()
{
return false;
}
}

View File

@ -42,9 +42,9 @@ public class ScraperProgressDialog extends JDialog
getContentPane().add(getTextLabel(), gbc_textLabel);
}
public void updateProgress()
public void updateProgress(String message)
{
getTextLabel().setText("Fetching Screenshots...");
getTextLabel().setText(message);
this.repaint();
}

View File

@ -1,5 +1,6 @@
package se.lantz.gui.scraper;
import java.util.Arrays;
import java.util.List;
import javax.swing.SwingWorker;
@ -27,8 +28,15 @@ public class ScraperWorker extends SwingWorker<Void, String>
{
scraperManager.scrapeGameInformation(fields);
publish("");
if (fields.isCover())
{
publish("Fetching Covers...");
scraperManager.scrapeCovers();
}
if (fields.isScreenshots())
{
publish("Fetching Screenshots...");
scraperManager.scrapeScreenshots();
}
return null;
@ -37,8 +45,15 @@ public class ScraperWorker extends SwingWorker<Void, String>
@Override
protected void process(List<String> chunks)
{
scraperManager.updateModelWithGamesInfo();
dialog.updateProgress();
if (chunks.get(0).isEmpty())
{
scraperManager.updateModelWithGamesInfo();
}
for (String chunk : chunks)
{
dialog.updateProgress(chunk);
}
}
@Override

View File

@ -26,6 +26,7 @@ public class ScraperManager
Scraper usedScraper = mobyScraper;
List<BufferedImage> screenshotsList = new ArrayList<>();
List<BufferedImage> coversList = new ArrayList<>();
private InfoModel infoModel;
private SystemModel systemModel;
private ScraperFields fields;
@ -74,6 +75,16 @@ public class ScraperManager
{
return screenshotsList;
}
public void scrapeCovers()
{
coversList = usedScraper.scrapeCovers();
}
public List<BufferedImage> getCovers()
{
return coversList;
}
public void updateModelWithGamesInfo()
{
@ -111,11 +122,7 @@ public class ScraperManager
{
infoModel.setComposer(usedScraper.getComposer());
}
if (fields.isCover())
{
infoModel.setCoverImage(usedScraper.getCover());
}
if (fields.isGame())
{
File scrapedFile = usedScraper.getGameFile();
@ -140,4 +147,9 @@ public class ScraperManager
infoModel.setScreen1Image(screen1);
infoModel.setScreen2Image(screen2);
}
public void updateModelWithCoverImage(BufferedImage cover)
{
infoModel.setCoverImage(cover);
}
}

View File

@ -5,6 +5,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -297,6 +298,13 @@ public class C64comScraper implements Scraper
return screensList;
}
@Override
public List<BufferedImage> scrapeCovers()
{
return Arrays.asList(scrapedCover);
}
@Override
public String getTitle()
{

View File

@ -8,6 +8,7 @@ import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -271,6 +272,12 @@ public class GamebaseScraper implements Scraper
}
return screensList;
}
@Override
public List<BufferedImage> scrapeCovers()
{
return new ArrayList<>();
}
private void scrapeGame(Document doc)
{

View File

@ -10,6 +10,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.imageio.ImageIO;
@ -25,6 +26,7 @@ import org.slf4j.LoggerFactory;
import se.lantz.model.data.ScraperFields;
import se.lantz.util.ExceptionHandler;
import se.lantz.util.FileManager;
public class MobyGamesScraper implements Scraper
{
@ -36,8 +38,8 @@ public class MobyGamesScraper implements Scraper
private String authorCssQuery = "#coreGameRelease > div:contains(Published)";
private String yearCssQuery = "#coreGameRelease > div:contains(Released)";
private String genreCssQuery = "#coreGameGenre > div > div:contains(Genre)";
private String coverCssQuery = "#coreGameCover > a > img";
private String screensCssQuery = ".thumbnail-image-wrapper > a";
private String coversCssQuery = ".thumbnail-image-wrapper > a";
Map<String, String> genreMap = new HashMap<>();
private String scrapedTitle = "";
@ -131,11 +133,6 @@ public class MobyGamesScraper implements Scraper
{
scrapedComposer = scrapeComposer(doc);
}
if (fields.isCover())
{
scrapedCover = scrapeCover(doc);
}
}
catch (IOException e)
{
@ -272,48 +269,11 @@ public class MobyGamesScraper implements Scraper
return value;
}
public BufferedImage scrapeCover(Document doc)
{
//Fetch the right element
Elements coverElements = doc.select(coverCssQuery);
if (coverElements.first() != null)
{
Element coverElement = coverElements.first();
String bigCoverUrl = coverElement.parent().attr("href");
return scrapeBigCover(bigCoverUrl);
}
return null;
}
private BufferedImage scrapeBigCover(String url)
{
String cssQuery = "#main > div > div:eq(1) > center > img"; //*[@id="main"]/div/div[2]/center/img
Document doc;
try
{
Connection.Response result = Jsoup.connect(url).method(Connection.Method.GET).execute();
doc = result.parse();
//Fetch the right element
Elements coverElements = doc.select(cssQuery);
if (coverElements.first() != null)
{
Element coverElement = coverElements.first();
String absoluteUrl = coverElement.absUrl("src");
URL imageUrl = new URL(absoluteUrl);
return ImageIO.read(imageUrl);
}
}
catch (IOException e)
{
ExceptionHandler.handleException(e, "Could not scrape cover");
}
return null;
}
@Override
public List<BufferedImage> scrapeScreenshots()
{
String cssQueryForBigScreenshot = "#main > div > div:eq(1) > div > div > img"; //*[@id="main"]/div/div[2]/div/div/img
List<BufferedImage> returnList = new ArrayList<>();
Document doc;
try
@ -330,7 +290,7 @@ public class MobyGamesScraper implements Scraper
{
String bigScreenUrl = coverElements.get(i).attr("href");
logger.debug("Screen URL = " + bigScreenUrl);
returnList.add(scrapeBigScreenshot(bigScreenUrl));
returnList.add(scrapeBigImage(bigScreenUrl, cssQueryForBigScreenshot));
}
}
catch (IOException e)
@ -340,9 +300,43 @@ public class MobyGamesScraper implements Scraper
return returnList;
}
private BufferedImage scrapeBigScreenshot(String url)
@Override
public List<BufferedImage> scrapeCovers()
{
String cssQueryForBigCover = "#main > div > div:eq(1) > center > img"; //*[@id="main"]/div/div[2]/center/img
List<BufferedImage> returnList = new ArrayList<>();
Document doc;
try
{
Connection.Response result =
Jsoup.connect(mobyGamesGameUrl + "/cover-art").method(Connection.Method.GET).execute();
doc = result.parse();
//Fetch the right element
Elements coverElements = doc.select(coversCssQuery);
logger.debug("Number of cover art found: {}", coverElements.size());
List<Element> filteredElements = coverElements.stream()
.filter(element -> element.attr("title").contains("Front Cover")).collect(Collectors.toList());
//Scrape max 6 covers
for (int i = 0; i < Math.min(6, filteredElements.size()); i++)
{
String bigScreenUrl = filteredElements.get(i).attr("href");
logger.debug("Screen URL = " + bigScreenUrl);
BufferedImage scrapedImage = FileManager.getScaledCoverImage(scrapeBigImage(bigScreenUrl, cssQueryForBigCover));
returnList.add(scrapedImage);
}
}
catch (IOException e)
{
ExceptionHandler.handleException(e, "Could not scrape cover");
}
return returnList;
}
private BufferedImage scrapeBigImage(String url, String cssQuery)
{
String cssQuery = "#main > div > div:eq(1) > div > div > img"; //*[@id="main"]/div/div[2]/div/div/img
Document doc;
try
{

View File

@ -30,6 +30,8 @@ public interface Scraper
BufferedImage getCover();
List<BufferedImage> scrapeScreenshots();
List<BufferedImage> scrapeCovers();
boolean isC64();

View File

@ -132,15 +132,18 @@ public class FileManager
{
try
{
Image coverToSave = cover.getScaledInstance(122, 175, Image.SCALE_SMOOTH);
BufferedImage copyOfImage =
new BufferedImage(coverToSave.getWidth(null), coverToSave.getHeight(null), BufferedImage.TYPE_INT_ARGB);
Graphics g = copyOfImage.createGraphics();
g.drawImage(coverToSave, 0, 0, null);
g.dispose();
BufferedImage imageToSave = cover;
if (cover.getWidth() != 122 || cover.getHeight() != 175)
{
Image scaledCoverImage = cover.getScaledInstance(122, 175, Image.SCALE_SMOOTH);
imageToSave =
new BufferedImage(scaledCoverImage.getWidth(null), scaledCoverImage.getHeight(null), BufferedImage.TYPE_INT_ARGB);
Graphics g = imageToSave.createGraphics();
g.drawImage(scaledCoverImage, 0, 0, null);
g.dispose();
}
File outputfile = new File(COVERS + coverFileName);
ImageIO.write(copyOfImage, "png", outputfile);
ImageIO.write(imageToSave, "png", outputfile);
}
catch (IOException e)
{
@ -351,7 +354,7 @@ public class FileManager
}
// Do the conversion
List<Character> forbiddenCharsList =
" ,:'-.!+*<>()/[]?|".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
" ,:'<EFBFBD>-.!+*<>()/[]?|".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
List<Character> newName =
title.chars().mapToObj(item -> (char) item).filter(character -> !forbiddenCharsList.contains(character))
@ -366,15 +369,14 @@ public class FileManager
logger.debug("Game title: \"{}\" ---- New fileName: \"{}\"", title, newNameString);
return newNameString;
}
public static String generateFileNameFromTitleForFileLoader(String title, int duplicateIndex)
{
List<Character> forbiddenCharsList =
":,'!+*<>()/[]?|".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
":,'<EFBFBD>!+*<>()/[]?|".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
List<Character> newName =
title.chars().mapToObj(item -> (char) item).filter(character -> !forbiddenCharsList.contains(character))
.collect(Collectors.toList());
List<Character> newName = title.chars().mapToObj(item -> (char) item)
.filter(character -> !forbiddenCharsList.contains(character)).collect(Collectors.toList());
String newNameString = newName.stream().map(String::valueOf).collect(Collectors.joining());
if (duplicateIndex > 0)
{
@ -391,11 +393,12 @@ public class FileManager
try
{
String filename = "";
if (fileLoader)
{
infoBuilder.append("Creating cjm file for " + gameDetails.getTitle() + "\n");
filename = generateFileNameFromTitleForFileLoader(gameDetails.getTitle(), gameDetails.getDuplicateIndex()) + ".cjm";
filename =
generateFileNameFromTitleForFileLoader(gameDetails.getTitle(), gameDetails.getDuplicateIndex()) + ".cjm";
}
else
{
@ -403,7 +406,7 @@ public class FileManager
//Add -ms to comply with the maxi game tool.
filename = generateFileNameFromTitle(gameDetails.getTitle(), gameDetails.getDuplicateIndex()) + "-ms.tsg";
}
writeGameInfoFile(filename, targetDir, gameDetails, fileLoader);
}
catch (Exception e)
@ -414,13 +417,14 @@ public class FileManager
}
}
public void writeGameInfoFile(String fileName, File targetDir, GameDetails gameDetails, boolean fileLoader) throws IOException
public void writeGameInfoFile(String fileName, File targetDir, GameDetails gameDetails, boolean fileLoader)
throws IOException
{
Path outDirPath = targetDir.toPath();
Path filePath = outDirPath.resolve(fileName);
filePath.toFile().createNewFile();
FileWriter fw = new FileWriter(filePath.toFile());
if (!fileLoader)
{
fw.write("T:" + gameDetails.getTitle() + "\n");
@ -444,7 +448,7 @@ public class FileManager
}
fw.write("E:" + gameDetails.getGenre() + "\n");
fw.write("Y:" + gameDetails.getYear() + "\n");
fw.write("F:" + "games/" + gameDetails.getGame() + "\n");
fw.write("C:" + "covers/" + gameDetails.getCover() + "\n");
if (!gameDetails.getScreen1().isEmpty())
@ -996,14 +1000,8 @@ public class FileManager
{
try
{
BufferedImage coverImage = ImageIO.read(coverImagePath.toFile());
Image newCover = coverImage.getScaledInstance(122, 175, Image.SCALE_SMOOTH);
BufferedImage copyOfImage =
new BufferedImage(newCover.getWidth(null), newCover.getHeight(null), BufferedImage.TYPE_INT_ARGB);
Graphics g = copyOfImage.createGraphics();
g.drawImage(newCover, 0, 0, null);
g.dispose();
ImageIO.write(copyOfImage, "png", coverImagePath.toFile());
BufferedImage coverImage = getScaledCoverImage(ImageIO.read(coverImagePath.toFile()));
ImageIO.write(coverImage, "png", coverImagePath.toFile());
}
catch (IOException e)
{
@ -1021,6 +1019,17 @@ public class FileManager
}
}
public static BufferedImage getScaledCoverImage(BufferedImage originalCoverImage)
{
Image newCover = originalCoverImage.getScaledInstance(122, 175, Image.SCALE_SMOOTH);
BufferedImage copyOfImage =
new BufferedImage(newCover.getWidth(null), newCover.getHeight(null), BufferedImage.TYPE_INT_ARGB);
Graphics g = copyOfImage.createGraphics();
g.drawImage(newCover, 0, 0, null);
g.dispose();
return copyOfImage;
}
public static void scaleScreenshotImageAndSave(Path screenshotImagePath, String gameName)
{
try
@ -1278,7 +1287,7 @@ public class FileManager
}
return filePath != null ? filePath.toFile() : file;
}
private static FileHeader getFirstMatchingRarEntry(Archive archive)
{
FileHeader fh = archive.nextFileHeader();