From f8348c0347b4604a96868709a7984337a4077f8b Mon Sep 17 00:00:00 2001 From: lantzelot-swe Date: Sun, 7 Mar 2021 22:47:10 +0100 Subject: [PATCH] feat: more gb import fixes --- .../se/lantz/gamebase/GamebaseImporter.java | 131 +++++++++----- .../java/se/lantz/gamebase/GbGameInfo.java | 166 ++++++++++++++++++ .../gui/imports/CarouselImportWorker.java | 3 +- .../gui/imports/GamebaseImportWorker.java | 63 ++++++- .../gui/imports/ImportProgressDialog.java | 17 +- .../gui/imports/ImportProgressPanel.java | 35 ++-- .../java/se/lantz/util/ExceptionHandler.java | 5 + src/main/java/se/lantz/util/FileManager.java | 2 +- 8 files changed, 359 insertions(+), 63 deletions(-) create mode 100644 src/main/java/se/lantz/gamebase/GbGameInfo.java diff --git a/src/main/java/se/lantz/gamebase/GamebaseImporter.java b/src/main/java/se/lantz/gamebase/GamebaseImporter.java index 17634b9..a0c6140 100644 --- a/src/main/java/se/lantz/gamebase/GamebaseImporter.java +++ b/src/main/java/se/lantz/gamebase/GamebaseImporter.java @@ -10,6 +10,10 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.Lists; import se.lantz.manager.ImportManager; import se.lantz.scraper.GamebaseScraper; @@ -18,6 +22,8 @@ import se.lantz.util.FileManager; public class GamebaseImporter { + private final int CHUNK_SIZE = 100; + public enum Options { ALL, FAVORITES, QUERY; @@ -28,6 +34,8 @@ public class GamebaseImporter private Path gbDatabasePath = Path.of("C://GameBase//GBC_V16//");//Path.of("C://GameBase//Vic20_v03//"); private boolean isC64 = true;//false; + private List gbGameInfoList = new ArrayList<>(); + public GamebaseImporter(ImportManager importManager) { this.importManager = importManager; @@ -36,19 +44,20 @@ public class GamebaseImporter public void setImportOptions(GamebaseOptions options) { -// this.gbDatabasePath = options.getGamebaseDbFile(); -// this.isC64 = options.isC64(); + // this.gbDatabasePath = options.getGamebaseDbFile(); + // this.isC64 = options.isC64(); } public StringBuilder importFromGamebase() { + gbGameInfoList.clear(); StringBuilder builder = new StringBuilder(); //Use the folder where the gamebase mdb file is located in the import manager -// importManager.setSelectedFolder(gbDatabasePath.getParent()); + // importManager.setSelectedFolder(gbDatabasePath.getParent()); importManager.setSelectedFolder(gbDatabasePath); //Just for test, use gbDatabasePath - "jdbc:ucanaccess:" + gbDatabasePath.toString() String vic20Test = "jdbc:ucanaccess://C://GameBase//Vic20_v03//Vic20_v03.mdb"; - + String databaseURL = "jdbc:ucanaccess://F://Github//PCUGameManager//GBC_v16.mdb"; String joyBase = ":JU,JD,JL,JR,JF,JF,SP,EN,,F1,F3,F5,,,"; @@ -57,15 +66,15 @@ public class GamebaseImporter { Statement statement = connection.createStatement(); //Get views -// -// String sql = "SELECT * FROM ViewData"; -// -// ResultSet result = statement.executeQuery(sql); -// while (result.next()) -// { -// String title = result.getString("Title"); -// System.out.println("view: " + title); -// } + // + // String sql = "SELECT * FROM ViewData"; + // + // ResultSet result = statement.executeQuery(sql); + // while (result.next()) + // { + // String title = result.getString("Title"); + // System.out.println("view: " + title); + // } String sql = "SELECT TOP 300 Games.Name, Musicians.Musician, Genres.Genre, Publishers.Publisher, Games.Filename, Games.ScrnshotFilename, Years.Year, Games.GA_Id, Games.Control, Games.V_PalNTSC, Games.V_TrueDriveEmu, Games.Gemus\r\n" + @@ -91,16 +100,16 @@ public class GamebaseImporter int palOrNtsc = result.getInt("V_PalNTSC"); int trueDriveEmu = result.getInt("V_TrueDriveEmu"); String gemus = result.getString("Gemus"); - + if (gamefile.isEmpty()) { builder.append("Ignoring " + title + " (No game file available)\n"); continue; } - + //Setup advanced string (system, sid, pal, truedrive etc) String advanced = constructAdvancedString(palOrNtsc, trueDriveEmu, gemus); - + //Control: 0=JoyPort2, 1=JoyPort1, 2=Keyboard, 3=PaddlePort2, 4=PaddlePort1, 5=Mouse, 6=LightPen, 7=KoalaPad, 8=LightGun //Setup joystick port String joy1config; @@ -117,11 +126,14 @@ public class GamebaseImporter joy1config = "J:1" + joyBase; joy2config = "J:2*" + joyBase; } - //Fix screenshots - screen1 = gbDatabasePath.toString() + "\\screenshots\\" + screen1; - String screen2 = getScreen2(screen1); + String screen2 = ""; + if (screen1 != null && !screen1.isEmpty()) + { + screen1 = gbDatabasePath.toString() + "\\screenshots\\" + screen1; + screen2 = getScreen2(screen1); + } //Map genre properly towards existing ones for the carousel genre = GamebaseScraper.mapGenre(genre); @@ -141,7 +153,7 @@ public class GamebaseImporter { coverFile = gbDatabasePath.toString() + "\\extras\\" + coverFile; } - + //Get cartridge if available, easyflash is preferred String cartridgeSql = "SELECT Extras.Name, Extras.Path\r\n" + "FROM Games INNER JOIN Extras ON Games.GA_Id = Extras.GA_Id\r\n" + @@ -162,27 +174,26 @@ public class GamebaseImporter cartridgePath = sqlResult.getString("Path"); } } - + if (!cartridgePath.isEmpty()) { gamefile = gbDatabasePath.toString() + "\\extras\\" + cartridgePath; } - - //Fix game file - gamefile = getFileToInclude(gbDatabasePath, gamefile); - - importManager.addFromGamebaseImporter(title, - year, - publisher, - musician, - genre, - gamefile, - coverFile, - screen1, - screen2, - joy1config, - joy2config, - advanced); + + GbGameInfo info = new GbGameInfo(title, + year, + publisher, + musician, + genre, + gamefile, + coverFile, + screen1, + screen2, + joy1config, + joy2config, + advanced); + + gbGameInfoList.add(info); gameCount++; } catch (Exception e) @@ -200,7 +211,42 @@ public class GamebaseImporter } return builder; } - + + public List> getGbGameInfoChunks() + { + return Lists.partition(gbGameInfoList, CHUNK_SIZE); + } + + public StringBuilder checkGameFileForGbGames(List gbGameList) + { + StringBuilder builder = new StringBuilder(); + for (GbGameInfo gbGameInfo : gbGameList) + { + try + { + String gameFile = getFileToInclude(gbDatabasePath, gbGameInfo.getGamefile()); + importManager.addFromGamebaseImporter(gbGameInfo.getTitle(), + gbGameInfo.getYear(), + gbGameInfo.getPublisher(), + gbGameInfo.getMusician(), + gbGameInfo.getGenre(), + gameFile, + gbGameInfo.getCoverFile(), + gbGameInfo.getScreen1(), + gbGameInfo.getScreen2(), + gbGameInfo.getJoy1config(), + gbGameInfo.getJoy2config(), + gbGameInfo.getAdvanced()); + } + catch (Exception e) + { + builder.append("Ignoring " + gbGameInfo.getTitle() + ", Could not check game file (file is corrupt?). Game is not imported.\n"); + ExceptionHandler.logException(e, "Could not check game file for " + gbGameInfo.getTitle() + ", game is not imported"); + } + } + return builder; + } + private String constructAdvancedString(int palOrNtsc, int trueDriveEmu, String gemus) { //Setup advanced string (system, sid, pal, truedrive etc) @@ -215,7 +261,7 @@ public class GamebaseImporter } //Setup video mode //0=PAL, 1=BOTH, 2=NTSC, 3=PAL[+NTSC?] - String video = (palOrNtsc == 2) ? "ntsc" : "pal"; + String video = (palOrNtsc == 2) ? "ntsc" : "pal"; advanced = advanced + "," + video; //Setup truedrive if (trueDriveEmu > 0 || "vte=yes".equalsIgnoreCase(gemus)) @@ -228,6 +274,7 @@ public class GamebaseImporter private String getScreen2(String screen1) { String returnValue = ""; + String screen2 = screen1.substring(0, screen1.lastIndexOf(".")) + "_1.png"; File screen2File = new File(screen2); if (screen2File.exists()) @@ -252,5 +299,9 @@ public class GamebaseImporter FileManager.compressGzip(selectedFile.toPath(), compressedFilePath); return compressedFilePath.toString(); } - + + public void clearAfterImport() + { + gbGameInfoList.clear(); + } } diff --git a/src/main/java/se/lantz/gamebase/GbGameInfo.java b/src/main/java/se/lantz/gamebase/GbGameInfo.java new file mode 100644 index 0000000..b7632bc --- /dev/null +++ b/src/main/java/se/lantz/gamebase/GbGameInfo.java @@ -0,0 +1,166 @@ +package se.lantz.gamebase; + +public class GbGameInfo +{ + + private String title; + private String year; + private String publisher; + private String musician; + private String genre; + private String gamefile; + private String coverFile; + private String screen1; + private String screen2; + private String joy1config; + private String joy2config; + private String advanced; + + public GbGameInfo(String title, + String year, + String publisher, + String musician, + String genre, + String gamefile, + String coverFile, + String screen1, + String screen2, + String joy1config, + String joy2config, + String advanced) + { + this.title = title; + this.year = year; + this.publisher = publisher; + this.musician = musician; + this.genre = genre; + this.gamefile = gamefile; + this.coverFile = coverFile; + this.screen1 = screen1; + this.screen2 = screen2; + this.joy1config = joy1config; + this.joy2config = joy2config; + this.advanced = advanced; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getYear() + { + return year; + } + + public void setYear(String year) + { + this.year = year; + } + + public String getPublisher() + { + return publisher; + } + + public void setPublisher(String publisher) + { + this.publisher = publisher; + } + + public String getMusician() + { + return musician; + } + + public void setMusician(String musician) + { + this.musician = musician; + } + + public String getGenre() + { + return genre; + } + + public void setGenre(String genre) + { + this.genre = genre; + } + + public String getGamefile() + { + return gamefile; + } + + public void setGamefile(String gamefile) + { + this.gamefile = gamefile; + } + + public String getCoverFile() + { + return coverFile; + } + + public void setCoverFile(String coverFile) + { + this.coverFile = coverFile; + } + + public String getScreen1() + { + return screen1; + } + + public void setScreen1(String screen1) + { + this.screen1 = screen1; + } + + public String getScreen2() + { + return screen2; + } + + public void setScreen2(String screen2) + { + this.screen2 = screen2; + } + + public String getJoy1config() + { + return joy1config; + } + + public void setJoy1config(String joy1config) + { + this.joy1config = joy1config; + } + + public String getJoy2config() + { + return joy2config; + } + + public void setJoy2config(String joy2config) + { + this.joy2config = joy2config; + } + + public String getAdvanced() + { + return advanced; + } + + public void setAdvanced(String advanced) + { + this.advanced = advanced; + } + +} diff --git a/src/main/java/se/lantz/gui/imports/CarouselImportWorker.java b/src/main/java/se/lantz/gui/imports/CarouselImportWorker.java index 9b4d6cd..3d978a7 100644 --- a/src/main/java/se/lantz/gui/imports/CarouselImportWorker.java +++ b/src/main/java/se/lantz/gui/imports/CarouselImportWorker.java @@ -41,7 +41,7 @@ public class CarouselImportWorker extends SwingWorker publish(importManager.copyFiles(false, copyList).toString()); } int numberOfGamesProcessed = importManager.clearAfterImport(); - publish("Imported " + numberOfGamesProcessed + " games."); + publish("Processed " + numberOfGamesProcessed + " games."); publish("Done!"); return null; } @@ -67,5 +67,6 @@ public class CarouselImportWorker extends SwingWorker ExceptionHandler.handleException(e, "Error during import"); } dialog.finish(); + importManager.clearAfterImport(); } } diff --git a/src/main/java/se/lantz/gui/imports/GamebaseImportWorker.java b/src/main/java/se/lantz/gui/imports/GamebaseImportWorker.java index 75deed0..d8eb7cf 100644 --- a/src/main/java/se/lantz/gui/imports/GamebaseImportWorker.java +++ b/src/main/java/se/lantz/gui/imports/GamebaseImportWorker.java @@ -7,6 +7,7 @@ import java.util.concurrent.ExecutionException; import javax.swing.SwingWorker; import se.lantz.gamebase.GamebaseImporter; +import se.lantz.gamebase.GbGameInfo; import se.lantz.manager.ImportManager; import se.lantz.util.ExceptionHandler; @@ -15,6 +16,10 @@ public class GamebaseImportWorker extends SwingWorker private ImportManager importManager; private ImportProgressDialog dialog; private final GamebaseImporter gbInporter; + + private volatile String progressValueString = ""; + private volatile int progressMaximum = 0; + private volatile int progressValue = 0; public GamebaseImportWorker(GamebaseImporter gamebaseImporter, ImportManager importManager, ImportProgressDialog dialog) { @@ -26,20 +31,53 @@ public class GamebaseImportWorker extends SwingWorker @Override protected Void doInBackground() throws Exception { - publish("Reading from gamebase db..."); + publish("Reading from gamebase db... this may take a while, be patient!"); publish(gbInporter.importFromGamebase().toString()); - publish("Importing to db..."); - for (List rowList : importManager.getDbRowReadChunks()) + progressValueString = "Checking game files..."; + + List> listChunks = gbInporter.getGbGameInfoChunks(); + progressMaximum = listChunks.size(); + progressValue = 0; + for (List gbInfoList : listChunks) { + if (dialog.isCancelled()) + { + progressValueString = "Cancelled"; + progressMaximum = 1; + progressValue = 1; + publish("Import cancelled, no games added to the db."); + return null; + } + progressValue++; + publish(gbInporter.checkGameFileForGbGames(gbInfoList).toString()); + } + + List> dbRowReadChunks = importManager.getDbRowReadChunks(); + progressValueString = "Importing to db, copying covers, screens and game files..."; + progressMaximum = dbRowReadChunks.size() + 1; + progressValue = 0; + publish("Importing to db, copying covers, screens and game files..."); + for (List rowList : dbRowReadChunks) + { + if (dialog.isCancelled()) + { + progressValueString = "Cancelled"; + progressMaximum = 1; + progressValue = 1; + publish("Import cancelled, some games where added to the db."); + return null; + } + progressValue++; //Copy the list to avoid modifying it when reading several chunks ArrayList copyList = new ArrayList<>(); copyList.addAll(rowList); publish(importManager.insertRowsIntoDb(copyList).toString()); - publish("Copying screenshots, covers and game files..."); publish(importManager.copyFiles(true, copyList).toString()); } int numberOfGamesProcessed = importManager.clearAfterImport(); - publish("Imported " + numberOfGamesProcessed + " games."); + publish("Processed " + numberOfGamesProcessed + " games."); + progressValueString = "Finished!"; + progressValue++; publish("Done!"); return null; } @@ -49,7 +87,18 @@ public class GamebaseImportWorker extends SwingWorker { for (String value : chunks) { - dialog.updateProgress(value + "\n"); + if (value.isEmpty()) + { + dialog.updateProgress(""); + } + else + { + dialog.updateProgress(value + "\n"); + } + if (!progressValueString.isEmpty()) + { + dialog.updateProgressBar(progressValueString, progressMaximum, progressValue); + } } } @@ -65,5 +114,7 @@ public class GamebaseImportWorker extends SwingWorker ExceptionHandler.handleException(e, "Error during import"); } dialog.finish(); + importManager.clearAfterImport(); + gbInporter.clearAfterImport(); } } diff --git a/src/main/java/se/lantz/gui/imports/ImportProgressDialog.java b/src/main/java/se/lantz/gui/imports/ImportProgressDialog.java index eaa0548..2d86c4d 100644 --- a/src/main/java/se/lantz/gui/imports/ImportProgressDialog.java +++ b/src/main/java/se/lantz/gui/imports/ImportProgressDialog.java @@ -10,6 +10,8 @@ public class ImportProgressDialog extends JDialog private static final long serialVersionUID = 1L; private ImportProgressPanel panel; + + private boolean cancelled = false; public ImportProgressDialog(Frame frame) { @@ -27,9 +29,17 @@ public class ImportProgressDialog extends JDialog this.repaint(); } + public void updateProgressBar(String valuestring, int maximum, int value) + { + getImportProgressPanel().updateProgressBar(valuestring, maximum, value); + this.repaint(); + } + public void finish() { getImportProgressPanel().finish(); + getImportProgressPanel().getCancelButton().setText("Close"); + getImportProgressPanel().getCancelButton().addActionListener(e -> setVisible(false)); } public ImportProgressPanel getImportProgressPanel() @@ -37,8 +47,13 @@ public class ImportProgressDialog extends JDialog if (panel == null) { panel = new ImportProgressPanel(); - panel.getCloseButton().addActionListener(e -> setVisible(false)); + panel.getCancelButton().addActionListener(e -> cancelled = true); } return panel; } + + public boolean isCancelled() + { + return cancelled; + } } diff --git a/src/main/java/se/lantz/gui/imports/ImportProgressPanel.java b/src/main/java/se/lantz/gui/imports/ImportProgressPanel.java index d3c165a..e0c401e 100644 --- a/src/main/java/se/lantz/gui/imports/ImportProgressPanel.java +++ b/src/main/java/se/lantz/gui/imports/ImportProgressPanel.java @@ -15,7 +15,7 @@ public class ImportProgressPanel extends JPanel private JProgressBar progressBar; private JTextArea textArea; private JScrollPane textScrollPane; - private JButton closeButton; + private JButton cancelButton; public ImportProgressPanel() { @@ -36,11 +36,11 @@ public class ImportProgressPanel extends JPanel gbc_textScrollPane.gridx = 0; gbc_textScrollPane.gridy = 1; add(getTextScrollPane(), gbc_textScrollPane); - GridBagConstraints gbc_closeButton = new GridBagConstraints(); - gbc_closeButton.insets = new Insets(0, 5, 5, 5); - gbc_closeButton.gridx = 0; - gbc_closeButton.gridy = 2; - add(getCloseButton(), gbc_closeButton); + GridBagConstraints gbc_cancelButton = new GridBagConstraints(); + gbc_cancelButton.insets = new Insets(0, 5, 5, 5); + gbc_cancelButton.gridx = 0; + gbc_cancelButton.gridy = 2; + add(getCancelButton(), gbc_cancelButton); } private JProgressBar getProgressBar() @@ -49,6 +49,7 @@ public class ImportProgressPanel extends JPanel { progressBar = new JProgressBar(); progressBar.setIndeterminate(true); + progressBar.setStringPainted(true); } return progressBar; } @@ -73,14 +74,13 @@ public class ImportProgressPanel extends JPanel return textScrollPane; } - JButton getCloseButton() + JButton getCancelButton() { - if (closeButton == null) + if (cancelButton == null) { - closeButton = new JButton("Close"); - closeButton.setEnabled(false); + cancelButton = new JButton("Cancel"); } - return closeButton; + return cancelButton; } void updateProgress(String infoText) @@ -88,9 +88,16 @@ public class ImportProgressPanel extends JPanel getTextArea().append(infoText); } + void updateProgressBar(String valuestring, int maximum, int value) + { + getProgressBar().setString(valuestring); + getProgressBar().setIndeterminate(false); + getProgressBar().setMaximum(maximum); + getProgressBar().setValue(value); + } + public void finish() { - getCloseButton().setEnabled(true); getProgressBar().setIndeterminate(false); getProgressBar().setValue(getProgressBar().getMaximum()); //Check for errors @@ -99,7 +106,7 @@ public class ImportProgressPanel extends JPanel int ignoreCount = text.length() - text.replace("Ignoring", "").length(); if (ignoreCount > 0) { - getTextArea().append("\n" + ignoreCount/8 + " games ignored.\n"); + getTextArea().append("\n" + ignoreCount/8 + " games ignored (missing or corrupted game files).\n"); } if (count > 0) { @@ -107,7 +114,7 @@ public class ImportProgressPanel extends JPanel } else { - getTextArea().append("\nImport completed successfully."); + getTextArea().append("\nImport completed."); } } } diff --git a/src/main/java/se/lantz/util/ExceptionHandler.java b/src/main/java/se/lantz/util/ExceptionHandler.java index 21ab9f5..71cc73b 100644 --- a/src/main/java/se/lantz/util/ExceptionHandler.java +++ b/src/main/java/se/lantz/util/ExceptionHandler.java @@ -139,6 +139,11 @@ public class ExceptionHandler extends JDialog ExceptionHandler.dialogIsOpen = visible; super.setVisible(visible); } + + public static void logException(final Throwable ex, final String message) + { + logger.error(message, ex); + } public static void handleException(final Throwable ex, final String message) { diff --git a/src/main/java/se/lantz/util/FileManager.java b/src/main/java/se/lantz/util/FileManager.java index 21baae3..801b263 100644 --- a/src/main/java/se/lantz/util/FileManager.java +++ b/src/main/java/se/lantz/util/FileManager.java @@ -316,7 +316,7 @@ public class FileManager } // Do the conversion List forbiddenCharsList = - " ,:'’-.!+*<>()/".chars().mapToObj(item -> (char) item).collect(Collectors.toList()); + " ,:'’-.!+*<>()/[]".chars().mapToObj(item -> (char) item).collect(Collectors.toList()); List newName = title.chars().mapToObj(item -> (char) item).filter(character -> !forbiddenCharsList.contains(character))