feat: continued work on gb inporter

pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>se.lantz</groupId>
 	<artifactId>PCUGameManager</artifactId>
-	<version>1.4.1</version>
+	<version>1.5.0</version>
 	<name>PCUGameManager</name>
 	<dependencies>
 		<dependency>
@@ -52,6 +52,16 @@
 			<artifactId>gson</artifactId>
 			<version>2.8.0</version>
 		</dependency>
+		<dependency>
+		  <groupId>com.google.guava</groupId>
+		  <artifactId>guava</artifactId>
+		  <version>30.1-jre</version>
+		</dependency>
+		<dependency>
+		    <groupId>net.sf.ucanaccess</groupId>
+		    <artifactId>ucanaccess</artifactId>
+		    <version>4.0.4</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<plugins>

src/main/java/se/lantz/gamebase/GamebaseImporter.java

@@ -13,6 +13,7 @@ import java.sql.Statement;
 
 import se.lantz.manager.ImportManager;
 import se.lantz.scraper.GamebaseScraper;
+import se.lantz.util.ExceptionHandler;
 import se.lantz.util.FileManager;
 
 public class GamebaseImporter
@@ -81,7 +82,7 @@ public class GamebaseImporter
       sql =
         "SELECT Games.Name, Musicians.Musician, Genres.Genre, Publishers.Publisher, Games.Filename, Games.ScrnshotFilename, Years.Year, Games.GA_Id, Games.Control, Games.V_PalNTSC, Games.V_TrueDriveEmu, Games.Gemus\r\n" +
           "FROM Years INNER JOIN (Publishers INNER JOIN ((Games INNER JOIN Musicians ON Games.MU_Id = Musicians.MU_Id) INNER JOIN Genres ON Games.GE_Id = Genres.GE_Id) ON Publishers.PU_Id = Games.PU_Id) ON Years.YE_Id = Games.YE_Id\r\n" +
-          "WHERE (((Games.Name)='Birds, The'));";
+          "WHERE (((Games.Name) LIKE 'alien attack'));";
 
       result = statement.executeQuery(sql);
       int gameCount = 0;
@@ -103,6 +104,12 @@ public class GamebaseImporter
           int trueDriveEmu = result.getInt("V_TrueDriveEmu");
           String gemus = result.getString("Gemus");
                     
+          if (gamefile.isEmpty())
+          {
+            builder.append("Ignoring " + title + " (No game file available)\n");
+            continue;
+          }
+          
           //Setup video mode
           //0=PAL, 1=BOTH, 2=NTSC, 3=PAL[+NTSC?]
           String video = (palOrNtsc == 2)  ? "ntsc" : "pal";
@@ -127,8 +134,7 @@ public class GamebaseImporter
             joy1config = "J:1" + joyBase;
             joy2config = "J:2*" + joyBase;
           }
-          //Fix game file, but only for C64?
+          
-          gamefile = getFileToInclude(gbDatabasePath, gamefile);//isC64 ? getFileToInclude(gbDatabasePath, gamefile) : gamefile;
 
           //Fix screenshots
           screen1 = gbDatabasePath.toString() + "\\screenshots\\" + screen1;
@@ -152,6 +158,38 @@ public class GamebaseImporter
           {
             coverFile = gbDatabasePath.toString() + "\\extras\\" + coverFile;
           }
+          
+                   
+          //Get cartridge if available, easyflash is preferred
+          String cartridgeSql =
+            "SELECT Extras.Name, Extras.Path\r\n" + "FROM Games INNER JOIN Extras ON Games.GA_Id = Extras.GA_Id\r\n" +
+              "WHERE (((Games.GA_Id)=" + gameId + ") AND ((Extras.Name) Like \"*Cartridge*\"));";
+
+          sqlResult = statement.executeQuery(cartridgeSql);
+          String cartridgePath = "";
+          while (sqlResult.next())
+          {
+            if (cartridgePath.isEmpty())
+            {
+              cartridgePath = sqlResult.getString("Path");
+            }
+            //Pick easyflash if available
+            String name = sqlResult.getString("Name");
+            if (name.contains("EasyFlash"))
+            {
+              cartridgePath = sqlResult.getString("Path");
+            }
+          }
+          
+          if (!cartridgePath.isEmpty())
+          {
+            gamefile = gbDatabasePath.toString() + "\\extras\\" + cartridgePath;
+          }
+          
+          //Fix game file
+          gamefile = getFileToInclude(gbDatabasePath, gamefile);
+          
+          
           importManager.addFromGamebaseImporter(title,
                                                 year,
                                                 publisher,
@@ -169,6 +207,7 @@ public class GamebaseImporter
         catch (Exception e)
         {
           builder.append("ERROR: Could not fetch all info for " + title + ":\n" + e.toString() + "\n");
+          ExceptionHandler.handleException(e, "Could not fetch all info for " + title);
         }
       }
       builder.append("Read " + gameCount + " games from the gamebase db.\n");
@@ -176,6 +215,7 @@ public class GamebaseImporter
     catch (SQLException ex)
     {
       builder.append("ERROR: Query failed : " + ex.toString() + "\n");
+      ExceptionHandler.handleException(ex, "Query failed");
     }
     return builder;
   }

src/main/java/se/lantz/gui/imports/CarouselImportWorker.java

@@ -32,7 +32,10 @@ public class CarouselImportWorker extends SwingWorker<Void, String>
     publish("Importing to db...");
     publish(importManager.insertRowsIntoDb().toString());
     publish("Copying screenshots, covers and game files...");
-    publish(importManager.copyFiles(false).toString());
+    for (List<String> rowList : importManager.getDbRowReadChunks())
+    {
+      publish(importManager.copyFiles(false, rowList).toString());
+    }
     importManager.clearAfterImport();
     publish("Done!");
     return null;

src/main/java/se/lantz/gui/imports/GamebaseImportWorker.java

@@ -30,7 +30,10 @@ public class GamebaseImportWorker extends SwingWorker<Void, String>
     publish("Importing to db...");
     publish(importManager.insertRowsIntoDb().toString());
     publish("Copying screenshots, covers and game files...");
-    publish(importManager.copyFiles(true).toString());
+    for (List<String> rowList : importManager.getDbRowReadChunks())
+    {
+      publish(importManager.copyFiles(true, rowList).toString());
+    }
     importManager.clearAfterImport();
     publish("Done!");
     return null;

src/main/java/se/lantz/gui/imports/ImportProgressPanel.java

@@ -96,6 +96,11 @@ public class ImportProgressPanel extends JPanel
     //Check for errors
     String text = getTextArea().getText();
     int count = text.length() - text.replace("ERROR:", "").length();
+    int ignoreCount = text.length() - text.replace("Ignoring", "").length();
+    if (ignoreCount > 0)
+    {
+      getTextArea().append("\n" + ignoreCount/8 + " games ignored.\n");
+    }
     if (count > 0)
     {
       getTextArea().append("\nImport ended with " + count/6 + " errors. See log file for details.");

src/main/java/se/lantz/manager/ImportManager.java

@@ -24,6 +24,8 @@ import javax.imageio.ImageIO;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Lists;
+
 import se.lantz.model.MainViewModel;
 import se.lantz.util.ExceptionHandler;
 import se.lantz.util.FileManager;
@@ -429,9 +431,9 @@ public class ImportManager
     if (selectedOption == Options.ADD)
     {
       duplicateIndex = uiModel.getDbConnector().getGameDuplicateIndexToUse(title);     
-      //Check any duplicates in added rows also
-      duplicateIndex = duplicateIndex + getDbRowDuplicate(title);
     }
+    //Check any duplicates in added rows, always use this otherwise duplicates uses same names.
+    duplicateIndex = duplicateIndex + getDbRowDuplicate(title);
     return duplicateIndex;
   }
 
@@ -463,11 +465,17 @@ public class ImportManager
     return lines;
   }
   
-  public StringBuilder copyFiles(boolean gamebaseImport)
+  
+  public List<List<String>> getDbRowReadChunks()
+  {
+    return Lists.partition(dbRowDataList, 50);
+  }
+
+  public StringBuilder copyFiles(boolean gamebaseImport, List<String> rowList)
   {
     StringBuilder infoBuilder = new StringBuilder();
     //Copy with the existing file names. At export convert to a name that works with the maxi tool if needed.
-    dbRowDataList.stream().forEach(dbRowData -> copyFiles(dbRowData, infoBuilder, gamebaseImport));
+    rowList.stream().forEach(dbRowData -> copyFiles(dbRowData, infoBuilder, gamebaseImport));
     return infoBuilder;
   }
 

src/main/java/se/lantz/util/FileManager.java

@@ -316,7 +316,7 @@ public class FileManager
     }
     // Do the conversion
     List<Character> forbiddenCharsList =
-      " ,:'’-.!+*<>()".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
+      " ,:'’-.!+*<>()/".chars().mapToObj(item -> (char) item).collect(Collectors.toList());
 
     List<Character> newName =
       title.chars().mapToObj(item -> (char) item).filter(character -> !forbiddenCharsList.contains(character))