translation files are now assumed UTF8 and the parser can distinguish between a UTF8 and an Unicode (UTF-16 LE) BOM

Sandboxie/msgs/parse.c

@@ -223,9 +223,11 @@ void ReadTextFile(const UCHAR *path, LIST *msgs)
 {
     HANDLE hFile;
     ULONG ByteSize, ReadSize;
-    WCHAR *Buffer, *BufPtr;
+    UCHAR* Buffer;
+    WCHAR *BufPtr;
     ULONG LineNum;
     ULONG i;
+    BOOLEAN isUTF8 = TRUE;
 
     //
     // read entire contents of text file
@@ -252,7 +254,7 @@ void ReadTextFile(const UCHAR *path, LIST *msgs)
 
     CloseHandle(hFile);
 
-    memset(&Buffer[ReadSize / sizeof(WCHAR)], 0, 16);
+    memset(&Buffer[ReadSize], 0, 16);
 
     //
     // build list of messages
@@ -262,9 +264,32 @@ void ReadTextFile(const UCHAR *path, LIST *msgs)
 
     LineNum = 1;
 
+
+    if (Buffer[0] == 0xFF && Buffer[1] == 0xFE) // Unicode BOM
+    {
+        BufPtr = &Buffer[2];
+        ReadSize -= 2;
+        isUTF8 = FALSE;
+    }
+    else if (Buffer[0] == 0xEF && Buffer[1] == 0xBB && Buffer[2] == 0xBF)  // UTF8 BOM
+    {
+        BufPtr = &Buffer[3];
+        ReadSize -= 3;
+        isUTF8 = TRUE;
+    }
+    else
         BufPtr = Buffer;
-    if (*BufPtr == 0xFEFF)
-        ++BufPtr;
+
+    if (isUTF8)
+    {
+        char* utf8 = BufPtr;
+
+        ByteSize *= 2;
+        Buffer = Alloc(ByteSize + 16);
+        BufPtr = Buffer;
+
+        MultiByteToWideChar(CP_UTF8, 0, utf8, ReadSize, BufPtr, ByteSize);
+    }
 
     while (1) {