Merge pull request #828 from Shelf-nu/814-bug-importing-csv-with-spec…

…ial-characters-breaks-in-the-ui

updating csv parser

app/utils/csv.server.ts

@@ -4,16 +4,26 @@ import {
   unstable_createMemoryUploadHandler,
   unstable_parseMultipartFormData,
 } from "@remix-run/node";
+import chardet from "chardet";
 import { parse } from "csv-parse";
+import iconv from "iconv-lite";
 import { fetchAssetsForExport } from "~/modules/asset";
 
 export type CSVData = [string[], ...string[][]] | [];
 
 /** Parses csv Data into an array with type {@link CSVData} */
-export const parseCsv = (csvData: string) => {
+export const parseCsv = (csvData: ArrayBuffer) => {
   const results = [] as CSVData;
+  /** Detect the file encoding */
+  const encoding = chardet.detect(Buffer.from(csvData));
+
+  /** Convert the file to utf-8 from the detected encoding */
+  const csv = iconv.decode(Buffer.from(csvData), encoding || "utf-8");
+
   return new Promise((resolve, reject) => {
     const parser = parse({
+      encoding: "utf-8", // Set encoding to utf-8
+      bom: true, // Handle BOM
       delimiter: ";", // Set delimiter to ; as this allows for commas in the data
       // quote: '"', // Set quote to " as this allows for commas in the data
       escape: "\\", // Set escape to \ as this allows for commas in the data
@@ -32,7 +42,7 @@ export const parseCsv = (csvData: string) => {
         resolve(results);
       });
 
-    parser.write(csvData);
+    parser.write(csv);
     parser.end();
   });
 };
@@ -46,7 +56,7 @@ export const csvDataFromRequest = async ({ request }: { request: Request }) => {
   );
 
   const csvFile = formData.get("file") as File;
-  const csvData = Buffer.from(await csvFile.arrayBuffer()).toString("utf-8"); // Convert Uint8Array to string
+  const csvData = await csvFile.arrayBuffer();
 
   return (await parseCsv(csvData)) as CSVData;
 };

package.json

@@ -64,6 +64,7 @@
     "@tremor/react": "^3.14.0",
     "@zxing/library": "^0.20.0",
     "changedpi": "^1.0.4",
+    "chardet": "^2.0.0",
     "cookie": "^0.6.0",
     "crisp-sdk-web": "^1.0.21",
     "csv-parse": "^5.5.3",
@@ -72,6 +73,7 @@
     "dom-to-image": "^2.6.0",
     "framer-motion": "^11.0.5",
     "hono": "^4.0.2",
+    "iconv-lite": "^0.6.3",
     "intl-parse-accept-language": "^1.0.0",
     "isbot": "^4.4.0",
     "jotai": "^2.6.4",