Merge pull request #1096 from Shelf-nu/improvement-performance-of-import

Improvement performance of import
Shelf-nu · Jun 25, 2024 · 63c4950 · 63c4950
2 parents 4ee9e3a + 0a9bedc
commit 63c4950
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 36 deletions.
diff --git a/app/components/assets/form.tsx b/app/components/assets/form.tsx
@@ -261,6 +261,7 @@ export const AssetForm = ({
                 variant="link"
                 icon="plus"
                 className="w-full justify-start pt-4"
+                target="_blank"
               >
                 Create new category
               </Button>
@@ -319,6 +320,7 @@ export const AssetForm = ({
                 variant="link"
                 icon="plus"
                 className="w-full justify-start pt-4"
+                target="_blank"
               >
                 Create new location
               </Button>

diff --git a/app/components/location/location-select.tsx b/app/components/location/location-select.tsx
@@ -40,6 +40,7 @@ export const LocationSelect = () => {
               variant="link"
               icon="plus"
               className="w-full justify-start pt-4"
+              target="_blank"
             >
               Create new location
             </Button>

diff --git a/app/modules/asset/service.server.ts b/app/modules/asset/service.server.ts
@@ -1792,43 +1792,59 @@ export async function createAssetsFromContentImport({
       userId,
     });
 
-    for (let asset of data) {
-      const customFieldsValues: ShelfAssetCustomFieldValueType[] =
-        Object.entries(asset).reduce((res, [key, val]) => {
-          if (key.startsWith("cf:") && val) {
-            const { name } = getDefinitionFromCsvHeader(key);
-            if (customFields[name].id) {
-              res.push({
-                id: customFields[name].id,
-                value: buildCustomFieldValue(
-                  { raw: asset[key] },
-                  customFields[name]
-                ),
-              } as ShelfAssetCustomFieldValueType);
-            }
+    const BATCH_SIZE = 10; // Adjust based on your server's capacity
+
+    for (let i = 0; i < data.length; i += BATCH_SIZE) {
+      const batch = data.slice(i, i + BATCH_SIZE);
+      await Promise.all(
+        batch.map(async (asset) => {
+          try {
+            // Your existing logic to process each asset
+            const customFieldsValues: ShelfAssetCustomFieldValueType[] =
+              Object.entries(asset).reduce((res, [key, val]) => {
+                if (key.startsWith("cf:") && val) {
+                  const { name } = getDefinitionFromCsvHeader(key);
+                  if (customFields[name].id) {
+                    res.push({
+                      id: customFields[name].id,
+                      value: buildCustomFieldValue(
+                        { raw: asset[key] },
+                        customFields[name]
+                      ),
+                    } as ShelfAssetCustomFieldValueType);
+                  }
+                }
+                return res;
+              }, [] as ShelfAssetCustomFieldValueType[]);
+            await createAsset({
+              organizationId,
+              title: asset.title,
+              description: asset.description || "",
+              userId,
+              categoryId: asset.category ? categories[asset.category] : null,
+              locationId: asset.location
+                ? locations[asset.location]
+                : undefined,
+              custodian: asset.custodian
+                ? teamMembers[asset.custodian]
+                : undefined,
+              tags:
+                asset.tags.length > 0
+                  ? {
+                      set: asset.tags
+                        .filter((t) => tags[t])
+                        .map((t) => ({ id: tags[t] })),
+                    }
+                  : undefined,
+              valuation: asset.valuation ? +asset.valuation : null,
+              customFieldsValues,
+            });
+          } catch (error) {
+            console.error("Error processing asset", error);
+            // Handle the error as needed
           }
-          return res;
-        }, [] as ShelfAssetCustomFieldValueType[]);
-
-      await createAsset({
-        organizationId,
-        title: asset.title,
-        description: asset.description || "",
-        userId,
-        categoryId: asset.category ? categories[asset.category] : null,
-        locationId: asset.location ? locations[asset.location] : undefined,
-        custodian: asset.custodian ? teamMembers[asset.custodian] : undefined,
-        tags:
-          asset.tags.length > 0
-            ? {
-                set: asset.tags
-                  .filter((t) => tags[t])
-                  .map((t) => ({ id: tags[t] })),
-              }
-            : undefined,
-        valuation: asset.valuation ? +asset.valuation : null,
-        customFieldsValues,
-      });
+        })
+      );
     }
   } catch (cause) {
     throw new ShelfError({