fix: transactions processed by row not in batch

Greenstand · Nov 13, 2024 · 1e4e19d · 1e4e19d
1 parent a974ef5
commit 1e4e19d
Show file tree

Hide file tree

Showing 30 changed files with 20 additions and 12 deletions.
diff --git a/lib/__pycache__/assign_new_trees_to_cluster.cpython-39.pyc b/lib/__pycache__/assign_new_trees_to_cluster.cpython-39.pyc
diff --git a/lib/__pycache__/assign_new_trees_to_cluster_test.cpython-39.pyc b/lib/__pycache__/assign_new_trees_to_cluster_test.cpython-39.pyc
diff --git a/lib/__pycache__/capture_export.cpython-39.pyc b/lib/__pycache__/capture_export.cpython-39.pyc
diff --git a/lib/__pycache__/capture_export_test.cpython-39.pyc b/lib/__pycache__/capture_export_test.cpython-39.pyc
diff --git a/lib/__pycache__/contracts_earnings_fcc.cpython-39.pyc b/lib/__pycache__/contracts_earnings_fcc.cpython-39.pyc
diff --git a/lib/__pycache__/contracts_earnings_fcc_test.cpython-39.pyc b/lib/__pycache__/contracts_earnings_fcc_test.cpython-39.pyc
diff --git a/lib/__pycache__/country_leader_board_test.cpython-39.pyc b/lib/__pycache__/country_leader_board_test.cpython-39.pyc
diff --git a/lib/__pycache__/create_authors_test.cpython-39.pyc b/lib/__pycache__/create_authors_test.cpython-39.pyc
diff --git a/lib/__pycache__/earning_export.cpython-39.pyc b/lib/__pycache__/earning_export.cpython-39.pyc
diff --git a/lib/__pycache__/earning_export_test.cpython-39.pyc b/lib/__pycache__/earning_export_test.cpython-39.pyc
diff --git a/lib/__pycache__/grower_exort_test.cpython-39.pyc b/lib/__pycache__/grower_exort_test.cpython-39.pyc
diff --git a/lib/__pycache__/grower_export.cpython-39.pyc b/lib/__pycache__/grower_export.cpython-39.pyc
diff --git a/lib/__pycache__/leader_board.cpython-39.pyc b/lib/__pycache__/leader_board.cpython-39.pyc
diff --git a/lib/__pycache__/leader_board_test.cpython-39.pyc b/lib/__pycache__/leader_board_test.cpython-39.pyc
diff --git a/lib/__pycache__/messaging.cpython-39.pyc b/lib/__pycache__/messaging.cpython-39.pyc
diff --git a/lib/__pycache__/planter_entity.cpython-39.pyc b/lib/__pycache__/planter_entity.cpython-39.pyc
diff --git a/lib/__pycache__/planter_entity_test.cpython-39.pyc b/lib/__pycache__/planter_entity_test.cpython-39.pyc
diff --git a/lib/__pycache__/pre_request.cpython-39.pyc b/lib/__pycache__/pre_request.cpython-39.pyc
diff --git a/lib/__pycache__/pre_request_map_clusters.cpython-39.pyc b/lib/__pycache__/pre_request_map_clusters.cpython-39.pyc
diff --git a/lib/__pycache__/pre_request_map_clusters_test.cpython-39.pyc b/lib/__pycache__/pre_request_map_clusters_test.cpython-39.pyc
diff --git a/lib/__pycache__/pre_request_test.cpython-39.pyc b/lib/__pycache__/pre_request_test.cpython-39.pyc
diff --git a/lib/__pycache__/sync_entity_stakeholder.cpython-39.pyc b/lib/__pycache__/sync_entity_stakeholder.cpython-39.pyc
diff --git a/lib/__pycache__/sync_entity_stakeholder_test.cpython-39.pyc b/lib/__pycache__/sync_entity_stakeholder_test.cpython-39.pyc
diff --git a/lib/__pycache__/upload_planter_info.cpython-39.pyc b/lib/__pycache__/upload_planter_info.cpython-39.pyc
diff --git a/lib/__pycache__/upload_planter_info_test.cpython-39.pyc b/lib/__pycache__/upload_planter_info_test.cpython-39.pyc
diff --git a/lib/__pycache__/utils.cpython-39.pyc b/lib/__pycache__/utils.cpython-39.pyc
diff --git a/lib/__pycache__/utils_test.cpython-39.pyc b/lib/__pycache__/utils_test.cpython-39.pyc
diff --git a/v1Tov2Migrations/migrate_approved_captures.js b/v1Tov2Migrations/migrate_approved_captures.js
@@ -46,8 +46,9 @@ async function migrate() {
       { width: 40, total: recordCount },
     );
 
-    const trx = await knex.transaction();
     ws._write = async (rawCapture, enc, next) => {
+      console.log('processing ', rawCapture.id);
+      const trx = await knex.transaction();
       try {
         const tree = await trx
           .select()
@@ -61,12 +62,13 @@ async function migrate() {
           [+tree.id],
         );
 
-        console.log('migrating ', tree.id)
+        console.log('migrating ', tree.id);
         await createCapture(rawCapture, tree, trx, treeTags.rows);
 
+        await trx.commit();
+        console.log('processed ', rawCapture.id);
         bar.tick();
         if (bar.complete) {
-          await trx.commit();
           console.log('Migration Complete');
           process.exit();
         }

diff --git a/v1Tov2Migrations/migrate_raw_captures.js b/v1Tov2Migrations/migrate_raw_captures.js
@@ -61,10 +61,10 @@ async function migrate() {
       { width: 40, total: recordCount },
     );
 
-    const trx = await knex.transaction();
     ws._write = async (tree, enc, next) => {
+      console.log('processing ', tree.id);
+      const trx = await knex.transaction();
       try {
-	console.log('processing ', tree.id);
         const planter = await trx
           .select()
           .table('public.planter')
@@ -137,9 +137,10 @@ async function migrate() {
 
         await createRawCapture(tree, treeAttributes, sessionId, trx);
 
+        await trx.commit();
+        console.log('processed ', tree.id);
         bar.tick();
         if (bar.complete) {
-          await trx.commit();
           console.log('Migration Complete');
           process.exit();
         }

diff --git a/v1Tov2Migrations/migrate_trees.js b/v1Tov2Migrations/migrate_trees.js
@@ -27,13 +27,17 @@ async function migrate() {
     }
     console.log(`Migrating ${recordCount} records`);
 
-    const bar = new ProgressBar('Migrating [:bar] :percent :etas :current/:total (:rate)', {
-      width: 40,
-      total: recordCount,
-    });
+    const bar = new ProgressBar(
+      'Migrating [:bar] :percent :etas :current/:total (:rate)',
+      {
+        width: 40,
+        total: recordCount,
+      },
+    );
 
-    const trx = await knex.transaction();
     ws._write = async (capture, enc, next) => {
+      console.log('processing ', capture.id);
+      const trx = await knex.transaction();
       try {
         // migrate capture_tags as well, do we want to?
         const captureTags = await trx.raw(
@@ -43,9 +47,10 @@ async function migrate() {
 
         await createTree(capture, trx, captureTags.rows);
 
+        await trx.commit();
+        console.log('processed ', capture.id);
         bar.tick();
         if (bar.complete) {
-          await trx.commit();
           console.log('Migration Complete');
           process.exit();
         }