From bf526b45229ca153ac1fefe6e3b42174790637e5 Mon Sep 17 00:00:00 2001
From: Syphax <gs_bouazzouni@esi.dz>
Date: Thu, 23 Jan 2025 04:28:50 +0100
Subject: [PATCH] simplify compare count to not do the benchmarks

---
 Gemfile.lock                                  |  8 +--
 bin/migrations/compare_counts.rb              | 56 +------------------
 .../import_metadata_graphs_to_store           |  3 +-
 3 files changed, 9 insertions(+), 58 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index cd86ea2..d44a410 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 8d108c23a043039e9675b36f8f444d29a87b11fe
+  revision: 27300f28ca6c656c7e78af65013d88b792a6312f
   branch: development
   specs:
     goo (0.0.2)
@@ -29,7 +29,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 0aa6219c44143b94135e01c78eb94ad99a5e8b32
+  revision: 6cb18910e322645e3cc3490951d10f19468da52f
   branch: development
   specs:
     ontologies_linked_data (0.0.1)
@@ -49,7 +49,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/sparql-client.git
-  revision: 24bccbd0f4a5150fa6ce2af50d7c378c681027ea
+  revision: 4364d34e9e4c411f1dd0ea706bf052465bf0b467
   branch: development
   specs:
     sparql-client (3.2.2)
@@ -209,7 +209,7 @@ GEM
     mutex_m (0.3.0)
     net-http-persistent (4.0.5)
       connection_pool (~> 2.2)
-    net-scp (4.0.0)
+    net-scp (4.1.0)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
diff --git a/bin/migrations/compare_counts.rb b/bin/migrations/compare_counts.rb
index 4c6be55..d1a5b2a 100755
--- a/bin/migrations/compare_counts.rb
+++ b/bin/migrations/compare_counts.rb
@@ -13,7 +13,6 @@
 PROCESSED_DIR = ARGV[0] || './processed_files'
 profile = ARGV[1]
 
-
 case profile
 when 'ag'
   # AllegroGraph backend
@@ -86,7 +85,7 @@ def build_graphs_file_hash(folder_path = PROCESSED_DIR)
     if filename.end_with?('.graph')
       file_path = File.join(folder_path, filename)
       line = File.open(file_path, "r").readlines.first
-      graphs[line.strip] = filename.to_s.gsub('.graph','')
+      graphs[line.strip] = filename.to_s.gsub('.graph', '')
     end
   end
   graphs
@@ -104,10 +103,10 @@ def compare_graphs_with_files(graph_triples)
       graph_filename = graphs_files[graph_uri]
 
       next csv << [graph_uri, triples_count, "Graph not found", "N/A"] unless graph_filename
-      
+
       # Construct the expected file name based on the graph URI
       file_name = "#{PROCESSED_DIR}/#{graph_filename}"
-      
+
       # puts "count lines of the file #{file_name} for the graph #{graph_uri}"
       if File.exist?(file_name)
         file_lines_count = count_file_lines(file_name)
@@ -128,58 +127,9 @@ def compare_graphs_with_files(graph_triples)
 end
 
 # Main execution
-
 Goo.sparql_query_client.cache.redis_cache.flushdb
 puts "Redis cache flushed"
 
 puts "Comparing graph triple counts with file lines and exporting to CSV..."
 graph_triples = get_all_graphs_counts
 compare_graphs_with_files(graph_triples)
-
-count = 0
-attr_ontology = []
-time = Benchmark.realtime do
-  attr_ontology = LinkedData::Models::Ontology.attributes(:all)
-  count = LinkedData::Models::Ontology.where.include(attr_ontology).all.count
-end
-puts "Ontologies count: #{count} with display=all in #{format("%.4f", time)}s"
-count = 0
-time = Benchmark.realtime do
-  count = LinkedData::Models::OntologySubmission.where.all.count
-end
-puts "Submissions count: #{count} with no display in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::OntologySubmission.attributes(:all)
-  attr << {ontology: attr_ontology}
-  count = LinkedData::Models::OntologySubmission.where.include(attr).all.count
-end
-puts "Submissions count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::Agent.attributes(:all)
-  count = LinkedData::Models::Agent.where.include(attr).all.count
-end
-puts "Agent count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::MappingCount.attributes(:all)
-  count = LinkedData::Models::MappingCount.where.include(attr).all.count
-end
-puts "MappingsCount count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  count += LinkedData::Models::RestBackupMapping.where.all.count
-end
-puts "RestMappings count: #{count} with no display in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::RestBackupMapping.attributes(:all) + LinkedData::Models::MappingProcess.attributes(:all)
-  count += LinkedData::Models::RestBackupMapping.where.include(attr).all.count
-end
-puts "RestMappings count: #{count} with display=all in #{format("%.4f", time)}s"
diff --git a/bin/migrations/import_metadata_graphs_to_store b/bin/migrations/import_metadata_graphs_to_store
index d278664..fe158bc 100755
--- a/bin/migrations/import_metadata_graphs_to_store
+++ b/bin/migrations/import_metadata_graphs_to_store
@@ -49,7 +49,7 @@ begin
     puts "Start importing #{file} into graph <#{graph_uri}> of line count #{line_count}"
     result = false
     time = Benchmark.realtime do
-      result = system("ruby bin/migrations/import_nt_file.rb #{file} #{graph_uri} #{profile} > /dev/null 2>&1")
+      result = system("ruby bin/migrations/import_nt_file.rb #{file} #{graph_uri} #{profile} >> ./process_log.log 2>&1")
     end
 
     file_count += 1
@@ -57,6 +57,7 @@ begin
 
     if !result
       puts "Error importing #{file} into graph <#{graph_uri}>"
+      exit 1
     else
       import_count += 1
       puts "Imported <#{graph_uri}> successfully in #{time.round(2)} seconds"