From 3a9db6778a2fc51394e1bf9d3c0bb6e89feb59d0 Mon Sep 17 00:00:00 2001 From: Syphax Date: Fri, 17 Jan 2025 23:35:31 +0100 Subject: [PATCH] add option to run import metadata graphs using docker for testing --- Gemfile.lock | 16 ++-- bin/migrations/compare_counts.rb | 6 ++ .../import_metadata_graphs_to_store | 9 ++ docker-compose.yml | 2 +- start_ontoportal_services.sh | 85 +++++++++++++++++++ 5 files changed, 109 insertions(+), 9 deletions(-) create mode 100755 start_ontoportal_services.sh diff --git a/Gemfile.lock b/Gemfile.lock index 052099c..cd86ea2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: f8ac7b00e8d8b46d1eea04de014175525c1cdd83 + revision: 8d108c23a043039e9675b36f8f444d29a87b11fe branch: development specs: goo (0.0.2) @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: e65d887616aaf4ae6f099437223d86515ffdca79 + revision: 0aa6219c44143b94135e01c78eb94ad99a5e8b32 branch: development specs: ontologies_linked_data (0.0.1) @@ -49,7 +49,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/sparql-client.git - revision: 59251e59346c9a69a67c88552ba55a1244eec602 + revision: 24bccbd0f4a5150fa6ce2af50d7c378c681027ea branch: development specs: sparql-client (3.2.2) @@ -101,7 +101,7 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.3.4) + concurrent-ruby (1.3.5) connection_pool (2.5.0) cube-ruby (0.0.3) dante (0.2.0) @@ -138,7 +138,7 @@ GEM google-cloud-errors (~> 1.0) google-apis-analytics_v3 (0.16.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.15.1) + google-apis-core (0.16.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -157,7 +157,7 @@ GEM google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.17.0) + googleapis-common-protos-types (1.18.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.11.2) faraday (>= 1.0, < 3.a) @@ -251,7 +251,7 @@ GEM rexml (~> 3.2) redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.23.1) + redis-client (0.23.2) connection_pool representable (3.2.0) declarative (< 0.1.0) @@ -348,4 +348,4 @@ DEPENDENCIES test-unit-minitest BUNDLED WITH - 2.3.14 + 2.3.3 diff --git a/bin/migrations/compare_counts.rb b/bin/migrations/compare_counts.rb index 485dff9..4c6be55 100755 --- a/bin/migrations/compare_counts.rb +++ b/bin/migrations/compare_counts.rb @@ -171,6 +171,12 @@ def compare_graphs_with_files(graph_triples) end puts "MappingsCount count: #{count} with display=all in #{format("%.4f", time)}s" +count = 0 +time = Benchmark.realtime do + count += LinkedData::Models::RestBackupMapping.where.all.count +end +puts "RestMappings count: #{count} with no display in #{format("%.4f", time)}s" + count = 0 time = Benchmark.realtime do attr = LinkedData::Models::RestBackupMapping.attributes(:all) + LinkedData::Models::MappingProcess.attributes(:all) diff --git a/bin/migrations/import_metadata_graphs_to_store b/bin/migrations/import_metadata_graphs_to_store index 3084ce9..d278664 100755 --- a/bin/migrations/import_metadata_graphs_to_store +++ b/bin/migrations/import_metadata_graphs_to_store @@ -14,6 +14,15 @@ begin # Optional profile to use for the import (vo: virtruoso, fs: 4store, gb: GraphDB) profile = ARGV[1] + docker = ARGV[2] == "docker" + + if docker + result = system("./start_ontoportal_services.sh #{profile}") + unless result + puts "Error starting services" + exit 1 + end + end # Check if processed_files directory exists unless Dir.exist?(processed_dir) puts "Processed files directory #{processed_dir} does not exist!" diff --git a/docker-compose.yml b/docker-compose.yml index 037e7d6..295d6bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,7 +130,7 @@ services: - ./test/data/virtuoso.ini:/opt/virtuoso-opensource/database/virtuoso.ini - ./test/data/virtuso_grant_write_permission.sh:/opt/virtuoso-opensource/initdb.d/virtuso_grant_write_permission.sh entrypoint: > - bash -c "cd ..; ./bin/virtuoso-t -c ./database/virtuoso.ini ; ./initdb.d/virtuso_grant_write_permission.sh ; tail -f database/virtuoso.log" + bash -c "cd ..;./bin/virtuoso-t +wait +configfile ./database/virtuoso.ini ; ./initdb.d/virtuso_grant_write_permission.sh ; tail -f ./database/virtuoso.log" healthcheck: test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] start_period: 10s diff --git a/start_ontoportal_services.sh b/start_ontoportal_services.sh new file mode 100755 index 0000000..9a8a982 --- /dev/null +++ b/start_ontoportal_services.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +profile=$1 +acronym=$2 +set -e + + +if [ -z "$profile" ]; then + echo "Usage: $0 " + exit 1 +fi + +BACKEND_TYPE=$profile +if [ "$BACKEND_TYPE" == "ag" ]; then + # AllegroGraph backend + export GOO_BACKEND_NAME="allegrograph" + export GOO_PORT="10035" + export GOO_PATH_QUERY="/repositories/ontoportal_test" + export GOO_PATH_DATA="/repositories/ontoportal_test/statements" + export GOO_PATH_UPDATE="/repositories/ontoportal_test/statements" + export COMPOSE_PROFILES="ag" + +elif [ "$BACKEND_TYPE" == "fs" ]; then + # 4store backend + export GOO_PORT="9000" + export COMPOSE_PROFILES="fs" + +elif [ "$BACKEND_TYPE" == "vo" ]; then + # Virtuoso backend + export GOO_BACKEND_NAME="virtuoso" + export GOO_PORT="8890" + export GOO_PATH_QUERY="/sparql" + export GOO_PATH_DATA="/sparql" + export GOO_PATH_UPDATE="/sparql" + export COMPOSE_PROFILES="vo" + +elif [ "$BACKEND_TYPE" == "gb" ]; then + # Graphdb backend + export GOO_BACKEND_NAME="graphdb" + export GOO_PORT="7200" + export GOO_PATH_QUERY="/repositories/ontoportal" + export GOO_PATH_DATA="/repositories/ontoportal/statements" + export GOO_PATH_UPDATE="/repositories/ontoportal/statements" +else + echo "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', or 'vo'." +fi + +echo "###########################################################################" +echo "Stop and remove all containers, networks, and volumes and start fresh" +docker compose --profile fs --profile vo --profile gb --profile ag down --volumes --remove-orphans && docker compose --profile "$profile" up -d + +echo "Waiting for all Docker services to start..." + +while true; do + # Get the status of all containers + container_status=$(docker compose --profile "$profile" ps -a --format '{{.Names}} {{.State}}') + + all_running=true + while read -r container state; do + if [ "$state" != "running" ] && [ "$state" != "exited" ]; then + all_running=false + break + fi + done <<< "$container_status" + + # If all containers are running, exit the loop + if [ "$all_running" = true ]; then + echo "All containers are running!" + break + fi + + # Wait before checking again + sleep 2 +done + +if [ -z "$acronym" ]; then + exit 0 +fi + +echo "###########################################################################" +echo "Create a new user and make it an admin" +bundle exec rake user:create[admin,admin@nodomain.org,password] +bundle exec rake user:adminify[admin] +echo "###########################################################################" +echo "Create a new ontology $acronym and import it from a remote server" +bin/ncbo_ontology_import --admin-user admin -o "$acronym" --from https://data.stageportal.lirmm.fr --from-apikey 82602563-4750-41be-9654-36f46056a0db