Feature: add index all data step to the processing queue (#21)

* update the docker-based running tests script * add :index_all_data step the process actions * update github action test CI to do it using all the backends
ontoportal-lirmm · Apr 26, 2024 · 488ed37 · 488ed37
1 parent 5bf8a23
commit 488ed37
Show file tree

Hide file tree

Showing 24 changed files with 405 additions and 5,751 deletions.
diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml
@@ -9,22 +9,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        backend: ['ncbo_cron'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
+        goo-slice: [ '20', '100', '500' ]
+        ruby-version: [ '2.7' ]
+        triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: copy config.rb file from template
-      run:  cp config/config.rb.sample config/config.rb
-    - name: Build docker-compose
-      run: docker-compose build
-    - name: Run unit tests
-      run: |
-        ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        flags: unittests
-        verbose: true
-        fail_ci_if_error: false # optional (default = false)
-
+      - uses: actions/checkout@v3
+      - name: create config.rb file
+        run: cp config/config.rb.sample config/config.rb
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get -y install raptor2-utils
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run unit tests
+        # unit tests are run inside a container
+        # http://docs.codecov.io/docs/testing-with-docker
+        run: |
+          ci_env=`bash <(curl -s https://codecov.io/env)`
+          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+          verbose: true
+          fail_ci_if_error: false # optional (default = false)
diff --git a/Gemfile b/Gemfile
@@ -21,14 +21,16 @@ gem 'redis'
 gem 'rest-client'
 gem 'sys-proctable'
 gem 'request_store'
+gem 'parallel'
+gem 'json-ld'
 
 # Monitoring
 gem 'cube-ruby', require: 'cube'
 
-gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'master'
+gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
 gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
-gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'master'
-gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'master'
+gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'development'
+gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'development'
 # Testing
 group :test do
   gem 'email_spec'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,12 +1,15 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 8ddd2d719617ad082c6964a9efdac153cdd2b48e
-  branch: master
+  revision: 0e554fce49713ce4d5a742a06c2fb59a547caf47
+  branch: development
   specs:
     goo (0.0.2)
       addressable (~> 2.8)
       pry
-      rdf (= 1.0.8)
+      rdf (= 3.2.11)
+      rdf-raptor
+      rdf-rdfxml
+      rdf-vocab
       redis
       rest-client
       rsolr
@@ -15,8 +18,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git
-  revision: 57204d8e54432ba660af4c49806e2a3019a23fa2
-  branch: master
+  revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0
+  branch: development
   specs:
     ncbo_annotator (0.0.1)
       goo
@@ -26,8 +29,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 95e77989e4e8ea2fde86cf4f048f7d6cd7a6829f
-  branch: master
+  revision: 135b0df0d6394f69162f496674c5b2f2a3dcba4c
+  branch: development
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -46,7 +49,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/sparql-client.git
-  revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0
+  revision: 180c818f7715baac64b2699bb452ef5c756f62c5
   branch: master
   specs:
     sparql-client (1.0.1)
@@ -79,9 +82,9 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     base64 (0.2.0)
     bcrypt (3.1.20)
-    bigdecimal (3.1.6)
-    binding_of_caller (1.0.0)
-      debug_inspector (>= 0.0.1)
+    bigdecimal (3.1.7)
+    binding_of_caller (1.0.1)
+      debug_inspector (>= 1.2.0)
     builder (3.2.4)
     coderay (1.1.3)
     concurrent-ruby (1.2.3)
@@ -101,7 +104,7 @@ GEM
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
-    faraday-retry (2.2.0)
+    faraday-retry (2.2.1)
       faraday (~> 2.0)
     ffi (1.16.3)
     gapic-common (0.21.1)
@@ -112,34 +115,35 @@ GEM
       googleapis-common-protos-types (>= 1.11.0, < 2.a)
       googleauth (~> 1.9)
       grpc (~> 1.59)
-    google-analytics-data (0.5.0)
+    google-analytics-data (0.6.0)
       google-analytics-data-v1beta (>= 0.11, < 2.a)
       google-cloud-core (~> 1.6)
-    google-analytics-data-v1beta (0.11.2)
+    google-analytics-data-v1beta (0.12.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
-    google-apis-analytics_v3 (0.14.0)
-      google-apis-core (>= 0.12.0, < 2.a)
-    google-apis-core (0.13.0)
+    google-apis-analytics_v3 (0.15.0)
+      google-apis-core (>= 0.14.0, < 2.a)
+    google-apis-core (0.14.1)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 1.9)
       httpclient (>= 2.8.1, < 3.a)
       mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
       rexml
-    google-cloud-core (1.6.1)
+    google-cloud-core (1.7.0)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
-    google-cloud-errors (1.3.1)
+    google-cloud-errors (1.4.0)
+    google-protobuf (3.25.3-x86_64-darwin)
     google-protobuf (3.25.3-x86_64-linux)
-    googleapis-common-protos (1.4.0)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (~> 1.2)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.12.0)
+    googleapis-common-protos (1.5.0)
+      google-protobuf (~> 3.18)
+      googleapis-common-protos-types (~> 1.7)
+      grpc (~> 1.41)
+    googleapis-common-protos-types (1.14.0)
       google-protobuf (~> 3.18)
     googleauth (1.11.0)
       faraday (>= 1.0, < 3.a)
@@ -148,7 +152,10 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.61.0-x86_64-linux)
+    grpc (1.62.0-x86_64-darwin)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.62.0-x86_64-linux)
       google-protobuf (~> 3.25)
       googleapis-common-protos-types (~> 1.0)
     htmlentities (4.3.4)
@@ -158,21 +165,31 @@ GEM
     httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
-    json_pure (2.7.1)
-    jwt (2.7.1)
+    json (2.7.2)
+    json-canonicalization (0.4.0)
+    json-ld (3.2.5)
+      htmlentities (~> 4.3)
+      json-canonicalization (~> 0.3, >= 0.3.2)
+      link_header (~> 0.0, >= 0.0.8)
+      multi_json (~> 1.15)
+      rack (>= 2.2, < 4)
+      rdf (~> 3.2, >= 3.2.10)
+    json_pure (2.7.2)
+    jwt (2.8.1)
+      base64
     launchy (2.5.2)
       addressable (~> 2.8)
-    libxml-ruby (5.0.2)
+    libxml-ruby (5.0.3)
+    link_header (0.0.8)
     logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     mail (2.6.6)
       mime-types (>= 1.16, < 4)
-    method_source (1.0.0)
+    method_source (1.1.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0206)
+    mime-types-data (3.2024.0305)
     mini_mime (1.1.5)
     minitest (4.7.5)
     mlanett-redis-lock (0.2.7)
@@ -185,22 +202,36 @@ GEM
     omni_logger (0.1.4)
       logger
     os (1.1.4)
+    parallel (1.24.0)
     parseconfig (1.1.2)
     pony (1.13.1)
       mail (>= 2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.4)
-    rack (3.0.9)
+    public_suffix (5.0.5)
+    rack (3.0.10)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rake (13.1.0)
-    rdf (1.0.8)
-      addressable (>= 2.2)
-    redis (5.1.0)
-      redis-client (>= 0.17.0)
-    redis-client (0.20.0)
+    rake (13.2.1)
+    rdf (3.2.11)
+      link_header (~> 0.0, >= 0.0.8)
+    rdf-raptor (3.2.0)
+      ffi (~> 1.15)
+      rdf (~> 3.2)
+    rdf-rdfxml (3.2.2)
+      builder (~> 3.2)
+      htmlentities (~> 4.3)
+      rdf (~> 3.2)
+      rdf-xsd (~> 3.2)
+    rdf-vocab (3.2.7)
+      rdf (~> 3.2, >= 3.2.4)
+    rdf-xsd (3.2.1)
+      rdf (~> 3.2)
+      rexml (~> 3.2)
+    redis (5.2.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.22.1)
       connection_pool
     representable (3.2.0)
       declarative (< 0.1.0)
@@ -215,7 +246,7 @@ GEM
       netrc (~> 0.8)
     retriable (3.1.2)
     rexml (3.2.6)
-    rsolr (2.5.0)
+    rsolr (2.6.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
     ruby-xxHash (0.4.0.2)
@@ -250,6 +281,7 @@ GEM
       macaddr (~> 1.0)
 
 PLATFORMS
+  x86_64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -260,13 +292,15 @@ DEPENDENCIES
   goo!
   google-analytics-data
   google-apis-analytics_v3
+  json-ld
   mail (= 2.6.6)
   minitest (< 5.0)
   multi_json
   ncbo_annotator!
   ncbo_cron!
   oj (~> 3.0)
   ontologies_linked_data!
+  parallel
   parseconfig
   pony
   pry
@@ -281,4 +315,4 @@ DEPENDENCIES
   test-unit-minitest
 
 BUNDLED WITH
-   2.3.14
+   2.4.22
diff --git a/config/config.rb.sample b/config/config.rb.sample
@@ -7,7 +7,7 @@ GOO_PATH_QUERY        = ENV.include?("GOO_PATH_QUERY")        ? ENV["GOO_PATH_QU
 GOO_PATH_UPDATE       = ENV.include?("GOO_PATH_UPDATE")       ? ENV["GOO_PATH_UPDATE"]       : "/update/"
 GOO_PORT              = ENV.include?("GOO_PORT")              ? ENV["GOO_PORT"]              : 9000
 MGREP_HOST            = ENV.include?("MGREP_HOST")            ? ENV["MGREP_HOST"]            : "localhost"
-MGREP_PORT            = ENV.include?("MGREP_PORT")            ? ENV["MGREP_PORT"]            : 55555
+MGREP_PORT            = ENV.include?("MGREP_PORT")            ? ENV["MGREP_PORT"]            : 55556
 MGREP_DICT_PATH       = ENV.include?("MGREP_DICT_PATH")       ? ENV["MGREP_DICT_PATH"]       : "./test/data/dictionary.txt"
 REDIS_GOO_CACHE_HOST  = ENV.include?("REDIS_GOO_CACHE_HOST")  ? ENV["REDIS_GOO_CACHE_HOST"]  : "localhost"
 REDIS_HTTP_CACHE_HOST = ENV.include?("REDIS_HTTP_CACHE_HOST") ? ENV["REDIS_HTTP_CACHE_HOST"] : "localhost"
@@ -16,8 +16,9 @@ REDIS_PORT            = ENV.include?("REDIS_PORT")            ? ENV["REDIS_PORT"
 REPORT_PATH           = ENV.include?("REPORT_PATH")           ? ENV["REPORT_PATH"]           : "./test/tmp/ontologies_report.json"
 REPOSITORY_FOLDER     = ENV.include?("REPOSITORY_FOLDER")     ? ENV["REPOSITORY_FOLDER"]     : "./test/data/ontology_files/repo"
 REST_URL_PREFIX       = ENV.include?("REST_URL_PREFIX")       ? ENV["REST_URL_PREFIX"]       : "http://localhost:9393"
-SOLR_PROP_SEARCH_URL  = ENV.include?("SOLR_PROP_SEARCH_URL")  ? ENV["SOLR_PROP_SEARCH_URL"]  : "http://localhost:8983/solr/prop_search_core1"
-SOLR_TERM_SEARCH_URL  = ENV.include?("SOLR_TERM_SEARCH_URL")  ? ENV["SOLR_TERM_SEARCH_URL"]  : "http://localhost:8983/solr/term_search_core1"
+SOLR_PROP_SEARCH_URL  = ENV.include?("SOLR_PROP_SEARCH_URL")  ? ENV["SOLR_PROP_SEARCH_URL"]  : "http://localhost:8983/solr"
+SOLR_TERM_SEARCH_URL  = ENV.include?("SOLR_TERM_SEARCH_URL")  ? ENV["SOLR_TERM_SEARCH_URL"]  : "http://localhost:8983/solr"
+GOO_SLICES = ENV["GOO_SLICES"] || 500
 
 LinkedData.config do |config|
   config.goo_backend_name              = GOO_BACKEND_NAME.to_s
@@ -73,8 +74,8 @@ NcboCron.config do |config|
   # do not deaemonize in docker
   config.daemonize  = false
 
-  config.search_index_all_url = "http://localhost:8983/solr/term_search_core2"
-  config.property_search_index_all_url = "http://localhost:8983/solr/prop_search_core2"
+  config.search_index_all_url = "http://localhost:8983/solr"
+  config.property_search_index_all_url = "http://localhost:8983/solr"
 
   # Google Analytics GA4 config
   config.analytics_path_to_key_file     = "config/your_analytics_key.json"

diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml