diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 8a420ba1..1ade7407 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -9,22 +9,35 @@ jobs: strategy: fail-fast: false matrix: - backend: ['ncbo_cron'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend + goo-slice: [ '20', '100', '500' ] + ruby-version: [ '2.7' ] + triplestore: [ 'fs', 'ag', 'vo', 'gb' ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: copy config.rb file from template - run: cp config/config.rb.sample config/config.rb - - name: Build docker-compose - run: docker-compose build - - name: Run unit tests - run: | - ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - flags: unittests - verbose: true - fail_ci_if_error: false # optional (default = false) - + - uses: actions/checkout@v3 + - name: create config.rb file + run: cp config/config.rb.sample config/config.rb + - name: Install Dependencies + run: sudo apt-get update && sudo apt-get -y install raptor2-utils + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run unit tests + # unit tests are run inside a container + # http://docs.codecov.io/docs/testing-with-docker + run: | + ci_env=`bash <(curl -s https://codecov.io/env)` + GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v" + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + flags: unittests + verbose: true + fail_ci_if_error: false # optional (default = false) diff --git a/Gemfile b/Gemfile index 54646cf4..7d616193 100644 --- a/Gemfile +++ b/Gemfile @@ -21,14 +21,16 @@ gem 'redis' gem 'rest-client' gem 'sys-proctable' gem 'request_store' +gem 'parallel' +gem 'json-ld' # Monitoring gem 'cube-ruby', require: 'cube' -gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'master' -gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'master' -gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'master' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' +gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'development' +gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'development' # Testing group :test do gem 'email_spec' diff --git a/Gemfile.lock b/Gemfile.lock index 8f40993b..ba322adb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,15 @@ GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 8ddd2d719617ad082c6964a9efdac153cdd2b48e - branch: master + revision: b2a635fb1e8206e6e3010be4dbe033b47eb58481 + branch: development specs: goo (0.0.2) addressable (~> 2.8) pry - rdf (= 1.0.8) + rdf (= 3.2.11) + rdf-raptor + rdf-rdfxml + rdf-vocab redis rest-client rsolr @@ -15,8 +18,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git - revision: 57204d8e54432ba660af4c49806e2a3019a23fa2 - branch: master + revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + branch: development specs: ncbo_annotator (0.0.1) goo @@ -26,8 +29,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 95e77989e4e8ea2fde86cf4f048f7d6cd7a6829f - branch: master + revision: 0855adfce5365c6ef7a494e3d3ad9af2611974d9 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -46,13 +49,12 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/sparql-client.git - revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0 - branch: master + revision: 59251e59346c9a69a67c88552ba55a1244eec602 + branch: development specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) + sparql-client (3.2.2) + net-http-persistent (~> 4.0, >= 4.0.2) + rdf (~> 3.2, >= 3.2.11) PATH remote: . @@ -79,9 +81,9 @@ GEM public_suffix (>= 2.0.2, < 6.0) base64 (0.2.0) bcrypt (3.1.20) - bigdecimal (3.1.6) - binding_of_caller (1.0.0) - debug_inspector (>= 0.0.1) + bigdecimal (3.1.8) + binding_of_caller (1.0.1) + debug_inspector (>= 1.2.0) builder (3.2.4) coderay (1.1.3) concurrent-ruby (1.2.3) @@ -101,7 +103,7 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -112,15 +114,15 @@ GEM googleapis-common-protos-types (>= 1.11.0, < 2.a) googleauth (~> 1.9) grpc (~> 1.59) - google-analytics-data (0.5.0) + google-analytics-data (0.6.0) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.11.2) + google-analytics-data-v1beta (0.12.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.14.0) - google-apis-core (>= 0.12.0, < 2.a) - google-apis-core (0.13.0) + google-apis-analytics_v3 (0.16.0) + google-apis-core (>= 0.15.0, < 2.a) + google-apis-core (0.15.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.1, < 3.a) @@ -128,27 +130,31 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.a) rexml - google-cloud-core (1.6.1) + google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) - google-cloud-errors (1.3.1) - google-protobuf (3.25.2-x86_64-linux) - googleapis-common-protos (1.4.0) - google-protobuf (~> 3.14) - googleapis-common-protos-types (~> 1.2) - grpc (~> 1.27) - googleapis-common-protos-types (1.11.0) + google-cloud-errors (1.4.0) + google-protobuf (3.25.3-x86_64-darwin) + google-protobuf (3.25.3-x86_64-linux) + googleapis-common-protos (1.5.0) google-protobuf (~> 3.18) - googleauth (1.10.0) + googleapis-common-protos-types (~> 1.7) + grpc (~> 1.41) + googleapis-common-protos-types (1.14.0) + google-protobuf (~> 3.18) + googleauth (1.11.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.61.0-x86_64-linux) + grpc (1.63.0-x86_64-darwin) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) htmlentities (4.3.4) @@ -158,55 +164,79 @@ GEM httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) - json_pure (2.7.1) - jwt (2.7.1) + json (2.7.2) + json-canonicalization (0.4.0) + json-ld (3.2.5) + htmlentities (~> 4.3) + json-canonicalization (~> 0.3, >= 0.3.2) + link_header (~> 0.0, >= 0.0.8) + multi_json (~> 1.15) + rack (>= 2.2, < 4) + rdf (~> 3.2, >= 3.2.10) + jwt (2.8.1) + base64 launchy (2.5.2) addressable (~> 2.8) - libxml-ruby (5.0.2) + libxml-ruby (5.0.3) + link_header (0.0.8) logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.6.6) mime-types (>= 1.16, < 4) - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0206) + mime-types-data (3.2024.0507) mini_mime (1.1.5) minitest (4.7.5) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - net-http-persistent (2.9.4) + net-http-persistent (4.0.2) + connection_pool (~> 2.2) netrc (0.11.0) oj (3.16.3) bigdecimal (>= 3.0) omni_logger (0.1.4) logger os (1.1.4) + parallel (1.24.0) parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) - rack (3.0.9) + public_suffix (5.0.5) + rack (3.0.11) rack-test (2.1.0) rack (>= 1.3) - rake (13.1.0) - rdf (1.0.8) - addressable (>= 2.2) - redis (5.0.8) - redis-client (>= 0.17.0) - redis-client (0.19.1) + rake (13.2.1) + rdf (3.2.11) + link_header (~> 0.0, >= 0.0.8) + rdf-raptor (3.2.0) + ffi (~> 1.15) + rdf (~> 3.2) + rdf-rdfxml (3.2.2) + builder (~> 3.2) + htmlentities (~> 4.3) + rdf (~> 3.2) + rdf-xsd (~> 3.2) + rdf-vocab (3.2.7) + rdf (~> 3.2, >= 3.2.4) + rdf-xsd (3.2.1) + rdf (~> 3.2) + rexml (~> 3.2) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.2) connection_pool representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) - request_store (1.5.1) + request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -214,8 +244,9 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.2.6) - rsolr (2.5.0) + rexml (3.2.8) + strscan (>= 3.0.9) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby-xxHash (0.4.0.2) @@ -223,7 +254,7 @@ GEM rubyzip (2.3.2) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) - signet (0.18.0) + signet (0.19.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -237,6 +268,7 @@ GEM simplecov (~> 0.19) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) + strscan (3.1.0) sys-proctable (1.3.0) ffi (~> 1.1) systemu (2.6.5) @@ -250,6 +282,7 @@ GEM macaddr (~> 1.0) PLATFORMS + x86_64-darwin-23 x86_64-linux DEPENDENCIES @@ -260,6 +293,7 @@ DEPENDENCIES goo! google-analytics-data google-apis-analytics_v3 + json-ld mail (= 2.6.6) minitest (< 5.0) multi_json @@ -267,6 +301,7 @@ DEPENDENCIES ncbo_cron! oj (~> 3.0) ontologies_linked_data! + parallel parseconfig pony pry @@ -281,4 +316,4 @@ DEPENDENCIES test-unit-minitest BUNDLED WITH - 2.3.23 + 2.4.22 diff --git a/config/config.rb.sample b/config/config.rb.sample index 10cf7e90..1387a9fc 100644 --- a/config/config.rb.sample +++ b/config/config.rb.sample @@ -7,7 +7,7 @@ GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QU GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost" -MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555 +MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55556 MGREP_DICT_PATH = ENV.include?("MGREP_DICT_PATH") ? ENV["MGREP_DICT_PATH"] : "./test/data/dictionary.txt" REDIS_GOO_CACHE_HOST = ENV.include?("REDIS_GOO_CACHE_HOST") ? ENV["REDIS_GOO_CACHE_HOST"] : "localhost" REDIS_HTTP_CACHE_HOST = ENV.include?("REDIS_HTTP_CACHE_HOST") ? ENV["REDIS_HTTP_CACHE_HOST"] : "localhost" @@ -16,8 +16,9 @@ REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT" REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/tmp/ontologies_report.json" REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" -SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" -SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr" +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr" +GOO_SLICES = ENV["GOO_SLICES"] || 500 LinkedData.config do |config| config.goo_backend_name = GOO_BACKEND_NAME.to_s @@ -73,8 +74,8 @@ NcboCron.config do |config| # do not deaemonize in docker config.daemonize = false - config.search_index_all_url = "http://localhost:8983/solr/term_search_core2" - config.property_search_index_all_url = "http://localhost:8983/solr/prop_search_core2" + config.search_index_all_url = "http://localhost:8983/solr" + config.property_search_index_all_url = "http://localhost:8983/solr" # Google Analytics GA4 config config.analytics_path_to_key_file = "config/your_analytics_key.json" diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d3..00000000 --- a/config/solr/property_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede77425..00000000 --- a/config/solr/property_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml deleted file mode 100644 index 20824ea6..00000000 --- a/config/solr/property_search/schema.xml +++ /dev/null @@ -1,1179 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml deleted file mode 100644 index 771a0f32..00000000 --- a/config/solr/property_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/config/solr/solr.xml b/config/solr/solr.xml deleted file mode 100644 index d9d089e4..00000000 --- a/config/solr/solr.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - ${solr.max.booleanClauses:500000} - ${solr.sharedLib:} - ${solr.allowPaths:} - - - - ${host:} - ${solr.port.advertise:0} - ${hostContext:solr} - - ${genericCoreNodeNames:true} - - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - - - ${socketTimeout:600000} - ${connTimeout:60000} - ${solr.shardsWhitelist:} - - - - - diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d3..00000000 --- a/config/solr/term_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede77425..00000000 --- a/config/solr/term_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml deleted file mode 100644 index fa95e127..00000000 --- a/config/solr/term_search/schema.xml +++ /dev/null @@ -1,1222 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml deleted file mode 100644 index 771a0f32..00000000 --- a/config/solr/term_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/docker-compose.yml b/docker-compose.yml index 9f5d628d..d554f11a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,46 +1,37 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ncbo_cron:0.0.2 + image: agroportal/ontologies_api:development environment: &env - BUNDLE_PATH: /srv/ontoportal/bundle # default bundle config resolves to /usr/local/bundle/config inside of the container # we are setting it to local app directory if we need to use 'bundle config local' - BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_cron/.bundle - COVERAGE: 'true' - GOO_REDIS_HOST: redis-ut - REDIS_GOO_CACHE_HOST: redis-ut - REDIS_HTTP_CACHE_HOST: redis-ut - REDIS_PERSISTENT_HOST: redis-ut + BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle + BUNDLE_PATH: /srv/ontoportal/bundle + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut REDIS_PORT: 6379 - SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 - MGREP_HOST: mgrep-ut - MGREP_PORT: 55556 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr stdin_open: true tty: true - command: "bundle exec rackup -o 0.0.0.0 --port 9393" + command: /bin/bash volumes: - # bundle volume for hosting gems installed by bundle; it helps in local development with gem udpates + # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development - bundle:/srv/ontoportal/bundle - # ncbo_cron code - - .:/srv/ontoportal/ncbo_cron + - .:/srv/ontoportal/ontologies_api # mount directory containing development version of the gems if you need to use 'bundle config local' #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal depends_on: &depends_on - solr-ut: + solr-prop-ut: condition: service_healthy - redis-ut: + solr-term-ut: condition: service_healthy - mgrep-ut: + redis-ut: condition: service_healthy services: - ncbo_cron: + api: <<: *app + env_file: + .env environment: <<: *env GOO_BACKEND_NAME: 4store @@ -52,33 +43,23 @@ services: profiles: - 4store depends_on: - <<: *depends_on - 4store-ut: - condition: service_started - - - ncbo_cron-agraph: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: ag - GOO_PORT: 10035 - GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/bioportal_test - GOO_PATH_DATA: /repositories/bioportal_test/statements - GOO_PATH_UPDATE: /repositories/bioportal_test/statements - profiles: - - agraph - depends_on: - <<: *depends_on - agraph-ut: - condition: service_healthy + - solr-ut + - redis-ut + - mgrep-ut + - 4store-ut + ports: + - "9393:9393" + mgrep-ut: + image: ontoportal/mgrep-ncbo:0.1 + ports: + - "55556:55555" redis-ut: image: redis ports: - - 6379:6379 + - "6379:6379" + command: [ "redis-server", "--save", "", "--appendonly", "no" ] healthcheck: test: redis-cli ping interval: 10s @@ -87,69 +68,91 @@ services: 4store-ut: image: bde2020/4store - platform: linux/amd64 - #volume: fourstore:/var/lib/4store + volumes: + - 4store:/var/lib/4store command: > - bash -c "4s-backend-setup --segments 4 ontoportal_kb - && 4s-backend ontoportal_kb - && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + bash -c "if [ ! -d '/var/lib/4store/ontoportal_kb' ]; then 4s-backend-setup --segments 4 ontoportal_kb; fi ; 4s-backend ontoportal_kb ; 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + + ports: + - "9000:9000" profiles: + - fs - 4store - ports: - - 9000:9000 solr-ut: image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - ports: - - "8983:8983" - command: > - bash -c "precreate-core term_search_core1 /configsets/term_search - && precreate-core prop_search_core1 /configsets/property_search - && solr-foreground" - healthcheck: - test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] - start_period: 3s - interval: 10s - timeout: 5s - retries: 5 - - mgrep-ut: - image: ontoportal/mgrep:0.0.2 - platform: linux/amd64 - healthcheck: - test: ["CMD", "nc", "-z", "-v", "localhost", "55556"] - start_period: 3s - interval: 10s - timeout: 5s - retries: 5 ports: - - 55556:55555 - + - 8983:8983 + command: bin/solr start -cloud -f + # volumes: + #- solr_data:/var/solr/data agraph-ut: - image: franzinc/agraph:v8.0.0 + image: franzinc/agraph:v8.1.0 platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test - AGRAPH_SUPER_PASSWORD=xyzzy shm_size: 1g - # ports: - # - 10035:10035 + ports: + # - 10035:10035 + - 10000-10035:10000-10035 + volumes: + - agdata:/agraph/data + # - ./agraph/etc:/agraph/etc command: > bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create bioportal_test + ; agtool repos create ontoportal_test --supersede ; agtool users add anonymous - ; agtool users grant anonymous root:bioportal_test:rw + ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log" + # healthcheck: + # test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] + # start_period: 10s + # interval: 10s + # timeout: 5s + # retries: 5 + profiles: + - ag + + virtuoso-ut: + image: tenforce/virtuoso:virtuoso7.2.5 + platform: linux/amd64 + environment: + - SPARQL_UPDATE=true + ports: + - 1111:1111 + - 8890:8890 + profiles: + - vo healthcheck: - test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"] - start_period: 20s + test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] + start_period: 10s interval: 60s timeout: 5s retries: 3 + + graphdb-ut: + image: ontotext/graphdb:10.3.3 + platform: linux/amd64 + privileged: true + environment: + GDB_HEAP_SIZE: 5G + GDB_JAVA_OPTS: >- + -Xms5g -Xmx5g + ports: + - 7200:7200 + - 7300:7300 + volumes: + - ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl + - ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt + + entrypoint: > + bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 " profiles: - - agraph + - gb volumes: bundle: + agdata: + 4store: + #solr_data: \ No newline at end of file diff --git a/lib/ncbo_cron/ontology_helper.rb b/lib/ncbo_cron/ontology_helper.rb index 4d1f0716..2014b17a 100644 --- a/lib/ncbo_cron/ontology_helper.rb +++ b/lib/ncbo_cron/ontology_helper.rb @@ -10,6 +10,7 @@ module OntologyHelper :process_rdf => true, :generate_labels => true, :extract_metadata => true, + :index_all_data => true, :index_search => true, :index_properties => true, :run_metrics => true, diff --git a/rakelib/annotator_management.rake b/rakelib/annotator_management.rake index 02f23d8e..622cf41a 100644 --- a/rakelib/annotator_management.rake +++ b/rakelib/annotator_management.rake @@ -3,32 +3,39 @@ # desc 'Annotator Utilities' namespace :annotator do - require 'bundler/setup' - # Configure the process for the current cron configuration. - require_relative '../lib/ncbo_cron' - config_exists = File.exist?(File.expand_path('../../config/config.rb', __FILE__)) - abort('Please create a config/config.rb file using the config/config.rb.sample as a template') unless config_exists - require_relative '../config/config' - annotator = Annotator::Models::NcboAnnotator.new - cur_inst = annotator.redis_current_instance - alt_inst = annotator.redis_default_alternate_instance + def init_annotator + require 'bundler/setup' + # Configure the process for the current cron configuration. + require_relative '../lib/ncbo_cron' + config_exists = File.exist?(File.expand_path('../../config/config.rb', __FILE__)) + abort('Please create a config/config.rb file using the config/config.rb.sample as a template') unless config_exists + require_relative '../config/config' + + annotator = Annotator::Models::NcboAnnotator.new + cur_inst = annotator.redis_current_instance + alt_inst = annotator.redis_default_alternate_instance + [annotator, cur_inst, alt_inst] + end namespace :redis_instance do desc 'Get current Annotator redis terms cache prefix' task :get do + annotator, cur_inst, alt_inst = init_annotator puts cur_inst end desc 'Delete Annotator term cache from the alternate instance' # use with caution!!! useful for reducing memory/disk footprint task :purge_alternate do + annotator, cur_inst, alt_inst = init_annotator puts "Cleared Annotator Redis alternate terms cache #{alt_inst}" annotator.delete_term_cache(alt_inst) end desc 'Swap Annotator Redis term cache instance from primary to alternate' task :switch_to_alternate do + annotator, cur_inst, alt_inst = init_annotator annotator.redis_switch_instance puts "Annotator Redis terms cache instance has been switched to #{alt_inst}" end diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake new file mode 100644 index 00000000..52af504c --- /dev/null +++ b/rakelib/docker_based_test.rake @@ -0,0 +1,120 @@ +# Rake tasks for running unit tests with backend services running as docker containers + +desc 'Run unit tests with docker based backend' +namespace :test do + namespace :docker do + task :up do + system("docker compose up -d") || abort("Unable to start docker containers") + unless system("curl -sf http://localhost:8983/solr || exit 1") + printf("waiting for Solr container to initialize") + sec = 0 + until system("curl -sf http://localhost:8983/solr || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + abort(" Solr container hasn't initialized properly") + end + end + printf("\n") + end + end + task :down do + #system("docker compose --profile fs --profile ag stop") + #system("docker compose --profile fs --profile ag kill") + end + desc "run tests with docker AG backend" + task :ag do + ENV["GOO_BACKEND_NAME"]="allegrograph" + ENV["GOO_PORT"]="10035" + ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test" + ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements" + ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal_test/statements" + ENV["COMPOSE_PROFILES"]="ag" + Rake::Task["test:docker:up"].invoke + # AG takes some time to start and create databases/accounts + # TODO: replace system curl command with native ruby code + unless system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") + printf("waiting for AllegroGraph container to initialize") + sec = 0 + until system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") do + sleep(1) + printf(".") + sec += 1 + end + end + puts + system("docker compose ps") # TODO: remove after GH actions troubleshooting is complete + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + desc "run tests with docker 4store backend" + task :fs do + ENV["GOO_PORT"]="9000" + ENV["COMPOSE_PROFILES"]='fs' + Rake::Task["test:docker:up"].invoke + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + desc "run tests with docker Virtuoso backend" + task :vo do + ENV["GOO_BACKEND_NAME"]="virtuoso" + ENV["GOO_PORT"]="8890" + ENV["GOO_PATH_QUERY"]="/sparql" + ENV["GOO_PATH_DATA"]="/sparql" + ENV["GOO_PATH_UPDATE"]="/sparql" + ENV["COMPOSE_PROFILES"]="vo" + Rake::Task["test:docker:up"].invoke + # + unless system("curl -sf http://localhost:8890/sparql || exit 1") + printf("waiting for Virtuoso container to initialize") + sec = 0 + until system("curl -sf http://localhost:8890/sparql || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + system("docker compose logs virtuoso-ut") + abort(" Virtuoso container hasn't initialized properly") + end + end + end + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + + desc "run tests with docker GraphDb backend" + task :gb do + ENV["GOO_BACKEND_NAME"]="graphdb" + ENV["GOO_PORT"]="7200" + ENV["GOO_PATH_QUERY"]="/repositories/ontoportal" + ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements" + ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal/statements" + ENV["COMPOSE_PROFILES"]="gb" + Rake::Task["test:docker:up"].invoke + + #system("docker compose cp ./test/data/graphdb-repo-config.ttl graphdb:/opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl") + #system("docker compose cp ./test/data/graphdb-test-load.nt graphdb:/opt/graphdb/dist/configs/templates/graphdb-test-load.nt") + #system('docker compose exec graphdb sh -c "importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/graphdb-test-load.nt ;"') + unless system("curl -sf http://localhost:7200/repositories || exit 1") + printf("waiting for Graphdb container to initialize") + sec = 0 + until system("curl -sf http://localhost:7200/repositories || exit 1") do + sleep(1) + printf(".") + sec += 1 + if sec > 30 + system("docker compose logs graphdb") + abort(" Graphdb container hasn't initialized properly") + end + end + end + Rake::Task["test"].invoke + Rake::Task["test:docker:down"].invoke + end + + end +end diff --git a/rakelib/group_management.rake b/rakelib/group_management.rake index 2125af60..5586f96a 100644 --- a/rakelib/group_management.rake +++ b/rakelib/group_management.rake @@ -7,10 +7,11 @@ namespace :group do require_relative '../lib/ncbo_cron' config_exists = File.exist?(File.expand_path('../../config/config.rb', __FILE__)) abort('Please create a config/config.rb file using the config/config.rb.sample as a template') unless config_exists - require_relative '../config/config' + desc 'Create a new ontology group' task :create, [:acronym, :name] do |_t, args| + require_relative '../config/config' checkgroup = LinkedData::Models::Group.find(args.acronym).first abort("FAILED: The Group #{args.groupname} already exists") unless checkgroup.nil? group = LinkedData::Models::Group.new @@ -24,6 +25,7 @@ namespace :group do end desc 'Add ontology to a group' task :add_ontology, [:group_acronym, :ontology_acronym] do |_t, args| + require_relative '../config/config' grp = LinkedData::Models::Group.find(args.group_acronym).first abort("FAILED: The Group #{args.group_acronym} does not exist") if grp.nil? ontology = LinkedData::Models::Ontology.find(args.ontology_acronym).first diff --git a/rakelib/purl_management.rake b/rakelib/purl_management.rake index 58cfadd7..b1ba50f8 100644 --- a/rakelib/purl_management.rake +++ b/rakelib/purl_management.rake @@ -7,10 +7,11 @@ namespace :purl do require_relative '../lib/ncbo_cron' config_exists = File.exist?(File.expand_path('../../config/config.rb', __FILE__)) abort('Please create a config/config.rb file using the config/config.rb.sample as a template') unless config_exists - require_relative '../config/config' + desc 'update purl for all ontologies' task :update_all do + require_relative '../config/config' purl_client = LinkedData::Purl::Client.new LinkedData::Models::Ontology.all.each do |ont| ont.bring(:acronym) diff --git a/rakelib/user_management.rake b/rakelib/user_management.rake index 71a7bce4..bf43dc6b 100644 --- a/rakelib/user_management.rake +++ b/rakelib/user_management.rake @@ -8,10 +8,10 @@ namespace :user do require_relative '../lib/ncbo_cron' config_exists = File.exist?(File.expand_path('../../config/config.rb', __FILE__)) abort('Please create a config/config.rb file using the config/config.rb.sample as a template') unless config_exists - require_relative '../config/config' desc 'Add administrator role to the user' task :adminify, [:username] do |t, args| + require_relative '../config/config' username = args.username user = LinkedData::Models::User.find(username).first abort("FAILED: The user #{args.username} does not exist") if user.nil? @@ -33,6 +33,7 @@ namespace :user do desc 'Reset all roles to LIBRARIAN for the user' task :resetroles, [:username] do |t, args| + require_relative '../config/config' username = args.username user = LinkedData::Models::User.find(username).first abort("FAILED: user #{args.username} does not exist") if user.nil? @@ -50,6 +51,7 @@ namespace :user do desc 'Reset password to a random value for the user' task :resetpassword, [:username] do |t, args| + require_relative '../config/config' username = args.username newpassword = SecureRandom.base64(15) user = LinkedData::Models::User.find(username).first @@ -63,6 +65,7 @@ namespace :user do desc 'Create a new user' task :create, [:username, :email, :password] do |t, args| + require_relative '../config/config' args.with_defaults(password: nil) password = args.password args.password.nil? && password = SecureRandom.base64(15) @@ -84,6 +87,7 @@ namespace :user do namespace :apikey do desc 'get APIKEY for the user' task :get, [:username] do |t, args| + require_relative '../config/config' user = LinkedData::Models::User.find(args.username).first abort("FAILED: The user #{args.username} does not exist") if user.nil? user.bring_remaining @@ -91,6 +95,7 @@ namespace :user do end desc 'reset APIKEY for the user to random value or to specified value if API key is provided' task :reset, [:username, :apikey] do |t, args| + require_relative '../config/config' user = LinkedData::Models::User.find(args.username).first abort("FAILED: The user #{args.username} does not exist") if user.nil? user.bring_remaining @@ -110,6 +115,7 @@ namespace :user do desc 'Show all artifacts administrered by the user' task :artifacts, [:username] do |t, args| + require_relative '../config/config' # most of the code is copied from /bin/ncbo_spam_deletion username = args.username diff --git a/test/data/graphdb-repo-config.ttl b/test/data/graphdb-repo-config.ttl new file mode 100644 index 00000000..9200da9a --- /dev/null +++ b/test/data/graphdb-repo-config.ttl @@ -0,0 +1,33 @@ +@prefix rdfs: . +@prefix rep: . +@prefix sail: . +@prefix xsd: . + +<#ontoportal> a rep:Repository; + rep:repositoryID "ontoportal"; + rep:repositoryImpl [ + rep:repositoryType "graphdb:SailRepository"; + [ + "http://example.org/owlim#"; + "false"; + ""; + "true"; + "false"; + "true"; + "true"; + "32"; + "10000000"; + ""; + "true"; + ""; + "0"; + "0"; + "false"; + "file-repository"; + "rdfsplus-optimized"; + "storage"; + "false"; + sail:sailType "owlim:Sail" + ] + ]; + rdfs:label "" . \ No newline at end of file diff --git a/test/data/graphdb-test-load.nt b/test/data/graphdb-test-load.nt new file mode 100644 index 00000000..e69de29b diff --git a/test/test_case.rb b/test/test_case.rb index 75bb0454..3f69aa02 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -60,12 +60,18 @@ def backend_triplestore_delete raise StandardError, 'Too many triples in KB, does not seem right to run tests' unless count_pattern('?s ?p ?o') < 400000 - LinkedData::Models::Ontology.where.include(:acronym).each do |o| - query = "submissionAcronym:#{o.acronym}" - LinkedData::Models::Ontology.unindexByQuery(query) - end + # LinkedData::Models::Ontology.where.include(:acronym).each do |o| + # query = "submissionAcronym:#{o.acronym}" + # LinkedData::Models::Ontology.unindexByQuery(query) + # end + # LinkedData::Models::Ontology.indexCommit - Goo.sparql_update_client.update('DELETE {?s ?p ?o } WHERE { ?s ?p ?o }') + + graphs = Goo.sparql_query_client.query("SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o . } }") + graphs.each_solution do |sol| + Goo.sparql_data_client.delete_graph(sol[:g]) + end + LinkedData::Models::SubmissionStatus.init_enum LinkedData::Models::OntologyFormat.init_enum LinkedData::Models::OntologyType.init_enum diff --git a/test/test_ontology_submission_parser.rb b/test/test_ontology_submission_parser.rb index d9572802..d7329284 100644 --- a/test/test_ontology_submission_parser.rb +++ b/test/test_ontology_submission_parser.rb @@ -40,7 +40,7 @@ def test_queue_submission assert_nil val parser.queue_submission(o1.submissions[0], { - dummy_action: true, index_search: false, metrics: true, + dummy_action: true, index_all_data: false, index_search: false, metrics: true, process_annotator: true, another_dummy_action: true, all: true}) val = @@redis.hget(NcboCron::Models::OntologySubmissionParser::QUEUE_HOLDER, parser.get_prefixed_id(o1.submissions[0].id.to_s)) options = MultiJson.load(val, symbolize_keys: true) @@ -49,11 +49,11 @@ def test_queue_submission o2 = @@ontologies[1] o2.bring(:submissions) if o2.bring?(:submissions) parser.queue_submission(o2.submissions[0], { - :process_rdf => false, :index_search => true, :metrics => true, + :process_rdf => false, :index_search => true, index_all_data: true, :metrics => true, :process_annotator => false}) val = @@redis.hget(NcboCron::Models::OntologySubmissionParser::QUEUE_HOLDER, parser.get_prefixed_id(o2.submissions[0].id.to_s)) options = MultiJson.load(val, symbolize_keys: true) - assert_equal({:process_rdf => false, :index_search => true, :process_annotator => false}, options) + assert_equal({:process_rdf => false, :index_search => true, index_all_data: true, :process_annotator => false}, options) end def test_parse_submissions @@ -76,13 +76,13 @@ def test_parse_submissions o2_sub2.bring(:submissionStatus) options_o1 = { - :dummy_action => true, :process_rdf => true, :index_search => true, :index_properties => true, :diff => true, + :dummy_action => true, :process_rdf => true, :index_all_data => true, :index_search => true, :index_properties => true, :diff => true, :dummy_metrics => false, :run_metrics => false, :process_annotator => false, :another_dummy_action => false, :all => false } options_o2 = { - dummy_action: false, process_rdf: true, index_search: false, index_properties: false, :diff => true, + dummy_action: false, process_rdf: true, index_search: false, :index_all_data => false, index_properties: false, :diff => true, dummy_metrics: true, run_metrics: false, process_annotator: true, another_dummy_action: true, all: false }