diff --git a/Gemfile b/Gemfile index e1050f63..d25c7660 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' +gem 'request_store' # Rack middleware gem 'ffi' @@ -40,13 +41,13 @@ gem 'unicorn-worker-killer' gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' -# NCBO -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' +# NCBO gems (can be from a local dev path or from rubygems/git) +gem 'ncbo_annotator', git: 'https://github.com/ncbo/ncbo_annotator.git', branch: 'master' +gem 'ncbo_cron', git: 'https://github.com/ncbo/ncbo_cron.git', branch: 'master' +gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'pr/sync-agroportal-bioportal' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 @@ -74,4 +75,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io + gem 'webmock', '~> 3.19.1' end diff --git a/Gemfile.lock b/Gemfile.lock index 642ef823..488c23ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,23 +1,7 @@ -GIT - remote: https://github.com/ncbo/goo.git - revision: 33583fd1c1d72b449cd6bb91815e3422b8448983 - branch: develop - specs: - goo (0.0.2) - addressable (~> 2.8) - pry - rdf (= 1.0.8) - redis - request_store - rest-client - rsolr - sparql-client - uuid - GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e - branch: develop + revision: 63c986880aa88c9384043e6611a682434a14aba7 + branch: master specs: ncbo_annotator (0.0.1) goo @@ -27,13 +11,11 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 26638bfdbec1e6b01f5eb6a8b655bfd70adac8ac - branch: develop + revision: f239b34230131cd7cfb9eec5b34068ba990b698b + branch: master specs: ncbo_cron (0.0.1) dante - faraday (~> 2) - faraday-follow_redirects (~> 0) goo google-analytics-data mlanett-redis-lock @@ -45,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: f92a42f660635522eb8709e618ff2e641aef0d17 - branch: develop + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master specs: ncbo_ontology_recommender (0.0.1) goo @@ -55,9 +37,35 @@ GIT redis GIT - remote: https://github.com/ncbo/ontologies_linked_data.git - revision: a3f6cf0493bb595011f98acf95efda33111c537b - branch: develop + remote: https://github.com/ncbo/sparql-client.git + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc + branch: master + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + +GIT + remote: https://github.com/ontoportal-lirmm/goo.git + revision: f2751fe9324e48a0a5c2a8b15580ab879fc53a2b + branch: pr/sync-agroportal-bioportal + specs: + goo (0.0.2) + addressable (~> 2.8) + pry + rdf (= 1.0.8) + redis + request_store + rest-client + rsolr + sparql-client + uuid + +GIT + remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git + revision: fc48343ad7a3f1867821c173be900cf16506b4b7 + branch: pr/sync-agroportal-ncbo specs: ontologies_linked_data (0.0.1) activesupport @@ -74,16 +82,6 @@ GIT rsolr rubyzip -GIT - remote: https://github.com/ncbo/sparql-client.git - revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc - branch: develop - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 @@ -132,6 +130,9 @@ GEM coderay (1.1.3) concurrent-ruby (1.2.3) connection_pool (2.4.1) + crack (1.0.0) + bigdecimal + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.4) @@ -142,8 +143,6 @@ GEM base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-follow_redirects (0.3.0) - faraday (>= 1, < 3) faraday-net_http (3.0.2) faraday-retry (2.2.1) faraday (~> 2.0) @@ -170,8 +169,6 @@ GEM google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3) - google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) @@ -188,12 +185,6 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.63.0) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.63.0-aarch64-linux) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) @@ -206,6 +197,7 @@ GEM haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.1.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) @@ -382,14 +374,15 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - aarch64-linux arm64-darwin-22 arm64-darwin-23 - ruby x86_64-darwin-18 - x86_64-darwin-21 x86_64-darwin-23 x86_64-linux @@ -431,6 +424,7 @@ DEPENDENCIES redis redis-rack-cache (~> 2.0) redis-store (~> 1.10) + request_store rubocop shotgun! simplecov @@ -441,6 +435,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock (~> 3.19.1) BUNDLED WITH 2.4.22 diff --git a/app.rb b/app.rb index 41ad56ac..e09178bd 100644 --- a/app.rb +++ b/app.rb @@ -27,6 +27,7 @@ require_relative 'lib/rack/cube_reporter' require_relative 'lib/rack/param_translator' require_relative 'lib/rack/slice_detection' +require_relative 'lib/rack/request_lang' # Logging setup require_relative "config/logging" @@ -34,6 +35,8 @@ # Inflector setup require_relative "config/inflections" +require 'request_store' + # Protection settings set :protection, :except => :path_traversal @@ -141,6 +144,9 @@ use Rack::PostBodyToParams use Rack::ParamTranslator +use RequestStore::Middleware +use Rack::RequestLang + use LinkedData::Security::Authorization use LinkedData::Security::AccessDenied diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/property_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/property_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml new file mode 100644 index 00000000..20824ea6 --- /dev/null +++ b/config/solr/property_search/schema.xml @@ -0,0 +1,1179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/property_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/config/solr/solr.xml b/config/solr/solr.xml new file mode 100644 index 00000000..d9d089e4 --- /dev/null +++ b/config/solr/solr.xml @@ -0,0 +1,60 @@ + + + + + + + + ${solr.max.booleanClauses:500000} + ${solr.sharedLib:} + ${solr.allowPaths:} + + + + ${host:} + ${solr.port.advertise:0} + ${hostContext:solr} + + ${genericCoreNodeNames:true} + + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + + + ${socketTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} + + + + + diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/term_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/term_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml new file mode 100644 index 00000000..73c75b31 --- /dev/null +++ b/config/solr/term_search/schema.xml @@ -0,0 +1,1224 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/term_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 182a18c9..55de76eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,10 +95,18 @@ services: - 4store solr-ut: - image: ontoportal/solr-ut:0.1.0 + image: solr:8 + volumes: + - ./test/solr/configsets:/configsets:ro + ports: + - "8983:8983" + command: > + bash -c "precreate-core term_search_core1 /configsets/term_search + && precreate-core prop_search_core1 /configsets/property_search + && solr-foreground" healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 3s + test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] + start_period: 5s interval: 10s timeout: 5s retries: 5 diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 499e61ac..3096ab47 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -84,6 +84,9 @@ def get_term_search_query(text, params={}) end end + lang = params["lang"] || params["language"] + lang_suffix = lang && !lang.eql?("all") ? "_#{lang}" : "" + query = "" params["defType"] = "edismax" params["stopwords"] = "true" @@ -100,25 +103,25 @@ def get_term_search_query(text, params={}) if params[EXACT_MATCH_PARAM] == "true" query = "\"#{solr_escape(text)}\"" - params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact^10 synonymExact #{QUERYLESS_FIELDS_STR_NO_IDS}" - params["hl.fl"] = "resource_id prefLabelExact synonymExact #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact#{lang_suffix}^10 synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR}" elsif params[SUGGEST_PARAM] == "true" || text[-1] == '*' text.gsub!(/\*+$/, '') query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" - params["qf"] = "prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["qf"] = " prefLabelExact#{lang_suffix}^100 prefLabelSuggestEdge#{lang_suffix}^50 synonymSuggestEdge#{lang_suffix}^10 prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" params["pf"] = "prefLabelSuggest^50" - params["hl.fl"] = "prefLabelExact prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "prefLabelExact#{lang_suffix} prefLabelSuggestEdge#{lang_suffix} synonymSuggestEdge#{lang_suffix} prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? query = '*' else query = solr_escape(text) end - params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact^90 prefLabel^70 synonymExact^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" params["bq"] = "idAcronymMatch:true^80" - params["hl.fl"] = "resource_id prefLabelExact prefLabel synonymExact synonym #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} prefLabel#{lang_suffix} synonymExact#{lang_suffix} synonym#{lang_suffix} #{QUERYLESS_FIELDS_STR}" params["hl.fl"] = "#{params["hl.fl"]} property" if params[INCLUDE_PROPERTIES_PARAM] == "true" end @@ -347,6 +350,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) doc[:submission] = old_class.submission doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = LinkedData::Models::Class.read_only(doc) + instance.prefLabel = instance.prefLabel.first if instance.prefLabel.is_a?(Array) classes_hash[ont_uri_class_uri] = instance end diff --git a/lib/rack/request_lang.rb b/lib/rack/request_lang.rb new file mode 100644 index 00000000..b2221041 --- /dev/null +++ b/lib/rack/request_lang.rb @@ -0,0 +1,16 @@ +module Rack + class RequestLang + + def initialize(app = nil, options = {}) + @app = app + end + + def call(env) + r = Rack::Request.new(env) + lang = r.params["lang"] || r.params["language"] + lang = lang.upcase.to_sym if lang + RequestStore.store[:requested_lang] = lang + @app.call(env) + end + end +end \ No newline at end of file diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 3b21b9e7..55d8fb66 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -16,7 +16,12 @@ def self.before_suite end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true, + process_options: { + process_rdf: true, + extract_metadata: false, + index_search: true + }) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) @@ -260,16 +265,16 @@ def test_default_properties_output assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].first.downcase <=> b["annotatedClass"]["prefLabel"].first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] - assert_equal "Aggregate Human Data", annotations.first["annotatedClass"]["prefLabel"] + assert_equal "Aggregate Human Data", Array(annotations.first["annotatedClass"]["prefLabel"]).first params = {text: text, include: "prefLabel,definition"} get "/annotator", params assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| Array(a["annotatedClass"]["prefLabel"]).first.downcase <=> Array(b["annotatedClass"]["prefLabel"]).first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] assert_equal ["A resource that provides data from clinical care that comprises combined data from multiple individual human subjects."], annotations.first["annotatedClass"]["definition"] end @@ -348,7 +353,7 @@ def self.mapping_test_set classes = [] class_id = terms_a[i] ont_acr = onts_a[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) @@ -356,7 +361,7 @@ def self.mapping_test_set classes << c class_id = terms_b[i] ont_acr = onts_b[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) diff --git a/test/controllers/test_batch_controller.rb b/test/controllers/test_batch_controller.rb index 55d9cee9..d1e9d144 100644 --- a/test/controllers/test_batch_controller.rb +++ b/test/controllers/test_batch_controller.rb @@ -2,8 +2,7 @@ class TestBatchController < TestCase def self.before_suite - LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) end def test_class_batch_one_ontology diff --git a/test/controllers/test_classes_controller.rb b/test/controllers/test_classes_controller.rb index a918ece0..42ccf29b 100644 --- a/test/controllers/test_classes_controller.rb +++ b/test/controllers/test_classes_controller.rb @@ -7,7 +7,9 @@ def self.before_suite submission_count: 3, submissions_to_process: [1, 2], process_submission: true, - random_submission_count: false} + random_submission_count: false, + process_options: {process_rdf: true, extract_metadata: false} + } return LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end diff --git a/test/controllers/test_metrics_controller.rb b/test/controllers/test_metrics_controller.rb index 1b8890a6..b27fa198 100644 --- a/test/controllers/test_metrics_controller.rb +++ b/test/controllers/test_metrics_controller.rb @@ -18,11 +18,12 @@ def self.before_suite "individuals"=>124, "properties"=>63, "maxDepth"=> 7 } - @@options = {ont_count: 2, - submission_count: 3, - submissions_to_process: [1, 2], - process_submission: true, - random_submission_count: false} + @@options = { ont_count: 2, + submission_count: 3, + submissions_to_process: [1, 2], + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, run_metrics: true, index_properties: true }, + random_submission_count: false } LinkedData::SampleData::Ontology.create_ontologies_and_submissions(@@options) end @@ -78,18 +79,18 @@ def test_metrics_missing get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(0, ontologies.length, msg='Failure to detect 0 ontologies with missing metrics.') + assert_equal(0, ontologies.length, msg = 'Failure to detect 0 ontologies with missing metrics.') # create ontologies with latest submissions that have no metrics delete_ontologies_and_submissions - options = {ont_count: 2, - submission_count: 1, - process_submission: false, - random_submission_count: false} + options = { ont_count: 2, + submission_count: 1, + process_submission: false, + random_submission_count: false } create_ontologies_and_submissions(options) get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(2, ontologies.length, msg='Failure to detect 2 ontologies with missing metrics.') + assert_equal(2, ontologies.length, msg = 'Failure to detect 2 ontologies with missing metrics.') # recreate the before_suite data (this test might not be the last one to run in the suite) delete_ontologies_and_submissions create_ontologies_and_submissions(@@options) diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4f61256b..a5f242bd 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -188,7 +188,9 @@ def test_download_ontology end def test_download_ontology_csv - num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) + num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, + process_submission: true, + process_options:{process_rdf: true, extract_metadata: true, index_search: true}) ont = onts.first acronym = created_ont_acronyms.first @@ -220,13 +222,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test1@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test2@example.org", password: "12345" }) blocked_user.save @@ -296,6 +298,32 @@ def test_on_demand_ontology_pull end end + def test_detach_a_view + view = Ontology.find(@@view_acronym).include(:viewOf).first + ont = view.viewOf + refute_nil view + refute_nil ont + + remove_view_of = {viewOf: ''} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(remove_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}" + onto = MultiJson.load(last_response.body) + assert_nil onto["viewOf"] + + + add_view_of = {viewOf: @@acronym} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(add_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}?include=all" + onto = MultiJson.load(last_response.body) + assert_equal onto["viewOf"], ont.id.to_s + end + private def start_server diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 40532cd0..ef356fcf 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test3@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" @@ -36,6 +39,12 @@ def self._create_onts ont.save end + def setup + delete_ontologies_and_submissions + ont = Ontology.new(acronym: @@acronym, name: @@name, administeredBy: [@@user]) + ont.save + end + def test_submissions_for_given_ontology num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 1) ontology = created_ont_acronyms.first @@ -156,13 +165,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test4@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test5@example.org", password: "12345" }) blocked_user.save @@ -235,5 +244,4 @@ def test_ontology_submissions_access_controller del.delete if del end end - end diff --git a/test/controllers/test_properties_controller.rb b/test/controllers/test_properties_controller.rb index 8248403c..cbf249b8 100644 --- a/test/controllers/test_properties_controller.rb +++ b/test/controllers/test_properties_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_properties_search_controller.rb b/test/controllers/test_properties_search_controller.rb index f93a90a1..2589a293 100644 --- a/test/controllers/test_properties_search_controller.rb +++ b/test/controllers/test_properties_search_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesSearchController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_recommender_controller.rb b/test/controllers/test_recommender_controller.rb index 29caf28c..58d6d942 100644 --- a/test/controllers/test_recommender_controller.rb +++ b/test/controllers/test_recommender_controller.rb @@ -14,7 +14,7 @@ def self.before_suite @@redis.del(mappings) end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 7b14a63d..3ac4862d 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -1,10 +1,10 @@ require_relative '../test_case' -class TestRecommenderController < TestCase +class TestRecommenderV1Controller < TestCase def self.before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) @@text = < "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}) + refute_equal 0, res["response"]["numFound"] + refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activity&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + end + + end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index d64075cc..b2aeccf5 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -631,6 +631,9 @@ Activity + Activity + ActivityEnglish + Activité Activity of interest that may be related to a BRO:Resource. activities diff --git a/test/middleware/test_rack_attack.rb b/test/middleware/test_rack_attack.rb index 42b1ddf2..47e30a52 100644 --- a/test/middleware/test_rack_attack.rb +++ b/test/middleware/test_rack_attack.rb @@ -18,14 +18,14 @@ def self.before_suite LinkedData::OntologiesAPI.settings.req_per_second_per_ip = 1 LinkedData::OntologiesAPI.settings.safe_ips = Set.new(["1.2.3.4", "1.2.3.5"]) - @@user = LinkedData::Models::User.new({username: "user", password: "test_password", email: "test_email@example.org"}) + @@user = LinkedData::Models::User.new({username: "user", password: "test_password", email: "test_email1@example.org"}) @@user.save - @@bp_user = LinkedData::Models::User.new({username: "ncbobioportal", password: "test_password", email: "test_email@example.org"}) + @@bp_user = LinkedData::Models::User.new({username: "ncbobioportal", password: "test_password", email: "test_email2@example.org"}) @@bp_user.save admin_role = LinkedData::Models::Users::Role.find("ADMINISTRATOR").first - @@admin = LinkedData::Models::User.new({username: "admin", password: "test_password", email: "test_email@example.org", role: [admin_role]}) + @@admin = LinkedData::Models::User.new({username: "admin", password: "test_password", email: "test_email3@example.org", role: [admin_role]}) @@admin.save # Redirect output or we get a bunch of noise from Rack (gets reset in the after_suite method). @@ -34,8 +34,8 @@ def self.before_suite $stdout = File.open("/dev/null", "w") $stderr = File.open("/dev/null", "w") - # http://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports - @@port1 = Random.rand(55000..65535) + + @@port1 = self.new('').unused_port # Fork the process to create two servers. This isolates the Rack::Attack configuration, which makes other tests fail if included. @@pid1 = fork do @@ -47,7 +47,7 @@ def self.before_suite Signal.trap("HUP") { Process.exit! } end - @@port2 = Random.rand(55000..65535) # http://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports + @@port2 = self.new('').unused_port @@pid2 = fork do require_relative '../../config/rack_attack' Rack::Server.start( diff --git a/test/solr/configsets/term_search/conf/schema.xml b/test/solr/configsets/term_search/conf/schema.xml index 6b18a2a1..73c75b31 100644 --- a/test/solr/configsets/term_search/conf/schema.xml +++ b/test/solr/configsets/term_search/conf/schema.xml @@ -128,11 +128,20 @@ - - - - - + + + + + + + + + + + + + + @@ -140,9 +149,20 @@ - + + + + + + + + + + + + + - @@ -151,8 +171,8 @@ - - + + @@ -251,8 +271,19 @@ - + + + + + + + + + + + + - - - + + + - - - - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -769,255 +800,255 @@ - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - + \ No newline at end of file diff --git a/test/solr/generate_ncbo_configsets.sh b/test/solr/generate_ncbo_configsets.sh index 893f7f3a..29134dad 100755 --- a/test/solr/generate_ncbo_configsets.sh +++ b/test/solr/generate_ncbo_configsets.sh @@ -2,18 +2,23 @@ # generates solr configsets by merging _default configset with config files in config/solr # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/ -pushd solr/configsets -ld_config='../../../../ontologies_linked_data/config/solr/' -#ld_config='../../../../config/solr/' -ls -l $ld_config -pwd -[ -d property_search ] && rm -Rf property_search -[ -d term_search ] && rm -Rf property_search -[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" -mkdir -p property_search/conf -mkdir -p term_search/conf -cp -a _default/conf/* property_search/conf/ -cp -a _default/conf/* term_search/conf/ -cp -a $ld_config/property_search/* property_search/conf -cp -a $ld_config/term_search/* term_search/conf -popd +#cd solr/configsets +ld_config='config/solr' +configsets='test/solr/configsets' +[ -d ${configsets}/property_search ] && rm -Rf ${configsets}/property_search +[ -d ${configsets}/term_search ] && rm -Rf ${configsets}/term_search +if [[ ! -d ${ld_config}/property_search ]]; then + echo 'cant find ld solr config sets' + exit 1 +fi +if [[ ! -d ${configsets}/_default/conf ]]; then + echo 'cant find default solr configset' + exit 1 +fi +mkdir -p ${configsets}/property_search/conf +mkdir -p ${configsets}/term_search/conf +cp -a ${configsets}/_default/conf/* ${configsets}/property_search/conf/ +cp -a ${configsets}/_default/conf/* ${configsets}/term_search/conf/ +cp -a $ld_config/property_search/* ${configsets}/property_search/conf +cp -a $ld_config/term_search/* ${configsets}/term_search/conf + diff --git a/test/test_case.rb b/test/test_case.rb index 98b02442..0e91aa69 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -21,7 +21,9 @@ require_relative 'test_log_file' require_relative '../app' require 'minitest/unit' +require 'webmock/minitest' MiniTest::Unit.autorun +WebMock.allow_net_connect! require 'rack/test' require 'multi_json' require 'oj' @@ -144,6 +146,9 @@ def app # @option options [TrueClass, FalseClass] :random_submission_count Use a random number of submissions between 1 and :submission_count # @option options [TrueClass, FalseClass] :process_submission Parse the test ontology file def create_ontologies_and_submissions(options = {}) + if options[:process_submission] && options[:process_options].nil? + options[:process_options] = { process_rdf: true, extract_metadata: false, generate_missing_labels: false } + end LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end @@ -214,4 +219,24 @@ def self.reset_to_not_admin(user) user.save end + def unused_port + max_retries = 5 + retries = 0 + server_port = Random.rand(55000..65535) + while port_in_use?(server_port) + retries += 1 + break if retries >= max_retries + server_port = Random.rand(55000..65535) + end + server_port + end + private + def port_in_use?(port) + server = TCPServer.new(port) + server.close + false + rescue Errno::EADDRINUSE + true + end + end diff --git a/views/documentation/documentation.haml b/views/documentation/documentation.haml index 527d781f..916eb1e7 100644 --- a/views/documentation/documentation.haml +++ b/views/documentation/documentation.haml @@ -151,6 +151,7 @@ %li include={prefLabel, synonym, definition, notation, cui, semanticType} // default = (see Common Parameters section) %li page={integer representing the page number} // default = 1 %li pagesize={integer representing the size of the returned page} // default = 50 + %li language={an ISO 639-1 language value, e.g 'fr' or 'en'} // by default search in all languages %h4#nav_search_subtree Subtree Search