From ceb9bcf6309e92f994366055e330f86bf8a803f9 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 29 Mar 2024 09:53:17 -0700 Subject: [PATCH 1/6] reset branch specifier to master --- Gemfile | 12 ++++++------ Gemfile.lock | 34 +++++++++++++++++----------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Gemfile b/Gemfile index e1050f63..ec1f76f3 100644 --- a/Gemfile +++ b/Gemfile @@ -41,12 +41,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'develop' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'develop' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' +gem 'goo', github: 'ncbo/goo', branch: 'master' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' +gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' +gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 523439b4..fc1b44ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 87e991b0e720c580a71e2c2c10f9ab44183db322 - branch: develop + revision: f01386fe55ffba39cbf5e0bf95c8e2dc18767711 + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -15,8 +15,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: fe0783c542479ed0a12da5663ef28e1673e80eba - branch: develop + revision: 63c986880aa88c9384043e6611a682434a14aba7 + branch: master specs: ncbo_annotator (0.0.1) goo @@ -26,8 +26,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 2f8b1c1cff9a00da06ff68c1449f049e3f59c5ce - branch: develop + revision: b5764cc314758dbfdbb855b2b63c1c53b1325f6f + branch: master specs: ncbo_cron (0.0.1) dante @@ -42,8 +42,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: d047de0a9650df321db47300eba8ff8c195e0854 - branch: develop + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master specs: ncbo_ontology_recommender (0.0.1) goo @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: a466d530877cb48cdfb811d0b61ecaabd2b835eb - branch: develop + revision: 4dc3b1123e871b53ba9eb51983ff69c5d2c75c18 + branch: master specs: ontologies_linked_data (0.0.1) activesupport @@ -73,8 +73,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 55e7dbf858eb571c767bc67868f9af61663859cb - branch: develop + revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 + branch: master specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -173,7 +173,7 @@ GEM google-protobuf (~> 3.18) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.13.0) + googleapis-common-protos-types (1.14.0) google-protobuf (~> 3.18) googleauth (1.11.0) faraday (>= 1.0, < 3.a) @@ -242,11 +242,11 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol net-ssh (7.2.1) netrc (0.11.0) - newrelic_rpm (9.7.1) + newrelic_rpm (9.8.0) oj (3.16.1) omni_logger (0.1.4) logger @@ -287,7 +287,7 @@ GEM redcarpet (3.6.0) redis (5.1.0) redis-client (>= 0.17.0) - redis-client (0.21.0) + redis-client (0.21.1) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -301,7 +301,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) rexml (3.2.6) - rsolr (2.5.0) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) rubocop (1.62.1) From 75b05fe7579f7358860f9395a45302e70923995c Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:20:00 +0200 Subject: [PATCH 2/6] add again SOLR config file as changed for the multilingual support --- config/solr/property_search/enumsconfig.xml | 12 + .../mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/property_search/schema.xml | 1179 +++++++++++++++ config/solr/property_search/solrconfig.xml | 1299 +++++++++++++++++ config/solr/solr.xml | 60 + config/solr/term_search/enumsconfig.xml | 12 + .../term_search/mapping-ISOLatin1Accent.txt | 246 ++++ config/solr/term_search/schema.xml | 1224 ++++++++++++++++ config/solr/term_search/solrconfig.xml | 1299 +++++++++++++++++ docker-compose.yml | 14 +- .../configsets/term_search/conf/schema.xml | 1015 ++++++------- test/solr/generate_ncbo_configsets.sh | 35 +- 12 files changed, 6131 insertions(+), 510 deletions(-) create mode 100644 config/solr/property_search/enumsconfig.xml create mode 100644 config/solr/property_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/property_search/schema.xml create mode 100644 config/solr/property_search/solrconfig.xml create mode 100644 config/solr/solr.xml create mode 100644 config/solr/term_search/enumsconfig.xml create mode 100644 config/solr/term_search/mapping-ISOLatin1Accent.txt create mode 100644 config/solr/term_search/schema.xml create mode 100644 config/solr/term_search/solrconfig.xml diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/property_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/property_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml new file mode 100644 index 00000000..20824ea6 --- /dev/null +++ b/config/solr/property_search/schema.xml @@ -0,0 +1,1179 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/property_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/config/solr/solr.xml b/config/solr/solr.xml new file mode 100644 index 00000000..d9d089e4 --- /dev/null +++ b/config/solr/solr.xml @@ -0,0 +1,60 @@ + + + + + + + + ${solr.max.booleanClauses:500000} + ${solr.sharedLib:} + ${solr.allowPaths:} + + + + ${host:} + ${solr.port.advertise:0} + ${hostContext:solr} + + ${genericCoreNodeNames:true} + + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + + + ${socketTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} + + + + + diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml new file mode 100644 index 00000000..72e7b7d3 --- /dev/null +++ b/config/solr/term_search/enumsconfig.xml @@ -0,0 +1,12 @@ + + + + ONTOLOGY + VALUE_SET_COLLECTION + + + ANNOTATION + DATATYPE + OBJECT + + \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/config/solr/term_search/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml new file mode 100644 index 00000000..73c75b31 --- /dev/null +++ b/config/solr/term_search/schema.xml @@ -0,0 +1,1224 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml new file mode 100644 index 00000000..771a0f32 --- /dev/null +++ b/config/solr/term_search/solrconfig.xml @@ -0,0 +1,1299 @@ + + + + + + + + + 8.8.2 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:500000} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + java.lang.String + text_general + + *_str + 256 + + + true + + + java.lang.Boolean + booleans + + + java.util.Date + pdates + + + java.lang.Long + java.lang.Integer + plongs + + + java.lang.Number + pdoubles + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/docker-compose.yml b/docker-compose.yml index 182a18c9..55de76eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,10 +95,18 @@ services: - 4store solr-ut: - image: ontoportal/solr-ut:0.1.0 + image: solr:8 + volumes: + - ./test/solr/configsets:/configsets:ro + ports: + - "8983:8983" + command: > + bash -c "precreate-core term_search_core1 /configsets/term_search + && precreate-core prop_search_core1 /configsets/property_search + && solr-foreground" healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 3s + test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] + start_period: 5s interval: 10s timeout: 5s retries: 5 diff --git a/test/solr/configsets/term_search/conf/schema.xml b/test/solr/configsets/term_search/conf/schema.xml index 6b18a2a1..73c75b31 100644 --- a/test/solr/configsets/term_search/conf/schema.xml +++ b/test/solr/configsets/term_search/conf/schema.xml @@ -128,11 +128,20 @@ - - - - - + + + + + + + + + + + + + + @@ -140,9 +149,20 @@ - + + + + + + + + + + + + + - @@ -151,8 +171,8 @@ - - + + @@ -251,8 +271,19 @@ - + + + + + + + + + + + + - - - + + + - - - - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -769,255 +800,255 @@ - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - + \ No newline at end of file diff --git a/test/solr/generate_ncbo_configsets.sh b/test/solr/generate_ncbo_configsets.sh index 893f7f3a..29134dad 100755 --- a/test/solr/generate_ncbo_configsets.sh +++ b/test/solr/generate_ncbo_configsets.sh @@ -2,18 +2,23 @@ # generates solr configsets by merging _default configset with config files in config/solr # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/ -pushd solr/configsets -ld_config='../../../../ontologies_linked_data/config/solr/' -#ld_config='../../../../config/solr/' -ls -l $ld_config -pwd -[ -d property_search ] && rm -Rf property_search -[ -d term_search ] && rm -Rf property_search -[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" -mkdir -p property_search/conf -mkdir -p term_search/conf -cp -a _default/conf/* property_search/conf/ -cp -a _default/conf/* term_search/conf/ -cp -a $ld_config/property_search/* property_search/conf -cp -a $ld_config/term_search/* term_search/conf -popd +#cd solr/configsets +ld_config='config/solr' +configsets='test/solr/configsets' +[ -d ${configsets}/property_search ] && rm -Rf ${configsets}/property_search +[ -d ${configsets}/term_search ] && rm -Rf ${configsets}/term_search +if [[ ! -d ${ld_config}/property_search ]]; then + echo 'cant find ld solr config sets' + exit 1 +fi +if [[ ! -d ${configsets}/_default/conf ]]; then + echo 'cant find default solr configset' + exit 1 +fi +mkdir -p ${configsets}/property_search/conf +mkdir -p ${configsets}/term_search/conf +cp -a ${configsets}/_default/conf/* ${configsets}/property_search/conf/ +cp -a ${configsets}/_default/conf/* ${configsets}/term_search/conf/ +cp -a $ld_config/property_search/* ${configsets}/property_search/conf +cp -a $ld_config/term_search/* ${configsets}/term_search/conf + From 9435c7efb50b69ebfb5496a5a50d5b40c28a670d Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:21:49 +0200 Subject: [PATCH 3/6] add request lang middleware to save as a global variable the request parameter "lang" or "language" --- Gemfile | 14 ++- Gemfile.lock | 145 +++++++++++++++---------- app.rb | 6 + lib/rack/request_lang.rb | 16 +++ views/documentation/documentation.haml | 1 + 5 files changed, 119 insertions(+), 63 deletions(-) create mode 100644 lib/rack/request_lang.rb diff --git a/Gemfile b/Gemfile index ec1f76f3..0e113266 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' +gem 'request_store' # Rack middleware gem 'ffi' @@ -40,13 +41,13 @@ gem 'unicorn-worker-killer' gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' -# NCBO -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' -gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' -gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' +# NCBO gems (can be from a local dev path or from rubygems/git) +gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' +gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' +gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'pr/sync-agroportal-bioportal' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'pr/sync-agroportal-ncbo' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 @@ -74,4 +75,5 @@ group :test do gem 'rack-test' gem 'simplecov', require: false gem 'simplecov-cobertura' # for codecov.io + gem 'webmock', '~> 3.19.1' end diff --git a/Gemfile.lock b/Gemfile.lock index fc1b44ac..79d8b762 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,22 +1,44 @@ GIT - remote: https://github.com/ncbo/goo.git - revision: f01386fe55ffba39cbf5e0bf95c8e2dc18767711 + remote: https://github.com/ncbo/ncbo_ontology_recommender.git + revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 + branch: master + specs: + ncbo_ontology_recommender (0.0.1) + goo + ncbo_annotator + ontologies_linked_data + redis + +GIT + remote: https://github.com/ncbo/sparql-client.git + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc branch: master + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + +GIT + remote: https://github.com/ontoportal-lirmm/goo.git + revision: f2751fe9324e48a0a5c2a8b15580ab879fc53a2b + branch: pr/sync-agroportal-bioportal specs: goo (0.0.2) addressable (~> 2.8) pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client uuid GIT - remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git + revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + branch: development specs: ncbo_annotator (0.0.1) goo @@ -25,14 +47,15 @@ GIT ruby-xxHash GIT - remote: https://github.com/ncbo/ncbo_cron.git - revision: b5764cc314758dbfdbb855b2b63c1c53b1325f6f + remote: https://github.com/ontoportal-lirmm/ncbo_cron.git + revision: 9ec0147203007cc368a5119ffe1a019fa8701c14 branch: master specs: ncbo_cron (0.0.1) dante goo google-analytics-data + google-apis-analytics_v3 mlanett-redis-lock multi_json ncbo_annotator @@ -41,20 +64,9 @@ GIT rufus-scheduler (~> 2.0.24) GIT - remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 013abea4af3b10910ec661dbb358a4b6cae198a4 - branch: master - specs: - ncbo_ontology_recommender (0.0.1) - goo - ncbo_annotator - ontologies_linked_data - redis - -GIT - remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4dc3b1123e871b53ba9eb51983ff69c5d2c75c18 - branch: master + remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git + revision: f70abe981c6fb3533b7785d58b7234855bcd5cf2 + branch: pr/sync-agroportal-ncbo specs: ontologies_linked_data (0.0.1) activesupport @@ -71,16 +83,6 @@ GIT rsolr rubyzip -GIT - remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 @@ -105,7 +107,7 @@ GEM multi_json (~> 1.0) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) - airbrussh (1.5.1) + airbrussh (1.5.2) sshkit (>= 1.6.1, != 1.7.0) ast (2.4.2) backports (3.25.0) @@ -129,9 +131,13 @@ GEM coderay (1.1.3) concurrent-ruby (1.2.3) connection_pool (2.4.1) + crack (1.0.0) + bigdecimal + rexml cube-ruby (0.0.3) dante (0.2.0) date (3.3.4) + declarative (0.0.20) docile (1.4.0) domain_name (0.6.20240107) ed25519 (1.3.0) @@ -140,7 +146,7 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -159,13 +165,22 @@ GEM google-analytics-data-v1beta (0.12.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) + google-apis-analytics_v3 (0.15.0) + google-apis-core (>= 0.14.0, < 2.a) + google-apis-core (0.14.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 1.9) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + rexml google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.3-aarch64-linux) google-protobuf (3.25.3-arm64-darwin) google-protobuf (3.25.3-x86_64-darwin) google-protobuf (3.25.3-x86_64-linux) @@ -182,30 +197,31 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0-aarch64-linux) + grpc (1.63.0-arm64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-arm64-darwin) + grpc (1.63.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-darwin) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt + hashdiff (1.1.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) + httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) + json (2.7.2) + json-ld (1.1.1) + rdf (>= 1.0.8) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.7.1) + json_pure (2.7.2) jwt (2.8.1) base64 kgio (2.11.4) @@ -219,7 +235,7 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) @@ -244,16 +260,16 @@ GEM net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.5.0) net-protocol - net-ssh (7.2.1) + net-ssh (7.2.3) netrc (0.11.0) - newrelic_rpm (9.8.0) + newrelic_rpm (9.9.0) oj (3.16.1) omni_logger (0.1.4) logger os (1.1.4) parallel (1.24.0) parseconfig (1.1.2) - parser (3.3.0.5) + parser (3.3.1.0) ast (~> 2.4.1) racc pony (1.13.1) @@ -261,7 +277,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) racc (1.7.3) rack (1.6.13) rack-accept (0.4.5) @@ -285,9 +301,9 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.1) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -295,16 +311,23 @@ GEM redis-store (1.10.0) redis (>= 4, < 6) regexp_parser (2.9.0) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + retriable (3.1.2) rexml (3.2.6) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.62.1) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -315,8 +338,8 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -350,7 +373,8 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.22.0) + sshkit (1.22.2) + base64 mutex_m net-scp (>= 1.1.2) net-sftp (>= 2.1.2) @@ -359,8 +383,10 @@ GEM temple (0.10.3) tilt (2.3.0) timeout (0.4.1) + trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + uber (0.1.0) unicode-display_width (2.5.0) unicorn (6.1.0) kgio (~> 2.6) @@ -370,13 +396,14 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - aarch64-linux arm64-darwin-22 - arm64-darwin-23 x86_64-darwin-18 - x86_64-darwin-21 x86_64-darwin-23 x86_64-linux @@ -393,6 +420,7 @@ DEPENDENCIES ffi goo! haml (~> 5.2.2) + json-ld json-schema (~> 2.0) minitest (~> 4.0) minitest-stub_any_instance @@ -403,6 +431,7 @@ DEPENDENCIES newrelic_rpm oj (~> 3.0) ontologies_linked_data! + parallel parseconfig rack rack-accept (~> 0.4) @@ -418,6 +447,7 @@ DEPENDENCIES redis redis-rack-cache (~> 2.0) redis-store (~> 1.10) + request_store rubocop shotgun! simplecov @@ -428,6 +458,7 @@ DEPENDENCIES sparql-client! unicorn unicorn-worker-killer + webmock (~> 3.19.1) BUNDLED WITH 2.4.22 diff --git a/app.rb b/app.rb index 41ad56ac..e09178bd 100644 --- a/app.rb +++ b/app.rb @@ -27,6 +27,7 @@ require_relative 'lib/rack/cube_reporter' require_relative 'lib/rack/param_translator' require_relative 'lib/rack/slice_detection' +require_relative 'lib/rack/request_lang' # Logging setup require_relative "config/logging" @@ -34,6 +35,8 @@ # Inflector setup require_relative "config/inflections" +require 'request_store' + # Protection settings set :protection, :except => :path_traversal @@ -141,6 +144,9 @@ use Rack::PostBodyToParams use Rack::ParamTranslator +use RequestStore::Middleware +use Rack::RequestLang + use LinkedData::Security::Authorization use LinkedData::Security::AccessDenied diff --git a/lib/rack/request_lang.rb b/lib/rack/request_lang.rb new file mode 100644 index 00000000..b2221041 --- /dev/null +++ b/lib/rack/request_lang.rb @@ -0,0 +1,16 @@ +module Rack + class RequestLang + + def initialize(app = nil, options = {}) + @app = app + end + + def call(env) + r = Rack::Request.new(env) + lang = r.params["lang"] || r.params["language"] + lang = lang.upcase.to_sym if lang + RequestStore.store[:requested_lang] = lang + @app.call(env) + end + end +end \ No newline at end of file diff --git a/views/documentation/documentation.haml b/views/documentation/documentation.haml index 527d781f..916eb1e7 100644 --- a/views/documentation/documentation.haml +++ b/views/documentation/documentation.haml @@ -151,6 +151,7 @@ %li include={prefLabel, synonym, definition, notation, cui, semanticType} // default = (see Common Parameters section) %li page={integer representing the page number} // default = 1 %li pagesize={integer representing the size of the returned page} // default = 50 + %li language={an ISO 639-1 language value, e.g 'fr' or 'en'} // by default search in all languages %h4#nav_search_subtree Subtree Search From f7eaf98320687e2c14c1bf8050de26d58f9c25db Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:22:40 +0200 Subject: [PATCH 4/6] update the search helper to add as a suffix the selected language --- helpers/search_helper.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 499e61ac..3096ab47 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -84,6 +84,9 @@ def get_term_search_query(text, params={}) end end + lang = params["lang"] || params["language"] + lang_suffix = lang && !lang.eql?("all") ? "_#{lang}" : "" + query = "" params["defType"] = "edismax" params["stopwords"] = "true" @@ -100,25 +103,25 @@ def get_term_search_query(text, params={}) if params[EXACT_MATCH_PARAM] == "true" query = "\"#{solr_escape(text)}\"" - params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact^10 synonymExact #{QUERYLESS_FIELDS_STR_NO_IDS}" - params["hl.fl"] = "resource_id prefLabelExact synonymExact #{QUERYLESS_FIELDS_STR}" + params["qf"] = "resource_id^20 notation^20 oboId^20 prefLabelExact#{lang_suffix}^10 synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} synonymExact#{lang_suffix} #{QUERYLESS_FIELDS_STR}" elsif params[SUGGEST_PARAM] == "true" || text[-1] == '*' text.gsub!(/\*+$/, '') query = "\"#{solr_escape(text)}\"" params["qt"] = "/suggest_ncbo" - params["qf"] = "prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["qf"] = " prefLabelExact#{lang_suffix}^100 prefLabelSuggestEdge#{lang_suffix}^50 synonymSuggestEdge#{lang_suffix}^10 prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" params["pf"] = "prefLabelSuggest^50" - params["hl.fl"] = "prefLabelExact prefLabelSuggestEdge synonymSuggestEdge prefLabelSuggestNgram synonymSuggestNgram resource_id #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "prefLabelExact#{lang_suffix} prefLabelSuggestEdge#{lang_suffix} synonymSuggestEdge#{lang_suffix} prefLabelSuggestNgram#{lang_suffix} synonymSuggestNgram#{lang_suffix} resource_id #{QUERYLESS_FIELDS_STR}" else if text.strip.empty? query = '*' else query = solr_escape(text) end - params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact^90 prefLabel^70 synonymExact^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" + params["qf"] = "resource_id^100 notation^100 oboId^100 prefLabelExact#{lang_suffix}^90 prefLabel#{lang_suffix}^70 synonymExact#{lang_suffix}^50 synonym^10 #{QUERYLESS_FIELDS_STR_NO_IDS}" params["qf"] << " property" if params[INCLUDE_PROPERTIES_PARAM] == "true" params["bq"] = "idAcronymMatch:true^80" - params["hl.fl"] = "resource_id prefLabelExact prefLabel synonymExact synonym #{QUERYLESS_FIELDS_STR}" + params["hl.fl"] = "resource_id prefLabelExact#{lang_suffix} prefLabel#{lang_suffix} synonymExact#{lang_suffix} synonym#{lang_suffix} #{QUERYLESS_FIELDS_STR}" params["hl.fl"] = "#{params["hl.fl"]} property" if params[INCLUDE_PROPERTIES_PARAM] == "true" end @@ -347,6 +350,7 @@ def populate_classes_from_search(classes, ontology_acronyms=nil) doc[:submission] = old_class.submission doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) instance = LinkedData::Models::Class.read_only(doc) + instance.prefLabel = instance.prefLabel.first if instance.prefLabel.is_a?(Array) classes_hash[ont_uri_class_uri] = instance end From b5360d242d26c4f216f72cfe688b462a732e489f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:29:07 +0200 Subject: [PATCH 5/6] refactor test to be faster by doing the indexation only if needed --- test/controllers/test_annotator_controller.rb | 17 ++++++---- test/controllers/test_batch_controller.rb | 3 +- test/controllers/test_classes_controller.rb | 4 ++- test/controllers/test_metrics_controller.rb | 23 +++++++------ .../controllers/test_ontologies_controller.rb | 34 +++++++++++++++++-- .../test_ontology_submissions_controller.rb | 16 ++++++--- .../controllers/test_properties_controller.rb | 2 ++ .../test_properties_search_controller.rb | 2 ++ .../test_recommender_controller.rb | 2 +- .../test_recommender_v1_controller.rb | 4 +-- test/middleware/test_rack_attack.rb | 12 +++---- test/test_case.rb | 25 ++++++++++++++ 12 files changed, 108 insertions(+), 36 deletions(-) diff --git a/test/controllers/test_annotator_controller.rb b/test/controllers/test_annotator_controller.rb index 3b21b9e7..55d8fb66 100644 --- a/test/controllers/test_annotator_controller.rb +++ b/test/controllers/test_annotator_controller.rb @@ -16,7 +16,12 @@ def self.before_suite end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true, + process_options: { + process_rdf: true, + extract_metadata: false, + index_search: true + }) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) @@ -260,16 +265,16 @@ def test_default_properties_output assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].first.downcase <=> b["annotatedClass"]["prefLabel"].first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] - assert_equal "Aggregate Human Data", annotations.first["annotatedClass"]["prefLabel"] + assert_equal "Aggregate Human Data", Array(annotations.first["annotatedClass"]["prefLabel"]).first params = {text: text, include: "prefLabel,definition"} get "/annotator", params assert last_response.ok? annotations = MultiJson.load(last_response.body) assert_equal 9, annotations.length - annotations.sort! { |a,b| a["annotatedClass"]["prefLabel"].downcase <=> b["annotatedClass"]["prefLabel"].downcase } + annotations.sort! { |a,b| Array(a["annotatedClass"]["prefLabel"]).first.downcase <=> Array(b["annotatedClass"]["prefLabel"]).first.downcase } assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Aggregate_Human_Data", annotations.first["annotatedClass"]["@id"] assert_equal ["A resource that provides data from clinical care that comprises combined data from multiple individual human subjects."], annotations.first["annotatedClass"]["definition"] end @@ -348,7 +353,7 @@ def self.mapping_test_set classes = [] class_id = terms_a[i] ont_acr = onts_a[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) @@ -356,7 +361,7 @@ def self.mapping_test_set classes << c class_id = terms_b[i] ont_acr = onts_b[i] - sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission + sub = LinkedData::Models::Ontology.find(ont_acr).first.latest_submission(status: :any) sub.bring(ontology: [:acronym]) c = LinkedData::Models::Class.find(RDF::URI.new(class_id)) .in(sub) diff --git a/test/controllers/test_batch_controller.rb b/test/controllers/test_batch_controller.rb index 55d9cee9..d1e9d144 100644 --- a/test/controllers/test_batch_controller.rb +++ b/test/controllers/test_batch_controller.rb @@ -2,8 +2,7 @@ class TestBatchController < TestCase def self.before_suite - LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) end def test_class_batch_one_ontology diff --git a/test/controllers/test_classes_controller.rb b/test/controllers/test_classes_controller.rb index a918ece0..42ccf29b 100644 --- a/test/controllers/test_classes_controller.rb +++ b/test/controllers/test_classes_controller.rb @@ -7,7 +7,9 @@ def self.before_suite submission_count: 3, submissions_to_process: [1, 2], process_submission: true, - random_submission_count: false} + random_submission_count: false, + process_options: {process_rdf: true, extract_metadata: false} + } return LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) end diff --git a/test/controllers/test_metrics_controller.rb b/test/controllers/test_metrics_controller.rb index 1b8890a6..b27fa198 100644 --- a/test/controllers/test_metrics_controller.rb +++ b/test/controllers/test_metrics_controller.rb @@ -18,11 +18,12 @@ def self.before_suite "individuals"=>124, "properties"=>63, "maxDepth"=> 7 } - @@options = {ont_count: 2, - submission_count: 3, - submissions_to_process: [1, 2], - process_submission: true, - random_submission_count: false} + @@options = { ont_count: 2, + submission_count: 3, + submissions_to_process: [1, 2], + process_submission: true, + process_options: { process_rdf: true, extract_metadata: false, run_metrics: true, index_properties: true }, + random_submission_count: false } LinkedData::SampleData::Ontology.create_ontologies_and_submissions(@@options) end @@ -78,18 +79,18 @@ def test_metrics_missing get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(0, ontologies.length, msg='Failure to detect 0 ontologies with missing metrics.') + assert_equal(0, ontologies.length, msg = 'Failure to detect 0 ontologies with missing metrics.') # create ontologies with latest submissions that have no metrics delete_ontologies_and_submissions - options = {ont_count: 2, - submission_count: 1, - process_submission: false, - random_submission_count: false} + options = { ont_count: 2, + submission_count: 1, + process_submission: false, + random_submission_count: false } create_ontologies_and_submissions(options) get '/metrics/missing' assert last_response.ok? ontologies = MultiJson.load(last_response.body) - assert_equal(2, ontologies.length, msg='Failure to detect 2 ontologies with missing metrics.') + assert_equal(2, ontologies.length, msg = 'Failure to detect 2 ontologies with missing metrics.') # recreate the before_suite data (this test might not be the last one to run in the suite) delete_ontologies_and_submissions create_ontologies_and_submissions(@@options) diff --git a/test/controllers/test_ontologies_controller.rb b/test/controllers/test_ontologies_controller.rb index 4f61256b..a5f242bd 100644 --- a/test/controllers/test_ontologies_controller.rb +++ b/test/controllers/test_ontologies_controller.rb @@ -188,7 +188,9 @@ def test_download_ontology end def test_download_ontology_csv - num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) + num_onts_created, created_ont_acronyms, onts = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, + process_submission: true, + process_options:{process_rdf: true, extract_metadata: true, index_search: true}) ont = onts.first acronym = created_ont_acronyms.first @@ -220,13 +222,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test1@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test2@example.org", password: "12345" }) blocked_user.save @@ -296,6 +298,32 @@ def test_on_demand_ontology_pull end end + def test_detach_a_view + view = Ontology.find(@@view_acronym).include(:viewOf).first + ont = view.viewOf + refute_nil view + refute_nil ont + + remove_view_of = {viewOf: ''} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(remove_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}" + onto = MultiJson.load(last_response.body) + assert_nil onto["viewOf"] + + + add_view_of = {viewOf: @@acronym} + patch "/ontologies/#{@@view_acronym}", MultiJson.dump(add_view_of), "CONTENT_TYPE" => "application/json" + + assert last_response.status == 204 + + get "/ontologies/#{@@view_acronym}?include=all" + onto = MultiJson.load(last_response.body) + assert_equal onto["viewOf"], ont.id.to_s + end + private def start_server diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 40532cd0..ef356fcf 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test3@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" @@ -36,6 +39,12 @@ def self._create_onts ont.save end + def setup + delete_ontologies_and_submissions + ont = Ontology.new(acronym: @@acronym, name: @@name, administeredBy: [@@user]) + ont.save + end + def test_submissions_for_given_ontology num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 1) ontology = created_ont_acronyms.first @@ -156,13 +165,13 @@ def test_download_acl_only begin allowed_user = User.new({ username: "allowed", - email: "test@example.org", + email: "test4@example.org", password: "12345" }) allowed_user.save blocked_user = User.new({ username: "blocked", - email: "test@example.org", + email: "test5@example.org", password: "12345" }) blocked_user.save @@ -235,5 +244,4 @@ def test_ontology_submissions_access_controller del.delete if del end end - end diff --git a/test/controllers/test_properties_controller.rb b/test/controllers/test_properties_controller.rb index 8248403c..cbf249b8 100644 --- a/test/controllers/test_properties_controller.rb +++ b/test/controllers/test_properties_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_properties_search_controller.rb b/test/controllers/test_properties_search_controller.rb index f93a90a1..2589a293 100644 --- a/test/controllers/test_properties_search_controller.rb +++ b/test/controllers/test_properties_search_controller.rb @@ -5,6 +5,7 @@ class TestPropertiesSearchController < TestCase def self.before_suite count, acronyms, bro = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "BROSEARCHTEST", name: "BRO Search Test", file_path: "./test/data/ontology_files/BRO_v3.2.owl", @@ -15,6 +16,7 @@ def self.before_suite count, acronyms, mccl = LinkedData::SampleData::Ontology.create_ontologies_and_submissions({ process_submission: true, + process_options:{process_rdf: true, extract_metadata: false, index_properties: true}, acronym: "MCCLSEARCHTEST", name: "MCCL Search Test", file_path: "./test/data/ontology_files/CellLine_OWL_BioPortal_v1.0.owl", diff --git a/test/controllers/test_recommender_controller.rb b/test/controllers/test_recommender_controller.rb index 29caf28c..58d6d942 100644 --- a/test/controllers/test_recommender_controller.rb +++ b/test/controllers/test_recommender_controller.rb @@ -14,7 +14,7 @@ def self.before_suite @@redis.del(mappings) end LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests() annotator.create_term_cache_from_ontologies(@@ontologies, false) diff --git a/test/controllers/test_recommender_v1_controller.rb b/test/controllers/test_recommender_v1_controller.rb index 7b14a63d..3ac4862d 100644 --- a/test/controllers/test_recommender_v1_controller.rb +++ b/test/controllers/test_recommender_v1_controller.rb @@ -1,10 +1,10 @@ require_relative '../test_case' -class TestRecommenderController < TestCase +class TestRecommenderV1Controller < TestCase def self.before_suite LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) @@text = <= max_retries + server_port = Random.rand(55000..65535) + end + server_port + end + private + def port_in_use?(port) + server = TCPServer.new(port) + server.close + false + rescue Errno::EADDRINUSE + true + end + end From 84f079ef8464b9c0e9916d344cbb30ad136b3f57 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 6 May 2024 14:33:46 +0200 Subject: [PATCH 6/6] add multilingual search unit tests --- test/controllers/test_search_controller.rb | 100 ++++++++++++++++----- test/data/ontology_files/BRO_v3.2.owl | 3 + 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/test/controllers/test_search_controller.rb b/test/controllers/test_search_controller.rb index 70bf246e..851616f7 100644 --- a/test/controllers/test_search_controller.rb +++ b/test/controllers/test_search_controller.rb @@ -85,7 +85,7 @@ def test_search_ontology_filter assert last_response.ok? results = MultiJson.load(last_response.body) doc = results["collection"][0] - assert_equal "cell line", doc["prefLabel"] + assert_equal "cell line", doc["prefLabel"].first assert doc["links"]["ontology"].include? acronym results["collection"].each do |doc| acr = doc["links"]["ontology"].split('/')[-1] @@ -103,7 +103,8 @@ def test_search_other_filters get "search?q=data&require_definitions=true" assert last_response.ok? results = MultiJson.load(last_response.body) - assert_equal 26, results["collection"].length + assert results["collection"].all? {|doc| !doc["definition"].nil? && doc.values.flatten.join(" ").include?("data") } + #assert_equal 26, results["collection"].length get "search?q=data&require_definitions=false" assert last_response.ok? @@ -115,10 +116,14 @@ def test_search_other_filters get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + + assert results["collection"].all? { |x| !x["obsolete"] } + count = results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=false" results = MultiJson.load(last_response.body) - assert_equal 22, results["collection"].length + assert_equal count, results["collection"].length + get "search?q=Integration%20and%20Interoperability&ontologies=#{acronym}&also_search_obsolete=true" results = MultiJson.load(last_response.body) assert_equal 29, results["collection"].length @@ -134,8 +139,14 @@ def test_search_other_filters # testing cui and semantic_types flags get "search?q=Funding%20Resource&ontologies=#{acronym}&include=prefLabel,synonym,definition,notation,cui,semanticType" results = MultiJson.load(last_response.body) - assert_equal 35, results["collection"].length - assert_equal "Funding Resource", results["collection"][0]["prefLabel"] + #assert_equal 35, results["collection"].length + assert results["collection"].all? do |r| + ["prefLabel", "synonym", "definition", "notation", "cui", "semanticType"].map {|x| r[x]} + .flatten + .join(' ') + .include?("Funding Resource") + end + assert_equal "Funding Resource", results["collection"][0]["prefLabel"].first assert_equal "T028", results["collection"][0]["semanticType"][0] assert_equal "X123456", results["collection"][0]["cui"][0] @@ -190,7 +201,7 @@ def test_search_provisional_class assert_equal 10, results["collection"].length provisional = results["collection"].select {|res| assert_equal ontology_type, res["ontologyType"]; res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_root.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_root.label, provisional[0]["prefLabel"].first # subtree root with provisional class test get "search?ontology=#{acronym}&subtree_root_id=#{CGI::escape(@@cls_uri.to_s)}&also_search_provisional=true" @@ -199,7 +210,7 @@ def test_search_provisional_class provisional = results["collection"].select {|res| res["provisional"]} assert_equal 1, provisional.length - assert_equal @@test_pc_child.label, provisional[0]["prefLabel"] + assert_equal @@test_pc_child.label, provisional[0]["prefLabel"].first end def test_search_obo_id @@ -254,11 +265,11 @@ def test_search_obo_id assert_equal ogms_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[0]["links"]["ontology"]) assert_equal cno_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[1]["links"]["ontology"]) assert_equal ncit_acronym, LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) - assert_equal 'realization', docs[1]["prefLabel"] - assert_equal 'realization', docs[2]["prefLabel"] - assert docs[3]["prefLabel"].upcase.include?('OGMS ') - assert docs[4]["prefLabel"].upcase.include?('OGMS ') - assert docs[5]["prefLabel"].upcase.include?('OGMS ') + assert_equal 'realization', docs[1]["prefLabel"].first + assert_equal 'realization', docs[2]["prefLabel"].first + assert docs[3]["prefLabel"].first.upcase.include?('OGMS ') + assert docs[4]["prefLabel"].first.upcase.include?('OGMS ') + assert docs[5]["prefLabel"].first.upcase.include?('OGMS ') get "/search?q=CNO:0000002" assert last_response.ok? @@ -270,10 +281,10 @@ def test_search_obo_id assert acr_1 === ncit_acronym || acr_1 === ogms_acronym acr_2= LinkedData::Utils::Triples.last_iri_fragment(docs[2]["links"]["ontology"]) assert acr_2 === ncit_acronym || acr_2 === ogms_acronym - assert docs[3]["prefLabel"].upcase.include?('CNO ') - assert docs[4]["prefLabel"].upcase.include?('CNO ') - assert docs[5]["prefLabel"].upcase.include?('CNO ') - assert docs[6]["prefLabel"].upcase.include?('CNO ') + assert docs[3]["prefLabel"].first.upcase.include?('CNO ') + assert docs[4]["prefLabel"].first.upcase.include?('CNO ') + assert docs[5]["prefLabel"].first.upcase.include?('CNO ') + assert docs[6]["prefLabel"].first.upcase.include?('CNO ') # mdorf, 3/2/2024, when the : is followed by a LETTER, as in NCIT:C20480, # then Solr does not split the query on the tokens, @@ -284,22 +295,22 @@ def test_search_obo_id results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"] + assert_equal 'Cellular Process', docs[0]["prefLabel"].first get "/search?q=NCIT:C20480" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] assert_equal 1, docs.size - assert_equal 'Cellular Process', docs[0]["prefLabel"] + assert_equal 'Cellular Process', docs[0]["prefLabel"].first get "/search?q=Leukocyte Apoptotic Process&ontologies=#{ncit_acronym}" assert last_response.ok? results = MultiJson.load(last_response.body) docs = results["collection"] - assert_equal 'Leukocyte Apoptotic Process', docs[0]["prefLabel"] - assert_equal 'Leukocyte Apoptotic Test Class', docs[1]["prefLabel"] - assert_equal 'Lymphocyte Apoptotic Process', docs[2]["prefLabel"] + assert_equal 'Leukocyte Apoptotic Process', docs[0]["prefLabel"].first + assert_equal 'Leukocyte Apoptotic Test Class', docs[1]["prefLabel"].first + assert_equal 'Lymphocyte Apoptotic Process', docs[2]["prefLabel"].first ensure ont = LinkedData::Models::Ontology.find(ncit_acronym).first ont.delete if ont @@ -357,4 +368,49 @@ def test_search_short_id end end + def test_multilingual_search + get "/search?q=Activity&ontologies=BROSEARCHTEST-0" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + + doc = res["collection"].select{|doc| doc["@id"].to_s.eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + refute_nil doc + + res = LinkedData::Models::Class.search("prefLabel_none:Activity", {:fq => "submissionAcronym:BROSEARCHTEST-0", :start => 0, :rows => 80}) + refute_equal 0, res["response"]["numFound"] + refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en" + res = MultiJson.load(last_response.body) + refute_equal 0, res["totalCount"] + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=ActivityEnglish&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activity&ontologies=BROSEARCHTEST-0&lang=en&require_exact_match=true" + res = MultiJson.load(last_response.body) + assert_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + get "/search?q=Activit%C3%A9&ontologies=BROSEARCHTEST-0&lang=fr&require_exact_match=true" + res = MultiJson.load(last_response.body) + refute_nil res["collection"].select{|doc| doc["@id"].eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first + + + end + + end diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index d64075cc..b2aeccf5 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -631,6 +631,9 @@ Activity + Activity + ActivityEnglish + Activité Activity of interest that may be related to a BRO:Resource. activities