From 46c6e285ac14b4bfc1d7487ec5508d51977d5cab Mon Sep 17 00:00:00 2001 From: Caleb Pomar Date: Thu, 21 Apr 2022 14:38:10 -0600 Subject: [PATCH] allow customers to append custom distributions to the distribution array --- CHANGELOG.md | 4 + example-app/package-lock.json | 407 ++++++++++++++++++-- src/dcat-us/_generate-distributions.test.ts | 148 ++++++- src/dcat-us/_generate-distributions.ts | 74 +++- src/dcat-us/dataset-formatter.test.ts | 49 ++- src/dcat-us/dataset-formatter.ts | 18 +- src/dcat-us/noneditable-fields.ts | 1 - 7 files changed, 635 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bccd749..685b8fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG.md +## Unreleased +Added +- If the `distribution` property of a site's dcat config is an array, those custom distributions will now be prepended to the distributions list [#23](https://github.com/koopjs/koop-output-dcat-us-11/pull/23) + ## 1.8.2 Fixed - Fixed access urls for downloadable distributions [#22](https://github.com/koopjs/koop-output-dcat-us-11/pull/22) diff --git a/example-app/package-lock.json b/example-app/package-lock.json index adfd454..522b982 100644 --- a/example-app/package-lock.json +++ b/example-app/package-lock.json @@ -233,11 +233,11 @@ "@esri/arcgis-rest-auth": "^3.2.1", "@esri/arcgis-rest-portal": "^3.2.1", "@esri/arcgis-rest-request": "^3.2.1", - "@esri/hub-common": "^9.17.1", - "@esri/hub-initiatives": "^9.17.1", - "@esri/hub-search": "^9.17.1", - "@esri/hub-sites": "^9.17.1", - "@esri/hub-teams": "^9.17.1", + "@esri/hub-common": "^9.27.0", + "@esri/hub-initiatives": "^9.27.0", + "@esri/hub-search": "^9.27.0", + "@esri/hub-sites": "^9.27.0", + "@esri/hub-teams": "^9.27.0", "config": "^3.3.6", "lodash": "^4.17.21", "tslib": "~2.3.0" @@ -296,12 +296,13 @@ "integrity": "sha512-lVOAhu0qvhQEm5BFSOp8VxqxkAd5dBa7ZNXhtuDNNyELy38R0C/0zt3ri4eQLpMIZG4oip/djinOLgzAdQZzBw==" }, "@esri/hub-common": { - "version": "9.17.1", - "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-9.17.1.tgz", - "integrity": "sha512-cJ6LB98WxRfdqBih2w21icQh9LXRlCuP8wmKoUpC10r8jNnourKq9wUFP/4HJcfZCrmQhmBRlMIrsdcQNhljaQ==", + "version": "9.29.0", + "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-9.29.0.tgz", + "integrity": "sha512-wKnkmv3RnXqzniXecnhQ1PX523l1fqaYHtgzhjyY4r0xei55Jmt989iVs2YwVqcIoFEzlqvHvOHYNV2wIQ/6gQ==", "requires": { "abab": "^2.0.5", "adlib": "^3.0.7", + "fast-xml-parser": "~3.2.4", "jsonapi-typescript": "^0.1.3", "tslib": "^1.13.0" }, @@ -314,9 +315,9 @@ } }, "@esri/hub-initiatives": { - "version": "9.17.1", - "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-9.17.1.tgz", - "integrity": "sha512-yDGMuGJQ8PYFR4WoTgWFTiAolN2tl2Qz3C7T7FrRWLCC3BZY7nly1h3j4OrNpf/6e7UYrtN1N/ZVw13CTAn0dg==", + "version": "9.29.0", + "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-9.29.0.tgz", + "integrity": "sha512-nRyR51X95d3iO3JmyfUzxE1tQV3wY3JLW/iq/7f6oUwUb7Wv/3EoZR1gjZuZreIzxVJBNfTwjaEjBVzRQ6tzRQ==", "requires": { "tslib": "^1.13.0" }, @@ -329,9 +330,9 @@ } }, "@esri/hub-search": { - "version": "9.17.1", - "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-9.17.1.tgz", - "integrity": "sha512-eXkx0EdOo9Paz3PSSwlwu0CRxJYYIZGQ1Mz3xbMW2U3KvUwxXkqkszWL5D9FcRheSyYhKlnNzjMsnXl9H/EezQ==", + "version": "9.29.0", + "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-9.29.0.tgz", + "integrity": "sha512-JSYuAiT5CBX8pgTF133qrYrf43Fv2uOJ0cOhxFf89GVYA16yRMSnNWzVRA4UhI6EZPjZYc9dxojVNWAavQVQLQ==", "requires": { "tslib": "^1.13.0" }, @@ -344,9 +345,9 @@ } }, "@esri/hub-sites": { - "version": "9.17.1", - "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-9.17.1.tgz", - "integrity": "sha512-i+po3aM7WBOu3s/9+oQJTXcVe2/cdMTz8geC/X8yBGCCa3vwhGXHX+1DRx1eu+ZatlZq2COkEw5xkt1j1sOYuw==", + "version": "9.29.0", + "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-9.29.0.tgz", + "integrity": "sha512-KkxwArXRZq8LID2adjADQ/uXskMWVWmPcwb+4PWVAIPJBRyAXms9aNsM/oY1pZdD97RXt8Y39OKZkpZsda62IA==", "requires": { "tslib": "^1.13.0" }, @@ -359,9 +360,9 @@ } }, "@esri/hub-teams": { - "version": "9.17.1", - "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-9.17.1.tgz", - "integrity": "sha512-e02dB/Hb+9HzRBefRZQ6CnSOVnnDp26Io6GSTp3C07ZkxfchB3SuF4g4LY4ACmrkcvPS2TrAYZNxC5JzK3HPNQ==", + "version": "9.29.0", + "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-9.29.0.tgz", + "integrity": "sha512-LryHH6DOPNY5ztdBSMUtouAnJ98sGz5dT7MjE3poSZoyks8RhKoOY/5FHuoMKhn+RTa5zRX3EHrgI2u0ScjoZw==", "requires": { "tslib": "^1.13.0" }, @@ -374,9 +375,9 @@ } }, "abab": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", - "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==" + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" }, "adlib": { "version": "3.0.7", @@ -386,6 +387,77 @@ "esm": "^3.2.25" } }, + "ansi-escapes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz", + "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4=" + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" + }, + "babel-polyfill": { + "version": "6.23.0", + "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.23.0.tgz", + "integrity": "sha1-g2TKYt+Or7gwSZ9pkXdGbDsDSZ0=", + "requires": { + "babel-runtime": "^6.22.0", + "core-js": "^2.4.0", + "regenerator-runtime": "^0.10.0" + } + }, + "babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", + "requires": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + }, + "dependencies": { + "regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + } + } + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "chardet": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", + "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=" + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" + }, "config": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/config/-/config-3.3.6.tgz", @@ -394,11 +466,118 @@ "json5": "^2.1.1" } }, + "core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==" + }, + "encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "requires": { + "iconv-lite": "^0.6.2" + }, + "dependencies": { + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, "esm": { "version": "3.2.25", "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" }, + "external-editor": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", + "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", + "requires": { + "chardet": "^0.4.0", + "iconv-lite": "^0.4.17", + "tmp": "^0.0.33" + } + }, + "fast-xml-parser": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-3.2.4.tgz", + "integrity": "sha512-19R4aaHzqsTe7gyL3wthWiyYyNFqVK1tzO6BMEAQEKRu4R3ptkl4FyuEtUMOPKdYsO6k6C/Ym3YJ6eCL49OdSw==", + "requires": { + "he": "~1.1.1", + "nimnjs": "^1.1.0", + "opencollective": "^1.0.3" + } + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "he": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inquirer": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.0.6.tgz", + "integrity": "sha1-4EqqnQW3o8ubD0B9BDdfBEcZA0c=", + "requires": { + "ansi-escapes": "^1.1.0", + "chalk": "^1.0.0", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^2.0.1", + "figures": "^2.0.0", + "lodash": "^4.3.0", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rx": "^4.1.0", + "string-width": "^2.0.0", + "strip-ansi": "^3.0.0", + "through": "^2.3.6" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, "json-typescript": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/json-typescript/-/json-typescript-1.1.2.tgz", @@ -425,11 +604,193 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" + }, "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=" + }, + "nimn-date-parser": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/nimn-date-parser/-/nimn-date-parser-1.0.0.tgz", + "integrity": "sha512-1Nf+x3EeMvHUiHsVuEhiZnwA8RMeOBVTQWfB1S2n9+i6PYCofHd2HRMD+WOHIHYshy4T4Gk8wQoCol7Hq3av8Q==" + }, + "nimn_schema_builder": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/nimn_schema_builder/-/nimn_schema_builder-1.1.0.tgz", + "integrity": "sha512-DK5/B8CM4qwzG2URy130avcwPev4uO0ev836FbQyKo1ms6I9z/i6EJyiZ+d9xtgloxUri0W+5gfR8YbPq7SheA==" + }, + "nimnjs": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/nimnjs/-/nimnjs-1.3.2.tgz", + "integrity": "sha512-TIOtI4iqkQrUM1tiM76AtTQem0c7e56SkDZ7sj1d1MfUsqRcq2ZWQvej/O+HBTZV7u/VKnwlKTDugK/75IRPPw==", + "requires": { + "nimn-date-parser": "^1.0.0", + "nimn_schema_builder": "^1.0.0" + } + }, + "node-fetch": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.6.3.tgz", + "integrity": "sha1-3CNO3WSJmC1Y6PDbT2lQKavNjAQ=", + "requires": { + "encoding": "^0.1.11", + "is-stream": "^1.0.1" + } + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "opencollective": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/opencollective/-/opencollective-1.0.3.tgz", + "integrity": "sha1-ruY3K8KBRFg2kMPKja7PwSDdDvE=", + "requires": { + "babel-polyfill": "6.23.0", + "chalk": "1.1.3", + "inquirer": "3.0.6", + "minimist": "1.2.0", + "node-fetch": "1.6.3", + "opn": "4.0.2" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + } + } + }, + "opn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/opn/-/opn-4.0.2.tgz", + "integrity": "sha1-erwi5kTf9jsKltWrfyeQwPAavJU=", + "requires": { + "object-assign": "^4.0.1", + "pinkie-promise": "^2.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "requires": { + "pinkie": "^2.0.0" + } + }, + "regenerator-runtime": { + "version": "0.10.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", + "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==" + }, + "rx": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/rx/-/rx-4.1.0.tgz", + "integrity": "sha1-pfE/957zt0D+MKqAP7CfmIBdR4I=" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==" + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "requires": { + "os-tmpdir": "~1.0.2" + } + }, "tslib": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", diff --git a/src/dcat-us/_generate-distributions.test.ts b/src/dcat-us/_generate-distributions.test.ts index 8f85195..40438f3 100644 --- a/src/dcat-us/_generate-distributions.test.ts +++ b/src/dcat-us/_generate-distributions.test.ts @@ -6,7 +6,7 @@ describe('_generateDistributions', () => { const getDownloadLink = (id: string) => `https://my-site.hub.arcgis.com/datasets/${id}`; it('add default distributions if dataset is a collection', () => { - const dataset = { + const hubDataset = { id: 'foo', // non-layer id url: serviceUrl }; @@ -27,14 +27,19 @@ describe('_generateDistributions', () => { }, ] - const distributions = _generateDistributions(dataset, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const distributions = _generateDistributions({ + hubDataset, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id), + }); expect(distributions.length).toBe(2) expect(distributions).toEqual(expected); }) it('adds default and csv distributions if dataset is a proxied csv', () => { - const dataset = { + const hubDataset = { id: 'foo', access: 'public', slug: 'nissan::skyline-gtr', @@ -64,14 +69,19 @@ describe('_generateDistributions', () => { } ] - const distributions = _generateDistributions(dataset, getLandingPage(dataset.slug), getDownloadLink(dataset.slug)); + const distributions = _generateDistributions({ + hubDataset, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.slug), + downloadLink: getDownloadLink(hubDataset.slug) + }); expect(distributions.length).toBe(3) expect(distributions).toEqual(expected); }); it('adds default distributions if dataset is a layer', () => { - const dataset = { + const hubDataset = { id: 'foo_0', // layer id url: serviceUrl }; @@ -106,14 +116,19 @@ describe('_generateDistributions', () => { } ] - const distributions = _generateDistributions(dataset, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const distributions = _generateDistributions({ + hubDataset, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(distributions.length).toBe(4) expect(distributions).toEqual(expected); }); it('adds KML and Shapefile if dataset is a layer AND has geometryType', () => { - const dataset = { + const hubDataset = { id: 'foo_0', // layer id url: serviceUrl, layer: { @@ -137,14 +152,19 @@ describe('_generateDistributions', () => { } ] - const distributions = _generateDistributions(dataset, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const distributions = _generateDistributions({ + hubDataset, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(distributions.length).toBe(6) expect(distributions.slice(4)).toEqual(expected); }); it('adds WFS and WMS if supported', () => { - const dataset = { + const hubDataset = { id: 'foo', // non-layer id url: serviceUrl }; @@ -165,21 +185,36 @@ describe('_generateDistributions', () => { accessURL: 'https://servicesqa.arcgis.com/Xj56SBi2udA78cC9/arcgis/services/Tahoe_Things/FeatureServer/WFSServer?request=GetCapabilities&service=WFS' }; - const justWFS = _generateDistributions({ ...dataset, supportedExtensions: 'WFSServer' }, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const justWFS = _generateDistributions({ + hubDataset: { ...hubDataset, supportedExtensions: 'WFSServer' }, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(justWFS.length).toBe(3); expect(justWFS.pop()).toEqual(expectedWFSDistribution); - const justWMS = _generateDistributions({ ...dataset, supportedExtensions: 'WMSServer' }, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const justWMS = _generateDistributions({ + hubDataset: { ...hubDataset, supportedExtensions: 'WMSServer' }, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(justWMS.length).toBe(3); expect(justWMS.pop()).toEqual(expectedWMSDistribution); - const allOGCServices = _generateDistributions({ ...dataset, supportedExtensions: 'WMSServer,WFSServer' }, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const allOGCServices = _generateDistributions({ + hubDataset: { ...hubDataset, supportedExtensions: 'WMSServer,WFSServer' }, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(allOGCServices.length).toBe(4); expect(allOGCServices.slice(2)).toEqual([expectedWFSDistribution, expectedWMSDistribution]); }); - it('adds custom distributions from metadata', () => { - const dataset = { + it('adds distributions from metadata', () => { + const hubDataset = { id: 'foo', // non-layer id url: serviceUrl, metadata: { @@ -206,7 +241,12 @@ describe('_generateDistributions', () => { } }; - const distributions = _generateDistributions(dataset, getLandingPage(dataset.id), getDownloadLink(dataset.id)); + const distributions = _generateDistributions({ + hubDataset, + dcatDataset: {}, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id) + }); expect(distributions.length).toBe(4); expect(distributions.slice(2)).toEqual([ @@ -226,4 +266,82 @@ describe('_generateDistributions', () => { } ]) }); + + it('does not add custom distributions if interpolated value is an empty array', () => { + const hubDataset = { + id: 'foo', // non-layer id + url: serviceUrl + }; + + const dcatDataset = { + distribution: [] + } + + const expected = [ + { + '@type': 'dcat:Distribution', + title: 'ArcGIS Hub Dataset', + format: 'Web Page', + mediaType: 'text/html', + accessURL: 'https://my-site.hub.arcgis.com/maps/foo' + }, + { + '@type': 'dcat:Distribution', + title: 'ArcGIS GeoService', + format: 'ArcGIS GeoServices REST API', + mediaType: 'application/json', + accessURL: 'https://servicesqa.arcgis.com/Xj56SBi2udA78cC9/arcgis/rest/services/Tahoe_Things/FeatureServer/0' + }, + ] + + const distributions = _generateDistributions({ + hubDataset, + dcatDataset, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id), + }); + + expect(distributions.length).toBe(2) + expect(distributions).toEqual(expected); + }); + + it('does not add custom distributions if interpolated value is an object', () => { + const hubDataset = { + id: 'foo', // non-layer id + url: serviceUrl + }; + + const dcatDataset = { + distribution: { + customKey: 'customValue' + } + } + + const expected = [ + { + '@type': 'dcat:Distribution', + title: 'ArcGIS Hub Dataset', + format: 'Web Page', + mediaType: 'text/html', + accessURL: 'https://my-site.hub.arcgis.com/maps/foo' + }, + { + '@type': 'dcat:Distribution', + title: 'ArcGIS GeoService', + format: 'ArcGIS GeoServices REST API', + mediaType: 'application/json', + accessURL: 'https://servicesqa.arcgis.com/Xj56SBi2udA78cC9/arcgis/rest/services/Tahoe_Things/FeatureServer/0' + }, + ] + + const distributions = _generateDistributions({ + hubDataset, + dcatDataset, + landingPage: getLandingPage(hubDataset.id), + downloadLink: getDownloadLink(hubDataset.id), + }); + + expect(distributions.length).toBe(2) + expect(distributions).toEqual(expected); + }) }); \ No newline at end of file diff --git a/src/dcat-us/_generate-distributions.ts b/src/dcat-us/_generate-distributions.ts index f2ed4f6..79cbb61 100644 --- a/src/dcat-us/_generate-distributions.ts +++ b/src/dcat-us/_generate-distributions.ts @@ -11,58 +11,92 @@ export const DISTRIBUTION_DEPENDENCIES = [ 'url', ]; +interface IGenerateDistributionParams { + hubDataset: Record; // dataset from the Hub API + dcatDataset: Record; // interpolated dataset from adlib + landingPage: string; // best guess at the canonical Hub URL for the dataset + downloadLink: string; // Hub download url for the dataset +} + +/** + * Generates distributions for a given dataset. + * + * A note about custom distributions. We now allow clients to define custom + * distributions on their DCAT Config objects via the `distribution` property. + * + * We will prepend the interpolated value of `distribution` onto the result if + * the value is an array. If the value not an array, it will be ignored. + * + * @param {IGenerateDistributionParams} params + * @returns all distributions + */ +export function _generateDistributions(params: IGenerateDistributionParams) { + const { + hubDataset, + dcatDataset, + landingPage, + downloadLink + } = params; + + const customDistributions = _.isArray(dcatDataset.distribution) ? dcatDataset.distribution : []; + const standardDistributions = _generateStandardDistributions(hubDataset, landingPage, downloadLink); + + return [...customDistributions, ...standardDistributions]; +} + /* -* Generate DCAT Distributions +* Generate Standard DCAT Distributions */ -export function _generateDistributions (dataset: any, landingPage: string, downloadLink: string) { +function _generateStandardDistributions (hubDataset: Record, landingPage: string, downloadLink: string) { + const distributionFns = [ getHubLandingPageDistribution, getEsriGeoServiceDistribution ]; - if (isProxiedCSV(dataset)) { + if (isProxiedCSV(hubDataset)) { distributionFns.push(getCSVDistribution); } - if (isLayer(dataset)) { + if (isLayer(hubDataset)) { distributionFns.push(getGeoJSONDistribution); distributionFns.push(getCSVDistribution); - if (_.has(dataset, 'layer.geometryType')) { + if (_.has(hubDataset, 'layer.geometryType')) { distributionFns.push(getKMLDistribution); distributionFns.push(getShapefileDistribution); } } - if (dataset.supportedExtensions?.includes(WFS_SERVER)) { + if (hubDataset.supportedExtensions?.includes(WFS_SERVER)) { distributionFns.push(getWFSDistribution); } - if (dataset.supportedExtensions?.includes(WMS_SERVER)) { + if (hubDataset.supportedExtensions?.includes(WMS_SERVER)) { distributionFns.push(getWMSDistribution); } const params: DistributionParameters = { landingPage, downloadLink, - serviceUrl: dataset.url, - downloadLinkFor: getDownloadLinkFn(downloadLink, dataset) + serviceUrl: hubDataset.url, + downloadLinkFor: getDownloadLinkFn(downloadLink, hubDataset) }; const distributions = distributionFns.map(fn => fn(params)); - const customDistributions = getCustomDistributions(dataset); + const metadataDistributions = getMetadataDistributions(hubDataset); - return [...distributions, ...customDistributions]; + return [...distributions, ...metadataDistributions]; } -function isLayer (dataset: any) { - return /_/.test(dataset.id); +function isLayer (hubDataset: any) { + return /_/.test(hubDataset.id); } -function isProxiedCSV(dataset: any) { +function isProxiedCSV(hubDataset: any) { const item = datasetToItem({ - id: dataset.id, - attributes: dataset + id: hubDataset.id, + attributes: hubDataset } as DatasetResource); const requestOptions: IHubRequestOptions = { isPortal: false }; @@ -70,8 +104,8 @@ function isProxiedCSV(dataset: any) { } // HUBJS CANDIDATE -function getDownloadLinkFn (downloadLink: string, dataset: any) { - const spatialReference = _.get(dataset, 'server.spatialReference'); +function getDownloadLinkFn (downloadLink: string, hubDataset: any) { + const spatialReference = _.get(hubDataset, 'server.spatialReference'); let queryStr = ''; @@ -87,9 +121,9 @@ function getDownloadLinkFn (downloadLink: string, dataset: any) { return (ext: string) => `${downloadLink}.${ext}${queryStr}`; } -function getCustomDistributions (dataset: any) { +function getMetadataDistributions (hubDataset: any) { const distros = []; - const data = _.get(dataset, 'metadata.metadata.distInfo.distTranOps.onLineSrc'); + const data = _.get(hubDataset, 'metadata.metadata.distInfo.distTranOps.onLineSrc'); if (Array.isArray(data)) { for (const dist of data) { diff --git a/src/dcat-us/dataset-formatter.test.ts b/src/dcat-us/dataset-formatter.test.ts index 003e95e..6640115 100644 --- a/src/dcat-us/dataset-formatter.test.ts +++ b/src/dcat-us/dataset-formatter.test.ts @@ -17,7 +17,6 @@ it('dcatHelper: it does not allow customizations to overwrite critical fields', landingPage: 'SABOTAGE', webService: 'SABOTAGE', spatial: 'SABOTAGE', - distribution: 'SABOTAGE' }; const template = buildDatasetTemplate(customizations); expect(template['@type']).not.toBe('SABOTAGE'); @@ -26,7 +25,6 @@ it('dcatHelper: it does not allow customizations to overwrite critical fields', expect(template.landingPage).not.toBe('SABOTAGE'); expect(template.webService).not.toBe('SABOTAGE'); expect(template.spatial).not.toBe('SABOTAGE'); - expect(template.distribution).not.toBe('SABOTAGE'); expect(template.title).toBe('{{metadata.metadata.name||item.title}}') expect(template.contactPoint.fn).toBe('{{item.owner}}'); expect(template.contactPoint.hasEmail).toBe('mailto:dcat.support@dc.gov'); @@ -798,7 +796,52 @@ describe('formatDcatDataset', () => { ).toBe(expectedKeyword); }); - it('should create custom distribution when data is supplied', () => { + it('should prepend custom distribution if template.distribution is a non-empty array', () => { + const hubDataset = { + owner: 'fpgis.CALFIRE', + created: 1570747289000, + modified: 1570747379000, + tags: ['Uno', 'Dos', 'Tres'], + extent: { + coordinates: [ + [-123.8832, 35.0024], + [-118.3281, 42.0122], + ], + type: 'envelope', + }, + name: 'DCAT_Test', + description: 'Some Description', + source: 'Test Source', + id: '00000000000000000000000000000000_0', + type: 'Feature Layer', + url: 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/DCAT_Test/FeatureServer/0', + layer: { + geometryType: 'esriGeometryPolygon', + }, + server: { + spatialReference: { + wkid: 3310, + }, + }, + }; + + const template = buildDatasetTemplate({ + distribution: [{ + interpolated: '{{name}}', + constant: 'myConstant', + }] + }); + + const expectedDistribution = { + interpolated: 'DCAT_Test', + constant: 'myConstant', + }; + + const actual = JSON.parse(formatDcatDataset(hubDataset, siteUrl, siteModel, template)); + expect(actual.distribution.shift()).toEqual(expectedDistribution); + }); + + it('should create distributions from metadata when data is supplied', () => { const dataset = { owner: 'fpgis.CALFIRE', created: 1570747289000, diff --git a/src/dcat-us/dataset-formatter.ts b/src/dcat-us/dataset-formatter.ts index da93f83..c710ff6 100644 --- a/src/dcat-us/dataset-formatter.ts +++ b/src/dcat-us/dataset-formatter.ts @@ -48,7 +48,8 @@ export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: st const dcatDataset = Object.assign({}, defaultDataset, adlib(datasetTemplate, hubDataset, transforms)); - setLeftoverInterpolations(dcatDataset); + // reset uninterpolated, non-editable fields since customers cannot remove them + resetUninterpolatedPaths(dcatDataset, nonEditableFieldPaths); if (!dcatDataset.license || dcatDataset.license.match(/{{.+}}/g)?.length) { dcatDataset.license = url || licenseInfo || ''; @@ -58,7 +59,12 @@ export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: st dcatDataset.keyword = ['ArcGIS Hub page']; } - dcatDataset.distribution = _generateDistributions(hubDataset, landingPage, downloadLink); + dcatDataset.distribution = _generateDistributions({ + hubDataset, + dcatDataset, + landingPage, + downloadLink + }); if (_.has(hubDataset, 'extent.coordinates')) { dcatDataset.spatial = hubDataset.extent.coordinates.join(','); @@ -99,8 +105,12 @@ export function buildDatasetTemplate (customizations: DcatDatasetTemplate = {}): return Object.assign({}, baseDatasetTemplate, customConfig); } -function setLeftoverInterpolations(dataset) { - nonEditableFieldPaths.forEach(path => { +/** + * If the value at each field path is an uninterpolated adlib string, + * change the field to the empty string. + */ +function resetUninterpolatedPaths(dataset, fieldPaths) { + fieldPaths.forEach(path => { const value = _.get(dataset, path, ''); if (typeof value === 'string' && value.match(/{{.+}}/)?.length) { _.set(dataset, path, ''); diff --git a/src/dcat-us/noneditable-fields.ts b/src/dcat-us/noneditable-fields.ts index e1faa8d..18df8f4 100644 --- a/src/dcat-us/noneditable-fields.ts +++ b/src/dcat-us/noneditable-fields.ts @@ -6,5 +6,4 @@ export const nonEditableFieldPaths = [ 'webService', 'spatial', 'contactPoint[@type]', - 'distribution' ];