initial commit

.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: Test
+on:
+  - pull_request
+  - push
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        node:
+          - '18'
+          - '20'
+          - '21'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+      - run: npm install
+      - run: npm test
+      - uses: codecov/codecov-action@v3

.gitignore

@@ -0,0 +1,3 @@
+coverage
+node_modules
+package-lock.json

LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Thomas Bergwinkl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,33 @@
+# rdf-ext-cli
+
+[![build status](https://img.shields.io/github/actions/workflow/status/rdf-ext/rdf-ext-cli/test.yaml?branch=master)](https://github.com/rdf-ext/rdf-ext-cli/actions/workflows/test.yaml)
+[![npm version](https://img.shields.io/npm/v/rdf-ext-cli.svg)](https://www.npmjs.com/package/rdf-ext-cli)
+
+A command line util for [RDF-Ext](https://rdf-ext.org/) to convert and validate RDF data.
+It supports reading different formats from the local file system, HTTP URLs, and SPARQL endpoints.
+The data can be validated with SHACL using [shacl-engine](https://github.com/rdf-ext/shacl-engine).
+
+## Install
+
+Use the following command to install the util with `npm`.
+A symlink will be added to the path.
+
+```bash
+npm install -g rdf-ext-cli
+```
+
+## Usage
+
+You can run the tool like this:
+
+```bash
+rdf-ext somePathOrUrl
+```
+
+This will read `somePathOrUrl` with content-type auto-detection and write the result in the default format to `stdout`.
+If `--shacl-url` is given, the input will be validated against the given SHACL shape, and the report will be written to the output.
+Run `rdf-ext` without any arguments to get a full list of supported parameters.
+
+## Examples
+
+See the examples folder for some example commands.

bin/rdf-ext-cli.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+
+import { promisify } from 'node:util'
+import formatsPretty from '@rdfjs/formats/pretty.js'
+import { Command } from 'commander'
+import rdf from 'rdf-ext'
+import { finished } from 'readable-stream'
+import createInputStream from '../lib/createInputStream.js'
+import createOutputStream from '../lib/createOutputStream.js'
+import createShaclStream from '../lib/createShaclStream.js'
+import transformMapNamespaceFunc from '../lib/transformMapNamespace.js'
+import transformToTripleFunc from '../lib/transformToTriple.js'
+
+const program = new Command()
+
+function collectMappings (value, mappings) {
+  const [, from, to] = value.match(/(.*)=(.*)/)
+
+  mappings.set(rdf.namedNode(from), rdf.namedNode(to))
+
+  return mappings
+}
+
+function collectPrefixes (value, prefixes) {
+  const [, prefix, namespace] = value.match(/(.*)=(.*)/)
+
+  prefixes.set(prefix, rdf.namedNode(namespace))
+
+  return prefixes
+}
+
+program
+  .argument('[input]', 'input')
+  .option('--input-endpoint <url>', 'input SPARQL endpoint url')
+  .option('--input-query <query>', 'input SPARQL query')
+  .option('--input-type <type>', 'input content type')
+  .option('--shacl-endpoint <url>', 'SHACL SPARQL endpoint url')
+  .option('--shacl-query <query>', 'SHACL SPARQL query')
+  .option('--shacl-type <type>', 'SHACL content type', 'text/turtle')
+  .option('--shacl-url <url>', 'SHACL URL')
+  .option('--shacl-debug', 'generate results for successful validations')
+  .option('--shacl-details', 'generate nested result details')
+  .option('--shacl-trace', 'generate results for path traversing')
+  .option('--transform-map-namespace <mapping>', 'map the given namespaces', collectMappings, rdf.termMap())
+  .option('--transform-to-triples', 'set graph to default graph')
+  .option('--output-prefix <prefix>', 'output prefix', collectPrefixes, new Map())
+  .option('--output-type <type>', 'output content type', 'text/turtle')
+  .option('--pretty', 'use pretty print serializer')
+  .action(async (input, {
+    inputType,
+    inputEndpoint,
+    inputQuery,
+    shaclType,
+    shaclEndpoint,
+    shaclQuery,
+    shaclUrl,
+    shaclDebug,
+    shaclDetails,
+    shaclTrace,
+    transformMapNamespace,
+    transformToTriple,
+    outputPrefix,
+    outputType,
+    pretty
+  }) => {
+    if (!input) {
+      return program.help()
+    }
+
+    if (pretty) {
+      rdf.formats.import(formatsPretty)
+    }
+
+    let stream = await createInputStream({
+      contentType: inputType,
+      defaultStream: process.stdin,
+      endpointUrl: inputEndpoint,
+      query: inputQuery,
+      url: input
+    })
+
+    if (shaclUrl) {
+      const shapeStream = await createInputStream({
+        contentType: shaclType,
+        endpointUrl: shaclEndpoint,
+        query: shaclQuery,
+        url: shaclUrl
+      })
+
+      const shaclStream = createShaclStream(shapeStream, {
+        debug: shaclDebug,
+        details: shaclDetails,
+        trace: shaclTrace
+      })
+
+      stream.pipe(shaclStream)
+      stream = shaclStream
+    }
+
+    if (transformMapNamespace) {
+      const toTripleStream = transformMapNamespaceFunc(transformMapNamespace)
+
+      stream.pipe(toTripleStream)
+      stream = toTripleStream
+    }
+
+    if (transformToTriple) {
+      const toTripleStream = transformToTripleFunc()
+
+      stream.pipe(toTripleStream)
+      stream = toTripleStream
+    }
+
+    const outputStream = await createOutputStream({
+      contentType: outputType,
+      prefixes: outputPrefix
+    })
+
+    stream.pipe(outputStream)
+
+    await promisify(finished)(outputStream)
+  })
+
+program.parse(process.argv)

examples/houseShape.ttl

@@ -0,0 +1,16 @@
+@prefix schema: <http://schema.org/>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+
+<PersonShape> a sh:NodeShape;
+  sh:targetClass schema:Person;
+  sh:property [
+    sh:path schema:birthDate;
+    sh:datatype xsd:date
+  ], [
+    sh:path schema:familyName;
+    sh:nodeKind sh:Literal
+  ], [
+    sh:path schema:givenName;
+    sh:nodeKind sh:Literal
+  ].

examples/url-to-turtle.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+./bin/rdf-ext-cli.js \
+  --pretty \
+  --output-prefix house=https://housemd.rdf-ext.org/person/ \
+  --output-prefix houseplace=https://housemd.rdf-ext.org/place/ \
+  --output-prefix schema=http://schema.org/ \
+  --output-prefix xsd=http://www.w3.org/2001/XMLSchema# \
+  https://housemd.rdf-ext.org/person/gregory-house

examples/validate-url.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+./bin/rdf-ext-cli.js \
+  --shacl-url=examples/houseShape.ttl \
+  --shacl-debug \
+  --shacl-details \
+  --shacl-trace \
+  --pretty \
+  --output-prefix sh=http://www.w3.org/ns/shacl# \
+  https://housemd.rdf-ext.org/person/gregory-house

lib/createInputStream.js

@@ -0,0 +1,30 @@
+import rdf from 'rdf-ext'
+import Client from 'sparql-http-client/StreamClient.js'
+
+async function createInputStream ({ contentType, defaultStream, endpointUrl, query, url }) {
+  if (endpointUrl) {
+    if (!query) {
+      query = `DESCRIBE <${url}>`
+    }
+
+    const client = new Client({ endpointUrl })
+
+    return client.query.construct(query)
+  }
+
+  if (url !== '-') {
+    const res = await rdf.fetch(url)
+
+    if (contentType) {
+      res.headers.set('content-type', contentType)
+    }
+
+    return res.quadStream()
+  }
+
+  const parser = rdf.formats.parsers.get(contentType)
+
+  return parser.import(defaultStream)
+}
+
+export default createInputStream

lib/createOutputStream.js

@@ -0,0 +1,14 @@
+import rdf from 'rdf-ext'
+import { PassThrough } from 'readable-stream'
+
+async function createOutputStream ({ contentType, prefixes }) {
+  const output = new PassThrough({ objectMode: true })
+  const serializer = rdf.formats.serializers.get(contentType)
+
+  const stream = serializer.import(output, { prefixes })
+  stream.pipe(process.stdout)
+
+  return output
+}
+
+export default createOutputStream

lib/createShaclStream.js

@@ -0,0 +1,22 @@
+import duplexify from 'duplexify'
+import rdf from 'rdf-ext'
+import { PassThrough } from 'readable-stream'
+import { Validator } from 'shacl-engine'
+
+function createShaclStream (shapeStream, { debug, details, trace } = {}) {
+  const input = new PassThrough({ objectMode: true })
+  const output = new PassThrough({ objectMode: true })
+
+  queueMicrotask(async () => {
+    const shape = await rdf.dataset().import(shapeStream)
+    const engine = new Validator(shape, { debug, details, factory: rdf, trace })
+    const dataset = await rdf.dataset().import(input)
+    const report = await engine.validate({ dataset })
+
+    report.dataset.toStream().pipe(output)
+  })
+
+  return duplexify.obj(input, output)
+}
+
+export default createShaclStream

lib/transformMapNamespace.js

@@ -0,0 +1,8 @@
+import rdf from 'rdf-ext'
+import { mapStream } from 'rdf-utils-namespace/map.js'
+
+function transformMapNamespace (mapping) {
+  return mapStream(mapping, { factory: rdf })
+}
+
+export default transformMapNamespace

lib/transformToTriple.js

@@ -0,0 +1,13 @@
+import rdf from 'rdf-ext'
+import { Transform } from 'readable-stream'
+
+function transformToTriple () {
+  return new Transform({
+    objectMode: true,
+    transform: (quad, encoding, callback) => {
+      callback(null, rdf.quad(quad.subject, quad.predicate, quad.object))
+    }
+  })
+}
+
+export default transformToTriple

package.json

@@ -0,0 +1,43 @@
+{
+  "name": "rdf-ext-cli",
+  "version": "0.0.0",
+  "description": "Command line tool for RDF-Ext",
+  "type": "module",
+  "main": "index.js",
+  "bin": {
+    "rdf-ext": "bin/rdf-ext-cli.js",
+    "rdf-ext-cli": "bin/rdf-ext-cli.js"
+  },
+  "scripts": {
+    "test": "stricter-standard"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/rdf-ext/rdf-ext-cli.git"
+  },
+  "keywords": [
+    "cli",
+    "rdf",
+    "rdfjs",
+    "rdf-ext"
+  ],
+  "author": "Thomas Bergwinkl <[email protected]> (https://www.bergnet.org/people/bergi/card#me)",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/rdf-ext/rdf-ext-cli/issues"
+  },
+  "homepage": "https://github.com/rdf-ext/rdf-ext-cli",
+  "dependencies": {
+    "@rdfjs/formats": "^4.0.0",
+    "commander": "^11.1.0",
+    "duplexify": "^4.1.2",
+    "rdf-ext": "^2.5.0",
+    "rdf-utils-namespace": "^0.2.1",
+    "readable-stream": "^4.4.2",
+    "shacl-engine": "^0.1.0",
+    "sparql-http-client": "^2.4.2"
+  },
+  "devDependencies": {
+    "stricter-standard": "^0.3.0"
+  }
+}