From 1761afc6cf7ed640dab9cbddc06328c52fff6325 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Sat, 24 Feb 2024 05:48:24 +0100 Subject: [PATCH] Have a proper structure for a Python project The directory structure is now as follows: ``` README.md pyproject.toml (project file) src qlever __init__.py (empty) __main__.py (the actual code) Qleverfiles (directory with Qleverfiles) ``` Update package on pypi.org as follows: ``` Increment version number in pyproject.toml rm -f dist/* && python3 -m build --wheel && ll dist twine upload -u __token__ -p dist/qlever-*.whl ``` --- README.md | 87 +- pyproject.toml | 12 +- qlever.OLD | 993 ------------------ .../qlever/Qleverfiles}/Qleverfile.dblp | 0 .../qlever/Qleverfiles}/Qleverfile.dblp-plus | 0 .../qlever/Qleverfiles}/Qleverfile.default | 0 .../qlever/Qleverfiles}/Qleverfile.dnb | 0 .../qlever/Qleverfiles}/Qleverfile.fbeasy | 0 .../qlever/Qleverfiles}/Qleverfile.freebase | 0 .../qlever/Qleverfiles}/Qleverfile.imdb | 0 .../qlever/Qleverfiles}/Qleverfile.olympics | 0 .../Qleverfiles}/Qleverfile.osm-country | 0 .../qlever/Qleverfiles}/Qleverfile.pubchem | 0 .../qlever/Qleverfiles}/Qleverfile.scientists | 0 .../qlever/Qleverfiles}/Qleverfile.uniprot | 0 .../qlever/Qleverfiles}/Qleverfile.vvz | 0 .../qlever/Qleverfiles}/Qleverfile.wikidata | 0 .../qlever/Qleverfiles}/Qleverfile.yago-4 | 0 src/qlever/__init__.py | 0 qlever => src/qlever/__main__.py | 10 +- 20 files changed, 60 insertions(+), 1042 deletions(-) delete mode 100755 qlever.OLD rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.dblp (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.dblp-plus (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.default (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.dnb (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.fbeasy (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.freebase (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.imdb (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.olympics (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.osm-country (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.pubchem (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.scientists (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.uniprot (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.vvz (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.wikidata (100%) rename {Qleverfiles => src/qlever/Qleverfiles}/Qleverfile.yago-4 (100%) create mode 100644 src/qlever/__init__.py rename qlever => src/qlever/__main__.py (99%) diff --git a/README.md b/README.md index 7324f2d0..67bc0076 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,61 @@ # QLever Control -This is a very small repository. Its main contents is a script `qlever` -that can control everything that QLever does. The script is supposed to be very -easy to use and pretty much self-explanatory as you use it. If you use docker, you -don't even have to download any QLever code (docker will pull anything it needs) -and the script is all you need. +This is a small repository, which provides a script `qlever` that can control +everything that QLever does. The script is supposed to be very easy to use and +pretty much self-explanatory as you use it. If you use Docker, you don't even +have to download any QLever code (Docker will pull the required images) and the +script is all you need. -# Directory structure +# Installation -We recommend that you have a directory "qlever" for all things QLever on your machine, -with subdirectories for the different components, in particular: "qlever-control" (this -repository), "qlever-indices" (with a subfolder for each of your datasets), and "qlever-code" -(only needed if you don't want to use docker, but compile the binaries on your machine). +Simply do `pip install qlever` and make sure that the directory where pip +installs the package is in your `PATH`. Typically, `pip` will warn you when +that is not the case and tell you what to do. -# Quickstart +# Usage -Create an empty directory (preferably as a subdirectory of "qlever-indices", go there, -and call the `qlever` script once with its full path and a dot and a space preceding it, -and the name of a preconfiguration as only argument. For example: - -```. /path/to/qlever olympics``` - -This will create a `Qleverfile` preconfigured for the -[120 Years of Olympics](https://github.com/wallscope/olympics-rdf) dataset, which is -a great dataset to get started because it's small. Other options are: -`scientists` (another small test collection), `dblp` (larger), `wikidata` (very large), -and more. If you leave out the argument, you get a default `Qleverfile`, which you need -to edit first to use for your own dataset (it should be self-explanatory, after you have -played around with and looked at one of the preconfigured Qleverfiles). - -Now you can call `qlever` without path and without a dot and a space preceding it and -with one or more actions as argument. To see the set of avaiable actions, just use the -autocompletion. When you are a first-timer, execute these commands one after the other -(without the comments): +First, create an empty directory, with a name corresponding to the dataset you +want to work with. For the following example, take `olympics`. Go to that +directory, and do the following: ``` -qlever get-data # Download the dataset -qlever index # Build a QLever index for your data -qlever start # Start a QLever server using that index -qlever example-query # Launch an example query +qlever # Basic help + lists of available pre-configs +qlever setup-config olympics # Get examplary Qleverfile (config file) +qlever get-data # Download the dataset (see below) +qlever index # Build index data structures for this dataset +qlever start # Start a QLever server using that index +qlever test-query # Launch a test query +qlever ui # Launch the QLever UI ``` -Each command will not only execute the respective action, but it will also show you -the exact command line it uses. That way you can learn, on the side, how QLever works -internally. If you just want to know the command used for a particular action, but -not execute it, you can append "show" like this: +This will create a SPARQL endpoint for the [120 Years of +Olympics](https://github.com/wallscope/olympics-rdf) dataset. It is a great +dataset for getting started because it is small, but not trivial (around 2 +million triples), and the downloading and indexing should only take a few +seconds. + +Each command will also show you the command line it uses. That way you can +learn, on the side, how QLever works internally. If you just want to know the +command line for a particular command, without executing it, you can append +"show" like this: ``` qlever index show ``` -You can also perform a sequence of actions with a single call, for example: +There are many more commands and options, see `qlever --help`. The script +supports autocompletion for all its commands and options. You can (and should) +activate it following the instructions given when you just type `qlever` +without any arguments. -``` -qlever stop remove-index index start -``` +# For developers + +The (Python) code for the script is in the `*.py` files in `src/qlever`. The +preconfigured Qleverfiles are in `src/qlever/Qleverfiles`. -There are many more actions. The script supports autocompletion. Just type "qlever " -and then TAB and you will get a list of all the available actions. +If you want to make changes to the script, git clone this repository, make any +changes you want, and run `pip install -e .`. Then you can use the script (with +whatever modifications you have made), just as if you had installed it via `pip +install qlever`. Note that unless you change the directory structure, you have +to execute `pip install -e .` only once (this local installation will not copy +your files but link to them). diff --git a/pyproject.toml b/pyproject.toml index 3be68b6d..7854014f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "qlever" description = "Script for using the QLever SPARQL engine." -version = "0.3.1" +version = "0.3.5" authors = [ { name = "Hannah Bast", email = "bast@cs.uni-freiburg.de" } ] @@ -18,11 +18,13 @@ classifiers = [ dependencies = [ "psutil", "termcolor" ] -[project.scripts] -"qlever" = "qlever:main" - [project.urls] Github = "https://github.com/ad-freiburg/qlever-control" +[project.scripts] +"qlever" = "qlever.__main__:main" + [tool.setuptools] -package-data = { "*" = ["*.py", "Qleverfiles/Qleverfile.*"] } +package-dir = { "" = "src" } +packages = [ "qlever" ] +package-data = { "qlever" = ["Qleverfiles/*"] } diff --git a/qlever.OLD b/qlever.OLD deleted file mode 100755 index b4675361..00000000 --- a/qlever.OLD +++ /dev/null @@ -1,993 +0,0 @@ -#!/bin/bash - -# This is the script that is called when calling "qlever" from the command line. -# The script is self-explanatory when you run it. Type ". qlever" for initial -# configuration and then "qlever" or "qlever help" for initial help. -# -# © 2022, University of Freiburg, Chair of Algorithms and Data Structures -# Author: Hannah Bast - - -### -### STRUCTURE OF THIS SCRIPT -### -### 1. Definition of available actions (as bash functions) -### 2. Setup code when called with ". qlever" -### 3. Handle calls "qlever", "qlever help", "qlever help-install" -### 4. Settings (default + from Qleverfile + derived settings) -### 5. Process actions -### - - -BLUE=$(printf "\033[34m") -RED=$(printf "\033[31m") -GRAY=$(printf "\033[37m") -MAGENTA=$(printf "\033[35m") -BOLD=$(printf "\033[1m") -NORMAL=$(printf "\033[0m") - -QLEVER_CMD="${BASH_SOURCE[0]}" # Command of this script -QLEVER_CMD_DIR=$(cd -- "$(dirname -- "${QLEVER_CMD}")" &> /dev/null && pwd) -QLEVER_BIN_DIR=$(which ServerMain | sed 's|/ServerMain$||') -QLEVERFILE_NEEDED=true -SIMPLE_ACTIONS=:update:status:help:help-install:help-install-ubuntu-18:help-install-ubuntu-20:show-configs: -if [[ ${SIMPLE_ACTIONS} == *:$*:* ]]; then QLEVERFILE_NEEDED=false; fi - -# Helper function to display simple markdown using ANSI escape codes. -function display_markdown { - sed -E "s/^# (.*)/${BOLD}\1${NORMAL}/g" \ - | sed -E "s/\`([^\`]+)\`/${BLUE}\1${NORMAL}/g" -} - - -### -### AVAILABLE ACTIONS (all available via autocompletion) -### - -function action_help { - cat << "EOT" | display_markdown -# QLEVER HELP - -The `qlever` script controls everything concerning QLever. Calling it without -arguments always displays this information. Before using the script the first -time in a particular shell and directory, type `. /path/to/qlever`. This will -enable autocompletion, update your PATH, and create a default Qleverfile. - -# Qleverfile -The Qleverfile configures everything: how to get the data, build an index, -start the server, and all kinds of settings. Modify the default Qleverfile as -you see fit. For some datasets, there are preconfigured QLeverfiles: to get one -of those, type `qlever show-configs` and follow the instructions. - -# Actions -On the command line, typing TAB after `qlever` gives you the available actions. -You can also execute several actions with one command, for example "qlever index -start log`. If you just want to see what an action does, but not exeute it, -append "show", for example `qlever index start show`. - -# Binaries -By default, the qlever script uses the official QLever docker image (settings -USE_DOCKER and QLEVER_DOCKER_IMAGE in the Qleverfile). If you want to use the -QLever binaries, you have to compile them on your machine. For instructions, -type `qlever install-binaries`. - -EOT -} - -function action_install_binaries { - cat << "EOT" | display_markdown -# QLEVER INSTALL BINARIES - -If you don't want to use docker for the actions "index" (building an index) and -"start" (starting a server), you have to compile the binaries "IndexBuilderMain" -and "ServerMain" on your machine. To use these binaries, in your Qleverfile set -USE_DOCKER=false and QLEVER_BIN_DIR to the directory with your binaries. - -# Get a copy of the QLever code -`git clone --recursive https://github.com/ad-freiburg/qlever qlever-code` -`cd qlever-code` - -# On Ubuntu 18.04, 20.04, or 22.04 -Just type `qlever help-install-ubuntu-22` (or ...-20 or ...-18) and follow the -instructions. The call will create and show an INSTALL script, which you can -then just execute. - -# On other operatings systems -We are working on an INSTALL script for MacOS and WSL, analogous to the ones for -Ubuntu above. In the meantime, look at one of the Ubuntu INSTALL scripts and -adapt them as good as you can. - -EOT -} - -SHOW= -NOSHOW= - -INSTALL_UBUNTU_HELP_TEXT=$(cat << "EOT" | display_markdown -The following command sequence has been written to a file INSTALL. If it looks -OK to you, execute the sequence with `source ./INSTALL`. If there is a conflict -with an existing libboost version, uninstall it and then run again. If you run -out of RAM, use fewer cores for the last call to ninja, e.g., `ninja -j 3`. -EOT -) - -function action_help_install_ubuntu_18 { - echo "${INSTALL_UBUNTU_HELP_TEXT}" - echo - $SHOW curl -Gs https://raw.githubusercontent.com/ad-freiburg/qlever/master/Dockerfiles/Dockerfile.Ubuntu18.04 \ - | sed -En 's/(add-apt|apt|tee)/sudo \1/g; s/^RUN //p' \ - | sed '/^cmake/q' \ - | sed -E 's/^(cmake.*)/mkdir -p build \&\& cd build\n\1\ncd ../' \ - > INSTALL - echo "${BLUE}$(cat INSTALL)${NORMAL}" -} - -function action_help_install_ubuntu_20 { - echo "${INSTALL_UBUNTU_HELP_TEXT}" - echo - $SHOW curl -Gs https://raw.githubusercontent.com/ad-freiburg/qlever/master/Dockerfiles/Dockerfile.Ubuntu20.04 \ - | sed -En 's/(add-apt|apt|tee)/sudo \1/g; s/^RUN //p' \ - | sed '/^cmake/q' \ - | sed -E 's/^(cmake.*)/mkdir -p build \&\& cd build\n\1\ncd ../' \ - > INSTALL - echo "${BLUE}$(cat INSTALL)${NORMAL}" -} - -function action_help_install_ubuntu_22 { - echo "${INSTALL_UBUNTU_HELP_TEXT}" - echo - $SHOW curl -Gs https://raw.githubusercontent.com/ad-freiburg/qlever/master/Dockerfile \ - | sed -En 's/(add-apt|apt|tee)/sudo \1/g; s/^RUN //p' \ - | sed '/^cmake/q' \ - | sed -E 's/^(cmake.*)/mkdir -p build \&\& cd build\n\1\ncd ../' \ - > INSTALL - echo "${BLUE}$(cat INSTALL)${NORMAL}" -} - -function action_show_configs { - cat << "EOT" | display_markdown -The following pre-configured Qleverfiles are available. To get one of these in -the current directory, type `. qlever `. Beware that this will -overwrite any Qleverfile that is already there. - -EOT - CONFIGS_AVAILABLE=$($SHOW ls ${QLEVER_CMD_DIR}/Qleverfiles/Qleverfile.* | sed 's/^.*\/Qleverfile\.//' $NOSHOW) - echo ${BLUE}${CONFIGS_AVAILABLE}${NORMAL} -} - -function action_remove_data { - echo "Removing files specified in RDF_FILES in Qleverfile ..." - echo - ls -l "${RDF_FILES}" - $SHOW eval ${REMOVE_DATA_CMD} -} - -function action_get_data { - if [ -z "${GET_DATA_CMD}" ]; then - echo "${RED}No GET_DATA_CMD specified in Qleverfile${NORMAL}" - else - echo "Getting data using GET_DATA_CMD from Qleverfile ..." - echo - eval $SHOW ${GET_DATA_CMD} - fi -} - -function action_index { - if ls ${DB}.index.* > /dev/null 2>&1; then - echo "${RED}Index exists, delete it first with \"qlever remove-index\" (add \"show\" to find out what gets deleted)${NORMAL}" - else - # # Prepend ulimit setting to INDEXER_CMD if input files are large. - # if [ "$(du -bc ${RDF_FILES} | tail -1 | cut -f1)" -gt 1000000000 ]; then - # export INDEXER_CMD="ulimit -Sn 1048576; ${INDEXER_CMD}" - # fi - # Create settings.json. - if jq --version > /dev/null 2>&1; then - echo ${SETTINGS_JSON} | jq -M '.' > ${DB}.settings.json - else - echo ${SETTINGS_JSON} > ${DB}.settings.json - fi - # Create prefix definitions - if [ ! -z "${EXTRACT_PREFIXES}" ]; then eval $SHOW ${EXTRACT_PREFIXES} $NOSHOW; fi - # Build index. - eval ${INDEXER_CMD_PREFIX} $SHOW ${INDEXER_CMD} - fi -} - -function action_remove_index { - $SHOW rm -f ${DB}.index.* ${DB}.vocabulary.* ${DB}.prefixes ${DB}.meta-data.json ${DB}.index-log.txt - echo "The index files have been removed." -} - -function action_index_stats { - if [ ! -e ${DB}.index-log.txt ]; then echo "${RED}Need file \"${DB}.index-log.txt\"${NORMAL}"; return; fi - $SHOW readarray -t T < <(sed -En '/INFO: (Processing|Done, total|Converting triples|Creating|Index build|Adding text index|DocsDB done)/p' ${DB}.index-log.txt | cut -c1-19) - if [ "${#T[@]}" -lt "4" ]; then echo "${RED}Missing key lines in \"${DB}.index-log.txt\"${NORMAL}"; return; fi - Ts=$(echo "($(date +%s -d "${T[-1]}") - $(date +%s -d "${T[0]}"))" | bc) - if [ "${Ts}" -gt "3600" ]; then Tdiv=3600; Tunit="h"; else Tdiv=60; Tunit="min"; fi - T1=$(echo "($(date +%s -d "${T[1]}") - $(date +%s -d "${T[0]}")) / ${Tdiv}" | bc -l) - T2=$(echo "($(date +%s -d "${T[2]}") - $(date +%s -d "${T[1]}")) / ${Tdiv}" | bc -l) - T3=$(echo "($(date +%s -d "${T[3]}") - $(date +%s -d "${T[2]}")) / ${Tdiv}" | bc -l) - T4=$(echo "($(date +%s -d "${T[4]}") - $(date +%s -d "${T[3]}")) / ${Tdiv}" | bc -l) - T5=$(echo "($(date +%s -d "${T[5]}") - $(date +%s -d "${T[4]}")) / ${Tdiv}" | bc -l) - T6=$(echo "($(date +%s -d "${T[6]}") - $(date +%s -d "${T[5]}")) / ${Tdiv}" | bc -l) - T7=$(echo "($(date +%s -d "${T[7]}") - $(date +%s -d "${T[6]}")) / ${Tdiv}" | bc -l) - T8=$(echo "($(date +%s -d "${T[8]}") - $(date +%s -d "${T[7]}")) / ${Tdiv}" | bc -l) - T9=$(echo "($(date +%s -d "${T[9]}") - $(date +%s -d "${T[8]}")) / ${Tdiv}" | bc -l) - TT=$(echo "($(date +%s -d "${T[-1]}") - $(date +%s -d "${T[0]}")) / ${Tdiv}" | bc -l) - echo "Time needed for the various phases of the index build" - echo - printf "%-23s : %5.1f ${Tunit}\n" "Parse input" ${T1} - printf "%-23s : %5.1f ${Tunit}\n" "Build vocabularies" $(echo "${T2} + ${T4}" | bc) - printf "%-23s : %5.1f ${Tunit}\n" "Convert to global IDs" ${T3} - if [ "${#T[@]}" -gt "5" ]; then printf "%-23s : %5.1f ${Tunit}\n" "PSO & POS permutations" ${T5}; fi - if [ "${#T[@]}" -gt "6" ]; then printf "%-23s : %5.1f ${Tunit}\n" "SPO & SOP permutations" ${T6}; fi - if [ "${#T[@]}" -gt "7" ]; then printf "%-23s : %5.1f ${Tunit}\n" "OSP & OPS permutations" ${T7}; fi - if [ "${#T[@]}" -gt "9" ]; then printf "%-23s : %5.1f ${Tunit}\n" "Add text index" ${T9}; fi - echo - printf "%-23s : %5.1f ${Tunit}\n" "TOTAL index build time" ${TT} -} - -function action_start { - if curl --silent http://localhost:${SERVER_PORT} > /dev/null; then - echo "${RED}Server already running on port ${SERVER_PORT}, use \"qlever restart\" or check SERVER_PORT in your Qleverfile${NORMAL}" - echo "${BLUE}" - ps -e -o "%p" -o " %U " -o start_time -o " %a" \ - | \grep "${SERVER_BINARY} .*-p ${SERVER_PORT}" \ - | \grep -v "grep.*${SERVER_BINARY}" - printf "${NORMAL}" - else - if [ -f "${DB}.index.pso" ] && [ -f "${DB}.index.pos" ]; then - echo "Starting the QLever server in the background and waiting until it's ready (Ctrl+C will not kill it) ..." - echo - rm -f ${DB}.server-log.txt && touch ${DB}.server-log.txt - eval $SHOW ${SERVER_CMD} - tail -f -n 10 ${DB}.server-log.txt & PID=$! - ALIVE_CHECK_URL="http://localhost:${SERVER_PORT}/ping?msg=from%20the%20qlever%20script" - while [ $(curl -s "${ALIVE_CHECK_URL}" > /dev/null; echo $?) != 0 ]; do sleep 1; done - # if [ ! -z "${INDEX_DESCRIPTION}${TEXT_DESCRIPTION}" ]; then echo; fi - if [ ! -z "${INDEX_DESCRIPTION}" ]; then - # echo "Setting index description to \"${INDEX_DESCRIPTION}\"" - curl -Gs ${SERVER_API} \ - --data-urlencode "index-description=${INDEX_DESCRIPTION}" \ - --data-urlencode "access-token=${ACCESS_TOKEN}" > /dev/null - fi - if [ ! -z "${TEXT_DESCRIPTION}" ]; then - # echo "Setting text description to \"${TEXT_DESCRIPTION}\"" - curl -Gs ${SERVER_API} \ - --data-urlencode "text-description=${TEXT_DESCRIPTION}" \ - --data-urlencode "access-token=${ACCESS_TOKEN}" > /dev/null - fi - kill ${PID} - wait ${PID} 2> /dev/null # Supress "Terminated ..." message. - else - echo "${RED}Index missing, first build one using \"qlever index\"${NORMAL}" - fi - fi -} - -function action_stop { - echo "Stop the QLever server (find and kill manually if it fails) ..." - echo - if printf ${BLUE} && pgrep -af "${SERVER_BINARY} -i [^ ]*${DB}" && printf ${NORMAL}; then - echo - eval $SHOW ${STOP_CMD} $NOSHOW && echo "Killed processes and removed \"${DB}.server-log.txt\"" - else - echo "${RED}Did not find a running server for \"${DB}\" or could not kill it, try \"qlever status\"${NORMAL}" - fi -} - -function action_restart { - $SHOW ${QLEVER_CMD_WITH_VARS} stop $NOSHOW | sed '1,3d' - $SHOW ${QLEVER_CMD_WITH_VARS} start $NOSHOW | sed '1,3d; $d' -} - -function action_wait { - tail -f -n 10 ${DB}.server-log.txt & PID=$! - while [ $(curl --silent http://localhost:$(SERVER_PORT) > /dev/null; echo $?) != 0 ]; do sleep 1; done - kill $PID -} - -function action_log { - echo "Showing the log (abort with Ctrl+C) ..." - echo - $SHOW tail -f -n 50 ${DB}.server-log.txt -} - -function action_log_until_server_up { - $SHOW tail -f -n 10 ${DB}.server-log.txt & PID=$! - while ! curl --silent http://localhost:${SERVER_PORT} > /dev/null; do sleep 1; done - kill ${PID} -} - -function action_status { - echo "Showing all QLever-related processes and docker containers on this machine ..." - echo "${BLUE}" - $SHOW ps -e -o %p -o %U -o start_time -o rss:10 -o %a | sed -E '1p; /\b('${SERVER_BINARY}'|'${INDEXER_BINARY}')/!d; /bash -c/d' $NOSHOW \ - | awk 'NR==1 {o=$0; a=match($0, $5)} NR > 1 {o=$0; $4=int($4/(1024*1024))"G";} {printf "%-7s %-8s %5s %5s %s\n", $1, $2, $3, $4, substr(o, a)}' - printf "${NORMAL}" - if [ $(docker ps --filter "name=qlever" | wc -l) -gt 1 ]; then \ - echo "${BLUE}" - $SHOW docker ps --filter "name=qlever" --format "table {{.Names}}\t{{.State}}\t{{.RunningFor}}\t{{.Ports}}" - printf "${NORMAL}" - fi -} - -function action_example_query { - echo "Launching SELECT query for 10 random triples (in TSV format) ..." - echo - $SHOW curl ${SERVER_API} -H "Accept: text/tab-separated-values" -H "Content-type: application/sparql-query" --data "SELECT * WHERE { ?s ?p ?o } LIMIT 10" -} - -function action_example_query_construct { - echo "Launching CONSTRUCT query for 10 random triples (in TSV format) ..." - echo - $SHOW curl ${SERVER_API} -H "Accept: text/turtle" -H "Content-type: application/sparql-query" --data "CONSTRuCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 10" -} - -function action_query { - echo "Launching GET query (results in TSV format) ..." - echo - if [ -z "${QUERY}" ]; then - echo "${RED}Empty query, specify one via the command line like this: qlever QUERY=\"...\" query${NORMAL}" - else - $SHOW "${QUERY_CMD[@]}" - fi -} - - -# Warmup queries for the autocompletion. -# -# TODO: This requires a running QLever UI and his currently hard-coded to use -# instances on particular machine (galera.informatik.privat). It will be fixed -# soon. -# -# NOTE: We need -tt here because the "qlever" script calls "make" via a pipe and -# so it does not have a terminal (the error message is "Pseudo-terminal will not -# be allocated because stdin is not a terminal." and "the input device is not a -# TTY"). The manpage for ssh says: "Multiple -t options force tty allocation, -# even if ssh has no local tty." and "-tt" is just a shorthand for "-t -t".in.master: -function action_pin_INTERNAL { - echo "Launch warmup queries for the QLever UI autocompletion (via galera) ..." - $SHOW ssh -tt -o LogLevel=QUIET galera docker exec -it qlever-ui bash -c \"python manage.py warmup ${SLUG} pin\" -} - -# Clear the cache. -function action_clear_cache { - echo "Clearing the cache (unpinned entries only) ..." - $SHOW curl -Gs ${SERVER_API} --data-urlencode "cmd=clear-cache" $NOSHOW > /dev/null - curl -Gs ${SERVER_API} --data-urlencode "cmd=cache-stats" \ - | sed 's/[{}",]//g; s/:/: /g; s/^\s\+//' | tr -s " " \ - | numfmt --field=2,5,8,11,14 --grouping -} - -function action_clear_cache_complete { - echo "Clearing the cache completely (including pinned entries) ..." - $SHOW curl -Gs ${SERVER_API} \ - --data-urlencode "cmd=clear-cache-complete" \ - --data-urlencode "access-token=${ACCESS_TOKEN}" $NOSHOW > /dev/null - curl -Gs ${SERVER_API} --data-urlencode "cmd=cache-stats" \ - | sed 's/[{}",]//g; s/:/: /g; s/^\s\+//' | tr -s " " \ - | numfmt --field=2,5,8,11,14 --grouping -} - -# Get various settings and statistics. -function action_disk_usage { - echo "Disk usage of all files in this QLever index, and the size of the input files" - echo - $SHOW du -hc $(ls ${DB}.* | sed -E '/(index\.|vocabulary|prefixes|meta-data|text)/!d') - echo - # echo && echo "For comparison, the size of the input files" && echo # - du -hc ${RDF_FILES} -} - -function action_cache_stats_and_settings { - echo "Show cache statistics (sizes are in bytes) and cache settings (sizes are in GB) ..." - echo ${BLUE} - $SHOW curl -Gs ${SERVER_API} --data-urlencode "cmd=cache-stats" $NOSHOW \ - | sed -n 's/[",]//g; s/:/: /g; s/^\s\+//p' | tr -s " " \ - | awk '{ if ($1 ~ "pinned-size:") { $2 = 8 * $2 } print }' \ - | numfmt --field=2 --grouping - $SHOW curl -Gs ${SERVER_API} --data-urlencode "cmd=get-settings" $NOSHOW \ - | sed 's/[{}",]//g; s/:/: /g; s/^\s\+//' | tr -s " " - printf ${NORMAL} -} - -function action_server_stats { - echo "Getting stats from \"${SERVER_API}\" ..." - echo ${BLUE} - $SHOW curl -Gs ${SERVER_API} --data-urlencode "cmd=stats" $NOSHOW \ - | sed -n 's/[",]//g; s/:/: /g; s/^\s\+//p' | tr -s " " \ - | sed -E 's/^((kb|text)index:\s*)(.*)$/\1"\3"/' \ - | numfmt --field=2 --grouping --invalid=ignore - printf ${NORMAL} -} - -function action_index_description { - echo "Setting index description to \"${INDEX_DESCRIPTION}\" ..." - $SHOW curl -Gs ${SERVER_API} --data-urlencode "index-description=${INDEX_DESCRIPTION}" --data-urlencode "access-token=${ACCESS_TOKEN}" $NOSHOW > /dev/null -} - -function action_text_description { - echo "Setting text description to \"${TEXT_DESCRIPTION}\" ..." - $SHOW curl -Gs ${SERVER_API} --data-urlencode "text-description=${TEXT_DESCRIPTION}" --data-urlencode "access-token=${ACCESS_TOKEN}" $NOSHOW > /dev/null -} - - -# QLever UI (work in progress). -LOG_FILE=qleverui-log.tmp -ERROR_CMD="(echo && echo \"${BLUE}An error occured, check ${LOG_FILE} for details, the last 10 lines are:${NORMAL}\" && tail ${LOG_FILE} && false)" -QLEVERUI_GIT=https://github.com/ad-freiburg/qlever-ui.git -QLEVERUI_DIR="qlever-ui" -function action_ui { - echo "Starting the QLever UI on ${HOSTNAME}:${QLEVERUI_PORT} ..." - echo - $SHOW docker rm -f ${QLEVERUI_DOCKER_CONTAINER} 2>&1 > /dev/null - $SHOW docker run -d -p ${QLEVERUI_PORT}:7000 --name ${QLEVERUI_DOCKER_CONTAINER} ${QLEVERUI_DOCKER_IMAGE} > /dev/null $NOSHOW || return - $SHOW docker exec -it ${QLEVERUI_DOCKER_CONTAINER} bash -c "python manage.py configure ${QLEVERUI_CONFIG} ${SERVER_API}" $NOSHOW || return - echo - echo "The QLever UI should now be up at http://${HOSTNAME}:${QLEVERUI_PORT} ." - echo "You can log in as QLever UI admin with username and passwort \"demo\"" -} - -function action_autocompletion_warmup { - echo "Launch warmup queries for the QLever UI autocompletion ..." - echo - if docker container inspect -f '{{.State.Running}}' ${QLEVERUI_DOCKER_CONTAINER} > /dev/null 2>&1; then - $SHOW docker exec -i qlever-ui bash -c "python manage.py warmup ${QLEVERUI_CONFIG} pin" | sed -En 's/^\x1b\[[0-9]+/\x1b\[34/p' - else - echo "${RED}The autocompletion queries are defined by the UI, run \"qlever ui\" first${NORMAL}" - fi -} - -# Update the qlever script (simply git pull from the GitHub repo). -function action_update { - echo "Updating \"qlever\" script ..." - echo - if [ -d "${QLEVER_CMD_DIR}/.git" ]; then - $SHOW cd ${QLEVER_CMD_DIR} && git pull - else - echo "${RED}Did not find directory \".git\" in \"${QLEVER_CMD_DIR}\"${NORMAL}" - echo - fi -} - -# Show the paths to "qlever", the QLever binaries, and the QLever UI. -function action_where { - cat << EOT -Show where everything is ... -${BLUE} -Directory with "qlever" script : ${QLEVER_CMD_DIR} -Directory with QLever binaries : ${QLEVER_BIN_DIR} -Directory with QLever UI repo : ${QLEVERUI_DIR} -${NORMAL} -EOT -} - -# Produce list of all input files. -function action_rdf_files { - eval $SHOW ls ${RDF_FILES} -} - -# Produce the input as fed to the index builder. -function action_cat_files { - eval $SHOW ${CAT_FILES} -} - -# Actions for turning docker on and off. -# -# TODO: Currently, there is one docker container, which is then used for for -# *all* docker actions using "docker exec". This makes it easier to mix actions, -# where some actions use docker and some don't. However, if the docker container -# is running, it blocks the server port even when nothing is running inside the -# container, so that when cannot do "qlever start" without docker then. -function action_docker_on_DEPRECATED { - if docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} > /dev/null 2>&1; then - echo "${MAGENTA}Docker container \"${QLEVER_DOCKER_CONTAINER}\" already running${NORMAL}" - else - docker rm -f "${QLEVER_DOCKER_CONTAINER}" > /dev/null 2> /dev/null - echo "${MAGENTA}Starting docker container \"${QLEVER_DOCKER_CONTAINER}\" ...${NORMAL}" - $SHOW docker run -d -u $(id -u):$(id -g) -it -v $(pwd):/index \ - -p ${SERVER_PORT}:${SERVER_PORT} \ - --entrypoint bash \ - --name ${QLEVER_DOCKER_CONTAINER} ${QLEVER_DOCKER_IMAGE} $NOSHOW \ - > /dev/null - while ! docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} > /dev/null 2>&1; do sleep 0.1; done - # $SHOW docker cp ${QLEVER_CMD_DIR}/qlever \ - # ${QLEVER_DOCKER_CONTAINER}:/index $NOSHOW \ - # > /dev/null # - fi - echo "${BLUE}" - docker ps --filter "name=${QLEVER_DOCKER_CONTAINER}" - printf "${NORMAL}" -} - -function action_docker_off_DEPRECATED { - $SHOW docker rm -f "${QLEVER_DOCKER_CONTAINER}" -} - - -### -### CONFIGURATION (when called with . qlever or if we need to) -### - -# If not called with ". qlever" and completions or the Qleverfile are missing, -# tell the user to run ". qlever" once for full functionality. Note that bash -# autocompletion can only be activated by "sourcing" the respective commands -# (the ". qlever" is equivalent to "source qlever"). If the Qleverfile has -# BASH_COMPLETION = 0, don't mind the autocompletion. -# -# Skip this when calling "qlever update" because it's not only not necessary -# then, but even annoying. -if [ ${BASH_SOURCE[0]} == $0 ] \ - && ( ( [ -z "${QLEVER_COMPLETIONS}" ] && [ "${BASH_COMPLETION}" != "0" ] ) || \ - ( [ ! -e Qleverfile.OLD ] && ${QLEVERFILE_NEEDED} ) ); then - echo - action_help - exit -fi - -# If called with ". qlever", setup what is not already setup: the PATH, the bash -# autocompletion (unless user don't want it), the QLeverfile. -if [ "${QLEVER_CMD}" != "$0" ]; then - echo - echo "${BOLD}QLEVER CONFIG${NORMAL}" - echo - - # PATH: Check if QLEVER_CMD_DIR (read from Qleverfile if it exists) is in the - # path and if not add it to the path. - # - # Note: The full processing of the QLeverfile comes *after* the code for the - # ". qlever" case because otherwise the shell from which ". qlever" was called - # would be full with variables from this script. The QLEVER_BIN_DIR is an - # exception because adding something to the path requires a call to ". qlever" - # and would not work with "qlever" (which runs in a subshell). - # - echo "${BOLD}Checking your PATH ...${NORMAL}" - if [ -s "Qleverfile.OLD" ] && [ -z "${QLEVER_BIN_DIR}" ]; then - export QLEVER_BIN_DIR="$(sed -En 's/^\s*QLEVER_BIN_DIR\s*=\s*(\S+)/\1/p' Qleverfile.OLD)" - fi - for DIR in ${QLEVER_CMD_DIR} ${QLEVER_BIN_DIR}; do - if [[ ":${PATH}:" == *":${DIR}:"* ]]; then - echo "The directory \"${DIR}\" is already contained in your PATH" - else - export PATH=${PATH}:${DIR} - echo "Added the directory \"${DIR}\" to your PATH" - fi - done - echo - - # If we called ". qlever path" (just to set the paths), exit here. - if [ "$*" == "path" ]; then - echo "${NORMAL}Called \". qlever path\", so leaving now${NORMAL}" - echo - return - fi - - # Bash autocompletion: Set completions with "complete -W" and store the - # completions in "QLEVER_COMPLETIONS". We need the latter so that we know that - # the completions are set (complete -p does not work in a subshell). - COMP_WORDBREAKS=${COMP_WORDBREAKS/=/} - COMPLETIONS_FILE=no_longer_needed - echo "${BOLD}Setting up bash autocompletion ...${NORMAL}" - COMPLETIONS=$(cat ${QLEVER_CMD} | sed -En '/DEPRECATED/d; s/^function action_([a-zA-Z0-9_.-]+).*/\1/p' | sed 's/_/-/g') - COMPLETIONS+=" \"USE_DOCKER=true\" \"USE_DOCKER=false\"" - export QLEVER_COMPLETIONS="$(echo "${COMPLETIONS}" | paste -sd" ")" - complete -W "${QLEVER_COMPLETIONS}" qlever - echo "Done, number of completions: $(echo "${COMPLETIONS}" | wc -l)" - echo ${NORMAL} - - # If we called ". qlever init" (for use in .bashrc, where we only want to set - # the PATH and activate the autocompletion), exit here. - if [ "$*" == "shell_init" ]; then - echo "${NORMAL}Called \". qlever shell_init\", so leaving now${NORMAL}" - echo - return - fi - - # QLeverfile: If it's not there yet, create a basic one (with good guesses for - # some basic settings). Don't overwriting an existing Qleverfile, unless the - # script was called with an explicit preconfiguration as argument, for example - # ". qlever olympics". - if [ -e Qleverfile.OLD ] && [ -z $* ]; then - echo "${BOLD}Checking Qleverfile ...${NORMAL}" - cat << EOT -There is already a QLeverfile in this directory. If you want a freshly generated -default Qleverfile, remove or move the existing one and run ". qlever" again. - -EOT - else - echo "${BOLD}Creating new Qleverfile ...${NORMAL}" - # If Qleverfiles/Qleverfile. exists, just copy that. Otherwise copy - # Qleverfiles/Qleverfile.default and fill out the %...% templates. - QLEVERFILE_NAME="${QLEVER_CMD_DIR}/Qleverfiles/Qleverfile.$1" - if [ "$1" != "default" ] && [ -f "${QLEVERFILE_NAME}" ]; then - cat ${QLEVER_CMD_DIR}/Qleverfiles/Qleverfile.$1 \ - | sed -E "s/^(QLEVER_BIN_DIR\s+=).*/\1 \"${QLEVER_BIN_DIR//\//\\\/}\"/" \ - | sed -E "s/%RANDOM%/${RANDOM}${RANDOM}/" \ - > Qleverfile.OLD - echo "Copied pre-configured Qleverfile for \"$1\" into current directory." - echo - else - # Default values for the %...% templates in Qleverfiles/Qleverfile.default. - FOLDER_NAME=$(pwd) - DATE=$(date +"%d.%m.%Y %H:%M") - DB=must_specify - RDF_FILES="\${DB}.ttl" - CAT_FILES="cat \${RDF_FILES}" - # If a file with suffix .ttl or .nt exists, use the basename as the basename - # for the index files (the DB variable in the configuration) and pick the - # right "cat" command according to the suffix. If several such files exist, - # take the largest one. If no such file exists, leave the basename empty and - # for the user to set. - FILE=$(ls -S | egrep "\.(ttl|nt)(\.(gz|bz|bz2|xz))?\$" | head -1) - if [ ! -z "$FILE" ]; then - DB="${FILE/.*/}" - RDF_FILES="$FILE" - if [[ ${RDF_FILES} =~ .gz$ ]]; then CAT_FILES="zcat \${RDF_FILES}"; fi - if [[ ${RDF_FILES} =~ .bz2?$ ]]; then CAT_FILES="bzcat \${RDF_FILES}"; fi - if [[ ${RDF_FILES} =~ .xz$ ]]; then CAT_FILES="xzcat \${RDF_FILES}"; fi - fi - # Copy QLeverfiles/Qleverfile.default and replace the %...% templates. - cat ${QLEVER_CMD_DIR}/Qleverfiles/Qleverfile.default \ - | sed -E "s/%FOLDER_NAME%/${FOLDER_NAME//\//\\\/}/" \ - | sed -E "s/%DATE%/${DATE}/" \ - | sed -E "s/%DB%/\"${DB}\"/" \ - | sed -E "s/%RDF_FILES%/\"${RDF_FILES}\"/" \ - | sed -E "s/%CAT_FILES%/\"${CAT_FILES}\"/" \ - | sed -E "s/%QLEVER_BIN_DIR%/\"${QLEVER_BIN_DIR//\//\\\/}\"/" \ - | sed -E "s/%RANDOM%/${RANDOM}${RANDOM}/" \ - > Qleverfile.OLD - # | sed -E "s/%FOLDER_NAME%/${FOLDER_NAME}/; s/%DATE%/${DATE}/" \ - cat << EOT -No pre-configuration name specified (as argument of ". qlever"). Copied default -Qleverfile to current directory, please edit and check. - -EOT - fi - fi - - # DONE: give the user a short heads up what they can do now. - cat << "EOT" | display_markdown -# Setup is complete -Type `qlever` and use autocompletion to see which actions are available. Add a -"show" in the end to see what an action does without executing it (for example, -`qlever index show`). Edit your local Qleverfile to change settings. A typical -sequence of actions if you have used a preconfigured Qleverfile is: - -`qlever get-data` -`qlever index` -`qlever start` -`qlever example-query` - -EOT - return -fi - - -### -### BASIC HELP AND INSTALL INFO -### -### Note that this must come after the code for ". qlever" (which is also called -### without arguments). -### - -if [ -z "$*" ] || [ "$*" == "help" ]; then - echo - action_help - exit -fi -if [ "$*" == "install-binaries" ]; then - echo - action_install_binaries - exit -fi - -# Leave a blank line at the beginning (the welcome message used to be here, but -# we found it redundant). -echo - -### -### VARIABLE ASSIGNMENTS -### -### 1. Default settings -### 2. Read from Qleverfile (space around = allowed) -### 3. Computed derived variables -### - -# Some default settings. -APP_DIR=$(pwd) # Where the script was called from -DB=must_specify -USE_DOCKER=false -WITH_TEXT_INDEX=false -QLEVER_DOCKER_IMAGE=adfreiburg/qlever -QLEVERUI_DOCKER_IMAGE=adfreiburg/qlever-ui -QLEVERUI_DOCKER_CONTAINER=qlever-ui -HOSTNAME=must_specify # backend host -SERVER_PORT=must_specify # backend port -QLEVERUI_PORT=7000 # frontend port -MEMORY_FOR_QUERIES=30 # in GB -CACHE_MAX_SIZE_GB=30 # in GB -CACHE_MAX_SIZE_GB_SINGLE_ENTRY=5 # in GB -CACHE_MAX_NUM_ENTRIES=1000 -ADMIN_TOKEN= # for actions requiring privilege -QUERY= - -# Read from Qleverfile (only lines that look like an assignment are considered, -# space around = allowed). -# echo "$(cat Qleverfile | sed -En 's/^([A-Z_]+)\s*=\s*(.*)$/\1=\2/p')" -# echo -if ${QLEVERFILE_NEEDED}; then - eval "$(cat Qleverfile.OLD | sed -En 's/^([A-Z_]+)\s*=\s*(.*)$/\1=\2/p')" -fi -# echo "CAT_FILES = \"${CAT_FILES}\"" - -# Check for variable assignments (of the form VARIABLE=value) in the command -# line. They override the setting from the Qleverfile. The remaining arguments -# stay in "$@". -VARS=() -for ARG do - shift - # [[ "${ARG}" =~ ^--$ ]] && OPT="${ARG}" && continue - [[ "${ARG}" =~ ^[A-Z_]+=[^[:space:]] ]] && VARS+=( "${ARG}" ) && continue - set -- "$@" "${ARG}" -done -if (( ${#VARS[@]} )); then - echo -e "\033[1mVariable settings specified (will override those from Qleverfile):\033[0m" - echo "${BLUE}" - for VAR in "${VARS[@]}"; do VAR="${VAR/=/=\"}\""; echo "${BLUE}${VAR}${NORMAL}"; eval "${VAR}"; done - echo "${NORMAL}" -fi -# for ARG in "$@"; do echo "ARG = \"${ARG}\""; done - -# Derived variables. TODO: Only if if not explicitly defined in Qleverfile. -QLEVER_CMD_WITH_VARS=qlever -if (( ${#VARS[@]} )); then QLEVER_CMD_WITH_VARS+=" ${VARS[*]}"; fi -QLEVER_DOCKER_CONTAINER=qlever.${DB} -SLUG=${QLEVERUI_CONFIG} # URL slug used in the QLever UI -INDEXER_BINARY=IndexBuilderMain -ULIMIT_CMD="ulimit -Sn 1048576; " -if [ ! -z "$(ls ${RDF_FILES} 2> /dev/null)" ] && \ - [ "$(du -Lbc $(eval ls ${RDF_FILES}) | tail -1 | cut -f1)" -lt 10000000000 ]; then ULIMIT_CMD=""; fi -INDEXER_OPTIONS="-F ttl -f - -i ${DB} -s ${DB}.settings.json" -if [[ :1:yes:true: == *:${PSO_AND_POS_ONLY}:* ]]; then - INDEXER_OPTIONS="${INDEXER_OPTIONS} --only-pso-and-pos-permutations --no-patterns" -fi -if [ "${STXXL_MEMORY_GB}" ]; then - INDEXER_OPTIONS="${INDEXER_OPTIONS} --stxxl-memory-gb ${STXXL_MEMORY_GB}" -fi -if [[ :from_text_records:from_text_records_and_literals: == *:${WITH_TEXT_INDEX}:* ]]; then - INDEXER_OPTIONS="${INDEXER_OPTIONS} -w ${DB}.wordsfile.tsv -d ${DB}.docsfile.tsv" -fi -if [[ :from_literals:from_text_records_and_literals: == *:${WITH_TEXT_INDEX}:* ]]; then - INDEXER_OPTIONS="${INDEXER_OPTIONS} --text-words-from-literals" -fi -if [[ :1:yes:true: == *:${USE_DOCKER}:* ]]; then - INDEXER_CMD_PREFIX="" - INDEXER_CMD="docker run -it --rm -u $(id -u):$(id -g) -v /etc/localtime:/etc/localtime:ro -v $(pwd):/index -w /index --entrypoint bash --name ${QLEVER_DOCKER_CONTAINER}.index-build ${QLEVER_DOCKER_IMAGE} -c \"${PIPEFAIL_CMD}${ULIMIT_CMD}${CAT_FILES} | ${INDEXER_BINARY} ${INDEXER_OPTIONS//\"/\\\"} | tee ${DB}.index-log.txt\"" -else - INDEXER_CMD_PREFIX="/bin/time -o >(xargs printf \"%d/1000000\n\" | bc -l | xargs printf \"\nMax RAM usage: %'.1f GB\n\n\" | tee -a ${DB}.index-log.txt) -f \"%M\"" - INDEXER_CMD="bash -c \"${ULIMIT_CMD}${CAT_FILES} | ${INDEXER_BINARY} ${INDEXER_OPTIONS//\"/\\\"} | tee ${DB}.index-log.txt\"" -fi -REMOVE_DATA_CMD="for RDF_FILE in ${RDF_FILES}; do rm -f \$RDF_FILE; done" -SERVER_BINARY=ServerMain -SERVER_OPTIONS="-i ${DB} -j 8 -p ${SERVER_PORT} -m ${MEMORY_FOR_QUERIES} -c ${CACHE_MAX_SIZE_GB} -e ${CACHE_MAX_SIZE_GB_SINGLE_ENTRY} -k ${CACHE_MAX_NUM_ENTRIES}" -if [ ! -z "${ACCESS_TOKEN}" ]; then SERVER_OPTIONS="${SERVER_OPTIONS} -a \"${ACCESS_TOKEN}\""; fi -if [[ :1:yes:true: == *:${PSO_AND_POS_ONLY}:* ]]; then - SERVER_OPTIONS="${SERVER_OPTIONS} --only-pso-and-pos-permutations --no-patterns" -fi -if [[ :1:yes:true: == *:${NO_PATTERNS}:* ]]; then - SERVER_OPTIONS="${SERVER_OPTIONS} --no-patterns" -fi -if [[ - :1:yes:true:from_text_records:from_literals:from_text_records_and_literals: == *:${WITH_TEXT_INDEX}:* ]]; then - SERVER_OPTIONS="${SERVER_OPTIONS} -t" -fi - -# Command for action "start" -if [[ :1:yes:true: == *:${USE_DOCKER}:* ]]; then - SERVER_CMD="docker run -d --restart unless-stopped -u $(id -u):$(id -g) -it -v /etc/localtime:/etc/localtime:ro -v $(pwd):/index -p ${SERVER_PORT}:${SERVER_PORT} -w /index --entrypoint bash --name ${QLEVER_DOCKER_CONTAINER} ${QLEVER_DOCKER_IMAGE} -c \"${SERVER_BINARY} ${SERVER_OPTIONS//\"/\\\"} > ${DB}.server-log.txt\" > /dev/null" -else - SERVER_CMD="${SERVER_BINARY} ${SERVER_OPTIONS} > ${DB}.server-log.txt &" -fi - -# Command for action "stop" -if [ "$(docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} 2> /dev/null)" == "true" ]; then - STOP_CMD="docker rm -f ${QLEVER_DOCKER_CONTAINER} > /dev/null" -elif [ ! -z "$(pgrep -af "${SERVER_BINARY} -i ${DB} ")" ]; then - STOP_CMD="pkill -f \"${SERVER_BINARY} -i [^ ]*${DB}\"" -elif [[ :1:yes:true: == *:${USE_DOCKER}:* ]]; then - STOP_CMD="docker rm -f ${QLEVER_DOCKER_CONTAINER} > /dev/null" -else - STOP_CMD="pkill -f \"${SERVER_BINARY} -i [^ ]*${DB}\"" -fi - -SERVER_API=http://${HOSTNAME}:${SERVER_PORT} - -# Command for action "query" -QUERY=${QUERY//\"/\\\"} -QUERY_CMD=(curl -Gs ${SERVER_API} -H "Accept: text/tab-separated-values" --data-urlencode "query=${QUERY}") - - -# Variant of make that reads the master Qleverfile (using GET_MASTER_QLEVERFILE) -# and the local Qleverfile (in the directory from which this script is called). -# The -s option stands for "silent" (do not show the commands while executing -# them). -function qlever_make { - QLEVERFILE_ARG= - if [ -e "${APP_DIR}/Qleverfile.OLD" ]; then - QLEVERFILE_ARG=" -f ${APP_DIR}/Qleverfile.OLD" - fi - get_master_qleverfile | make -s -f - ${QLEVERFILE_ARG} "$@" -} - -# Like the above, but only show what would be executed (using the -n option of -# "make"), in blue. Leave out command lines that end with #. -function qlever_make_show { - printf "${BLUE}" - get_master_qleverfile \ - | make -f - -f ${APP_DIR}/Qleverfile.OLD -n "$@" \ - | egrep -v "#.?\$" | sed 's/^\s*//; s/\s?*\\$//' \ - | sed 's/^: #/#/' - printf "${NORMAL}" -} - -# Execute action function (given as argument $1). -function execute_action { - ACTION_FUNCTION=$1 - if [ "$(declare -f ${ACTION_FUNCTION})" ]; then - # echo "Executing function \"${ACTION_FUNCTION}\" ..." - # echo - eval ${ACTION_FUNCTION} - else - echo "${RED}Action function \"$1\" not defined${NORMAL}" - fi -} - -# Show selected code of action function (given as argument $1). -function show_action { - QUOTE=\" - DOLLAR=\$ - BACKSLASH=\\ - ACTION_FUNCTION="$1" - if [ "$(declare -f ${ACTION_FUNCTION})" ]; then - printf "${BLUE}" - # This miracle code show the code of the action function. It works as - # follows: - # 1. The declare -t gives the full code (as written above). - # 2. The first sed filters out the lines starting with $SHOW and removing any - # parts after $NOSHOW (this gives us control which part to show). - # 3. The second sed temporarily replaces all $ that are not followed by a { - # and all " by ${DOLLAR} and ${QUOTE}, respsectively. That way, these will - # be be preserved (and not get interpreted somehow). - # 4. The last part expands all the ${...} variables (without the eval, no - # expansion would happen), including the ${DOLLAR} and ${QUOTE} introduced - # in the previous step. - # 5. Remove empty lines (these correpond to commands which are only exeuted - # under certain circumstances) - declare -f ${ACTION_FUNCTION} \ - | sed -En 's/\s+\$NOSHOW.*$//; s/;$//; s/^.*\$SHOW //p' \ - | sed -En 's/"/${QUOTE}/g; s/\$([^{])/${DOLLAR}\1/g; s/\\/${BACKSLASH}/g; p' \ - | while read LINE; do eval "echo \"${LINE}\""; done \ - | sed -E '/^\s*$/d' - printf "${NORMAL}" - else - echo "${RED}Action function \"$1\" not defined${NORMAL}" - fi -} - -# Command to leave the script ("return" when the script is sourced, "exit" when -# it is called normally, that is, within its own subshell). -EXIT=exit -if [ ${QLEVER_CMD} != $0 ]; then EXIT=return; fi -function exit_qlever { eval ${EXIT}; } - - - -# # Special command "show-config" (for debugging, not one of the completions). -# if [ "$*" == "show-config" ]; then -# echo "${BOLD}${MAGENTA}Secret action \"show-config\":${NORMAL}" -# echo -# cat << EOT -# Showing the settings of some variables after they went through "make" and then -# exported to the "qlever" script again (for debugging). If you call the script -# with a variable assignment as in "qlever USE_DOCKER=0 ...", that value should -# overrule the one from the QLeverfile. -# EOT -# echo "${BLUE}" -# qlever_make "${VARS[@]}" export.HIDDEN -# echo "${NORMAL}" -# eval ${EXIT} -# fi -# -# # Actually do these assignments. NO: this is already done above! -# $(qlever_make "${VARS[@]}" export.HIDDEN) -# eval $(qlever_make "${VARS[@]}" export.HIDDEN) - - - - -### -### WHAT THE SCRIPT ACTUALLY DOES -### - -# For selected actions: If USE_DOCKER=1 check if docker container is running -# and if not run it. -USE_DOCKER_ACTIONS=none -# USE_DOCKER_ACTIONS=:start:index: -if [[ ${USE_DOCKER_ACTIONS} == *:"$ARG":* ]]; then - if [ "${USE_DOCKER}" == "1" ]; then - if [ "$(docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} 2> /dev/null)" == "true" ]; then - echo "${MAGENTA}Using the already running docker container \"${QLEVER_DOCKER_CONTAINER}\"${NORMAL}" - echo - else - echo "${MAGENTA}Starting new docker container \"${QLEVER_DOCKER_CONTAINER}\"${NORMAL}" - echo - qlever_make_show "${VARS[@]}" QLEVER_CMD_DIR="${QLEVER_CMD_DIR}" docker-run.HIDDEN - echo - qlever_make QLEVER_CMD_DIR="${QLEVER_CMD_DIR}" "${VARS[@]}" docker-run.HIDDEN - fi - else - if [ "$(docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} 2> /dev/null)" == "true" ]; then - echo "${MAGENTA}USE_DOCKER=0 but found docker container \"${QLEVER_DOCKER_CONTAINER}\", removing it ...${NORMAL}" - echo - qlever_make_show "${VARS[@]}" docker-remove.HIDDEN - echo - qlever_make "${VARS[@]}" docker-remove.HIDDEN - fi - fi -fi - - -# If the last argument is "show", just show the commands, but don't execute -# them. The first for loop is a portable hack to get the last argument. -for LAST; do true; done -if [ "${LAST}" == "show" ]; then - echo "${BOLD}Just showing what would be executed${NORMAL}" - echo - for ARG in "$@"; do - if [ "${ARG}" != "show" ]; then - ACTION_FUNCTION=action_$(echo ${ARG} | sed 's/-/_/g') - if [ "$(declare -f ${ACTION_FUNCTION})" ]; then - echo "\"${ARG}\" would execute the following:" - echo - show_action "${ACTION_FUNCTION}" - # qlever_make_show "${VARS[@]}" "${ARG}" - echo - else - echo "${RED}Action \"${ARG}\" not defined, use the autocompletion${NORMAL}" - echo - fi - fi - done - exit -fi - -# For each action, check whether the corresponding action function exists, and -# if yes, show the command and then execute it. -for ARG in "$@"; do - # If USE_DOCKER=1 and action requires docker, make sure it's running. - USE_DOCKER_ACTIONS=none - # USE_DOCKER_ACTIONS=:start:index: - if [[ ${USE_DOCKER_ACTIONS} == *:"$ARG":* ]]; then - if [[ :1:yes:true: == *:${USE_DOCKER}:* ]]; then - if ! docker container inspect -f '{{.State.Running}}' ${QLEVER_DOCKER_CONTAINER} > /dev/null 2>&1; then - execute_action action_docker_on - echo - fi - fi - fi - # Now execute the action. - echo "${BOLD}Executing \"${ARG}\":${NORMAL}" - echo - ACTION_FUNCTION=action_$(echo ${ARG} | sed 's/-/_/g') - if [ "$(declare -f ${ACTION_FUNCTION})" ]; then - if show_action "${ACTION_FUNCTION}"; then - echo - execute_action "${ACTION_FUNCTION}" - echo - fi - else - echo "${RED}Action \"${ARG}\" not defined, use the autocompletion${NORMAL}" - echo - fi -done diff --git a/Qleverfiles/Qleverfile.dblp b/src/qlever/Qleverfiles/Qleverfile.dblp similarity index 100% rename from Qleverfiles/Qleverfile.dblp rename to src/qlever/Qleverfiles/Qleverfile.dblp diff --git a/Qleverfiles/Qleverfile.dblp-plus b/src/qlever/Qleverfiles/Qleverfile.dblp-plus similarity index 100% rename from Qleverfiles/Qleverfile.dblp-plus rename to src/qlever/Qleverfiles/Qleverfile.dblp-plus diff --git a/Qleverfiles/Qleverfile.default b/src/qlever/Qleverfiles/Qleverfile.default similarity index 100% rename from Qleverfiles/Qleverfile.default rename to src/qlever/Qleverfiles/Qleverfile.default diff --git a/Qleverfiles/Qleverfile.dnb b/src/qlever/Qleverfiles/Qleverfile.dnb similarity index 100% rename from Qleverfiles/Qleverfile.dnb rename to src/qlever/Qleverfiles/Qleverfile.dnb diff --git a/Qleverfiles/Qleverfile.fbeasy b/src/qlever/Qleverfiles/Qleverfile.fbeasy similarity index 100% rename from Qleverfiles/Qleverfile.fbeasy rename to src/qlever/Qleverfiles/Qleverfile.fbeasy diff --git a/Qleverfiles/Qleverfile.freebase b/src/qlever/Qleverfiles/Qleverfile.freebase similarity index 100% rename from Qleverfiles/Qleverfile.freebase rename to src/qlever/Qleverfiles/Qleverfile.freebase diff --git a/Qleverfiles/Qleverfile.imdb b/src/qlever/Qleverfiles/Qleverfile.imdb similarity index 100% rename from Qleverfiles/Qleverfile.imdb rename to src/qlever/Qleverfiles/Qleverfile.imdb diff --git a/Qleverfiles/Qleverfile.olympics b/src/qlever/Qleverfiles/Qleverfile.olympics similarity index 100% rename from Qleverfiles/Qleverfile.olympics rename to src/qlever/Qleverfiles/Qleverfile.olympics diff --git a/Qleverfiles/Qleverfile.osm-country b/src/qlever/Qleverfiles/Qleverfile.osm-country similarity index 100% rename from Qleverfiles/Qleverfile.osm-country rename to src/qlever/Qleverfiles/Qleverfile.osm-country diff --git a/Qleverfiles/Qleverfile.pubchem b/src/qlever/Qleverfiles/Qleverfile.pubchem similarity index 100% rename from Qleverfiles/Qleverfile.pubchem rename to src/qlever/Qleverfiles/Qleverfile.pubchem diff --git a/Qleverfiles/Qleverfile.scientists b/src/qlever/Qleverfiles/Qleverfile.scientists similarity index 100% rename from Qleverfiles/Qleverfile.scientists rename to src/qlever/Qleverfiles/Qleverfile.scientists diff --git a/Qleverfiles/Qleverfile.uniprot b/src/qlever/Qleverfiles/Qleverfile.uniprot similarity index 100% rename from Qleverfiles/Qleverfile.uniprot rename to src/qlever/Qleverfiles/Qleverfile.uniprot diff --git a/Qleverfiles/Qleverfile.vvz b/src/qlever/Qleverfiles/Qleverfile.vvz similarity index 100% rename from Qleverfiles/Qleverfile.vvz rename to src/qlever/Qleverfiles/Qleverfile.vvz diff --git a/Qleverfiles/Qleverfile.wikidata b/src/qlever/Qleverfiles/Qleverfile.wikidata similarity index 100% rename from Qleverfiles/Qleverfile.wikidata rename to src/qlever/Qleverfiles/Qleverfile.wikidata diff --git a/Qleverfiles/Qleverfile.yago-4 b/src/qlever/Qleverfiles/Qleverfile.yago-4 similarity index 100% rename from Qleverfiles/Qleverfile.yago-4 rename to src/qlever/Qleverfiles/Qleverfile.yago-4 diff --git a/src/qlever/__init__.py b/src/qlever/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qlever b/src/qlever/__main__.py similarity index 99% rename from qlever rename to src/qlever/__main__.py index dbedf02a..96d9c5db 100755 --- a/qlever +++ b/src/qlever/__main__.py @@ -20,6 +20,7 @@ import subprocess import sys import time +import pkg_resources from termcolor import colored import traceback @@ -1341,6 +1342,12 @@ def setup_autocompletion_cmd(): def main(): + # Get the version. + try: + version = pkg_resources.get_distribution("qlever").version + except Exception as e: + log.error(f"Could not determine package version: {e}") + version = "unknown" # If the script is called without argument, say hello and provide some # help to get started. if len(sys.argv) == 1 or \ @@ -1348,7 +1355,8 @@ def main(): (len(sys.argv) == 2 and sys.argv[1] == "--help") or \ (len(sys.argv) == 2 and sys.argv[1] == "-h"): log.info("") - log.info(f"{BOLD}Hello, I am the qlever script{NORMAL}") + log.info(f"{BOLD}Hello, I am the qlever script" + f" (version {version}){NORMAL}") log.info("") if os.path.exists("Qleverfile"): log.info("I see that you already have a \"Qleverfile\" in the "