Skip to content

Commit

Permalink
#28 add/change docker-compose files
Browse files Browse the repository at this point in the history 
change setup_prod.sh
adapt the README.md
  • Loading branch information
[email protected] authored and [email protected] committed Oct 29, 2024
1 parent 5a16ec8 commit 29ff0c6
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 34 deletions.
17 changes: 12 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,13 @@ Repeat for port 443.
### Sample data

For the search to work you will need some data that we can use.

#### Workflow container
This will install a docker container that will take care of gathering and processing the data.

This will require a SSL certificate, a VHost setting in proxy.conf and an entry in the DNS to point to that VHost.

#### Sample data
@todo : how do people get that sample data???
Unzip the sample data file (sample-solr-data) on the machine to /tmp.

Expand Down Expand Up @@ -311,20 +318,20 @@ chmod -R 777 /data/oih-ui-docker/api/solr/sample-solr-data

**_WARNING_** depending on the version of Docker installed on your system, you will need to use either **_docker compose_** or **_docker-compose_**.

#### for local installation
#### installation ***without*** the workflow container

```bash

docker compose up -d

docker compose -f docker-compose.noWorkflow.yml up -d

```

#### production installation
#### installation ***with*** the workflow container
**_WARNING_** do not forget to uncomment and correct the lines in the nginx/conf.d/letsencrypt_user_data.conf file

```bash

docker compose -f docker-compose.prod.yml up -d
docker compose -f docker-compose.workflow.yml up -d

```

Expand Down
96 changes: 80 additions & 16 deletions docker-compose.workflow.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ volumes:
le-data:
# index_test_solr-data:
# external: true
dagster-postgres:
driver: local

networks:
default:
Expand All @@ -14,6 +16,7 @@ networks:
internal: true
solrbridge:
driver: host
dagster_host:

services:
nginx-proxy:
Expand All @@ -39,6 +42,7 @@ services:
- "/var/run/docker.sock:/tmp/docker.sock:ro"
labels:
com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: "true"

letsencrypt-nginx-proxy-companion:
restart: always
image: "nginxproxy/acme-companion"
Expand Down Expand Up @@ -147,19 +151,79 @@ services:
- solr-data:/var/solr/data/ckan/data:rw
- "./api/solr/sample-solr-data/:/var/solr/data/ckan/data/index"

#workflow:
#restart: always
#image: "solr:8"
#ports:
# - "8983:8983"
#logging:
# driver: "json-file"
# options:
# max-size: 50m
#networks:
# solr:
#environment:
# - SOLR_JAVA_MEM=-Xms1g -Xmx1g
# - SOLR_OPTS=-Dlog4j2.formatMsgNoLookups=true
#volumes:
# - "./api/solr:/var/solr/data/ckan/"
# This is for a single dagster instance, that does not use an externally defined network
# this NEEDS following environment variables from the .env file
# $HOST (already defined)
# $PROJECT default eco
# CONTAINER_TAG default latest


#remove lines after check


dagster-dagit:
restart: always
image: "docker.io/nsfearthcube/dagster-${PROJECT:-eco}:${CONTAINER_TAG:-latest}"
environment: &env
- DEBUG=${DEBUG:-false}
- PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
- PORTAINER_URL=${PORTAINER_URL}
- PORTAINER_KEY=${PORTAINER_KEY}
- GLEANERIO_GLEANER_IMAGE=${GLEANERIO_GLEANER_IMAGE}
- GLEANERIO_NABU_IMAGE=${GLEANERIO_NABU_IMAGE}
- GLEANERIO_LOG_PREFIX=${GLEANERIO_LOG_PREFIX}
- GLEANER_MINIO_ADDRESS=${GLEANERIO_MINIO_ADDRESS}
- GLEANER_MINIO_PORT=${GLEANERIO_MINIO_PORT}
- GLEANER_MINIO_USE_SSL=${GLEANERIO_MINIO_USE_SSL}
- GLEANER_MINIO_BUCKET=${GLEANERIO_MINIO_BUCKET}
- GLEANER_MINIO_ACCESS_KEY=${GLEANERIO_MINIO_ACCESS_KEY}
- GLEANER_MINIO_SECRET_KEY=${GLEANERIO_MINIO_SECRET_KEY}
- GLEANER_HEADLESS_ENDPOINT=${GLEANERIO_HEADLESS_ENDPOINT}
- GLEANER_HEADLESS_NETWORK=${GLEANERIO_HEADLESS_NETWORK}
- GLEANER_GRAPH_URL=${GLEANERIO_GRAPH_URL}
- GLEANER_GRAPH_NAMESPACE=${GLEANERIO_GRAPH_NAMESPACE}
- GLEANERIO_NABU_CONFIG_PATH=${GLEANERIO_NABU_CONFIG_PATH:-/configs/gleaner/nabuconfig.yaml}
- GLEANERIO_GLEANER_CONFIG_PATH=${GLEANERIO_GLEANER_CONFIG_PATH:-/configs/gleaner/gleanerconfig.yaml}
- GLEANERIO_NABU_DOCKER_CONFIG=${GLEANERIO_NABU_DOCKER_CONFIG:-nabu}
- GLEANERIO_GLEANER_DOCKER_CONFIG=${GLEANERIO_GLEANER_DOCKER_CONFIG:-gleaner}
ports:
- 3000:3000
networks:
- dagster_host
depends_on:
- dagster-postgres

dagster-daemon:
restart: always
image: docker.io/nsfearthcube/dagster-${PROJECT:-eco}:${CONTAINER_TAG:-latest}
environment: *env
command: "dagster-daemon run"
depends_on:
- dagster-postgres
networks:
- dagster_host

dagster-postgres:
restart: always
image: postgres:13.3
container_name: dagster-postgres
ports:
- 5432:5432
environment:
- POSTGRES_PASSWORD=secret
volumes:
- dagster-postgres:/var/lib/postgresql/data
networks:
- dagster_host

headless:
# image: chromedp/headless-shell:stable
# stable after 105 causes "devtool: CreateURL: Using unsafe HTTP verb GET to invoke /json/new. This action supports only PUT verb.",
image: chromedp/headless-shell:105.0.5195.127
restart: unless-stopped
shm_size: "2gb"
ports:
- 9222:9222
environment:
- SERVICE_PORTS=9222

95 changes: 82 additions & 13 deletions setup_prod.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ if [[ "$installDir" =~ ^\/[[:alnum:]] && "$installDir" =~ [[:alnum:]]\/$ ]]; the
fi

else
printf "${RED}ERROR${NC}: $installDir is not a valid complete path, should be like /data/oih-ui-docker/\n"
printf "${RED}ERROR${NC}: $installDir is not a valid complete path, should be something like /data/oih-ui-docker/\n"
exit 2
fi

Expand All @@ -75,15 +75,15 @@ if [[ "$solrDir" =~ ^\/[[:alnum:]] && "$solrDir" =~ [[:alnum:]]\/$ ]]; then
exit 5
fi
else
printf "${RED}ERROR${NC}: $solrDir is not a valid complete path, should be like /data/solr_data/\n"
printf "${RED}ERROR${NC}: $solrDir is not a valid complete path, should be something like /data/solr_data/\n"
exit 3
fi

#url must be a correct url
if [[ "$url" =~ ^https?\:\/\/([[:alnum:]]+\.)+[[:alnum:]]+\/?$ ]]; then
printf "$url is a valid url\n"
else
printf "${RED}ERROR${NC}: $url is not a valid url, should be like http(s)://devsearch.oceaninfohub.org/\n"
printf "${RED}ERROR${NC}: $url is not a valid url, should be something like http(s)://devsearch.oceaninfohub.org/\n"
exit 4
fi

Expand All @@ -93,12 +93,23 @@ if [[ $url =~ $re ]]; then
host=${BASH_REMATCH[1]}
fi

#do we want to install the worklfow container?
printf "\nDo you want to install the workflow container (${YELLOW}Y${NC}/n):\n"
read useWorkflowContainer
if [ "$useWorkflowContainer" = "n" ] ;then
printf "no workflow container will be installed\n"
else
printf "workflow container will be installed\n"
useWorkflowContainer="y"
fi

printf "\nUsage: $0 -d installDir -s solrDir -u url \n"
printf "for more info: $0 -h \n"
printf "\nare these the correct settings (${YELLOW}Y${NC}/n):\n"
printf "\tinstall dir : $installDir\n"
printf "\tsolr test data : $solrDir\n"
printf "\turl : $url\n\n"
printf "\turl : $url\n"
printf "\tworkflow : $useWorkflowContainer\n\n"

read answer4

Expand All @@ -109,6 +120,71 @@ fi

printf "start installation\n\n"


# Create the .env file in the project directory
cd $installDir
touch $installDir/.env

# Add the HOST variable to the .env file
echo "HOST=$host" > $installDir/.env

if [ "$useWorkflowContainer" = "n" ] ;then
dockerComposeFile="docker-compose.noWorkflow.yml"
printf "using ${YELLOW}$dockerComposeFile${NC}\n"
else
dockerComposeFile="docker-compose.workflow.yml"

#copy the old file and add a date to the name
cp $installDir/nginx/conf.d/letsencrypt_user_data.conf $installDir/nginx/conf.d/letsencrypt_user_data.conf_$(date +%Y%m%d%H%M%S)

#add the correct lines for the SSL certificates
#our current ip
localIp=$(ifconfig -a | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | grep 192 | grep -v 255 | head -1)
workflowHost="workflow.${host}"
echo "LETSENCRYPT_STANDALONE_CERTS=('workflow')" >> $installDir/nginx/conf.d/letsencrypt_user_data.conf
echo "LETSENCRYPT_workflow_HOST=('$workflowHost')" >> $installDir/nginx/conf.d/letsencrypt_user_data.conf

#copy the old proxy file and add a date to the name
cp $installDir/nginx/conf.d/proxy.conf $installDir/nginx/conf.d/proxy.conf_$(date +%Y%m%d%H%M%S)

#add the correct lines for the nginx proxy
echo "server {" >> $installDir/nginx/conf.d/proxy.conf
echo " server_name $workflowHost;" >> $installDir/nginx/conf.d/proxy.conf
echo " listen 80;" >> $installDir/nginx/conf.d/proxy.conf
echo " access_log /var/log/nginx/workflow.search.log vhost;" >> $installDir/nginx/conf.d/proxy.conf;
echo " location / {" >> $installDir/nginx/conf.d/proxy.conf
echo " proxy_pass http://$localIp:3000;" >> $installDir/nginx/conf.d/proxy.conf;
echo " }" >> $installDir/nginx/conf.d/proxy.conf
echo "}" >> $installDir/nginx/conf.d/proxy.conf

printf "using ${YELLOW}$dockerComposeFile${NC}\n"
printf "access workflow via ${YELLOW}https://$workflowHost${NC}\n"

#copy the old .env file and add a date to the name
cp $installDir/.env $installDir/.env_$(date +%Y%m%d%H%M%S)

#change the .env file, add some lines for the workflow container
echo "#settings for the workflow" >> $installDir/.env
echo "PROJECT=eco" >> $installDir/.env
echo "GLEANERIO_GLEANER_IMAGE=nsfearthcube/gleaner:dev_ec" >> $installDir/.env
echo "GLEANERIO_NABU_IMAGE=nsfearthcube/nabu:dev_eco" >> $installDir/.env
echo "GLEANERIO_GLEANER_CONFIG_PATH=/gleaner/gleanerconfig.yaml" >> $installDir/.env
echo "GLEANERIO_NABU_CONFIG_PATH=/nabu/nabuconfig.yaml" >> $installDir/.env
echo "GLEANERIO_LOG_PREFIX=scheduler/logs/" >> $installDir/.env
echo "GLEANERIO_MINIO_ADDRESS=ossapi.provisium.io" >> $installDir/.env
echo "GLEANERIO_MINIO_PORT=" >> $installDir/.env
echo "GLEANERIO_MINIO_USE_SSL=true" >> $installDir/.env
echo "GLEANERIO_MINIO_BUCKET=gleaner" >> $installDir/.env
echo "GLEANERIO_MINIO_ACCESS_KEY=minioadmin" >> $installDir/.env
echo "GLEANERIO_MINIO_SECRET_KEY=6EcDLmMiXsAPjc9kttAE7PMXitxrnyqxEefCYPoy" >> $installDir/.env
echo "GLEANERIO_HEADLESS_ENDPOINT=http://workstation.lan:9222" >> $installDir/.env
echo "GLEANERIO_GRAPH_URL=http://nas.lan:49153/blazegraph" >> $installDir/.env
echo "GLEANERIO_GRAPH_NAMESPACE=earthcube" >> $installDir/.env

fi

exit 0

# stop and remove old containers
printf " \n${YELLOW}stopping and removing old containers${NC}\n"
# stop running container(s)
Expand Down Expand Up @@ -152,21 +228,14 @@ cd $installDir/api
git fetch
git checkout feature/restyling

# Create the .env file by copying the env.sample file
cd $installDir
touch $installDir/.env

# Add the HOST variable to the .env file
echo "HOST=$host" > $installDir/.env

# Copy Solr data to the target directory and change its permissions (change the root with the right one)
printf " \n${YELLOW}copying Solr data to the target directory and changing its permissions${NC}\n"
cp -r $solrDir $installDir/api/solr/sample-solr-data/
chmod -R 777 $installDir/api/solr/sample-solr-data/

# Start the Docker containers in production mode
printf " \n${YELLOW}starting the Docker containers in production mode${NC}\n"
docker compose -f docker-compose.prod.yml up -d --remove-orphans
docker compose -f $dockerComposeFile up -d --remove-orphans

# Wait for 10 seconds before continue
printf " \n${YELLOW}sleep for 10s${NC}\n"
Expand All @@ -185,7 +254,7 @@ echo -ne "\r\033[K"

# Change permissions for the Solr directory
printf " \n${YELLOW}changing permissions for the Solr directory${NC}\n"
docker compose -f docker-compose.prod.yml run -u root solr chown solr:solr /var/solr/data/ckan/data
docker compose -f $dockerComposeFile run -u root solr chown solr:solr /var/solr/data/ckan/data

# Restart the containers
printf " \n${YELLOW}restarting the containers${NC}\n"
Expand Down

0 comments on commit 29ff0c6

Please sign in to comment.