diff --git a/.gitignore b/.gitignore index 6a57c1dd..98a93c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .*un~ home/.* home/*.log +home/cache/ home/fingerprints/ home/identity.key.enc home/init.groovy.d/ @@ -35,3 +36,4 @@ slave/.matplotlib slave/.oracle_jre_usage slave/.local slave/.ssh +web/.jenkins \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 2dffb1cf..e12e8723 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,29 @@ language: java services: - docker +env: + global: + - DOCKER_VERSION=1.10.1-0~trusty + - DOCKER_COMPOSE_VERSION=1.6.0 + before_install: -- ./rename.py travis + - sudo apt-get update + - sudo DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" install docker-engine=${DOCKER_VERSION} + - sudo gpasswd -a $USER docker + - export USER_ID=`id -u $USER` + - docker -v + + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + - docker-compose -v + + - ./rename.py travis install: -- docker-compose build + - docker-compose -f docker-compose.yml -f docker-compose.unix.yml build script: -- docker-compose up -d slave -- docker-compose stop + - docker-compose -f docker-compose.yml -f docker-compose.unix.yml up -d + - docker-compose -f docker-compose.yml -f docker-compose.unix.yml stop diff --git a/docker-compose.osx.yml b/docker-compose.osx.yml index 8091cac9..ac892c4d 100644 --- a/docker-compose.osx.yml +++ b/docker-compose.osx.yml @@ -5,3 +5,21 @@ jenkins: user: "1000" slave: user: "1000" +omero: + privileged: true +web: + ports: + - "4080:4080" + privileged: true +nginx: + ports: + - "8888:80" + privileged: true +redis: + ports: + - "6379:6379" +seleniumhub: + ports: + - "4444:4444" +robot: + privileged: true diff --git a/docker-compose.unix.yml b/docker-compose.unix.yml new file mode 100644 index 00000000..3f7fe2c9 --- /dev/null +++ b/docker-compose.unix.yml @@ -0,0 +1,22 @@ +jenkins: + user: ${USER_ID} + +omero: + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run + +web: + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run + +nginx: + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run + +robot: + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run diff --git a/docker-compose.yml b/docker-compose.yml index ae58c46b..f9b92dfa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,18 +21,84 @@ slave: - jenkins volumes: - ./slave:/home/slave - volumes_from: - - git environment: - JAVA_HOME=/etc/alternatives/java_sdk - - SERVICE_NAME=SPACENAME-omero - DOCKER_FIX=' ' + command: -labels centos6 -labels slave -executors 4 -nginx: - image: nginx +omero: + build: server links: + - jenkins + - pg + volumes: + - ./server:/home/omero + environment: + - JENKINS_LABEL=/etc/alternatives/java_sdk + - JENKINS_MODE=exclusive + - SERVICE_NAME=SPACENAME-omero + +web: + build: web + links: + - jenkins + - redis - slave + - omero volumes: - - ./slave:/home/jenkins-slave + - ./web:/home/omero + - ./nginx/conf.d:/home/omero/nginx + environment: + - JENKINS_LABEL=/etc/alternatives/java_sdk + - JENKINS_MODE=exclusive + +nginx: + build: nginx + links: + - jenkins + - web + volumes: + - ./nginx/conf.d:/etc/nginx/conf.d + - ./web/static/web:/home/omero/static/web + environment: + - JENKINS_LABEL=/etc/alternatives/java_sdk + - JENKINS_MODE=exclusive + - SERVICE_NAME=SPACENAME-nginx + +redis: + image: redis + environment: + - SERVICE_NAME=SPACENAME-redis + +seleniumhub: + image: selenium/hub + environment: + - "JAVA_OPTS: -d64 -Xmx512m -Djava.security.egd=file:/dev/./urandom" + - SERVICE_NAME=SPACENAME-seleniumhub + - GRID_BROWSER_TIMEOUT=5000 + - GRID_TIMEOUT=5000 +seleniumfirefox: + image: selenium/node-firefox + links: + - seleniumhub:hub + environment: + - "JAVA_OPTS: -d64 -Xmx512m -Djava.security.egd=file:/dev/./urandom" +seleniumchrome: + image: selenium/node-chrome + links: + - seleniumhub:hub + environment: + - "JAVA_OPTS: -d64 -Xmx512m -Djava.security.egd=file:/dev/./urandom" + +robot: + build: robot + links: + - jenkins + - slave + - omero + - nginx + - seleniumhub environment: - - SERVICE_NAME=SPACENAME-web + - JENKINS_LABEL=/etc/alternatives/java_sdk + - JENKINS_MODE=exclusive + - SERVICE_NAME=SPACENAME-robot diff --git a/ds b/ds index 61410697..7b0c79eb 100755 --- a/ds +++ b/ds @@ -1,4 +1,4 @@ #!/usr/bin/env bash -EXTRA=${EXTRA:-"docker-compose.osx.yml"} -exec docker-compose -f docker-compose.yml -f "$EXTRA" "$@" +EXTRA=${EXTRA:-"docker-compose.unix.yml"} +exec env USER_ID=$UID docker-compose -f docker-compose.yml -f "$EXTRA" "$@" diff --git a/home/hudson.plugins.git.GitSCM.xml b/home/hudson.plugins.git.GitSCM.xml index a1194cb3..4ef1af63 100644 --- a/home/hudson.plugins.git.GitSCM.xml +++ b/home/hudson.plugins.git.GitSCM.xml @@ -4,4 +4,4 @@ false - + \ No newline at end of file diff --git a/home/hudson.plugins.git.GitTool.xml b/home/hudson.plugins.git.GitTool.xml index 658d3b87..67e71af2 100644 --- a/home/hudson.plugins.git.GitTool.xml +++ b/home/hudson.plugins.git.GitTool.xml @@ -7,4 +7,4 @@ - + \ No newline at end of file diff --git a/home/hudson.tasks.Shell.xml b/home/hudson.tasks.Shell.xml index df44a4d9..6e560390 100644 --- a/home/hudson.tasks.Shell.xml +++ b/home/hudson.tasks.Shell.xml @@ -1,2 +1,4 @@ - \ No newline at end of file + + /bin/bash + \ No newline at end of file diff --git a/home/jobs/BIOFORMATS-maven/config.xml b/home/jobs/BIOFORMATS-maven/config.xml index 530a2496..b6190ded 100644 --- a/home/jobs/BIOFORMATS-maven/config.xml +++ b/home/jobs/BIOFORMATS-maven/config.xml @@ -20,7 +20,8 @@ - true + slave + false false false false diff --git a/home/jobs/BIOFORMATS-push/config.xml b/home/jobs/BIOFORMATS-push/config.xml index 33d4ee7b..a14ec350 100644 --- a/home/jobs/BIOFORMATS-push/config.xml +++ b/home/jobs/BIOFORMATS-push/config.xml @@ -16,9 +16,9 @@ - success-only no-error none + success-only @@ -57,7 +57,8 @@ - true + slave + false false false false diff --git a/home/jobs/OMERO-build/config.xml b/home/jobs/OMERO-build/config.xml index cc74a6ef..b2cc61a7 100644 --- a/home/jobs/OMERO-build/config.xml +++ b/home/jobs/OMERO-build/config.xml @@ -30,7 +30,8 @@ - true + slave + false false false false @@ -54,6 +55,7 @@ source docs/hudson/OMERO.sh false false true + true diff --git a/home/jobs/OMERO-push/config.xml b/home/jobs/OMERO-push/config.xml index ccca2b8e..06dd60ab 100644 --- a/home/jobs/OMERO-push/config.xml +++ b/home/jobs/OMERO-push/config.xml @@ -53,7 +53,8 @@ - true + slave + false false false false @@ -68,4 +69,4 @@ $HOME/.local/bin/scc $MERGE_COMMAND -S $STATUS --push $PUSH_BRANCH - + \ No newline at end of file diff --git a/home/jobs/OMERO-robot/config.xml b/home/jobs/OMERO-robot/config.xml new file mode 100644 index 00000000..68f45944 --- /dev/null +++ b/home/jobs/OMERO-robot/config.xml @@ -0,0 +1,99 @@ + + + + + false + + + + + HOSTNAME + + omero + + + + + + robot + false + false + false + false + + false + + + rm -rf src + + + OMERO-build + src/target/openmicroscopy*zip + + + + true + false + + + OMERO-build + src/target/OMERO.server*zip + + + + true + false + + + SRC=$WORKSPACE/src + +# Setup configuration for robot job +ZIP_SRC=$(ls openmicroscopy*zip) +unzip $ZIP_SRC +rm -f $ZIP_SRC +mv $WORKSPACE/${ZIP_SRC%.zip} $SRC + +ZIP_DIST=$(ls OMERO.server*.zip) +unzip $ZIP_DIST +rm -f $ZIP_DIST +mv $WORKSPACE/${ZIP_DIST%.zip} $SRC/dist + +source $HOME/omero-virtualenv/bin/activate + +pip install --upgrade --no-deps --force-reinstall -r $SRC/dist/share/web/requirements-py27-nginx.txt + +$SRC/dist/bin/omero config set omero.web.prefix "/web" + +cd $SRC/dist +bash ../components/tests/ui/robot_setup.sh +mv $SRC/dist/robot_ice.config $SRC/components/tests/ui/resources/ +export ICE_CONFIG=$SRC/components/tests/ui/resources/robot_ice.config +$SRC/dist/bin/omero --path $SRC/components/tests/ui/plugins robot config --protocol http --webhost 'SPACENAME-nginx.docker.openmicroscopy.org' --remoteurl 'http://seleniumhub:4444/wd/hub' --dc 'browserName:${BROWSER},javascriptEnabled:True' > $SRC/components/tests/ui/resources/config.txt + +echo Running the UI tests + +$SRC/build.py -f components/tests/ui/build.xml -Dtest.with.fail=false ui-test-web + +#aggregate the various output files. +$SRC/build.py -f components/tests/ui/build.xml aggregate-results + + + + + + $WORKSPACE/src/components/tests/ui/target/reports + report.html + log.html + output.xml + false + 100.0 + 80.0 + + + + true + true + + + + \ No newline at end of file diff --git a/home/jobs/OMERO-server/config.xml b/home/jobs/OMERO-server/config.xml new file mode 100644 index 00000000..d52d16d5 --- /dev/null +++ b/home/jobs/OMERO-server/config.xml @@ -0,0 +1,82 @@ + + + + + false + + + server + false + false + false + false + + false + + + OMERO_INSTALL=/tmp/omero-install/linux +source $OMERO_INSTALL/settings.env + +OMERO_DIST=$HOME/OMERO.server +if [ -e $OMERO_DIST ]; then + #eval $(bash /opt/multi-config.sh ice3.5) + sudo sudo service omero stop + sleep 5 + sudo sudo service omero status || true + rm -rf $OMERO_DIST + source $OMERO_INSTALL/settings.env + dropdb -h $OMERO_DB_HOST -U $OMERO_DB_USER $OMERO_DB_NAME || echo Already exists maybe +else + sudo service omero stop + source $OMERO_INSTALL/settings.env + dropdb -h $OMERO_DB_HOST -U $OMERO_DB_USER $OMERO_DB_NAME || echo Already exists maybe +fi + +mkdir -p $OMERO_DATA_DIR +rm -rf $OMERO_DATA_DIR/* + + + + OMERO-build + src/target/OMERO.server*zip + + + + true + false + + + OMERO_DIST=$HOME/OMERO.server +OMERO_INSTALL=/tmp/omero-install/linux + +ZIP=$(ls OMERO.server*.zip) +DIST=${ZIP%.zip} +unzip $ZIP +rm -f $ZIP +mv $WORKSPACE/$DIST $OMERO_DIST + +source $OMERO_INSTALL/settings.env + +createdb -h $OMERO_DB_HOST -U $OMERO_DB_USER $OMERO_DB_NAME + + +# CONFIG HERE. +$OMERO_DIST/bin/omero config set omero.db.name $OMERO_DB_NAME +$OMERO_DIST/bin/omero config set omero.db.host $OMERO_DB_HOST +$OMERO_DIST/bin/omero config set omero.db.user $OMERO_DB_USER +$OMERO_DIST/bin/omero config set omero.data.dir $OMERO_DATA_DIR +# DONE + + +# PURGE +$OMERO_DIST/bin/omero db script -f dbsetup.sql "" "" $OMERO_ROOT_PASS +psql -h $OMERO_DB_HOST -U $OMERO_DB_USER -d $OMERO_DB_NAME -f dbsetup.sql +rm dbsetup.sql + +sudo sudo service omero start + + + + + + \ No newline at end of file diff --git a/home/jobs/OMERO-start/config.xml b/home/jobs/OMERO-test-integration/config.xml similarity index 98% rename from home/jobs/OMERO-start/config.xml rename to home/jobs/OMERO-test-integration/config.xml index 98bd78aa..3b447f42 100644 --- a/home/jobs/OMERO-start/config.xml +++ b/home/jobs/OMERO-test-integration/config.xml @@ -5,7 +5,8 @@ false - true + slave + false false false false diff --git a/home/jobs/OMERO-web/config.xml b/home/jobs/OMERO-web/config.xml new file mode 100644 index 00000000..447a36c5 --- /dev/null +++ b/home/jobs/OMERO-web/config.xml @@ -0,0 +1,98 @@ + + + + + false + + + web + false + false + false + false + + false + + + +OMERO_DIST=$HOME/OMERO.server +if [ -e $OMERO_DIST ]; then + if [ -e $HOME/omero-virtualenv ]; then + #eval $(bash /opt/multi-config.sh ice3.5) + sudo sudo service omero-web stop + sleep 5 + sudo sudo service omero-web status || true + fi + rm -rf $OMERO_DIST +else + sudo service omero-web stop + virtualenv $HOME/omero-virtualenv --system-site-packages +fi + + + + OMERO-build + src/target/OMERO.py*zip + + + + true + false + + + +OMERO_DIST=$HOME/OMERO.server +OMERO_INSTALL=/tmp/omero-install/linux + +ZIP=$(ls OMERO.py*.zip) +DIST=${ZIP%.zip} +unzip $ZIP +rm -f $ZIP +mv $WORKSPACE/$DIST $OMERO_DIST + +source $HOME/omero-virtualenv/bin/activate + +source $OMERO_INSTALL/settings.env + +# redis IP +pip install --upgrade --no-deps --force-reinstall -r $OMERO_DIST/share/web/requirements-py27-nginx.txt + +pip install django-redis-cache>=1.6.5 + +#start-config +$OMERO_DIST/bin/omero config set omero.web.application_server 'wsgi-tcp' +$OMERO_DIST/bin/omero config set omero.web.application_server.max_requests 0 + +$OMERO_DIST/bin/omero config set omero.web.prefix '/web' +$OMERO_DIST/bin/omero config set omero.web.static_url '/web/static/' +$OMERO_DIST/bin/omero config set omero.web.static_root '/home/omero/static/web' + +$OMERO_DIST/bin/omero config set omero.web.session_engine 'django.contrib.sessions.backends.cache' +$OMERO_DIST/bin/omero config set omero.web.caches '{"default": {"BACKEND": "redis_cache.RedisCache","LOCATION": "redis:6379"}}' + +$OMERO_DIST/bin/omero config set omero.web.server_list '[["omero", 4064, "omero"], ["slave", 4064, "slave"]]' + + +$OMERO_DIST/bin/omero config set omero.web.application_server.host web +$OMERO_DIST/bin/omero config set omero.web.application_server.port 4080 + +$OMERO_DIST/bin/omero web config nginx --http "$OMERO_WEB_PORT" >$OMERO_DIST/nginx.conf.tmp + +sudo -u root cp $OMERO_DIST/nginx.conf.tmp $HOME/nginx/omero-web.conf + +cat $HOME/nginx/omero-web.conf + +$OMERO_DIST/bin/omero config set omero.web.application_server.host 0.0.0.0 + +#eval $(bash /opt/multi-config.sh ice3.5) + +deactivate + +sudo sudo service omero-web start + + + + + + + \ No newline at end of file diff --git a/home/jobs/Trigger/config.xml b/home/jobs/Trigger/config.xml index 01e8f4b3..cee457b5 100644 --- a/home/jobs/Trigger/config.xml +++ b/home/jobs/Trigger/config.xml @@ -13,7 +13,15 @@ build job: 'OMERO-push' build job: 'OMERO-build' -build job: 'OMERO-start', wait: false +build job: 'OMERO-server' + +build job: 'OMERO-web' + +build job: 'nginx' + +build job: 'OMERO-robot', wait: false + +build job: 'OMERO-test-integration', wait: false false diff --git a/home/jobs/nginx/config.xml b/home/jobs/nginx/config.xml new file mode 100644 index 00000000..9bffff4f --- /dev/null +++ b/home/jobs/nginx/config.xml @@ -0,0 +1,23 @@ + + + + + false + + + nginx + false + false + false + false + + false + + + sudo service nginx reload +sudo service nginx status + + + + + \ No newline at end of file diff --git a/home/org.jenkinsci.plugins.gitclient.JGitTool.xml b/home/org.jenkinsci.plugins.gitclient.JGitTool.xml index 0bc93da5..f4c5945f 100644 --- a/home/org.jenkinsci.plugins.gitclient.JGitTool.xml +++ b/home/org.jenkinsci.plugins.gitclient.JGitTool.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/jenkins/Dockerfile b/jenkins/Dockerfile index 9653a9bd..bf9344e3 100644 --- a/jenkins/Dockerfile +++ b/jenkins/Dockerfile @@ -1,5 +1,8 @@ FROM jenkins:1.642.2 MAINTAINER OME +# Temp fix robot test results +ENV JAVA_OPTS "-Dhudson.model.DirectoryBrowserSupport.CSP=" + COPY plugins.txt /usr/share/jenkins/ref/plugins.txt RUN /usr/local/bin/plugins.sh /usr/share/jenkins/ref/plugins.txt diff --git a/jenkins/plugins.txt b/jenkins/plugins.txt index acb91fc2..0d3dcec3 100644 --- a/jenkins/plugins.txt +++ b/jenkins/plugins.txt @@ -1,4 +1,4 @@ -swarm:1.24 +swarm:2.0 git:2.4.0 mailer:1.15 scm-api:0.2 @@ -38,3 +38,6 @@ workflow-job:1.11 workflow-scm-step:1.11 workflow-step-api:1.11 workflow-support:1.11 + +# Robot +robot:1.6.4 diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 00000000..05d44d31 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,45 @@ +FROM openmicroscopy/omero-ssh-systemd + +MAINTAINER OME + +ENV JENKINS_SWARM_VERSION 2.0 +ENV LANG en_US.UTF-8 + +# To avoid error: sudo: sorry, you must have a tty to run sudo +RUN sed -i -e "s/Defaults requiretty.*/ #Defaults requiretty/g" /etc/sudoers + +RUN yum install -y initscripts \ + && yum clean all + +RUN chmod a+X /home/omero + +RUN yum -y localinstall http://nginx.org/packages/rhel/7/noarch/RPMS/nginx-release-rhel-7-0.el7.ngx.noarch.rpm \ + && yum clean all +RUN yum -y install nginx \ + && yum clean all +RUN mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.disabled + +VOLUME ["/etc/nginx/conf.d", "/var/log/nginx"] + +WORKDIR /etc/nginx + +RUN systemctl enable nginx + +EXPOSE 80 443 + +RUN yum install yum -y install java-1.8.0-openjdk && yum clean all + +RUN yum install -y http://download-aws.ej-technologies.com/exe4j/exe4j_linux_5_0_1.rpm && yum clean all + +USER omero +RUN curl --create-dirs -sSLo /tmp/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar http://maven.jenkins-ci.org/content/repositories/releases/org/jenkins-ci/plugins/swarm-client/$JENKINS_SWARM_VERSION/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar + +USER root + +ADD ./jenkins-slave.sh /tmp/jenkins-slave.sh +RUN chmod +x /tmp/jenkins-slave.sh +ADD ./jenkins.service /etc/systemd/system/jenkins.service +RUN systemctl enable jenkins.service + + +#ENTRYPOINT ["/tmp/jenkins-slave.sh"] diff --git a/nginx/jenkins-slave.sh b/nginx/jenkins-slave.sh new file mode 100755 index 00000000..e9023222 --- /dev/null +++ b/nginx/jenkins-slave.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# https://raw.githubusercontent.com/carlossg/jenkins-swarm-slave-docker/master/jenkins-slave.sh + +eval `(sudo cat /proc/1/environ; echo) | tr '\000' '\n' | while read line; do if [[ $line == JENKINS_PORT_8080_TCP_* ]] ; then echo export $line; fi done` + +# if `docker run` first argument start with `-` the user is passing jenkins swarm launcher arguments +if [[ $# -lt 1 ]] || [[ "$1" == "-"* ]]; then + + # jenkins swarm slave + JAR=`ls -1 /tmp/swarm-client-*.jar | tail -n 1` + + # if -master is not provided and using --link jenkins:jenkins + if [[ "$@" != *"-master "* ]] && [ ! -z "$JENKINS_PORT_8080_TCP_ADDR" ]; then + PARAMS="-master http://$JENKINS_PORT_8080_TCP_ADDR:$JENKINS_PORT_8080_TCP_PORT" + fi + + echo Running java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" + exec java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" +fi + +# As argument is not jenkins, assume user want to run his own process, for sample a `bash` shell to explore this image +exec "$@" + + diff --git a/nginx/jenkins.service b/nginx/jenkins.service new file mode 100644 index 00000000..bb55407d --- /dev/null +++ b/nginx/jenkins.service @@ -0,0 +1,15 @@ +[Unit] +Description=Jenkins slave +Documentation=https://developer.cloudbees.com/bin/view/DEV/On-Premise+Executors +After=multi-user.target + +[Service] +Type=simple +ExecStart=/tmp/jenkins-slave.sh -labels centos7 -labels nginx -executors 4 +Restart=always +RestartSec=60 +StartLimitInterval=0 +User=omero + +[Install] +WantedBy=multi-user.target diff --git a/rename.py b/rename.py index 487e151c..5825d442 100755 --- a/rename.py +++ b/rename.py @@ -33,5 +33,5 @@ def replace(name, branch, uid): branch = name # This number will need to be updated when new changes are commited. - assert 16 == replace(name, branch, ns.uid) + assert 26 == replace(name, branch, ns.uid) print "Done. You may want to review and commit your changes now" diff --git a/robot/Dockerfile b/robot/Dockerfile new file mode 100644 index 00000000..06139ef0 --- /dev/null +++ b/robot/Dockerfile @@ -0,0 +1,46 @@ +FROM openmicroscopy/omero-ssh-systemd + +MAINTAINER OME + +ENV JENKINS_SWARM_VERSION 2.0 +ENV LANG en_US.UTF-8 +ARG JAVAVER=openjdk18 + +# To avoid error: sudo: sorry, you must have a tty to run sudo +RUN sed -i -e "s/Defaults requiretty.*/ #Defaults requiretty/g" /etc/sudoers + +RUN yum install -y initscripts \ + && yum clean all + +# Download and run omero-install. +ENV OMERO_INSTALL /tmp/omero-install/linux + +RUN yum install -y git \ + && git clone https://github.com/ome/omero-install.git /tmp/omero-install \ + && yum clean all + +# Customize: run particular scripts from omero-install +RUN bash $OMERO_INSTALL/step01_centos7_init.sh +RUN bash $OMERO_INSTALL/step01_centos_java_deps.sh +RUN bash $OMERO_INSTALL/step01_centos7_deps.sh +RUN export ICEVER=ice35-devel; bash /tmp/omero-install/linux/step01_centos7_ice_deps.sh + +RUN yum install -y http://download-aws.ej-technologies.com/exe4j/exe4j_linux_5_0_1.rpm && yum clean all + +USER omero +RUN curl --create-dirs -sSLo /tmp/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar http://maven.jenkins-ci.org/content/repositories/releases/org/jenkins-ci/plugins/swarm-client/$JENKINS_SWARM_VERSION/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar + +RUN virtualenv /home/omero/omero-virtualenv --system-site-packages +RUN source /home/omero/omero-virtualenv/bin/activate; pip install robotframework +RUN source /home/omero/omero-virtualenv/bin/activate; pip install robotframework-selenium2library + +USER root + +WORKDIR /home/omero + +ADD ./jenkins-slave.sh /tmp/jenkins-slave.sh +RUN chmod +x /tmp/jenkins-slave.sh +ADD ./jenkins.service /etc/systemd/system/jenkins.service +RUN systemctl enable jenkins.service + +#ENTRYPOINT ["/tmp/jenkins-slave.sh"] diff --git a/robot/jenkins-slave.sh b/robot/jenkins-slave.sh new file mode 100755 index 00000000..e9023222 --- /dev/null +++ b/robot/jenkins-slave.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# https://raw.githubusercontent.com/carlossg/jenkins-swarm-slave-docker/master/jenkins-slave.sh + +eval `(sudo cat /proc/1/environ; echo) | tr '\000' '\n' | while read line; do if [[ $line == JENKINS_PORT_8080_TCP_* ]] ; then echo export $line; fi done` + +# if `docker run` first argument start with `-` the user is passing jenkins swarm launcher arguments +if [[ $# -lt 1 ]] || [[ "$1" == "-"* ]]; then + + # jenkins swarm slave + JAR=`ls -1 /tmp/swarm-client-*.jar | tail -n 1` + + # if -master is not provided and using --link jenkins:jenkins + if [[ "$@" != *"-master "* ]] && [ ! -z "$JENKINS_PORT_8080_TCP_ADDR" ]; then + PARAMS="-master http://$JENKINS_PORT_8080_TCP_ADDR:$JENKINS_PORT_8080_TCP_PORT" + fi + + echo Running java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" + exec java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" +fi + +# As argument is not jenkins, assume user want to run his own process, for sample a `bash` shell to explore this image +exec "$@" + + diff --git a/robot/jenkins.service b/robot/jenkins.service new file mode 100644 index 00000000..7ed67168 --- /dev/null +++ b/robot/jenkins.service @@ -0,0 +1,15 @@ +[Unit] +Description=Jenkins slave +Documentation=https://developer.cloudbees.com/bin/view/DEV/On-Premise+Executors +After=multi-user.target + +[Service] +Type=simple +ExecStart=/tmp/jenkins-slave.sh -labels centos7 -labels robot -executors 4 +Restart=always +RestartSec=60 +StartLimitInterval=0 +User=omero + +[Install] +WantedBy=multi-user.target diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 00000000..3985b166 --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,71 @@ +FROM openmicroscopy/omero-ssh-systemd + +MAINTAINER OME + +ENV JENKINS_SWARM_VERSION 2.0 +ENV LANG en_US.UTF-8 +ARG JAVAVER=openjdk18 +ARG ICEVER=ice35-devel + + +# To avoid error: sudo: sorry, you must have a tty to run sudo +RUN sed -i -e "s/Defaults requiretty.*/ #Defaults requiretty/g" /etc/sudoers + +RUN yum install -y initscripts \ + && yum clean all + +# Download and run omero-install. +ENV OMERO_INSTALL /tmp/omero-install/linux + +RUN yum install -y git \ + && git clone https://github.com/ome/omero-install.git /tmp/omero-install \ + && yum clean all + +ADD ./settings.env $OMERO_INSTALL/settings.env +RUN chmod +x $OMERO_INSTALL/settings.env + +# Customize: run particular scripts from omero-install +RUN rm -f $OMERO_INSTALL/setup_centos_selinux.sh +RUN touch $OMERO_INSTALL/setup_centos_selinux.sh +# db will be managed by the job +RUN rm -f $OMERO_INSTALL/setup_omero_db.sh +RUN touch $OMERO_INSTALL/setup_omero_db.sh +RUN rm -f $OMERO_INSTALL/step03_all_postgres.sh +RUN touch $OMERO_INSTALL/step03_all_postgres.sh +# omeroweb will be installed by the job +RUN rm -f $OMERO_INSTALL/step04_all_omero.sh +RUN touch $OMERO_INSTALL/step04_all_omero.sh +# remove nginx +RUN rm -rf $OMERO_INSTALL/step05_centos7_nginx.sh +RUN touch $OMERO_INSTALL/step05_centos7_nginx.sh + +RUN sed -i.bak -re 's/(.*postgresql-9.4.service.*)/#\1/' $OMERO_INSTALL/omero-systemd.service + +RUN sed -i.bak -re 's/(.*omero-web.service.*)/#\1/' $OMERO_INSTALL/step06_centos7_daemon.sh +RUN sed -i.bak -re 's/(systemctl daemon-reload)/#\1/' $OMERO_INSTALL/step06_centos7_daemon.sh + +WORKDIR /tmp/omero-install/linux + +RUN JAVAVER=$JAVAVER ICEVER=$ICEVER PGVER=nopg bash install_centos7_nginx.sh + +RUN yum -y install http://yum.postgresql.org/9.4/redhat/rhel-7-x86_64/pgdg-centos94-9.4-2.noarch.rpm +RUN yum -y install postgresql94 + +RUN yum install -y http://download-aws.ej-technologies.com/exe4j/exe4j_linux_5_0_1.rpm + +WORKDIR /tmp + +USER omero +RUN curl --create-dirs -sSLo /tmp/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar http://maven.jenkins-ci.org/content/repositories/releases/org/jenkins-ci/plugins/swarm-client/$JENKINS_SWARM_VERSION/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar + +USER root + +ADD ./jenkins-slave.sh /tmp/jenkins-slave.sh +RUN chmod +x /tmp/jenkins-slave.sh +ADD ./jenkins.service /etc/systemd/system/jenkins.service +RUN systemctl enable jenkins.service + +EXPOSE 4080 + + +#ENTRYPOINT ["/tmp/jenkins-slave.sh"] diff --git a/server/jenkins-slave.sh b/server/jenkins-slave.sh new file mode 100755 index 00000000..e9023222 --- /dev/null +++ b/server/jenkins-slave.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# https://raw.githubusercontent.com/carlossg/jenkins-swarm-slave-docker/master/jenkins-slave.sh + +eval `(sudo cat /proc/1/environ; echo) | tr '\000' '\n' | while read line; do if [[ $line == JENKINS_PORT_8080_TCP_* ]] ; then echo export $line; fi done` + +# if `docker run` first argument start with `-` the user is passing jenkins swarm launcher arguments +if [[ $# -lt 1 ]] || [[ "$1" == "-"* ]]; then + + # jenkins swarm slave + JAR=`ls -1 /tmp/swarm-client-*.jar | tail -n 1` + + # if -master is not provided and using --link jenkins:jenkins + if [[ "$@" != *"-master "* ]] && [ ! -z "$JENKINS_PORT_8080_TCP_ADDR" ]; then + PARAMS="-master http://$JENKINS_PORT_8080_TCP_ADDR:$JENKINS_PORT_8080_TCP_PORT" + fi + + echo Running java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" + exec java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" +fi + +# As argument is not jenkins, assume user want to run his own process, for sample a `bash` shell to explore this image +exec "$@" + + diff --git a/server/jenkins.service b/server/jenkins.service new file mode 100644 index 00000000..7f8ed94d --- /dev/null +++ b/server/jenkins.service @@ -0,0 +1,15 @@ +[Unit] +Description=Jenkins slave +Documentation=https://developer.cloudbees.com/bin/view/DEV/On-Premise+Executors +After=multi-user.target + +[Service] +Type=simple +ExecStart=/tmp/jenkins-slave.sh -labels centos7 -labels server -executors 4 +Restart=always +RestartSec=60 +StartLimitInterval=0 +User=omero + +[Install] +WantedBy=multi-user.target diff --git a/server/settings.env b/server/settings.env new file mode 100644 index 00000000..54180357 --- /dev/null +++ b/server/settings.env @@ -0,0 +1,9 @@ +OMERO_DB_HOST=pg +OMERO_DB_USER=postgres +OMERO_DB_PASS= +OMERO_DB_NAME=omero_database +OMERO_ROOT_PASS=omero +OMERO_DATA_DIR=/OMERO + +export OMERO_DB_USER OMERO_DB_PASS OMERO_DB_NAME OMERO_ROOT_PASS OMERO_DATA_DIR + diff --git a/slave/Dockerfile b/slave/Dockerfile index 2e2d358b..872d2875 100644 --- a/slave/Dockerfile +++ b/slave/Dockerfile @@ -2,7 +2,7 @@ FROM openmicroscopy/omero-build-c6-multienv MAINTAINER OME -ENV JENKINS_SWARM_VERSION 1.24 +ENV JENKINS_SWARM_VERSION 2.0 ENV LANG en_US.UTF-8 RUN yum install -y http://download-aws.ej-technologies.com/exe4j/exe4j_linux_5_0_1.rpm diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 00000000..80def4f9 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,73 @@ +FROM openmicroscopy/omero-ssh-systemd + +MAINTAINER OME + +ENV JENKINS_SWARM_VERSION 2.0 +ENV LANG en_US.UTF-8 +ARG JAVAVER=openjdk18 + + +# To avoid error: sudo: sorry, you must have a tty to run sudo +RUN sed -i -e "s/Defaults requiretty.*/ #Defaults requiretty/g" /etc/sudoers + +RUN yum install -y initscripts \ + && yum clean all + +# Download and run omero-install. +ENV OMERO_INSTALL /tmp/omero-install/linux + +RUN yum install -y git \ + && git clone https://github.com/ome/omero-install.git /tmp/omero-install \ + && yum clean all + +ADD ./settings.env $OMERO_INSTALL/settings.env +RUN chmod +x $OMERO_INSTALL/settings.env + +# Customize: run particular scripts from omero-install +RUN rm -f $OMERO_INSTALL/setup_centos_selinux.sh +RUN touch $OMERO_INSTALL/setup_centos_selinux.sh +RUN rm -f $OMERO_INSTALL/setup_omero_db.sh +RUN touch $OMERO_INSTALL/setup_omero_db.sh +# do not create user +RUN rm -f $OMERO_INSTALL/step02_all_setup.sh +RUN touch $OMERO_INSTALL/step02_all_setup.sh +# omeroweb will be isntalled by the job +RUN rm -f $OMERO_INSTALL/step04_all_omero.sh +RUN touch $OMERO_INSTALL/step04_all_omero.sh +# remove nginx +RUN mv $OMERO_INSTALL/step05_centos7_nginx.sh $OMERO_INSTALL/_step05_centos7_nginx.sh +RUN touch $OMERO_INSTALL/step05_centos7_nginx.sh + +RUN sed -i.bak -re 's/(Requires=nginx.service)/#\1/' $OMERO_INSTALL/omero-web-systemd.service +# TODO: remove when omero-install support virtualenv +# custom omero-web.service that support virtualenv +RUN rm -f $OMERO_INSTALL/omero-web-systemd.service +ADD ./omeroweb.service $OMERO_INSTALL/omero-web-systemd.service +#RUN chmod a $OMERO_INSTALL/omero-web-systemd.service + +RUN sed -i.bak -re 's/(.*omero.service.*)/#\1/' $OMERO_INSTALL/step06_centos7_daemon.sh +RUN sed -i.bak -re 's/(systemctl daemon-reload)/#\1/' $OMERO_INSTALL/step06_centos7_daemon.sh + +WORKDIR /tmp/omero-install/linux + +RUN JAVAVER=$JAVAVER PGVER=nopg bash install_centos7_nginx.sh +RUN yum install -y python-redis && yum clean all + +RUN yum install -y http://download-aws.ej-technologies.com/exe4j/exe4j_linux_5_0_1.rpm + +WORKDIR /tmp + +USER omero +RUN curl --create-dirs -sSLo /tmp/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar http://maven.jenkins-ci.org/content/repositories/releases/org/jenkins-ci/plugins/swarm-client/$JENKINS_SWARM_VERSION/swarm-client-$JENKINS_SWARM_VERSION-jar-with-dependencies.jar + +USER root + +ADD ./jenkins-slave.sh /tmp/jenkins-slave.sh +RUN chmod +x /tmp/jenkins-slave.sh +ADD ./jenkins.service /etc/systemd/system/jenkins.service +RUN systemctl enable jenkins.service + +EXPOSE 4080 + + +#ENTRYPOINT ["/tmp/jenkins-slave.sh"] diff --git a/web/jenkins-slave.sh b/web/jenkins-slave.sh new file mode 100755 index 00000000..e9023222 --- /dev/null +++ b/web/jenkins-slave.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# https://raw.githubusercontent.com/carlossg/jenkins-swarm-slave-docker/master/jenkins-slave.sh + +eval `(sudo cat /proc/1/environ; echo) | tr '\000' '\n' | while read line; do if [[ $line == JENKINS_PORT_8080_TCP_* ]] ; then echo export $line; fi done` + +# if `docker run` first argument start with `-` the user is passing jenkins swarm launcher arguments +if [[ $# -lt 1 ]] || [[ "$1" == "-"* ]]; then + + # jenkins swarm slave + JAR=`ls -1 /tmp/swarm-client-*.jar | tail -n 1` + + # if -master is not provided and using --link jenkins:jenkins + if [[ "$@" != *"-master "* ]] && [ ! -z "$JENKINS_PORT_8080_TCP_ADDR" ]; then + PARAMS="-master http://$JENKINS_PORT_8080_TCP_ADDR:$JENKINS_PORT_8080_TCP_PORT" + fi + + echo Running java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" + exec java $JAVA_OPTS -jar $JAR -fsroot $HOME $PARAMS "$@" +fi + +# As argument is not jenkins, assume user want to run his own process, for sample a `bash` shell to explore this image +exec "$@" + + diff --git a/web/jenkins.service b/web/jenkins.service new file mode 100644 index 00000000..80715c28 --- /dev/null +++ b/web/jenkins.service @@ -0,0 +1,15 @@ +[Unit] +Description=Jenkins slave +Documentation=https://developer.cloudbees.com/bin/view/DEV/On-Premise+Executors +After=multi-user.target + +[Service] +Type=simple +ExecStart=/tmp/jenkins-slave.sh -labels centos7 -labels web -executors 4 +Restart=always +RestartSec=60 +StartLimitInterval=0 +User=omero + +[Install] +WantedBy=multi-user.target diff --git a/web/omeroweb.service b/web/omeroweb.service new file mode 100644 index 00000000..ac2eda70 --- /dev/null +++ b/web/omeroweb.service @@ -0,0 +1,16 @@ +[Unit] +Description=OMERO.web + +[Service] +User=omero +Type=forking +PIDFile=/home/omero/OMERO.server/var/django.pid +Restart=on-failure +RestartSec=10 +Environment="VENVDIR=/home/omero/omero-virtualenv" "BINDIR=/home/omero/OMERO.server/bin" +ExecStart=/usr/bin/bash -c "source $VENVDIR/bin/activate; $BINDIR/omero web start" +ExecStop=/usr/bin/bash -c "source $VENVDIR/bin/activate; $BINDIR/omero web stop" +Restart=/usr/bin/bash -c "source $VENVDIR/bin/activate; $BINDIR/omero web restart" + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/web/settings.env b/web/settings.env new file mode 100644 index 00000000..6f581680 --- /dev/null +++ b/web/settings.env @@ -0,0 +1,4 @@ +OMERO_WEB_PORT=80 +OMERO_DATA_DIR=/OMERO + +export OMERO_WEB_PORT OMERO_DATA_DIR \ No newline at end of file