Merge pull request #94 from dice-group/develop

Develop - release 0.3
dice-group · Jan 11, 2019 · f5e798c · f5e798c
2 parents 11c4ace + dfd6c3f
commit f5e798c
Show file tree

Hide file tree

Showing 49 changed files with 1,015 additions and 679 deletions.
diff --git a/Dockerfile.frontier b/Dockerfile.frontier
@@ -3,12 +3,12 @@ FROM openjdk:8u151-jdk
 RUN apt-get update && apt-get install -y netcat
 
 COPY ./squirrel.frontier/target/squirrel.frontier.jar /data/squirrel/squirrel.jar
-COPY ./spring-config/default-config.xml /data/squirrel/default-config.xml
+COPY ./spring-config/ /data/squirrel/spring-config
 WORKDIR /data/squirrel
 
 #ADD entrypoint.sh /entrypoint.sh
 #RUN chmod +x /entrypoint.sh
 
 VOLUME ["/var/squirrel/data"]
 
-CMD java -cp squirrel.jar:. org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.FrontierComponent
+CMD java -cp squirrel.jar:. org.hobbit.core.run.ComponentStarter org.dice_research.squirrel.components.FrontierComponent
diff --git a/Dockerfile.web b/Dockerfile.web
@@ -0,0 +1,16 @@
+FROM openjdk:8u151-jdk
+
+RUN apt-get update && apt-get install -y netcat
+
+COPY ./squirrel.web/target/squirrel.web.jar /data/squirrel/squirrel.web.jar
+COPY ./squirrel.web/target/squirrel.web.jar.original /data/squirrel/squirrel.web.jar.original
+COPY ./squirrel.web/WEB-INF /data/squirrel/WEB-INF 
+WORKDIR /data/squirrel
+
+#ADD entrypoint.sh /entrypoint.sh
+#RUN chmod +x /entrypoint.sh
+
+VOLUME ["/var/squirrel/data"]
+
+CMD java -cp squirrel.web.jar:. com.squirrel.Application
+
diff --git a/Dockerfile.worker b/Dockerfile.worker
@@ -3,7 +3,7 @@ FROM openjdk:8u151-jdk
 RUN apt-get update && apt-get install -y netcat
 
 COPY ./squirrel.worker/target/squirrel.worker.jar /data/squirrel/squirrel.jar
-COPY ./spring-config/default-config.xml /data/squirrel/default-config.xml
+COPY ./spring-config /data/squirrel/spring-config
 WORKDIR /data/squirrel
 
 #ADD entrypoint.sh /entrypoint.sh

diff --git a/Makefile b/Makefile
@@ -1,12 +1,13 @@
 default: build
 
 build:
-	docker-compose -f docker-compose-sparql.yml down
+	docker-compose -f docker-compose.yml down
 	mvn clean install -U -DskipTests -Dmaven.javadoc.skip=true
 
 dockerize:
 	docker build -f Dockerfile.frontier -t squirrel.frontier .
 	docker build -f Dockerfile.worker -t squirrel.worker .
+	docker build -f Dockerfile.web -t squirrel.web .
 
 start: dockerize
 	docker-compose -f docker-compose-sparql.yml up

diff --git a/build-squirrel b/build-squirrel
@@ -0,0 +1,15 @@
+#!/bin/bash
+echo "Building Squirrel..."
+cd squirrel.web-api
+mvn clean install
+cd ..
+mvn clean install -DskipTests
+clear
+echo "Creating Frontier image..."
+docker build -f Dockerfile.frontier -t squirrel.frontier .
+echo "Creating Worker image..."
+docker build -f Dockerfile.worker -t squirrel.worker .
+echo "Creating Web image..."
+docker build -f Dockerfile.web -t squirrel.web .
+clear
+echo "Finished"
diff --git a/docker-compose-sparql.yml b/docker-compose-sparql.yml
@@ -130,8 +130,8 @@ services:
       DEDUPLICATION_ACTIVE: "true"
       HOBBIT_RABBIT_HOST: rabbit
       OUTPUT_FOLDER: /var/squirrel/data
-      RDB_HOST_NAME: rethinkdb
-      RDB_PORT: 28015
+      MDB_HOST_NAME: mongodb
+      MDB_PORT: 27017
       SPARQL_HOST_NAME: sparqlhost
       SPARQL_HOST_PORT: 3030
       SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"

diff --git a/docker-compose-sparql-web.yml → docker-compose-web.yml b/docker-compose-sparql-web.yml → docker-compose-web.yml
@@ -19,14 +19,14 @@ services:
   #  ports:
 
   frontier:
-    image: squirrel:latest
+    image: squirrel.frontier:latest
     container_name: frontier
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
       - SEED_FILE=/var/squirrel/seeds.txt
       - URI_WHITELIST_FILE=/var/squirrel/whitelist.txt
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
       - COMMUNICATION_WITH_WEBSERVICE=true
       - VISUALIZATION_OF_CRAWLED_GRAPH=true
     volumes:
@@ -36,10 +36,13 @@ services:
     command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.dice_research.squirrel.components.FrontierComponent
 
   web:
-    image: squirrel/webimage:latest
+    image: squirrel.web:latest
     container_name: web
+    environment:
+      - HOST=rabbit
     ports:
       - "8080:8080"
+    command: java -jar squirrel.web.jar
 
   sparqlhost:
     image: stain/jena-fuseki
@@ -70,19 +73,19 @@ services:
       - "5672:5672"
 
   worker1:
-    image: squirrel:latest
+    image: squirrel.worker:latest
     container_name: worker1
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
-      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
+      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context.xml
       - SPARQL_HOST_NAME=sparqlhost
       #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
       - SPARQL_HOST_PORT=3030
       - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
       #-CKAN_PORT=
     volumes:
         - ./data/worker1:/var/squirrel/data
@@ -92,19 +95,19 @@ services:
     command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
 
   worker2:
-    image: squirrel:latest
+    image: squirrel.worker:latest
     container_name: worker2
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
-      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
+      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context.xml
       - SPARQL_HOST_NAME=sparqlhost
       #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
       - SPARQL_HOST_PORT=3030
       - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
       #-CKAN_PORT=
     volumes:
         - ./data/worker2:/var/squirrel/data
@@ -114,19 +117,19 @@ services:
     command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
 
   worker3:
-    image: squirrel:latest
+    image: squirrel.worker:latest
     container_name: worker3
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
-      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-sparqlStoreBased.xml
+      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context.xml
       - SPARQL_HOST_NAME=sparqlhost
       #- CKAN_WHITELIST_FILE=/var/squirrel/ckanwhitelist.txt
       - SPARQL_HOST_PORT=3030
       - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
       #-CKAN_PORT=
     volumes:
         - ./data/worker3:/var/squirrel/data
@@ -142,8 +145,8 @@ services:
       DEDUPLICATION_ACTIVE: "true"
       HOBBIT_RABBIT_HOST: rabbit
       OUTPUT_FOLDER: /var/squirrel/data
-      RDB_HOST_NAME: rethinkdb
-      RDB_PORT: 28015
+      MDB_HOST_NAME: mongodb
+      MDB_PORT: 27017
       SPARQL_HOST_NAME: sparqlhost
       SPARQL_HOST_PORT: 3030
       SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,19 +14,30 @@ services:
     container_name: frontier
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
-      - SEED_FILE=/var/squirrel/seeds.txt
       - URI_WHITELIST_FILE=/var/squirrel/whitelist.txt
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - SEED_FILE=/var/squirrel/seeds.txt
       - MDB_HOST_NAME=mongodb
       - MDB_PORT=27017
       - COMMUNICATION_WITH_WEBSERVICE=false
       - VISUALIZATION_OF_CRAWLED_GRAPH=false
+      - JVM_ARGS=-Xmx8g
     volumes:
       - ./data/frontier:/var/squirrel/data
       - ./seed/seeds.txt:/var/squirrel/seeds.txt:ro
-      - ./whitelist/ckanwhitelist.txt:/var/squirrel/whitelist.txt:ro
-#    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.dice_research.squirrel.components.FrontierComponent
+      - ./whitelist/whitelist.txt:/var/squirrel/whitelist.txt:ro
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.dice_research.squirrel.components.FrontierComponent
+
+  sparqlhost:
+    image: stain/jena-fuseki
+    container_name: sparqlhost
+    ports:
+      - "3030:3030"
+    volumes:
+      - ./data/sparqlhost/sparqlhost_data:/fuseki
+    environment:
+      - ADMIN_PASSWORD=pw123
+      - JVM_ARGS=-Xmx2g
+
   mongodb:
     image: mongo:4.0.0
     volumes:
@@ -39,7 +50,8 @@ services:
     volumes:
       - ./data/rethinkdb:/data
     ports:
-      - "28015:28015"
+      - "8080:8080"
+    command: rethinkdb --bind all
 
   # message bus
   rabbit:
@@ -58,15 +70,18 @@ services:
       - HOBBIT_RABBIT_HOST=rabbit
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
-      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-fileBased.xml
-      - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context.xml
+      - SPARQL_HOST_NAME=sparqlhost
+      - SPARQL_HOST_PORT=3030
+      - DEDUPLICATION_ACTIVE=false
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
+      - JVM_ARGS=-Xmx8g
     volumes:
         - ./data/worker1:/var/squirrel/data
         - ./yaml:/var/squirrel/yaml
         - ./spring-config:/var/squirrel/spring-config
-#    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
+    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
 
   worker2:
     image: squirrel.worker:latest
@@ -76,14 +91,17 @@ services:
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
       - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-fileBased.xml
-      - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - SPARQL_HOST_NAME=sparqlhost
+      - SPARQL_HOST_PORT=3030
+      - DEDUPLICATION_ACTIVE=false
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
+      - JVM_ARGS=-Xmx8g
     volumes:
         - ./data/worker2:/var/squirrel/data
         - ./yaml:/var/squirrel/yaml
         - ./spring-config:/var/squirrel/spring-config
-#    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
+    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
 
   worker3:
     image: squirrel.worker:latest
@@ -93,28 +111,30 @@ services:
       - OUTPUT_FOLDER=/var/squirrel/data
       - HTML_SCRAPER_YAML_PATH=/var/squirrel/yaml
       - CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/context-fileBased.xml
+      - SPARQL_HOST_NAME=sparqlhost
+      - SPARQL_HOST_PORT=3030
       - DEDUPLICATION_ACTIVE=true
-      - RDB_HOST_NAME=rethinkdb
-      - RDB_PORT=28015
+      - MDB_HOST_NAME=mongodb
+      - MDB_PORT=27017
+      - JVM_ARGS=-Xmx8g
     volumes:
         - ./data/worker3:/var/squirrel/data
         - ./yaml:/var/squirrel/yaml
         - ./spring-config:/var/squirrel/spring-config
-#    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
-
-#  deduplicator:
-#    image: squirrel
-#    container_name: deduplicator
-#    environment:
-#      DEDUPLICATION_ACTIVE: "true"
-#      HOBBIT_RABBIT_HOST: rabbit
-#      OUTPUT_FOLDER: /var/squirrel/data
-#      RDB_HOST_NAME: rethinkdb
-#      RDB_PORT: 28015
-#      SPARQL_HOST_NAME: sparqlhost
-#      SPARQL_HOST_PORT: 3030
-#      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
-#    volumes:
-#        - ./data/deduplicator:/var/squirrel/data
-#    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.dice_research.squirrel.components.DeduplicatorComponent
+    command: java -cp squirrel.jar org.dice_research.squirrel.components.WorkerComponentStarter
 
+  deduplicator:
+    image: squirrel
+    container_name: deduplicator
+    environment:
+      DEDUPLICATION_ACTIVE: "true"
+      HOBBIT_RABBIT_HOST: rabbit
+      OUTPUT_FOLDER: /var/squirrel/data
+      MDB_HOST_NAME: mongodb
+      MDB_PORT: 27017
+      SPARQL_HOST_NAME: sparqlhost
+      SPARQL_HOST_PORT: 3030
+      SERVICE_PRECONDITION: "rethinkdb:28015 rabbit:5672"
+    volumes:
+        - ./data/deduplicator:/var/squirrel/data
+    command: java -cp squirrel.jar org.hobbit.core.run.ComponentStarter org.aksw.simba.squirrel.components.DeduplicatorComponent
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.dice-research</groupId>
 	<artifactId>squirrel</artifactId>
-	<version>0.3.0-SNAPSHOT</version>
+	<version>0.3.0</version>
 	<packaging>pom</packaging>
 	<inceptionYear>2017</inceptionYear>
 	<name>Squirrel</name>
@@ -33,12 +33,12 @@
 
 	<!-- MODULES -->
 	<modules>
+		<module>squirrel.web-api</module>
+		<module>squirrel.web</module>
 		<module>squirrel.api</module>
 		<module>squirrel.deduplication</module>
 		<module>squirrel.frontier</module>
 		<module>squirrel.mockup</module>
-		<module>squirrel.web</module>
-		<module>squirrel.web-api</module>
 		<module>squirrel.worker</module>
 	</modules>
 

diff --git a/seed/seeds.txt b/seed/seeds.txt
@@ -1,3 +1,3 @@
 https://dbpedia.org/resource/New_York
 https://dbpedia.org/resource/Moscow
-https://dbpedia.org/resource/Brazil
+https://dbpedia.org/resource/China