Developed version 0.5.0

Makefile

@@ -1,19 +1,40 @@
 default: build
 
+write-version:
+	mvn org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version -Doutput=version.txt
+
 build:
 	docker-compose -f docker-compose.yml down
 	mvn clean install -U -DskipTests -Dmaven.javadoc.skip=true
 
-dockerize:
-	docker build -f Dockerfile.frontier -t dicegroup/squirrel.frontier .
-	docker build -f Dockerfile.worker -t dicegroup/squirrel.worker .
-	docker build -f Dockerfile.web -t squirrel.web .
-	docker build -f Dockerfile.mockup -t dicegroup/squirrel.mockup .
+dockerize: write-version
+	docker build -f Dockerfile.frontier -t dicegroup/squirrel.frontier:"$$(cat version.txt)" .
+	docker build -f Dockerfile.worker -t dicegroup/squirrel.worker:"$$(cat version.txt)" .
+#	docker build -f Dockerfile.web -t squirrel.web:"$$(cat version.txt)" .
+	docker build -f Dockerfile.mockup -t dicegroup/squirrel.mockup:"$$(cat version.txt)" .
+
+push-images: write-version
+	docker push dicegroup/squirrel.frontier:"$$(cat version.txt)"
+	docker push dicegroup/squirrel.worker:"$$(cat version.txt)"
+	docker push dicegroup/squirrel.mockup:"$$(cat version.txt)"
+
+tag-orca-images: write-version
+	docker tag dicegroup/squirrel.frontier:"$$(cat version.txt)" git.project-hobbit.eu:4567/ldcbench/ldcbench-squirrel-adapter/squirrel-frontier:"$$(cat version.txt)"
+	docker tag dicegroup/squirrel.worker:"$$(cat version.txt)" git.project-hobbit.eu:4567/ldcbench/ldcbench-squirrel-adapter/squirrel-worker:"$$(cat version.txt)"
+
+push-orca-images: write-version
+	docker push git.project-hobbit.eu:4567/ldcbench/ldcbench-squirrel-adapter/squirrel-frontier:"$$(cat version.txt)"
+	docker push git.project-hobbit.eu:4567/ldcbench/ldcbench-squirrel-adapter/squirrel-worker:"$$(cat version.txt)"
+
+tag-latest: write-version
+	docker tag dicegroup/squirrel.frontier:"$$(cat version.txt)" dicegroup/squirrel.frontier:latest
+	docker tag dicegroup/squirrel.worker:"$$(cat version.txt)" dicegroup/squirrel.worker:latest
+	docker tag dicegroup/squirrel.mockup:"$$(cat version.txt)" dicegroup/squirrel.mockup:latest
 
-push-images:
-	docker push dicegroup/squirrel.frontier
-	docker push dicegroup/squirrel.worker
-	docker push dicegroup/squirrel.mockup
+push-latest-images:
+	docker push dicegroup/squirrel.frontier:latest
+	docker push dicegroup/squirrel.worker:latest
+	docker push dicegroup/squirrel.mockup:latest
 
 start: dockerize
 	docker-compose -f docker-compose-sparql.yml up

docker-compose-sparql.yml

@@ -28,8 +28,8 @@ services:
       - QUEUE_FILTER_PERSIST=true
       - COMMUNICATION_WITH_WEBSERVICE=false
       - VISUALIZATION_OF_CRAWLED_GRAPH=false
-      - JVM_ARGS=-Xmx8g 
-      
+      - JVM_ARGS=-Xmx8g
+
     volumes:
       - ./data/frontier:/var/squirrel/data
       - ./seed/seeds.csv:/var/squirrel/seeds.csv:ro

docker-compose.yml

@@ -15,8 +15,9 @@ services:
     environment:
       - HOBBIT_RABBIT_HOST=rabbit
       - URI_WHITELIST_FILE=/var/squirrel/whitelist.txt
+      - URI_BLACKLIST_FILE=/var/squirrel/blacklist.txt
       - FRONTIER_CONTEXT_CONFIG_FILE=/var/squirrel/spring-config/frontier-context.xml
-      - SEED_FILE=/var/squirrel/seeds.csv
+      - SEED_FILE=/var/squirrel/seeds.txt
       - MDB_HOST_NAME=mongodb
       - MDB_PORT=27017
       - MDB_CONNECTION_TIME_OUT=5000
@@ -26,12 +27,13 @@ services:
       - COMMUNICATION_WITH_WEBSERVICE=false
       - VISUALIZATION_OF_CRAWLED_GRAPH=false
       - JVM_ARGS=-Xmx8g 
-
     volumes:
       - ./data/frontier:/var/squirrel/data
       - ./seed/seeds.csv:/var/squirrel/seeds.csv:ro
       - ./seed/seeds.txt:/var/squirrel/seeds.txt:ro
       - ./whitelist/whitelist.txt:/var/squirrel/whitelist.txt:ro
+      - ./whitelist/blacklist.txt:/var/squirrel/blacklist.txt:ro
+      - ./spring-config:/var/squirrel/spring-config
     command: java -cp squirrel.jar org.dice_research.squirrel.components.FrontierComponentStarter
 
 

pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.dice-research</groupId>
     <artifactId>squirrel</artifactId>
-    <version>0.4.0</version>
+    <version>0.5.0</version>
     <packaging>pom</packaging>
     <inceptionYear>2017</inceptionYear>
     <name>Squirrel</name>
@@ -345,11 +345,17 @@
                 <version>4.12</version>
                 <scope>test</scope>
             </dependency>
+            <!-- System rules for setting environment variables 
+            <dependency>
+              <groupId>com.github.stefanbirkner</groupId>
+              <artifactId>system-rules</artifactId>
+              <scope>test</scope> -->
             <!-- System Lambda for setting environment variables -->
             <dependency>
                 <groupId>com.github.stefanbirkner</groupId>
                 <artifactId>system-lambda</artifactId>
                 <version>1.0.0</version>
+                <scope>test</scope>
             </dependency>
             <!-- ~~~~~~~~~~~~~~~~~~~ End Testing ~~~~~~~~~~~~~~~~~~~~~~ -->
 
@@ -486,6 +492,17 @@
                             </transformer>
                             <transformer
                                 implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                            <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                <resource>META-INF/spring.handlers</resource>
+                            </transformer>
+                            <!-- Important for Spring to be able to map XML 
+                                Schema files to local files. Otherwise, Spring may abort the processing of 
+                                XML files because of unavailable XML Schema files -->
+                            <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                <resource>META-INF/spring.schemas</resource>
+                            </transformer>
                         </transformers>
                     </configuration>
                     <executions>

seed/seeds.txt

@@ -0,0 +1 @@
+http://dbpedia.org/resource/New_York

spring-config/frontier-context.xml

@@ -72,10 +72,12 @@
 		<constructor-arg index="0" ref="mongoDBKnowUriFilter" />
 		<constructor-arg index="1">
 			<list>
-				<!-- <ref bean="depthFilter" /> -->
+                <ref bean="schemeFilter" />
+				<ref bean="whiteFilter" />
+                <ref bean="blackFilter" />
 			</list>
 		</constructor-arg>
-		<constructor-arg index="2" value="OR" />
+		<constructor-arg index="2" value="AND" />
 	</bean>
 
 	<!-- Triple Store sparql implementation -->
@@ -98,19 +100,33 @@
 
 
 	<bean id="mongoDBKnowUriFilter"
-		class="org.dice_research.squirrel.data.uri.filter.MongoDBKnowUriFilter">
+		class="org.dice_research.squirrel.data.uri.filter.MongoDBKnowUriFilter"
+        init-method="open">
 		<constructor-arg index="0"
 			value="#{systemEnvironment['MDB_HOST_NAME']}" />
 		<constructor-arg index="1"
 			value="#{systemEnvironment['MDB_PORT']}" />
-
 	</bean>
 
-
-	<bean id="depthFilter"
+    <bean id="schemeFilter"
+        class="org.dice_research.squirrel.data.uri.filter.SchemeBasedUriFilter">
+    </bean>
+    <bean id="whiteFilter"
+        class="org.dice_research.squirrel.data.uri.filter.RegexBasedWhiteListFilter">
+        <constructor-arg index="0" value="#{systemEnvironment['URI_WHITELIST_FILE']}" />
+        <constructor-arg index="1" value="false" />
+    </bean>
+    <bean id="blackFilter"
+        class="org.dice_research.squirrel.data.uri.filter.RegexBasedBlackListFilter">
+        <constructor-arg index="0" value="#{systemEnvironment['URI_BLACKLIST_FILE']}" />
+        <constructor-arg index="1" value="false" />
+    </bean>
+
+
+	<!-- <bean id="depthFilter"
 		class="org.dice_research.squirrel.data.uri.filter.DepthFilter">
 		<constructor-arg index="0" value="3" />
-	</bean>
+	</bean>-->
 
 
 

squirrel.api/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.dice-research</groupId>
         <artifactId>squirrel</artifactId>
-        <version>0.4.0</version>
+        <version>0.5.0</version>
     </parent>
     <artifactId>squirrel.api</artifactId>
     <packaging>jar</packaging>
@@ -182,7 +182,7 @@
             <artifactId>hsqldb</artifactId>
         </dependency>
 
-        <!-- We use the simpleframework to use local HTTP servers for our 
+        <!-- We use the simpleframework to use local HTTP servers for our
             JUnit tests and test scenarios -->
         <dependency>
             <groupId>org.simpleframework</groupId>
@@ -228,4 +228,4 @@
         <!-- ~~~~~~~~~~~~~~~~~~~ End Logging ~~~~~~~~~~~~~~~~~~~~~~ -->
     </dependencies>
 
-</project>
+</project>

squirrel.api/src/main/java/org/dice_research/squirrel/Constants.java

@@ -41,6 +41,7 @@ public class Constants {
     public static final String URI_HASH_KEY = "HashValue";
     public static final String UUID_KEY = "UUID";
 
+
     /**
      * The preferred date for recrawling a URI is assumed to be a timestamp (in ms
      * from 1st January 1970).
@@ -79,4 +80,5 @@ public class Constants {
 
     public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
     public static final String DEFAULT_USER_AGENT = "Squirrel";
+    public static final int JAVASCRIPT_WAIT_TIME = 30 * 1000; //30 seconds
 }

squirrel.api/src/main/java/org/dice_research/squirrel/data/uri/filter/AbstractKnownUriFilterDecorator.java

@@ -55,4 +55,9 @@ public void close() throws IOException {
         }
     }
 
+    @Override
+    public void open() {
+        decorated.open();
+    }
+
 }

squirrel.api/src/main/java/org/dice_research/squirrel/data/uri/filter/KnownUriFilter.java

@@ -5,49 +5,63 @@
 import org.dice_research.squirrel.data.uri.CrawleableUri;
 
 /**
- * A {@link UriFilter} that works like a blacklist filter and contains only those
- * URIs on its blacklist that the crawler already has seen before.
+ * A {@link UriFilter} that works like a blacklist filter and contains only
+ * those URIs on its blacklist that the crawler already has seen before.
  *
  * @author Michael Röder ([email protected])
  */
 public interface KnownUriFilter extends UriFilter {
 
     /**
-     * Adds the given URI to the list of already known URIs. Works like calling {@link #add(CrawleableUri, long)} with the current system time.
+     * Adds the given URI to the list of already known URIs. Works like calling
+     * {@link #add(CrawleableUri, long)} with the current system time.
      *
      * @param uri the URI that should be added to the list.
-     * @param nextCrawlTimestamp The time at which the given URI should be crawled next.
+     * 
      */
-    public void add(CrawleableUri uri, long nextCrawlTimestamp);
+    default void add(CrawleableUri uri) {
+        add(uri, System.currentTimeMillis()); // TODO This does not make much sense, since it will set the
+                                              // nextCrawlTimestamp to the current time!
+    }
 
     /**
-     * Adds the given URI to the list of already known URIs together with the the time at which it has been crawled.
+     * Adds the given URI to the list of already known URIs. Works like calling
+     * {@link #add(CrawleableUri, long)} with the current system time.
      *
-     * @param uri       the URI that should be added to the list.
+     * @param uri                the URI that should be added to the list.
+     * @param nextCrawlTimestamp The time at which the given URI should be crawled
+     *                           next.
+     */
+    void add(CrawleableUri uri, long nextCrawlTimestamp);
+
+    /**
+     * Adds the given URI to the list of already known URIs together with the the
+     * time at which it has been crawled.
+     *
+     * @param uri                the URI that should be added to the list.
      * @param lastCrawlTimestamp the time at which the given URI has eben crawled.
-     * @param nextCrawlTimestamp The time at which the given URI should be crawled next.
+     * @param nextCrawlTimestamp The time at which the given URI should be crawled
+     *                           next.
      */
     void add(CrawleableUri uri, long lastCrawlTimestamp, long nextCrawlTimestamp);
-
-    public default void add(CrawleableUri uri) {
-    	add(uri, System.currentTimeMillis());
-    }
 
     /**
-     * Returns all {@link CrawleableUri}s which have to be recrawled. This means their time to next crawl has passed.
+     * Returns all {@link CrawleableUri}s which have to be recrawled. This means
+     * their time to next crawl has passed.
      *
      * @return The outdated {@link CrawleableUri}s.
      */
-    public List<CrawleableUri> getOutdatedUris();
+    List<CrawleableUri> getOutdatedUris();
 
     /**
      * count the numbers of known URIs
+     * 
      * @return the number of entries from a open queue
      */
     long count();
-    
+
     /**
      * Opens the queue and allocates necessary resources.
      */
-    public void open();
+    void open();
 }

squirrel.api/src/main/java/org/dice_research/squirrel/data/uri/filter/UriFilter.java

@@ -21,13 +21,4 @@ public interface UriFilter {
      *         requirements imposed by this filter. Otherwise false is returned.
      */
     public boolean isUriGood(CrawleableUri uri);
-
-
-    /**
-     * Adds the given URI to the list of already known URIs. Works like calling {@link #add(CrawleableUri, long)} with the current system time.
-     *
-     * @param uri the URI that should be added to the list.
-     * 
-     */    
-    public void add(CrawleableUri uri);
 }

squirrel.api/src/main/java/org/dice_research/squirrel/data/uri/filter/UriFilterComposer.java

@@ -5,14 +5,14 @@
  * This class represents a composition of two or more filters,
  * requiring at least one @link {org.dice_research.squirrel.data.uri.filter.KnownUriFilter}
  * 
- *  * @author Geraldo de Souza Junior ([email protected])
+ * @author Geraldo de Souza Junior ([email protected])
  * 
  */
 
 public interface UriFilterComposer extends UriFilter {
 
 	/**
-	 * Returnsthe KnowUriFilter from this {@link UriFilterComposer}
+	 * Returns the KnowUriFilter from this {@link UriFilterComposer}
 	 * 
 	 * @return KnownUriFilter
 	 */
@@ -25,5 +25,7 @@ public interface UriFilterComposer extends UriFilter {
 	 * @param knownUriFilter
 	 */
 	public void setKnownUriFilter(KnownUriFilter knownUriFilter);
+
+
 
 }