Adding protection against infinite loops

lucidworks · Sep 1, 2020 · b5a8647 · b5a8647
1 parent d2e7d04
commit b5a8647
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/main/java/com/lucidworks/fusion/connector/service/DrupalContentCrawler.java b/src/main/java/com/lucidworks/fusion/connector/service/DrupalContentCrawler.java
@@ -54,8 +54,8 @@ public void startCrawling() {
                     String responseBody = drupalHttpClient.getContent(url);
                     if (responseBody != null) {
                         currentStepContent.put(url, responseBody);
-                        urlsVisitedInCurrentStep.add(url);
                     }
+                    urlsVisitedInCurrentStep.add(url);
                 });
 
                 drupalUrls.removeAll(urlsVisitedInCurrentStep);