Merge pull request #496 from boudicca-events/abl/migrate-eventcollectors

migrate all eventcollectors
boudicca-events · Sep 9, 2024 · 6d184dd · 6d184dd
2 parents 48365ba + ec3efda
commit 6d184dd
Show file tree

Hide file tree

Showing 42 changed files with 869 additions and 644 deletions.
diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/format/UrlUtils.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/format/UrlUtils.kt
@@ -0,0 +1,26 @@
+package base.boudicca.format
+
+import java.net.URI
+
+object UrlUtils {
+    fun parse(string: String?): URI? {
+        if (string.isNullOrEmpty()) {
+            return null
+        }
+        val trimmed = string.trim()
+        return try {
+            URI.create(string)
+        } catch (e: IllegalArgumentException) {
+            val fixedUrl = tryFixUrl(trimmed)
+            URI.create(fixedUrl)
+        }
+    }
+
+    //invalid urls everywhere -.-
+    private fun tryFixUrl(url: String): String {
+        return url
+            .replace("[", "%5B")
+            .replace("]", "%5D")
+            .replace(" ", "%20")
+    }
+}
diff --git a/.../common-model/src/main/kotlin/base/boudicca/model/structured/AbstractStructuredBuilder.kt b/.../common-model/src/main/kotlin/base/boudicca/model/structured/AbstractStructuredBuilder.kt
@@ -2,10 +2,10 @@ package base.boudicca.model.structured
 
 import base.boudicca.Property
 
-abstract class AbstractStructuredBuilder<T>(protected val data: MutableMap<Key, String> = mutableMapOf()) {
+abstract class AbstractStructuredBuilder<T, B : AbstractStructuredBuilder<T, B>>(protected val data: MutableMap<Key, String> = mutableMapOf()) {
 
     @Throws(IllegalArgumentException::class)
-    fun <P> withProperty(property: Property<P>, value: P?): AbstractStructuredBuilder<T> {
+    fun <P> withProperty(property: Property<P>, value: P?): B {
         return withProperty(property, null, value)
     }
 
@@ -14,9 +14,10 @@ abstract class AbstractStructuredBuilder<T>(protected val data: MutableMap<Key,
         property: Property<P>,
         language: String?,
         value: P?
-    ): AbstractStructuredBuilder<T> {
+    ): B {
         if (value == null) {
-            return this
+            @Suppress("UNCHECKED_CAST")
+            return this as B
         }
         return withKeyValuePair(
             property.getKey(language),
@@ -27,13 +28,15 @@ abstract class AbstractStructuredBuilder<T>(protected val data: MutableMap<Key,
     fun withKeyValuePair(
         key: Key,
         value: String?
-    ): AbstractStructuredBuilder<T> {
+    ): B {
         if (value.isNullOrEmpty()) {
-            return this
+            @Suppress("UNCHECKED_CAST")
+            return this as B
         }
 
         data[key] = value
-        return this
+        @Suppress("UNCHECKED_CAST")
+        return this as B
     }
 
     abstract fun build(): T

diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEntry.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEntry.kt
@@ -4,6 +4,7 @@ import base.boudicca.Property
 import base.boudicca.keyfilters.KeyFilters
 import base.boudicca.keyfilters.KeySelector
 import base.boudicca.model.Entry
+import base.boudicca.model.structured.StructuredEvent.StructuredEventBuilder
 import java.util.*
 
 
@@ -52,8 +53,12 @@ fun structuredEntryBuilder() : StructuredEntryBuilder {
     return StructuredEntryBuilder()
 }
 
-class StructuredEntryBuilder(data: Map<Key, String> = emptyMap()) : AbstractStructuredBuilder<StructuredEntry>(data.toMutableMap()) {
+class StructuredEntryBuilder(data: Map<Key, String> = emptyMap()) : AbstractStructuredBuilder<StructuredEntry, StructuredEntryBuilder>(data.toMutableMap()) {
     override fun build(): StructuredEntry {
         return data.toMap()
     }
+
+    fun copy(): StructuredEntryBuilder {
+        return StructuredEntryBuilder(data.toMutableMap())
+    }
 }
diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt
@@ -98,18 +98,43 @@ data class StructuredEvent(val name: String, val startDate: OffsetDateTime, val
             return Optional.of(StructuredEvent(name, startDate, data))
         }
 
+        fun builder(): StructuredEventBuilder {
+            return StructuredEventBuilder(null, null)
+        }
+
         fun builder(name: String, startDate: OffsetDateTime): StructuredEventBuilder {
             return StructuredEventBuilder(name, startDate)
         }
     }
 
     class StructuredEventBuilder internal constructor(
-        private val name: String,
-        private val startDate: OffsetDateTime,
+        private var name: String?,
+        private var startDate: OffsetDateTime?,
         data: Map<Key, String> = emptyMap()
-    ) : AbstractStructuredBuilder<StructuredEvent>(data.toMutableMap()) {
+    ) : AbstractStructuredBuilder<StructuredEvent, StructuredEventBuilder>(data.toMutableMap()) {
+
+        fun withName(name: String): StructuredEventBuilder {
+            this.name = name
+            return this
+        }
+
+        fun withStartDate(startDate: OffsetDateTime): StructuredEventBuilder {
+            this.startDate = startDate
+            return this
+        }
+
+        fun copy(): StructuredEventBuilder {
+            return StructuredEventBuilder(name, startDate, data.toMutableMap())
+        }
+
         override fun build(): StructuredEvent {
-            return StructuredEvent(name, startDate, data.toMap())
+            if (name == null) {
+                throw IllegalStateException("name cannot be null for an event!")
+            }
+            if (startDate == null) {
+                throw IllegalStateException("startDate cannot be null for an event!")
+            }
+            return StructuredEvent(name!!, startDate!!, data.toMap())
         }
     }
 }
diff --git a/...ollector-client/src/main/kotlin/base/boudicca/api/eventcollector/EventCollectionRunner.kt b/...ollector-client/src/main/kotlin/base/boudicca/api/eventcollector/EventCollectionRunner.kt
@@ -69,7 +69,7 @@ class EventCollectionRunner(
             if (event.name.isBlank()) {
                 LOG.warn("event has empty name: $event")
             }
-            if (event.data[SemanticKeys.SOURCES].isNullOrBlank()) {
+            if (event.toStructuredEvent().getProperty(SemanticKeys.SOURCES_PROPERTY).isEmpty()) {
                 LOG.error("event has no sources: $event")
             }
             for (entry in event.data.entries) {

diff --git a/...ector-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/IcalCollector.kt b/...ector-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/IcalCollector.kt
@@ -3,6 +3,7 @@ package base.boudicca.api.eventcollector.collectors
 import base.boudicca.api.eventcollector.EventCollector
 import base.boudicca.api.eventcollector.collectors.util.IcalParser
 import base.boudicca.model.Event
+import base.boudicca.model.structured.StructuredEvent
 import biweekly.component.VEvent
 import java.util.*
 
@@ -18,7 +19,7 @@ abstract class IcalCollector(private val name: String) : EventCollector {
         return name
     }
 
-    override fun collectEvents(): List<Event> {
+    override fun collectStructuredEvents(): List<StructuredEvent> {
         val icalResources = getAllIcalResources()
 
         return icalResources
@@ -36,14 +37,14 @@ abstract class IcalCollector(private val name: String) : EventCollector {
     /**
      * maps one VEvent to an (optional) Event. implementations can override this method to for example extract additional properties from the VEvent
      */
-    open fun mapVEventToEvent(vEvent: VEvent): Optional<Event> {
+    open fun mapVEventToEvent(vEvent: VEvent): Optional<StructuredEvent> {
         return IcalParser.mapVEventToEvent(vEvent)
     }
 
     /**
      * postProcess the Event. can be overridden to add for example static additional properties to the Event.
      */
-    open fun postProcess(event: Event): Event {
+    open fun postProcess(event: StructuredEvent): StructuredEvent {
         return event
     }
 

diff --git a/...tor-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParser.kt b/...tor-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParser.kt
@@ -1,16 +1,17 @@
 package base.boudicca.api.eventcollector.collectors.util
 
 import base.boudicca.SemanticKeys
-import base.boudicca.model.Event
+import base.boudicca.TextProperty
+import base.boudicca.model.structured.StructuredEvent
 import biweekly.Biweekly
 import biweekly.component.VEvent
 import biweekly.property.DateEnd
 import biweekly.property.DateStart
 import biweekly.util.ICalDate
 import org.slf4j.LoggerFactory
+import java.net.URI
 import java.time.OffsetDateTime
 import java.time.ZoneId
-import java.time.format.DateTimeFormatter
 import java.util.*
 
 /**
@@ -24,7 +25,7 @@ object IcalParser {
      * parses an icalResource (aka the string contents of a .ics file) to vEvents and maps them to Events
      * @param icalResource the ics file to parse and map
      */
-    fun parseAndMapToEvents(icalResource: String): List<Event> {
+    fun parseAndMapToEvents(icalResource: String): List<StructuredEvent> {
         val vEvents = parseToVEvents(icalResource)
         return mapVEventsToEvents(vEvents)
     }
@@ -43,7 +44,7 @@ object IcalParser {
     /**
      * maps a collection of vEvents to Events
      */
-    fun mapVEventsToEvents(vEvents: List<VEvent>): List<Event> {
+    fun mapVEventsToEvents(vEvents: List<VEvent>): List<StructuredEvent> {
         return vEvents
             .map { mapVEventToEvent(it) } //map to optional events
             .filter { it.isPresent } //filter only successful ones
@@ -53,7 +54,7 @@ object IcalParser {
     /**
      * maps a single vEvent to an Event. returns an optional which is empty when the vEvent does not include the required data for creating an Event
      */
-    fun mapVEventToEvent(vEvent: VEvent): Optional<Event> {
+    fun mapVEventToEvent(vEvent: VEvent): Optional<StructuredEvent> {
         if (vEvent.dateStart == null) {
             LOG.warn("event with uid ${vEvent.uid} and url ${vEvent.url} has no startDate!")
             return Optional.empty()
@@ -62,28 +63,29 @@ object IcalParser {
         val name = vEvent.summary.value
         val startDate = getStartDate(vEvent.dateStart)
 
-        val data = mutableMapOf<String, String>()
+        val builder = StructuredEvent
+            .builder(name, startDate)
         if (vEvent.location != null) {
-            data[SemanticKeys.LOCATION_NAME] = vEvent.location.value
+            builder.withProperty(SemanticKeys.LOCATION_NAME_PROPERTY, vEvent.location.value)
         }
         if (vEvent.description != null) {
-            data[SemanticKeys.DESCRIPTION] = vEvent.description.value
+            builder.withProperty(SemanticKeys.DESCRIPTION_TEXT_PROPERTY, vEvent.description.value)
         }
         if (vEvent.url != null) {
-            data[SemanticKeys.URL] = vEvent.url.value
+            builder.withProperty(SemanticKeys.URL_PROPERTY, URI.create(vEvent.url.value))
         }
         if (vEvent.uid != null) {
-            data["ics.event.uid"] = vEvent.uid.value
+            builder.withProperty(TextProperty("ics.event.uid"), vEvent.uid.value)
         }
         if (vEvent.dateEnd != null) {
-            data[SemanticKeys.ENDDATE] = getEndDate(vEvent.dateEnd)
+            builder.withProperty(SemanticKeys.ENDDATE_PROPERTY, getEndDate(vEvent.dateEnd))
         }
 
-        return Optional.of(Event(name, startDate, data))
+        return Optional.of(builder.build())
     }
 
-    private fun getEndDate(dateEnd: DateEnd): String {
-        return DateTimeFormatter.ISO_DATE_TIME.format(getDate(dateEnd.value))
+    private fun getEndDate(dateEnd: DateEnd): OffsetDateTime {
+        return getDate(dateEnd.value)
     }
 
     private fun getStartDate(dateStart: DateStart): OffsetDateTime {

diff --git a/...client/src/test/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParserTest.kt b/...client/src/test/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParserTest.kt
@@ -53,7 +53,7 @@ class IcalParserTest {
         val data = event.data
         assertEquals("Other title", event.name)
         assertEquals(OffsetDateTime.of(2007, 12, 14, 1, 0, 0, 0, ZoneOffset.ofHours(1)), event.startDate)
-        assertEquals("2007-12-14T01:10:00+01:00", data[SemanticKeys.ENDDATE])
+        assertEquals("2007-12-14T01:10:00+01:00", data[SemanticKeys.ENDDATE + ":format=date"])
         assertEquals(
             "a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at",
             data["ics.event.uid"]
@@ -135,15 +135,15 @@ class IcalParserTest {
     }
 
     private fun mapVEventToEvent(vEvent: VEvent): Event {
-        return IcalParser.mapVEventToEvent(vEvent).get()
+        return IcalParser.mapVEventToEvent(vEvent).get().toFlatEvent()
     }
 
     private fun tryMapVEventToEvent(vEvent: VEvent): Optional<Event> {
-        return IcalParser.mapVEventToEvent(vEvent)
+        return IcalParser.mapVEventToEvent(vEvent).map { it.toFlatEvent() }
     }
 
     private fun loadAndParseAndMapEvents(testFile: String): List<Event> {
-        return IcalParser.parseAndMapToEvents(loadTestData(testFile))
+        return IcalParser.parseAndMapToEvents(loadTestData(testFile)).map { it.toFlatEvent() }
     }
 
     private fun loadTestData(testFile: String) =

diff --git a/...her-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt b/...her-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt
@@ -92,6 +92,7 @@ class EventService @Autowired constructor(
             queryParts.add(equals(flag!!, "true"))
         }
         if (!searchDTO.bandName.isNullOrBlank()) {
+            //TODO maybe change sometime to equals?
             queryParts.add(contains(SemanticKeys.CONCERT_BANDLIST, searchDTO.bandName!!))
         }
         if (searchDTO.includeRecurring != true) {

diff --git a/...lectors/src/main/kotlin/events/boudicca/eventcollector/collectors/AlpenvereinCollector.kt b/...lectors/src/main/kotlin/events/boudicca/eventcollector/collectors/AlpenvereinCollector.kt
@@ -4,6 +4,7 @@ import base.boudicca.SemanticKeys
 import base.boudicca.TextProperty
 import base.boudicca.api.eventcollector.Fetcher
 import base.boudicca.api.eventcollector.TwoStepEventCollector
+import base.boudicca.format.UrlUtils
 import base.boudicca.model.structured.StructuredEvent
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
@@ -48,11 +49,14 @@ class AlpenvereinCollector : TwoStepEventCollector<String>("alpenverein") {
             .builder(name, startDate)
             .withProperty(SemanticKeys.NAME_PROPERTY, name)
             .withProperty(SemanticKeys.ENDDATE_PROPERTY, endDate)
-            .withProperty(SemanticKeys.URL_PROPERTY, URI.create(event))
+            .withProperty(SemanticKeys.URL_PROPERTY, UrlUtils.parse(event))
             .withProperty(SemanticKeys.SOURCES_PROPERTY, listOf(event))
             .withProperty(SemanticKeys.TYPE_PROPERTY, "sport")
             .withProperty(TextProperty("sport.participation"), "active")
-            .withProperty(SemanticKeys.DESCRIPTION_TEXT_PROPERTY, eventSite.select("div.elementBoxSheet div.elementText").text())
+            .withProperty(
+                SemanticKeys.DESCRIPTION_TEXT_PROPERTY,
+                eventSite.select("div.elementBoxSheet div.elementText").text()
+            )
             .withProperty(SemanticKeys.PICTURE_URL_PROPERTY, getPictureUrl(eventSite))
             .withProperty(SemanticKeys.LOCATION_CITY_PROPERTY, getLocationCity(eventSite))
             .build()
@@ -71,7 +75,7 @@ class AlpenvereinCollector : TwoStepEventCollector<String>("alpenverein") {
         if (imageElements.isNotEmpty()) {
             val pictureUrl = imageElements.first()!!.attr("src")
             if (!pictureUrl.isNullOrEmpty()) {
-                return URI.create(normalizeUrl(pictureUrl))
+                return UrlUtils.parse(normalizeUrl(pictureUrl))
             }
         }
         return null