Merge pull request #459 from boudicca-events/abl/ical-refactoring

Abl/ical refactoring

.github/workflows/gradle.yml

@@ -10,6 +10,7 @@ on:
 
 permissions:
   contents: read
+  checks: write
 
 jobs:
   build-gradle-project:
@@ -28,3 +29,8 @@ jobs:
         uses: gradle/actions/setup-gradle@v3
       - name: Execute Gradle build
         run: ./gradlew test
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: '**/build/test-results/test/TEST-*.xml'

boudicca.base/eventcollector-client/build.gradle.kts

@@ -5,6 +5,7 @@ plugins {
 
 dependencies {
     api(project(":boudicca.base:semantic-conventions"))
+    api("net.sf.biweekly:biweekly:0.6.8")
     implementation(project(":boudicca.base:publisher-client"))
     implementation(project(":boudicca.base:ingest-client"))
     implementation(project(":boudicca.base:enricher-client"))

boudicca.base/eventcollector-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/IcalCollector.kt

@@ -0,0 +1,53 @@
+package base.boudicca.api.eventcollector.collectors
+
+import base.boudicca.api.eventcollector.EventCollector
+import base.boudicca.api.eventcollector.collectors.util.IcalParser
+import base.boudicca.model.Event
+import biweekly.component.VEvent
+import java.util.*
+
+/**
+ * EventCollector implementation which will collect events from ical resources.
+ * implementations need to overwrite the #getAllIcalResources method and return fetched .ics files from there
+ * implementations also can overwrite the #mapVEventToEvent method to read additional properties of the VEvent
+ * implementations also can overwrite the #postProcess method to add custom properties or similar to the parsed events
+ */
+abstract class IcalCollector(private val name: String) : EventCollector {
+
+    override fun getName(): String {
+        return name
+    }
+
+    override fun collectEvents(): List<Event> {
+        val icalResources = getAllIcalResources()
+
+        return icalResources
+            .flatMap { parseSingleIcalResource(it) }
+            .map { mapVEventToEvent(it) }
+            .filter { it.isPresent }
+            .map { postProcess(it.get()) }
+    }
+
+    /**
+     * method which should return all ical resources (.ics files) as strings.
+     */
+    abstract fun getAllIcalResources(): List<String>
+
+    /**
+     * maps one VEvent to an (optional) Event. implementations can override this method to for example extract additional properties from the VEvent
+     */
+    open fun mapVEventToEvent(vEvent: VEvent): Optional<Event> {
+        return IcalParser.mapVEventToEvent(vEvent)
+    }
+
+    /**
+     * postProcess the Event. can be overridden to add for example static additional properties to the Event.
+     */
+    open fun postProcess(event: Event): Event {
+        return event
+    }
+
+    private fun parseSingleIcalResource(icalResource: String): List<VEvent> {
+        return IcalParser.parseToVEvents(icalResource)
+    }
+}

boudicca.base/eventcollector-client/src/main/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParser.kt

@@ -0,0 +1,96 @@
+package base.boudicca.api.eventcollector.collectors.util
+
+import base.boudicca.SemanticKeys
+import base.boudicca.model.Event
+import biweekly.Biweekly
+import biweekly.component.VEvent
+import biweekly.property.DateEnd
+import biweekly.property.DateStart
+import biweekly.util.ICalDate
+import org.slf4j.LoggerFactory
+import java.time.OffsetDateTime
+import java.time.ZoneId
+import java.time.format.DateTimeFormatter
+import java.util.*
+
+/**
+ * utility class for parsing and mapping ical resources to VEvents and then to Events
+ */
+object IcalParser {
+
+    private val LOG = LoggerFactory.getLogger(this::class.java)
+
+    /**
+     * parses an icalResource (aka the string contents of a .ics file) to vEvents and maps them to Events
+     * @param icalResource the ics file to parse and map
+     */
+    fun parseAndMapToEvents(icalResource: String): List<Event> {
+        val vEvents = parseToVEvents(icalResource)
+        return mapVEventsToEvents(vEvents)
+    }
+
+    /**
+     * parses an icalResource (aka the string contents of a .ics file) to vEvents
+     * @param icalResource the ics file to parse and map
+     */
+    fun parseToVEvents(icalResource: String): List<VEvent> {
+        val allCalendars = Biweekly.parse(icalResource).all()
+        val vEvents = allCalendars
+            .flatMap { it.events }
+        return vEvents
+    }
+
+    /**
+     * maps a collection of vEvents to Events
+     */
+    fun mapVEventsToEvents(vEvents: List<VEvent>): List<Event> {
+        return vEvents
+            .map { mapVEventToEvent(it) } //map to optional events
+            .filter { it.isPresent } //filter only successful ones
+            .map { it.get() }
+    }
+
+    /**
+     * maps a single vEvent to an Event. returns an optional which is empty when the vEvent does not include the required data for creating an Event
+     */
+    fun mapVEventToEvent(vEvent: VEvent): Optional<Event> {
+        if (vEvent.dateStart == null) {
+            LOG.warn("event with uid ${vEvent.uid} and url ${vEvent.url} has no startDate!")
+            return Optional.empty()
+        }
+
+        val name = vEvent.summary.value
+        val startDate = getStartDate(vEvent.dateStart)
+
+        val data = mutableMapOf<String, String>()
+        if (vEvent.location != null) {
+            data[SemanticKeys.LOCATION_NAME] = vEvent.location.value
+        }
+        if (vEvent.description != null) {
+            data[SemanticKeys.DESCRIPTION] = vEvent.description.value
+        }
+        if (vEvent.url != null) {
+            data[SemanticKeys.URL] = vEvent.url.value
+        }
+        if (vEvent.uid != null) {
+            data["ics.event.uid"] = vEvent.uid.value
+        }
+        if (vEvent.dateEnd != null) {
+            data[SemanticKeys.ENDDATE] = getEndDate(vEvent.dateEnd)
+        }
+
+        return Optional.of(Event(name, startDate, data))
+    }
+
+    private fun getEndDate(dateEnd: DateEnd): String {
+        return DateTimeFormatter.ISO_DATE_TIME.format(getDate(dateEnd.value))
+    }
+
+    private fun getStartDate(dateStart: DateStart): OffsetDateTime {
+        return getDate(dateStart.value)
+    }
+
+    private fun getDate(iCalDate: ICalDate): OffsetDateTime {
+        return iCalDate.toInstant().atZone(ZoneId.of("Europe/Vienna")).toOffsetDateTime()
+    }
+}

boudicca.base/eventcollector-client/src/test/kotlin/base/boudicca/api/eventcollector/collectors/util/IcalParserTest.kt

@@ -0,0 +1,151 @@
+package base.boudicca.api.eventcollector.collectors.util
+
+import base.boudicca.SemanticKeys
+import base.boudicca.model.Event
+import biweekly.component.VEvent
+import biweekly.property.DateStart
+import biweekly.property.Summary
+import biweekly.property.Uid
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import java.time.OffsetDateTime
+import java.time.ZoneId
+import java.time.ZoneOffset
+import java.time.ZonedDateTime
+import java.util.*
+
+class IcalParserTest {
+
+    @Test
+    fun testEmptyIcal() {
+        assertTrue(IcalParser.parseAndMapToEvents("").isEmpty())
+    }
+
+    @Test
+    fun testInvalidIcal() {
+        assertTrue(IcalParser.parseAndMapToEvents("THIS IS NOT AN ICAL FILE").isEmpty())
+    }
+
+    @Test
+    fun testSimpleIcsFile() {
+        val events = loadAndParseAndMapEvents("test1.ics")
+
+        assertEquals(1, events.size)
+        val event = events[0]
+        val data = event.data
+        assertEquals("No title", event.name)
+        assertEquals(OffsetDateTime.of(2007, 12, 14, 1, 0, 0, 0, ZoneOffset.ofHours(1)), event.startDate)
+        assertEquals(
+            "a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at",
+            data["ics.event.uid"]
+        )
+        assertEquals("https://valug.at/events/2007-12-14/", data[SemanticKeys.URL])
+        assertFalse(data.containsKey(SemanticKeys.DESCRIPTION))
+        assertFalse(data.containsKey(SemanticKeys.ENDDATE))
+    }
+
+    @Test
+    fun testSecondSimpleIcsFile() {
+        val events = loadAndParseAndMapEvents("test2.ics")
+
+        assertEquals(1, events.size)
+        val event = events[0]
+        val data = event.data
+        assertEquals("Other title", event.name)
+        assertEquals(OffsetDateTime.of(2007, 12, 14, 1, 0, 0, 0, ZoneOffset.ofHours(1)), event.startDate)
+        assertEquals("2007-12-14T01:10:00+01:00", data[SemanticKeys.ENDDATE])
+        assertEquals(
+            "a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at",
+            data["ics.event.uid"]
+        )
+        assertEquals("https://valug.at/events/2007-12-14/", data[SemanticKeys.URL])
+        assertEquals("Some description", data[SemanticKeys.DESCRIPTION])
+    }
+
+    @Test
+    fun testMultipleEvents() {
+        val events = loadAndParseAndMapEvents("test_multiple_events.ics")
+
+        assertEquals(2, events.size)
+        assertEquals("event1", events[0].name)
+        assertEquals("event2", events[1].name)
+    }
+
+    @Test
+    fun testMultipleCalendars() {
+        val events = loadAndParseAndMapEvents("test_multiple_calendars.ics")
+
+        assertEquals(4, events.size)
+        assertEquals("event1_1", events[0].name)
+        assertEquals("event1_2", events[1].name)
+        assertEquals("event2_1", events[2].name)
+        assertEquals("event2_2", events[3].name)
+    }
+
+    @Test
+    fun testParseIcalResource() {
+        val vEvents = IcalParser.parseToVEvents(loadTestData("test1.ics"))
+
+        assertEquals(1, vEvents.size)
+        assertEquals(
+            "a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at",
+            vEvents[0].uid.value
+        )
+    }
+
+    @Test
+    fun testMapVEventToEvent() {
+        val now = ZonedDateTime.now(ZoneId.of("Europe/Vienna")).withNano(0) //conversion to date does not keep nanos
+            .toOffsetDateTime()
+        val vEvent = VEvent()
+        vEvent.uid = Uid("myUid")
+        vEvent.summary = Summary("mySummary")
+        vEvent.dateStart = DateStart(Date(now.toInstant().toEpochMilli()))
+
+        val event = mapVEventToEvent(vEvent)
+
+        assertEquals("myUid", event.data["ics.event.uid"])
+        assertEquals("mySummary", event.name)
+        assertEquals(now, event.startDate)
+    }
+
+    @Test
+    fun testMapInvalidVEventToEvent() {
+        val vEvent = VEvent()
+
+        val event = tryMapVEventToEvent(vEvent)
+
+        assertFalse(event.isPresent)
+    }
+
+    @Test
+    fun testMapVEventsToEvent() {
+        val now = OffsetDateTime.now().withNano(0) //conversion to date does not keep nanos
+        val vEvent = VEvent()
+        vEvent.uid = Uid("myUid")
+        vEvent.summary = Summary("mySummary")
+        vEvent.dateStart = DateStart(Date(now.toInstant().toEpochMilli()))
+
+        val invalidVEvent = VEvent()
+        val vEvents = listOf(vEvent, vEvent, invalidVEvent)
+
+        val events = IcalParser.mapVEventsToEvents(vEvents)
+
+        assertEquals(2, events.size)
+    }
+
+    private fun mapVEventToEvent(vEvent: VEvent): Event {
+        return IcalParser.mapVEventToEvent(vEvent).get()
+    }
+
+    private fun tryMapVEventToEvent(vEvent: VEvent): Optional<Event> {
+        return IcalParser.mapVEventToEvent(vEvent)
+    }
+
+    private fun loadAndParseAndMapEvents(testFile: String): List<Event> {
+        return IcalParser.parseAndMapToEvents(loadTestData(testFile))
+    }
+
+    private fun loadTestData(testFile: String) =
+        String(this.javaClass.getResourceAsStream("/ical/$testFile").readAllBytes())
+}

boudicca.base/eventcollector-client/src/test/resources/ical/test1.ics

@@ -0,0 +1,14 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//valug-org//event calendar//DE
+METHOD:PUBLISH
+BEGIN:VEVENT
+ORGANIZER;CN="Voralpen Linux User Group":mailto:[email protected]
+SUMMARY:No title
+UID:a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at
+DTSTAMP:20230724T090555Z
+STATUS:CONFIRMED
+DTSTART:20071214T000000Z
+URL:https://valug.at/events/2007-12-14/
+END:VEVENT
+END:VCALENDAR

boudicca.base/eventcollector-client/src/test/resources/ical/test2.ics

@@ -0,0 +1,16 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//valug-org//event calendar//DE
+METHOD:PUBLISH
+BEGIN:VEVENT
+ORGANIZER;CN="Voralpen Linux User Group":mailto:[email protected]
+SUMMARY:Other title
+DESCRIPTION:Some description
+UID:a840b839819203073326e820176eb4ba757cc96cca71f43f8d34946a917dafe6@events.valug.at
+DTSTAMP:20230724T090555Z
+STATUS:CONFIRMED
+DTSTART:20071214T000000Z
+DTEND:20071214T001000Z
+URL:https://valug.at/events/2007-12-14/
+END:VEVENT
+END:VCALENDAR