From fdbca425780a200a65e8a922543c70b6dc2f7b57 Mon Sep 17 00:00:00 2001 From: abl Date: Fri, 6 Sep 2024 21:02:30 +0200 Subject: [PATCH] have some structured data support in eventdb --- .../boudicca/model/structured/KeyUtils.kt | 1 + .../model/structured/StructuredEvent.kt | 1 + .../boudicca/eventdb/service/EntryService.kt | 52 +++++++++++++------ .../event/html/service/EventService.kt | 4 +- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/KeyUtils.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/KeyUtils.kt index eef686b6..bd6cd401 100644 --- a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/KeyUtils.kt +++ b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/KeyUtils.kt @@ -4,6 +4,7 @@ object KeyUtils { @Throws(IllegalArgumentException::class) fun toStructuredKeyValuePairs(map: Map): Map { return map.mapKeys { + //TODO make more smart to include duplicate key detection (detect non canonical forms) val (propertyName, variants) = parseKey(it.key) Key(propertyName, variants) } diff --git a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt index 6ad0e54a..d7f16ea6 100644 --- a/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt +++ b/boudicca.base/common-model/src/main/kotlin/base/boudicca/model/structured/StructuredEvent.kt @@ -9,6 +9,7 @@ import base.boudicca.model.Event import java.time.OffsetDateTime import java.util.* +//TODO should data be Map or maybe something like Map, where the data is preparsed? data class StructuredEvent(val name: String, val startDate: OffsetDateTime, val data: Map) { constructor(event: Event) : this(event.name, event.startDate, KeyUtils.toStructuredKeyValuePairs(event.data)) diff --git a/boudicca.base/eventdb/src/main/kotlin/base/boudicca/eventdb/service/EntryService.kt b/boudicca.base/eventdb/src/main/kotlin/base/boudicca/eventdb/service/EntryService.kt index 01fbe77a..ac46c102 100644 --- a/boudicca.base/eventdb/src/main/kotlin/base/boudicca/eventdb/service/EntryService.kt +++ b/boudicca.base/eventdb/src/main/kotlin/base/boudicca/eventdb/service/EntryService.kt @@ -1,11 +1,14 @@ package base.boudicca.eventdb.service -import base.boudicca.model.Entry -import base.boudicca.model.Event import base.boudicca.SemanticKeys import base.boudicca.eventdb.BoudiccaEventDbProperties import base.boudicca.eventdb.model.EntryKey import base.boudicca.eventdb.model.InternalEventProperties +import base.boudicca.keyfilters.KeySelector +import base.boudicca.model.Entry +import base.boudicca.model.Event +import base.boudicca.model.structured.* +import base.boudicca.model.toStructuredEntry import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.DatabindException import com.fasterxml.jackson.databind.json.JsonMapper @@ -29,6 +32,7 @@ import java.util.concurrent.locks.ReentrantLock import kotlin.io.path.exists import kotlin.io.path.readBytes import kotlin.io.path.writeBytes +import kotlin.jvm.optionals.getOrNull @Service @@ -87,7 +91,7 @@ class EntryService @Autowired constructor( storeRead.forEach { val entry = Event.toEntry(it.first) - entries[getEntryKey(entry)] = Pair(entry, it.second) + entries[getEntryKey(entry.toStructuredEntry())] = Pair(entry, it.second) } needsPersist.set(true) //to write in the new format @@ -99,7 +103,7 @@ class EntryService @Autowired constructor( object : TypeReference>>() {}) storeRead.forEach { - entries[getEntryKey(it.first)] = it + entries[getEntryKey(it.first.toStructuredEntry())] = it } needsPersist.set(false) @@ -110,12 +114,19 @@ class EntryService @Autowired constructor( } fun add(entry: Entry) { - val eventKey = getEntryKey(entry) + val structuredEntry = entry.toStructuredEntry() - entries[eventKey] = Pair(entry, InternalEventProperties(System.currentTimeMillis())) - if (entry.containsKey(SemanticKeys.COLLECTORNAME)) { - lastSeenCollectors[entry[SemanticKeys.COLLECTORNAME]!!] = System.currentTimeMillis() + val eventKey = getEntryKey(structuredEntry) + + //TODO should we do format validation here? + //we reflatten the entry to make sure keys are canonical + entries[eventKey] = Pair(structuredEntry.toFlatEntry(), InternalEventProperties(System.currentTimeMillis())) + + val collectorName = structuredEntry.getProperty(SemanticKeys.COLLECTORNAME_PROPERTY) + if (collectorName.isNotEmpty()) { + lastSeenCollectors[collectorName.first().second] = System.currentTimeMillis() } + needsPersist.set(true) } @@ -123,19 +134,20 @@ class EntryService @Autowired constructor( @Scheduled(fixedRate = 1, timeUnit = TimeUnit.DAYS) fun cleanup() { - val toRemoveEvents = entries.values + val toRemoveEvents = entries.entries .filter { - if (it.first.containsKey(SemanticKeys.COLLECTORNAME)) { - val collectorName = it.first[SemanticKeys.COLLECTORNAME]!! - it.second.timeAdded + MAX_AGE < (lastSeenCollectors[collectorName] ?: Long.MIN_VALUE) + val entry = it.value.first + if (entry.containsKey(SemanticKeys.COLLECTORNAME)) { + val collectorName = entry[SemanticKeys.COLLECTORNAME]!! + it.value.second.timeAdded + MAX_AGE < (lastSeenCollectors[collectorName] ?: Long.MIN_VALUE) } else { false } } toRemoveEvents.forEach { - LOG.debug("removing event because it got too old: {}", it) - entries.remove(getEntryKey(it.first)) + LOG.debug("removing event because it got too old: {}", it.value.first) + entries.remove(it.key) needsPersist.set(true) } } @@ -167,6 +179,7 @@ class EntryService @Autowired constructor( if (needsPersist.get()) { val bytes = objectMapper.writeValueAsBytes(entries.values) try { + //TODO make more resilient saving, aka save then move Path.of(boudiccaEventDbProperties.store.path) .writeBytes(bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) } catch (e: IOException) { @@ -179,15 +192,20 @@ class EntryService @Autowired constructor( } } - private fun getEntryKey(entry: Entry): EntryKey { + private fun getEntryKey(entry: StructuredEntry): EntryKey { val keys = if (!boudiccaEventDbProperties.entryKeyNames.isNullOrEmpty()) { boudiccaEventDbProperties.entryKeyNames } else { - entry.keys + entry.keys.map { it.toKeyString() } } @Suppress("UNCHECKED_CAST") - return keys.map { it to entry[it] }.filter { it.second != null }.toMap() as EntryKey + return keys + .map { + it to entry.filterKeys(Key.parse(it)).firstOrNull()?.second + } + .filter { it.second != null } + .toMap() as EntryKey } } diff --git a/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt b/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt index 296c4408..0df33e95 100644 --- a/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt +++ b/boudicca.base/publisher-event-html/src/main/kotlin/base/boudicca/publisher/event/html/service/EventService.kt @@ -239,8 +239,8 @@ class EventService @Autowired constructor( val list = mutableListOf() for (keyValuePair in event.data) { if (keyValuePair.key.name.startsWith("accessibility.")) { - val accessibilityValue = getTextProperty(event, keyValuePair.key.name) - if (accessibilityValue != null && accessibilityValue.equals("true", true)) { + val accessibilityValue = keyValuePair.value + if (accessibilityValue.toBoolean()) { list.add( when (keyValuePair.key.name) { SemanticKeys.ACCESSIBILITY_ACCESSIBLETOILETS -> "Barrierefreie Toiletten"