Merge branch 'main' into nano/publisher-event-html-a11ytests

.github/workflows/deploy.yml

@@ -21,7 +21,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation[email protected]
+        uses: gradle/actions/wrapper-validation@v3
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
       - name: Execute Gradle build with tests

.github/workflows/gradle.yml

@@ -23,7 +23,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation[email protected]
+        uses: gradle/actions/wrapper-validation@v3
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
       - name: Execute Gradle build

.github/workflows/publish.yml

@@ -20,7 +20,7 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
       - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation[email protected]
+        uses: gradle/actions/wrapper-validation@v3
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
       - name: Execute Gradle build with tests

boudicca.base/eventcollector-client/build.gradle.kts

@@ -11,7 +11,7 @@ dependencies {
     implementation(project(":boudicca.base:remote-collector:remote-collector-client"))
     implementation("org.apache.velocity:velocity-engine-core:2.3")
     implementation("org.apache.velocity.tools:velocity-tools-generic:3.1")
-    implementation("ch.qos.logback:logback-classic:1.5.3")
-    implementation("org.slf4j:slf4j-api:2.0.12")
+    implementation("ch.qos.logback:logback-classic:1.5.5")
+    implementation("org.slf4j:slf4j-api:2.0.13")
     testImplementation("org.junit.jupiter:junit-jupiter:5.10.2")
 }

boudicca.base/publisher-event-html/src/main/resources/templates/events/events.hbs

@@ -2,7 +2,7 @@
     .events-grid {
         gap: 18px;
         display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(min(250px, 100%),1fr));
+        grid-template-columns: repeat(auto-fill, minmax(min(250px, 100%),1fr));
     }
 
     .event {

boudicca.base/publisher-event-ical/src/main/kotlin/base/boudicca/publisher/event/ical/CalendarService.kt

@@ -82,7 +82,7 @@ class CalendarService @Autowired constructor(@Value("\${boudicca.search.url}") p
     }
 
     fun getEvents(query: String): ByteArray {
-        val events = searchClient.queryEvents(QueryDTO(query, 100))
+        val events = searchClient.queryEvents(QueryDTO(query, 0, Int.MAX_VALUE))
         return createCalendar(events.result)
     }
 }

boudicca.base/publisher-event-ical/src/main/kotlin/base/boudicca/publisher/event/ical/IcalResource.kt

@@ -14,8 +14,8 @@ class IcalResource @Autowired constructor(private val calendarService: CalendarS
         "/calendar.ics",
         produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE],
     )
-    fun getAllEvents(@RequestParam query: String): ResponseEntity<ByteArray> {
-        val calendarFile = calendarService.getEvents(query)
+    fun getAllEvents(@RequestParam(required = false) query: String?): ResponseEntity<ByteArray> {
+        val calendarFile = calendarService.getEvents(query ?: "")
         return ResponseEntity.ok()
             .header("Content-Disposition", "attachment;filename=calendar.ics")
             .body(calendarFile)

boudicca.base/query-lib/build.gradle.kts

@@ -1,9 +1,16 @@
 plugins {
     id("boudicca-kotlin")
     id("boudicca-publish")
+    id ("me.champeau.jmh") version "0.7.2"
 }
 
 dependencies {
     api(project(":boudicca.base:semantic-conventions"))
     testImplementation("org.junit.jupiter:junit-jupiter:5.10.2")
+    jmh(project(":boudicca.base:publisher-client"))
+    jmh(project(":boudicca.base:ingest-client"))
+    jmh("com.fasterxml.jackson.core:jackson-core:2.17.0")
+    jmh("com.fasterxml.jackson.module:jackson-module-kotlin:2.17.0")
+    jmh("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0")
+    jmh("com.fasterxml.jackson.core:jackson-databind:2.17.0")
 }

boudicca.base/query-lib/src/jmh/kotlin/base/boudicca/query/EvaluatorTest.kt

@@ -0,0 +1,140 @@
+package base.boudicca.query
+
+import base.boudicca.model.Entry
+import base.boudicca.query.evaluator.*
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.databind.json.JsonMapper
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.fasterxml.jackson.module.kotlin.KotlinModule
+import org.openjdk.jmh.annotations.*
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import kotlin.io.path.exists
+import kotlin.io.path.readBytes
+import kotlin.time.measureTime
+
+
+@State(Scope.Benchmark)
+open class EvaluatorTest {
+
+//    @Param(
+//        """ "name" contains "rock" """,
+//        """ "description" contains "rock" """,
+//        """ "whatever" contains "rock" """,
+//    )
+//    var query: String? = null
+
+//    @Param(
+//        """ "category" equals "music" """,
+//        """ "name" equals "music" """,
+//        """ "whatever" equals "music" """,
+//    )
+//    var query: String? = null
+
+    @Param(
+        """ "category" equals "music" """,
+        """ "name" contains "rock" """,
+        """ "description" contains "rock" """,
+    )
+    var query: String? = null
+
+    var expression: Expression? = null
+
+    @Param(/*"noop", */"simple", "optimizing")
+    var mode: String? = null
+
+    @Param("5000", "20000", "100000")
+    var testDataSize: Int? = null
+
+    var evaluator: Evaluator? = null
+
+    @Setup
+    fun setup() {
+        expression = BoudiccaQueryRunner.parseQuery(query!!)
+        evaluator = when (mode) {
+            "noop" -> NoopEvaluator()
+            "simple" -> SimpleEvaluator(loadTestData(testDataSize))
+            "optimizing" -> OptimizingEvaluator(loadTestData(testDataSize))
+            else -> throw IllegalArgumentException("illegal mode $mode")
+        }
+    }
+
+    @Benchmark
+    @Fork(3)
+//    @BenchmarkMode(Mode.AverageTime)
+//    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    @Warmup(iterations = 2, time = 5000, timeUnit = TimeUnit.MILLISECONDS)
+    @Measurement(iterations = 2, time = 5000, timeUnit = TimeUnit.MILLISECONDS)
+    fun testEvaluator(): QueryResult {
+        return evaluator!!.evaluate(expression!!, PAGE_ALL)
+    }
+}
+
+fun main() {
+
+//    Thread.sleep(20000)
+    val testData: List<Map<String, String>> = loadTestData(100_000)
+
+
+    val evaluator = OptimizingEvaluator(testData.toList())
+//    val evaluator = SimpleEvaluator(testData.toList())
+
+
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "name" contains "rock" """)
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "description" contains "rock" """)
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "whatever" contains "rock" """)
+
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "category" equals "music" """)
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "name" equals "music" """)
+//    val expression = BoudiccaQueryRunner.parseQuery(""" "whatever" equals "music" """)
+
+    val expression = BoudiccaQueryRunner.parseQuery(""" ("startDate" after "2024-04-12") and (duration "startDate" "endDate" shorter 720.0) and ((not (hasField "recurrence.type")) or ("recurrence.type" equals "ONCE")) and "*" contains "rock" """)
+
+    println("search took:" + measureTime {
+        val queryResult = evaluator.evaluate(expression, PAGE_ALL)
+        println(queryResult.totalResults)
+    })
+    println("second search took:" + measureTime {
+        val queryResult = evaluator.evaluate(expression, PAGE_ALL)
+        println(queryResult.totalResults)
+    })
+    var sum = 0
+    while (sum != 1) {
+        val result = evaluator.evaluate(expression, PAGE_ALL)
+        sum += result.result.hashCode()
+    }
+
+    println(sum)
+
+
+}
+
+
+private fun loadTestData(testDataSize: Int? = null): List<Map<String, String>> {
+//        return listOf(
+//            mapOf("name" to "what","description" to "what","test" to "what"),
+//            mapOf("name" to "rock","description" to "what","test" to "what"),
+//            mapOf("name" to "asd","description" to "rock","test" to "what"),
+//            mapOf("name" to "what","description" to "what","test" to "what"),
+//        )
+
+    val objectMapper = JsonMapper.builder().addModule(JavaTimeModule())
+        .addModule(KotlinModule.Builder().build()).build()
+
+    var path = Path.of("testdata.dump")
+    if (!path.exists()) {
+        path = Path.of("../../testdata.dump")
+    }
+    val testData = objectMapper.readValue(
+        path.readBytes(),
+        object : TypeReference<List<Entry>>() {})
+
+
+//    val testData = EventDbPublisherClient("https://eventdb.boudicca.events").getAllEntries()
+
+    if(testDataSize != null){
+        return testData.take(testDataSize)
+    }else{
+        return testData
+    }
+}

boudicca.base/query-lib/src/jmh/kotlin/base/boudicca/query/LoadTestDataIntoLocalEventDb.kt

@@ -0,0 +1,24 @@
+package base.boudicca.query
+
+import base.boudicca.api.eventdb.ingest.EventDbIngestClient
+import base.boudicca.model.Entry
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.databind.json.JsonMapper
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.fasterxml.jackson.module.kotlin.KotlinModule
+import java.io.FileInputStream
+import java.io.ObjectInputStream
+import java.nio.file.Path
+import kotlin.io.path.readBytes
+
+fun main() {
+    val objectMapper = JsonMapper.builder().addModule(JavaTimeModule())
+        .addModule(KotlinModule.Builder().build()).build()
+
+    val storeRead = objectMapper.readValue(
+        Path.of("testdata.dump").readBytes(),
+        object : TypeReference<List<Entry>>() {})
+
+    val ingestClient = EventDbIngestClient("http://localhost:8081", "ingest", "ingest")
+    ingestClient.ingestEntries(storeRead)
+}

boudicca.base/query-lib/src/jmh/kotlin/base/boudicca/query/OrderAfterPerformanceTest.kt

@@ -0,0 +1,81 @@
+package base.boudicca.query
+
+import base.boudicca.model.Entry
+import base.boudicca.query.evaluator.OptimizingEvaluator
+import base.boudicca.query.evaluator.PAGE_ALL
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.databind.json.JsonMapper
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.fasterxml.jackson.module.kotlin.KotlinModule
+import org.openjdk.jmh.annotations.*
+import java.nio.file.Path
+import java.time.OffsetDateTime
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.TimeUnit
+import kotlin.io.path.exists
+import kotlin.io.path.readBytes
+import kotlin.time.measureTime
+
+@State(Scope.Benchmark)
+open class OrderAfterPerformanceTest {
+
+    @Param("5000", "20000", "100000")
+    var testDataSize: Int? = null
+
+    @Param("20", "30", "50", "70", "90")
+    var resultIsEveryXItem: Int? = null
+
+    var testData: List<Entry>? = null
+    var startDateCache: ConcurrentHashMap<String, OffsetDateTime>? = null
+
+    @Setup
+    fun setup() {
+        startDateCache = ConcurrentHashMap<String, OffsetDateTime>()
+//        testData = Utils.order(loadTestData(testDataSize), startDateCache!!)
+        testData = loadTestData(testDataSize)
+    }
+
+//    @Benchmark
+    @Fork(3)
+    @Warmup(iterations = 2, time = 5000, timeUnit = TimeUnit.MILLISECONDS)
+    @Measurement(iterations = 2, time = 5000, timeUnit = TimeUnit.MILLISECONDS)
+    fun test(): List<Entry> {
+        val result = mutableSetOf<Int>()
+        var i = 0
+        while (i < testData!!.size) {
+            result.add(i)
+            i += resultIsEveryXItem!!
+        }
+
+        return Utils.order(result.map { testData!![it] }, startDateCache!!)
+    }
+}
+
+private fun loadTestData(testDataSize: Int? = null): List<Map<String, String>> {
+//        return listOf(
+//            mapOf("name" to "what","description" to "what","test" to "what"),
+//            mapOf("name" to "rock","description" to "what","test" to "what"),
+//            mapOf("name" to "asd","description" to "rock","test" to "what"),
+//            mapOf("name" to "what","description" to "what","test" to "what"),
+//        )
+
+    val objectMapper = JsonMapper.builder().addModule(JavaTimeModule())
+        .addModule(KotlinModule.Builder().build()).build()
+
+    var path = Path.of("testdata.dump")
+    if (!path.exists()) {
+        path = Path.of("../../testdata.dump")
+    }
+    val testData = objectMapper.readValue(
+        path.readBytes(),
+        object : TypeReference<List<Entry>>() {})
+
+
+//    val testData = EventDbPublisherClient("https://eventdb.boudicca.events").getAllEntries()
+
+    if (testDataSize != null) {
+        return testData.take(testDataSize)
+    } else {
+        return testData
+    }
+}