Skip to content

Commit

Permalink
re-enabling package-benchmark and updating to reference CRDT benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@heckj
heckj committed Oct 13, 2024
1 parent 3225f92 commit a7fb51b
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 110 deletions.
209 changes: 101 additions & 108 deletions ExternalBenchmarks/Benchmarks/ExternalBenchmarks/ExternalBenchmarks.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,6 @@ import ZippyJSON

@main extension BenchmarkRunner {} // Required for the main() definition to no get linker errors

/*

Interesting comparison benchmark for parsing the JSON, turns out that hand-parsing is a
LOT faster than even ExtrasJSON Decodable conformance implementations. ExtrasJSON is still
notably faster than Foundation, but the hand-parse optimization was surprising to me.

swift package benchmark --grouping metric

ExternalBenchmarks
============================================================================================================================

Throughput (scaled / s)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace │ 82 │ 79 │ 78 │ 76 │ 75 │ 73 │ 68 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace │ 6 │ 6 │ 6 │ 6 │ 6 │ 5 │ 5 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace │ 8 │ 8 │ 8 │ 7 │ 7 │ 7 │ 7 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════
Time (wall clock)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace (ms) │ 12 │ 13 │ 13 │ 13 │ 13 │ 14 │ 15 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace (ms) │ 176 │ 176 │ 177 │ 178 │ 180 │ 184 │ 184 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace (ms) │ 823 │ 823 │ 825 │ 825 │ 825 │ 825 │ 825 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace (ms) │ 132 │ 132 │ 133 │ 134 │ 134 │ 136 │ 136 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════╛
*/
enum TextOp {
case insert(cursor: UInt32, value: String)
case delete(cursor: UInt32, count: UInt32)
Expand Down Expand Up @@ -160,84 +124,114 @@ func parseJSONIntoTrace(topOfTrace: JSONValue) async -> Trace {

@_dynamicReplacement(for: registerBenchmarks) // And this is how we register our benchmarks
func benchmarks() {
Benchmark.defaultConfiguration.maxIterations = .count(300)
Benchmark.defaultConfiguration.maxDuration = .seconds(5)

Benchmark("Loading JSON trace data",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
await blackHole(loadEditingTrace())
}
}

Benchmark("parse JSON with Swift Extras parser",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(parseDataIntoJSON(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("Custom parse JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
let jsonValue = await parseDataIntoJSON(data: data)
benchmark.startMeasurement()
await blackHole(parseJSONIntoTrace(topOfTrace: jsonValue))
benchmark.stopMeasurement()
}
}

Benchmark("Foundation decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("ExtrasJSON decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeXIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("ZippyJSON decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeZIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

// Benchmark("Loading JSON trace data",
// configuration: .init(metrics: [.throughput, .wallClock], desiredIterations: 20)) { benchmark in
// for _ in benchmark.scaledIterations {
// blackHole(await loadEditingTrace())
// }
// }
//
// Benchmark("parse JSON with Swift Extras parser",
// configuration: .init(metrics: [.throughput, .wallClock], desiredIterations: 50)) { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// blackHole(await parseDataIntoJSON(data: data))
// benchmark.stopMeasurement()
// }
// }

// Benchmark("Custom parse JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// let jsonValue = await parseDataIntoJSON(data: data)
// benchmark.startMeasurement()
// await blackHole(parseJSONIntoTrace(topOfTrace: jsonValue))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("Foundation decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("ExtrasJSON decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeXIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("ZippyJSON decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeZIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }

/*
Interesting comparison benchmark for parsing the JSON, turns out that hand-parsing is a
LOT faster than even ExtrasJSON Decodable conformance implementations. ExtrasJSON is still
notably faster than Foundation, but the hand-parse optimization was surprising to me.

swift package benchmark --grouping metric

ExternalBenchmarks
============================================================================================================================

Throughput (scaled / s)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace │ 82 │ 79 │ 78 │ 76 │ 75 │ 73 │ 68 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace │ 6 │ 6 │ 6 │ 6 │ 6 │ 5 │ 5 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace │ 8 │ 8 │ 8 │ 7 │ 7 │ 7 │ 7 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════
Time (wall clock)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace (ms) │ 12 │ 13 │ 13 │ 13 │ 13 │ 14 │ 15 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace (ms) │ 176 │ 176 │ 177 │ 178 │ 180 │ 184 │ 184 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace (ms) │ 823 │ 823 │ 825 │ 825 │ 825 │ 825 │ 825 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace (ms) │ 132 │ 132 │ 133 │ 134 │ 134 │ 136 │ 136 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════╛
*/

Benchmark("Create single-character List CRDT",
configuration: .init(metrics: BenchmarkMetric.all, scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
blackHole(blackHole(List(actorId: "a", ["a"])))
}
}

Benchmark("List six-character append",
configuration: .init(metrics: [.throughput, .wallClock], scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
var mylist = List(actorId: "a", ["a"])
benchmark.startMeasurement()
Expand All @@ -247,8 +241,7 @@ func benchmarks() {
}

Benchmark("List six-character append, individual characters",
configuration: .init(metrics: [.throughput, .wallClock], scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
var mylist = List(actorId: "a", ["a"])
let appendList = [" ", "h", "e", "l", "l", "o"]
Expand Down
1 change: 0 additions & 1 deletion ExternalBenchmarks/Benchmarks/ExternalBenchmarks/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,3 @@ General format of the JSON:

ref article for decoding in an enum w/ associated value in swift:
https://www.donnywals.com/splitting-a-json-object-into-an-enum-and-an-associated-object-with-codable/

Loading

0 comments on commit a7fb51b

Please sign in to comment.