Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

re-enabling package-benchmark and updating to reference CRDT benchmarks #44

Merged
merged 1 commit into from
Oct 13, 2024
Merged

re-enabling package-benchmark and updating to reference CRDT benchmarks #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 101 additions & 108 deletions ExternalBenchmarks/Benchmarks/ExternalBenchmarks/ExternalBenchmarks.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,42 +6,6 @@ import ZippyJSON

@main extension BenchmarkRunner {} // Required for the main() definition to no get linker errors

/*

Interesting comparison benchmark for parsing the JSON, turns out that hand-parsing is a
LOT faster than even ExtrasJSON Decodable conformance implementations. ExtrasJSON is still
notably faster than Foundation, but the hand-parse optimization was surprising to me.

swift package benchmark --grouping metric

ExternalBenchmarks
============================================================================================================================

Throughput (scaled / s)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace │ 82 │ 79 │ 78 │ 76 │ 75 │ 73 │ 68 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace │ 6 │ 6 │ 6 │ 6 │ 6 │ 5 │ 5 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace │ 8 │ 8 │ 8 │ 7 │ 7 │ 7 │ 7 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════
Time (wall clock)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace (ms) │ 12 │ 13 │ 13 │ 13 │ 13 │ 14 │ 15 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace (ms) │ 176 │ 176 │ 177 │ 178 │ 180 │ 184 │ 184 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace (ms) │ 823 │ 823 │ 825 │ 825 │ 825 │ 825 │ 825 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace (ms) │ 132 │ 132 │ 133 │ 134 │ 134 │ 136 │ 136 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════╛
*/
enum TextOp {
case insert(cursor: UInt32, value: String)
case delete(cursor: UInt32, count: UInt32)
Expand Down Expand Up @@ -160,84 +124,114 @@ func parseJSONIntoTrace(topOfTrace: JSONValue) async -> Trace {

@_dynamicReplacement(for: registerBenchmarks) // And this is how we register our benchmarks
func benchmarks() {
Benchmark.defaultConfiguration.maxIterations = .count(300)
Benchmark.defaultConfiguration.maxDuration = .seconds(5)

Benchmark("Loading JSON trace data",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
await blackHole(loadEditingTrace())
}
}

Benchmark("parse JSON with Swift Extras parser",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(parseDataIntoJSON(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("Custom parse JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
let jsonValue = await parseDataIntoJSON(data: data)
benchmark.startMeasurement()
await blackHole(parseJSONIntoTrace(topOfTrace: jsonValue))
benchmark.stopMeasurement()
}
}

Benchmark("Foundation decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("ExtrasJSON decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeXIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

Benchmark("ZippyJSON decode JSON into trace",
configuration: .init(metrics: [.throughput, .wallClock]))
{ benchmark in
for _ in benchmark.scaledIterations {
let data = await loadEditingTrace()
benchmark.startMeasurement()
await blackHole(decodeZIntoTrace(data: data))
benchmark.stopMeasurement()
}
}

// Benchmark("Loading JSON trace data",
// configuration: .init(metrics: [.throughput, .wallClock], desiredIterations: 20)) { benchmark in
// for _ in benchmark.scaledIterations {
// blackHole(await loadEditingTrace())
// }
// }
//
// Benchmark("parse JSON with Swift Extras parser",
// configuration: .init(metrics: [.throughput, .wallClock], desiredIterations: 50)) { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// blackHole(await parseDataIntoJSON(data: data))
// benchmark.stopMeasurement()
// }
// }

// Benchmark("Custom parse JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// let jsonValue = await parseDataIntoJSON(data: data)
// benchmark.startMeasurement()
// await blackHole(parseJSONIntoTrace(topOfTrace: jsonValue))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("Foundation decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("ExtrasJSON decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeXIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }
//
// Benchmark("ZippyJSON decode JSON into trace",
// configuration: .init(metrics: [.throughput, .wallClock]))
// { benchmark in
// for _ in benchmark.scaledIterations {
// let data = await loadEditingTrace()
// benchmark.startMeasurement()
// await blackHole(decodeZIntoTrace(data: data))
// benchmark.stopMeasurement()
// }
// }

/*
Interesting comparison benchmark for parsing the JSON, turns out that hand-parsing is a
LOT faster than even ExtrasJSON Decodable conformance implementations. ExtrasJSON is still
notably faster than Foundation, but the hand-parse optimization was surprising to me.

swift package benchmark --grouping metric

ExternalBenchmarks
============================================================================================================================

Throughput (scaled / s)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace │ 82 │ 79 │ 78 │ 76 │ 75 │ 73 │ 68 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace │ 6 │ 6 │ 6 │ 6 │ 6 │ 5 │ 5 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace │ 8 │ 8 │ 8 │ 7 │ 7 │ 7 │ 7 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════
Time (wall clock)
╒════════════════════════════════════════╤═════╤═════╤══════╤═════╤═════╤═════╤══════╤═════════╕
│ Test │ p0 │ p25 │ p50 │ p75 │ p90 │ p99 │ p100 │ Samples │
╞════════════════════════════════════════╪═════╪═════╪══════╪═════╪═════╪═════╪══════╪═════════╡
│ Custom parse JSON into trace (ms) │ 12 │ 13 │ 13 │ 13 │ 13 │ 14 │ 15 │ 300 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ExtrasJSON decode JSON into trace (ms) │ 176 │ 176 │ 177 │ 178 │ 180 │ 184 │ 184 │ 29 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ Foundation decode JSON into trace (ms) │ 823 │ 823 │ 825 │ 825 │ 825 │ 825 │ 825 │ 7 │
├────────────────────────────────────────┼─────┼─────┼──────┼─────┼─────┼─────┼──────┼─────────┤
│ ZippyJSON decode JSON into trace (ms) │ 132 │ 132 │ 133 │ 134 │ 134 │ 136 │ 136 │ 38 │
╘════════════════════════════════════════╧═════╧═════╧══════╧═════╧═════╧═════╧══════╧═════════╛
*/

Benchmark("Create single-character List CRDT",
configuration: .init(metrics: BenchmarkMetric.all, scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
blackHole(blackHole(List(actorId: "a", ["a"])))
}
}

Benchmark("List six-character append",
configuration: .init(metrics: [.throughput, .wallClock], scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
var mylist = List(actorId: "a", ["a"])
benchmark.startMeasurement()
Expand All @@ -247,8 +241,7 @@ func benchmarks() {
}

Benchmark("List six-character append, individual characters",
configuration: .init(metrics: [.throughput, .wallClock], scalingFactor: .kilo))
{ benchmark in
configuration: .init(timeUnits: .microseconds, scalingFactor: .kilo)) { benchmark in
for _ in benchmark.scaledIterations {
var mylist = List(actorId: "a", ["a"])
let appendList = [" ", "h", "e", "l", "l", "o"]
Expand Down
1 change: 0 additions & 1 deletion ExternalBenchmarks/Benchmarks/ExternalBenchmarks/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,3 @@ General format of the JSON:

ref article for decoding in an enum w/ associated value in swift:
https://www.donnywals.com/splitting-a-json-object-into-an-enum-and-an-associated-object-with-codable/

Loading
Loading