-
Notifications
You must be signed in to change notification settings - Fork 17
190 lines (165 loc) · 7.19 KB
/
lt-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
name: Load Test CI
on:
push:
branches:
- main
workflow_dispatch:
pull_request:
branches:
- main
paths:
- 'intent_brokering/common/**'
- 'intent_brokering/src/**'
- 'intent_brokering/tests/**'
- 'intent_brokering/proto/**'
- 'intent_brokering/proto.rs/**'
- 'intent_brokering/examples/**'
- '.github/workflows/lt-ci.yml'
- 'Cargo.lock'
- 'Cargo.toml'
- 'rust-toolchain.toml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
permissions:
contents: write
deployments: write
pull-requests: write
env:
# Push the results to gh-pages only on merge to main.
AUTO_PUSH: ${{ github.event_name == 'push' }}
# Comment the performance numbers of the load tests only on PRs and if the PR
# comes from within the Eclipse organization.
COMMENT_ALWAYS: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'eclipse-chariott/chariott' }}
jobs:
benchmark:
name: Execute load tests and memory profiling
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install stable toolchain
run: rustup show
- name: Cache Dependencies
uses: Swatinem/rust-cache@v1
- name: Install dependencies for build
run: sudo apt-get install -y protobuf-compiler xmlstarlet
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build intent brokering container
uses: docker/build-push-action@v3
with:
context: .
file: Dockerfile.intent_brokering.amd64
load: true
tags: intent_brokering:1
- name: Build valgrind enabled intent brokering container
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile.intent_brokering.valgrind
load: true
tags: intent_brokering.valgrind:1
- name: Build lt-provider-app container
uses: docker/build-push-action@v3
with:
context: .
file: ./intent_brokering/examples/applications/Dockerfile.lt-provider-app.ci
build-args: APP_NAME=lt-provider-app
load: true
tags: lt-provider-app:1
- name: Start intent_brokering and lt-provider-app
run: |
docker run --init --detach --name intent_brokering --network host intent_brokering:1
docker run --init --detach --name lt-provider-app --network host lt-provider-app:1
- name: Run load tests
run: |
mkdir -p lt-output
cargo build --release -p lt-consumer-app
COLLECT_DOCKER_STATS=true TARGET_INVOKE_COUNT=90000 TARGET_RATE=3000 ./target/release/lt-consumer-app > lt-output/lt-consumer-app.log 2>&1
- name: Stop containers
run: |
docker kill --signal=SIGINT lt-provider-app
docker kill --signal=SIGINT intent_brokering
docker stop lt-provider-app
docker stop intent_brokering
- name: Print performance numbers
run: |
cat lt-output/app.out | jq
- name: Print docker memory numbers
run: |
cat lt-output/docker.out | jq
- name: Print log output
if: ${{ always() }}
run: |
docker logs intent_brokering
docker logs lt-provider-app
for f in lt-output/*.log; do
echo -e "Printing output of file $f:\n"
cat "$f";
done
- name: Start intent_brokering and lt-provider-app
run: |
docker rm lt-provider-app
docker rm intent_brokering
mkdir -p valgrind-output
sudo chown -R 10001:10001 valgrind-output
sudo chmod -R 777 valgrind-output
docker run --init --detach --name intent_brokering.valgrind --network host --volume $(pwd)/valgrind-output:/output intent_brokering.valgrind:1
docker run --init --detach --name lt-provider-app --network host lt-provider-app:1
- name: Run Memory Profiling using load tests
run: |
mv lt-output/app.out lt-output/loadtest.out
sleep 10
TARGET_INVOKE_COUNT=100 TARGET_RATE=5 ./target/release/lt-consumer-app > lt-output/lt-consumer-app.log 2>&1
rm lt-output/app.out
- name: Stop containers
run: |
docker kill --signal=SIGINT lt-provider-app
docker kill --signal=SIGINT intent_brokering.valgrind
docker stop lt-provider-app
docker stop intent_brokering.valgrind
- name: Create valgrind benchmark report
run: |
VALGRIND_OUTPUT=$(pwd)/valgrind-output/out.xml
DEFINITELY_LOST_BYTES=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_DefinitelyLost']//xwhat/leakedbytes)" $VALGRIND_OUTPUT)
DEFINTIELY_LOST_BLOCKS=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_DefinitelyLost']//xwhat/leakedblocks)" $VALGRIND_OUTPUT)
INDIRECTLY_LOST_BYTES=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_IndirectlyLost']//xwhat/leakedbytes)" $VALGRIND_OUTPUT)
INDIRECTLY_LOST_BLOCKS=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_IndirectlyLost']//xwhat/leakedblocks)" $VALGRIND_OUTPUT)
POSSIBLY_LOST_BYTES=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_PossiblyLost']//xwhat/leakedbytes)" $VALGRIND_OUTPUT)
POSSIBLY_LOST_BLOCKS=$(xmlstarlet sel -t -v "sum(//error[kind='Leak_PossiblyLost']//xwhat/leakedblocks)" $VALGRIND_OUTPUT)
json_string=$(jq -n \
--arg dby "$DEFINITELY_LOST_BYTES" \
--arg dbl "$DEFINTIELY_LOST_BLOCKS" \
--arg iby "$INDIRECTLY_LOST_BYTES" \
--arg ibl "$INDIRECTLY_LOST_BLOCKS" \
--arg pby "$POSSIBLY_LOST_BYTES" \
--arg pbl "$POSSIBLY_LOST_BLOCKS" \
'[{name: "Definitely Lost Bytes", unit: "bytes", value: $dby}, {name: "Definitely Lost Blocks", unit: "blocks", value: $dbl}, {name: "Indirectly Lost Bytes", unit: "bytes", value: $iby}, {name: "Indirectly Lost Blocks", unit: "blocks", value: $ibl}, {name: "Possibly Lost Bytes", unit: "bytes", value: $pby}, {name: "Possibly Lost Blocks", unit: "blocks", value: $pbl}]')
echo $json_string > lt-output/valgrind.out
- name: Concatenate all benchmarks
run: |
find -type f -name "*.out" -exec cat {} \; | jq '.[]' | jq -s > lt-output/all.json
- name: Store combined benchmark result
env:
GIT_LFS_SKIP_SMUDGE: 1
uses: benchmark-action/github-action-benchmark@v1
if: ${{ github.event_name != 'workflow_dispatch' }}
with:
name: Intent Brokering Benchmark
tool: 'customSmallerIsBetter'
output-file-path: lt-output/all.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: ${{ env.AUTO_PUSH }}
comment-always: ${{ env.COMMENT_ALWAYS }}
- name: Print valgrind numbers
run: |
cat lt-output/valgrind.out | jq
- name: Print log output
if: ${{ always() }}
run: |
docker logs intent_brokering.valgrind
docker logs lt-provider-app
for f in lt-output/*.log; do
echo -e "Printing output of file $f:\n"
cat "$f";
done