Skip to content

Commit

Permalink
Optimized serialization modules
Browse files Browse the repository at this point in the history
Fixed issues in libpq async support
Added techempower tests for libpq async profiled sceanrios
[docker_push][skip_build]
  • Loading branch information
sumeetchhetri committed Nov 28, 2020
1 parent 2a14c07 commit 4e98d8b
Show file tree
Hide file tree
Showing 56 changed files with 1,962 additions and 727 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/ci-win-cygwin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ jobs:
choco install -y cyg-get
cyg-get gcc gcc-g++ autmake patch bzip2 perl tar xz autoconf automake m4 libtool make zlib-devel zlib cmake git libpq-devel curl libcurl-devel libuuid-devel gdb libssl-devel wget dos2unix unzip
- name: Set ENV
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
run: |
echo '::set-env name=PATH::C:\tools\cygwin\bin;C:\tools\cygwin\usr\bin'
- name: Install dependencies and ffead-cpp
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/ci-win-mingw.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Set ENV
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
run: |
echo '::set-env name=PATH::C:\msys64\mingw64\bin;C:\msys64\usr\bin'
- name: Install toolchain
Expand Down
4 changes: 2 additions & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -427,13 +427,13 @@ fi

AC_ARG_ENABLE(mod_binserialize,
AS_HELP_STRING([--enable-mod_binserialize],
[enable mod_binserialize, default: yes]),
[enable mod_binserialize, default: no]),
[case "${enableval}" in
yes) mod_binserialize=yes ;;
no) mod_binserialize=no ;;
*) AC_MSG_ERROR([bad value ${enableval} for --enable-mod_binserialize]) ;;
esac],
[mod_binserialize=yes])
[mod_binserialize=no])
AM_CONDITIONAL(MOD_BINSERIALIZE, test "x$mod_binserialize" = "xyes")
if test "x$mod_binserialize" = "xyes"; then
AC_DEFINE(INC_BINSER, [], [enable binary serilaization module])
Expand Down
8 changes: 8 additions & 0 deletions docker/build-te-images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,11 @@ docker push sumeetchhetri/ffead-cpp-5.0-sql-raw-profiled-base:5.2
docker rmi -f sumeetchhetri/ffead-cpp-5.0-sql-raw-clibpqb-profiled-base:5.2
docker build -f ffead-cpp-sql-raw-clibpqb-profiled-base.dockerfile -t sumeetchhetri/ffead-cpp-5.0-sql-raw-clibpqb-profiled-base:5.2 .
docker push sumeetchhetri/ffead-cpp-5.0-sql-raw-clibpqb-profiled-base:5.2

docker rmi -f sumeetchhetri/ffead-cpp-5.0-sql-raw-async-profiled-base:5.2
docker build -f ffead-cpp-sql-raw-async-profiled-base.dockerfile -t sumeetchhetri/ffead-cpp-5.0-sql-raw-async-profiled-base:5.2 .
docker push sumeetchhetri/ffead-cpp-5.0-sql-raw-async-profiled-base:5.2

docker rmi -f sumeetchhetri/ffead-cpp-5.0-sql-raw-async-clibpqb-profiled-base:5.2
docker build -f ffead-cpp-sql-raw-async-clibpqb-profiled-base.dockerfile -t sumeetchhetri/ffead-cpp-5.0-sql-raw-async-clibpqb-profiled-base:5.2 .
docker push sumeetchhetri/ffead-cpp-5.0-sql-raw-async-clibpqb-profiled-base:5.2
51 changes: 45 additions & 6 deletions docker/techempower-config/benchmark_config.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
"query_url": "/te-benchmark-um/queries?queries=",
"fortune_url": "/te-benchmark-um/fortunes",
"update_url": "/te-benchmark-um/updates?queries=",
"cached_query_url": "/te-benchmark-um/cached-worlds?count=",
"port": 8080,
"approach": "Realistic",
"classification": "Fullstack",
Expand Down Expand Up @@ -474,7 +473,7 @@
"display_name": "ffead-cpp-mysql",
"notes": "",
"versus": "",
"tags": []
"tags": ["broken"]
},
"postgresql": {
"json_url": "/te-benchmark-um/json",
Expand All @@ -483,7 +482,6 @@
"query_url": "/te-benchmark-um/queries?queries=",
"fortune_url": "/te-benchmark-um/fortunes",
"update_url": "/te-benchmark-um/updates?queries=",
"cached_query_url": "/te-benchmark-um/cached-worlds?count=",
"port": 8080,
"approach": "Realistic",
"classification": "Fullstack",
Expand Down Expand Up @@ -549,7 +547,7 @@
"db_url": "/te-benchmark-um-pq/db",
"query_url": "/te-benchmark-um-pq/queries?queries=",
"fortune_url": "/te-benchmark-um-pq/fortunes",
"update_url": "/te-benchmark-um-pq/bupdates?queries=",
"update_url": "/te-benchmark-um-pq/updates?queries=",
"port": 8080,
"approach": "Realistic",
"classification": "Fullstack",
Expand All @@ -572,7 +570,7 @@
"db_url": "/te-benchmark-um-pq-async/db",
"query_url": "/te-benchmark-um-pq-async/queries?queries=",
"fortune_url": "/te-benchmark-um-pq-async/fortunes",
"update_url": "/te-benchmark-um-pq-async/updates?queries=",
"update_url": "/te-benchmark-um-pq-async/bupdates?queries=",
"cached_query_url": "/te-benchmark-um-pq-async/cached-worlds?count=",
"port": 8080,
"approach": "Realistic",
Expand All @@ -590,6 +588,48 @@
"versus": "",
"tags": []
},
"postgresql-raw-async-profiled": {
"db_url": "/te-benchmark-um-pq-async/db",
"query_url": "/te-benchmark-um-pq-async/queries?queries=",
"fortune_url": "/te-benchmark-um-pq-async/fortunes",
"update_url": "/te-benchmark-um-pq-async/bupdates?queries=",
"port": 8080,
"approach": "Realistic",
"classification": "Fullstack",
"database": "postgres",
"framework": "ffead-cpp",
"language": "C++",
"orm": "Raw",
"platform": "None",
"webserver": "ffead-cpp",
"os": "Linux",
"database_os": "Linux",
"display_name": "ffead-cpp-postgresql-raw-async-profiled",
"notes": "memory profiled",
"versus": "",
"tags": []
},
"postgresql-raw-async-clibpqb-profiled": {
"db_url": "/te-benchmark-um-pq-async/db",
"query_url": "/te-benchmark-um-pq-async/queries?queries=",
"fortune_url": "/te-benchmark-um-pq-async/fortunes",
"update_url": "/te-benchmark-um-pq-async/bupdates?queries=",
"port": 8080,
"approach": "Realistic",
"classification": "Fullstack",
"database": "postgres",
"framework": "ffead-cpp",
"language": "C++",
"orm": "Raw",
"platform": "None",
"webserver": "ffead-cpp",
"os": "Linux",
"database_os": "Linux",
"display_name": "ffead-cpp-postgresql-raw-async-clibpqb-profiled",
"notes": "memory libpq batch patch profiled",
"versus": "",
"tags": []
},
"nginx": {
"json_url": "/te-benchmark-um/json",
"plaintext_url": "/plaintext",
Expand Down Expand Up @@ -630,4 +670,3 @@
}
}]
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM sumeetchhetri/ffead-cpp-5.0-sql-raw-async-clibpqb-profiled-base:5.2

ENV IROOT=/installs

WORKDIR /

CMD ./run_ffead.sh ffead-cpp-5.0-sql emb postgresql-raw-async memory
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM sumeetchhetri/ffead-cpp-5.0-sql-raw-async-profiled-base:5.2

ENV IROOT=/installs

WORKDIR /

CMD ./run_ffead.sh ffead-cpp-5.0-sql emb postgresql-raw-async memory
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
FROM sumeetchhetri/ffead-cpp-5.0-base:5.2
LABEL maintainer="Sumeet Chhetri"
LABEL version="5.2"
LABEL description="SQL Raw Custom libpq batch patched Base ffead-cpp docker image with commit id - master"

WORKDIR /tmp
RUN mkdir postgresql

COPY postgresql/* /tmp/postgresql/

#POSTGRESQL
WORKDIR /tmp/postgresql/

# prepare PostgreSQL APT repository
RUN apt-get -yqq update && apt-get -yqq install locales gnupg lsb-release

RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list

ENV PG_VERSION 13
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENV DEBIAN_FRONTEND noninteractive

# install postgresql on database machine
RUN apt-get -yqq update && apt-get -yqq install postgresql-13 postgresql-contrib-13 && rm -rf /var/lib/apt/lists/*

# Make sure all the configuration files in main belong to postgres
RUN mv postgresql.conf /etc/postgresql/${PG_VERSION}/main/postgresql.conf
RUN mv pg_hba.conf /etc/postgresql/${PG_VERSION}/main/pg_hba.conf

RUN chown -Rf postgres:postgres /etc/postgresql/${PG_VERSION}/main

RUN mkdir /ssd
RUN cp -R -p /var/lib/postgresql/${PG_VERSION}/main /ssd/postgresql
RUN cp /etc/postgresql/${PG_VERSION}/main/postgresql.conf /ssd/postgresql
RUN mv 60-postgresql-shm.conf /etc/sysctl.d/60-postgresql-shm.conf

RUN chown -Rf postgres:postgres /var/run/postgresql
RUN chmod 2777 /var/run/postgresql
RUN chown postgres:postgres /etc/sysctl.d/60-postgresql-shm.conf
RUN chown postgres:postgres create-postgres*
RUN chown -Rf postgres:postgres /ssd

ENV PGDATA=/ssd/postgresql

USER postgres

# We have to wait for postgres to start before we can use the cli
RUN service postgresql start && \
until psql -c "\q"; do sleep 1; done && \
psql < create-postgres-database.sql && \
psql -a hello_world < create-postgres.sql && \
service postgresql stop
#POSTGRESQL

USER root

#WRK
WORKDIR /tmp/wrk
RUN apt-get -yqq update && apt-get -yqq install libluajit-5.1-dev libssl-dev luajit && rm -rf /var/lib/apt/lists/* && \
curl -sL https://github.com/wg/wrk/archive/4.1.0.tar.gz | tar xz --strip-components=1
ENV LDFLAGS="-O3 -march=native -flto"
ENV CFLAGS="-I /usr/include/luajit-2.1 $LDFLAGS"
RUN make WITH_LUAJIT=/usr WITH_OPENSSL=/usr -j "$(nproc)"
RUN cp wrk /usr/local/bin

ENV name name
ENV server_host server_host
ENV levels levels
ENV duration duration
ENV max_concurrency max_concurrency
ENV max_threads max_threads
ENV pipeline pipeline
ENV accept accept
#WRK

WORKDIR ${IROOT}

COPY sql-async-profiled-install-clang.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
RUN chmod 755 ${IROOT}/sql-async-profiled-install-clang.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh

RUN ./sql-async-profiled-install-clang.sh batch
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
FROM sumeetchhetri/ffead-cpp-5.0-base:5.2
LABEL maintainer="Sumeet Chhetri"
LABEL version="5.2"
LABEL description="SQL Raw Base ffead-cpp docker image with commit id - master"

WORKDIR /tmp
RUN mkdir postgresql

COPY postgresql/* /tmp/postgresql/

#POSTGRESQL
WORKDIR /tmp/postgresql/

# prepare PostgreSQL APT repository
RUN apt-get -yqq update && apt-get -yqq install locales gnupg lsb-release

RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list

ENV PG_VERSION 13
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENV DEBIAN_FRONTEND noninteractive

# install postgresql on database machine
RUN apt-get -yqq update && apt-get -yqq install postgresql-13 postgresql-contrib-13 && rm -rf /var/lib/apt/lists/*

# Make sure all the configuration files in main belong to postgres
RUN mv postgresql.conf /etc/postgresql/${PG_VERSION}/main/postgresql.conf
RUN mv pg_hba.conf /etc/postgresql/${PG_VERSION}/main/pg_hba.conf

RUN chown -Rf postgres:postgres /etc/postgresql/${PG_VERSION}/main

RUN mkdir /ssd
RUN cp -R -p /var/lib/postgresql/${PG_VERSION}/main /ssd/postgresql
RUN cp /etc/postgresql/${PG_VERSION}/main/postgresql.conf /ssd/postgresql
RUN mv 60-postgresql-shm.conf /etc/sysctl.d/60-postgresql-shm.conf

RUN chown -Rf postgres:postgres /var/run/postgresql
RUN chmod 2777 /var/run/postgresql
RUN chown postgres:postgres /etc/sysctl.d/60-postgresql-shm.conf
RUN chown postgres:postgres create-postgres*
RUN chown -Rf postgres:postgres /ssd

ENV PGDATA=/ssd/postgresql

USER postgres

# We have to wait for postgres to start before we can use the cli
RUN service postgresql start && \
until psql -c "\q"; do sleep 1; done && \
psql < create-postgres-database.sql && \
psql -a hello_world < create-postgres.sql && \
service postgresql stop
#POSTGRESQL

USER root

#WRK
WORKDIR /tmp/wrk
RUN apt-get -yqq update && apt-get -yqq install libluajit-5.1-dev libssl-dev luajit && rm -rf /var/lib/apt/lists/* && \
curl -sL https://github.com/wg/wrk/archive/4.1.0.tar.gz | tar xz --strip-components=1
ENV LDFLAGS="-O3 -march=native -flto"
ENV CFLAGS="-I /usr/include/luajit-2.1 $LDFLAGS"
RUN make WITH_LUAJIT=/usr WITH_OPENSSL=/usr -j "$(nproc)"
RUN cp wrk /usr/local/bin

ENV name name
ENV server_host server_host
ENV levels levels
ENV duration duration
ENV max_concurrency max_concurrency
ENV max_threads max_threads
ENV pipeline pipeline
ENV accept accept
#WRK

WORKDIR ${IROOT}

COPY sql-async-profiled-install.sh install_ffead-cpp-sql-raw-profiled.sh ${IROOT}/
RUN chmod 755 ${IROOT}/sql-async-profiled-install.sh ${IROOT}/install_ffead-cpp-sql-raw-profiled.sh

RUN ./sql-async-profiled-install.sh
2 changes: 1 addition & 1 deletion docker/techempower-config/ffead-cpp-v-base.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ RUN git clone https://github.com/vlang/v && cd v && make && ./v symlink

WORKDIR ${IROOT}/lang-server-backends/v/vweb
#COPY vweb.v ${IROOT}/lang-server-backends/v/vweb/
RUN chmod +x *.sh && ./build.sh && cp vweb $IROOT/
#RUN chmod +x *.sh && ./build.sh && cp vweb $IROOT/

WORKDIR ${IROOT}/lang-server-backends/v/pico.v
#COPY main.v ${IROOT}/lang-server-backends/v/pico.v/
Expand Down
18 changes: 13 additions & 5 deletions docker/techempower-config/install_ffead-cpp-sql-raw-profiled.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

chmod +x $IROOT/ffead-cpp-sql-raw/*.sh

SUFFIX=""
if [ "$1" = "async" ]
then
SUFFIX="-async"
fi

cp $IROOT/ffead-cpp-sql-raw/server.sh /server_orig.sh

cd $IROOT/ffead-cpp-sql-raw
Expand Down Expand Up @@ -42,16 +48,18 @@ service postgresql start

sed -i 's|EVH_SINGLE=false|EVH_SINGLE=true|g' resources/server.prop
nohup bash -c "./server.sh > ffead.log &"
sleep 5
sleep 10
echo "ffead-cpp with sql-raw support launched"
wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' \
-H 'Connection: keep-alive' --latency -d 15 -c 256 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq/fortunes"
-H 'Connection: keep-alive' --latency -d 5 -c 256 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq${SUFFIX}/fortunes"
wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' \
-H 'Connection: keep-alive' --latency -d 5 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq${SUFFIX}/db"
wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' \
-H 'Connection: keep-alive' --latency -d 15 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq/db"
-H 'Connection: keep-alive' --latency -d 5 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq${SUFFIX}/queries?queries=20"
wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' \
-H 'Connection: keep-alive' --latency -d 15 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq/queries?queries=20"
-H 'Connection: keep-alive' --latency -d 5 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq${SUFFIX}/updates?queries=20"
wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' \
-H 'Connection: keep-alive' --latency -d 15 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq/updates?queries=20"
-H 'Connection: keep-alive' --latency -d 5 -c 512 --timeout 8 -t 2 "http://localhost:8080/te-benchmark-um-pq${SUFFIX}/bupdates?queries=20"
echo "normal shutdown"
rm -f serv.ctrl
pkill ffead-cpp
Expand Down
Loading

0 comments on commit 4e98d8b

Please sign in to comment.