From 39f98c18d72d384bc848f764269d3e6c37b2724c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Fri, 25 Aug 2023 22:37:12 +0200 Subject: [PATCH 01/19] Create .github/actions/run_e2e --- .github/actions/run_e2e/action.yaml | 36 +++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 13 +++-------- Dockerfile | 2 +- dev.py | 2 ++ 4 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 .github/actions/run_e2e/action.yaml diff --git a/.github/actions/run_e2e/action.yaml b/.github/actions/run_e2e/action.yaml new file mode 100644 index 00000000..fdea64fc --- /dev/null +++ b/.github/actions/run_e2e/action.yaml @@ -0,0 +1,36 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + python_version: + required: true + default: '3.8' + streamlit_version: + required: true + default: '1.25.0' + streamlit_path: + default: '' + +runs: + using: "composite" + steps: + - run: export + shell: bash + + - run: echo "${GH_JSON}" + shell: bash + env: + GH_JSON: ${{ toJson(github) }} + + - name: Build docker images + run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + shell: bash + env: + STREAMLIT_VERSION: ${{ inputs.streamlit_version }} + PYTHON_VERSION: ${{ inputs.python_version }} + + - name: Run e2e tests + run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + shell: bash + env: + STREAMLIT_VERSION: ${{ inputs.streamlit_version }} + PYTHON_VERSION: ${{ inputs.python_version }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4931eab0..dbc44380 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,17 +110,10 @@ jobs: path: dist/*.whl if-no-files-found: error - - name: Set up Docker Buildx + - name: Run E2E tests if: matrix.node_version == '19.x' - uses: docker/setup-buildx-action@7703e82fbced3d0c9eec08dff4429c023a5fd9a9 # v2.9.1 - - - name: Build docker images - if: matrix.node_version == '19.x' - run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" - - - name: Run e2e tests - if: matrix.node_version == '19.x' - run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + uses: + ./.github/actions/run_e2e build-cookiecutter: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 73fed010..19dc1082 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.4 -ARG PYTHON_VERSION="3.11.4" +ARG PYTHON_VERSION FROM python:${PYTHON_VERSION}-slim-bullseye SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] diff --git a/dev.py b/dev.py index b97a529e..10df75d7 100755 --- a/dev.py +++ b/dev.py @@ -23,6 +23,8 @@ THIS_DIRECTORY / "template", THIS_DIRECTORY / "template-reactless", ] +# TODO(kbregula): Enable tests for examples +EXAMPLE_DIRECTORIES = [] # Utilities function From 3e524ffbb469baf5d5acf08b12a73b20e91df6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Fri, 25 Aug 2023 22:48:08 +0200 Subject: [PATCH 02/19] Disable a few tests --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dbc44380..42729cd5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,11 +16,11 @@ jobs: fail-fast: false matrix: node_version: - # For details, see: https://nodejs.dev/en/about/releases/ - # Maintenance LTS. End Of Life: 2023-09-11 - - 16.x - # Active LTS. End Of Life: 2025-04-30 - - 18.x +# # For details, see: https://nodejs.dev/en/about/releases/ +# # Maintenance LTS. End Of Life: 2023-09-11 +# - 16.x +# # Active LTS. End Of Life: 2025-04-30 +# - 18.x # Current version - 19.x component_lib_version: From 67f6c119b3ce44b1b0ab7111b6ecbeb6ca4da57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Fri, 25 Aug 2023 22:57:52 +0200 Subject: [PATCH 03/19] Checkout all files to subdirectory --- .github/workflows/ci.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 42729cd5..97bc60b3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,7 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false + path: component-template - name: Use Node.js ${{ env.NODE_VERSION }} uses: actions/setup-node@v3 @@ -50,9 +51,11 @@ jobs: - name: Check dependencies for examples run: ./dev.py examples-check-deps + working-directory: ./component-template/ - name: Check e2e utils files run: ./dev.py e2e-utils-check + working-directory: ./component-template/ - name: Checkout streamlit/streamlit if: matrix.component_lib_version == 'develop' @@ -77,6 +80,7 @@ jobs: - name: Link develop version of streamlit-component-lib if: matrix.component_lib_version == 'develop' + working-directory: ./component-template/ env: COMPONENT_LIB_DIR: ${{ github.workspace }}/streamlit/component-lib/ run: | @@ -91,12 +95,15 @@ jobs: done - name: Install node dependencies + working-directory: ./component-template/ run: ./dev.py all-npm-install - name: Build frontend code + working-directory: ./component-template/ run: ./dev.py all-npm-build - name: Build wheel packages + working-directory: ./component-template/ run: | find examples template template-reactless -maxdepth 3 -name '__init__.py' |\ xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/'; @@ -107,13 +114,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: all-wheel - path: dist/*.whl + path: ./component-template/dist/*.whl if-no-files-found: error - name: Run E2E tests if: matrix.node_version == '19.x' - uses: - ./.github/actions/run_e2e + uses: ./component-template/.github/actions/run_e2e build-cookiecutter: runs-on: ubuntu-latest From dc427bd138b03a69fd8808e9bf9368ecc601e7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Fri, 25 Aug 2023 23:01:36 +0200 Subject: [PATCH 04/19] fixup! Checkout all files to subdirectory --- .github/actions/run_e2e/action.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/actions/run_e2e/action.yaml b/.github/actions/run_e2e/action.yaml index fdea64fc..3024ff96 100644 --- a/.github/actions/run_e2e/action.yaml +++ b/.github/actions/run_e2e/action.yaml @@ -13,16 +13,9 @@ inputs: runs: using: "composite" steps: - - run: export - shell: bash - - - run: echo "${GH_JSON}" - shell: bash - env: - GH_JSON: ${{ toJson(github) }} - - name: Build docker images run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + working-directory: ${{ github.action_path }}/../../../ shell: bash env: STREAMLIT_VERSION: ${{ inputs.streamlit_version }} @@ -31,6 +24,7 @@ runs: - name: Run e2e tests run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" shell: bash + working-directory: ${{ github.action_path }}/../../../ env: STREAMLIT_VERSION: ${{ inputs.streamlit_version }} PYTHON_VERSION: ${{ inputs.python_version }} From 7bc42278f784566eb8860edfef68eb42305863a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Fri, 25 Aug 2023 23:26:24 +0200 Subject: [PATCH 05/19] Cache playwright and browsers --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 19dc1082..b2d99893 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,12 @@ ENV PIP_VERSION=${PIP_VERSION} RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version +# Setup Playwright +ARG PLAYWRIGHT_VERSION="1.36.0" +ENV PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION} + +RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps + ENV PYTHONUNBUFFERED=1 ENV PIP_ROOT_USER_ACTION=ignore RUN mkdir /component From da002c7ed021849f1aca51c91b85cd8badcd3ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 00:01:53 +0200 Subject: [PATCH 06/19] Modularize workflows --- .../build_component_wheels/action.yaml | 56 ++++++++++++++++ .../action.yaml | 54 ++++++++++++++++ .github/workflows/ci.yaml | 64 ++++--------------- 3 files changed, 123 insertions(+), 51 deletions(-) create mode 100644 .github/actions/build_component_wheels/action.yaml create mode 100644 .github/actions/build_streamlit_component_library/action.yaml diff --git a/.github/actions/build_component_wheels/action.yaml b/.github/actions/build_component_wheels/action.yaml new file mode 100644 index 00000000..aab89a18 --- /dev/null +++ b/.github/actions/build_component_wheels/action.yaml @@ -0,0 +1,56 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + custom_streamlit_component_lib_file: + required: false + description: '' + default: '' + +outputs: + output_directory: + description: '' + value: ${{ steps.final_step.outputs.output_directory }} + +runs: + using: "composite" + steps: + - name: Link develop version of streamlit-component-lib + if: inputs.custom_streamlit_component_lib_file != '' + working-directory: ${{ github.action_path }}/../../../ + shell: bash + env: + STREAMLIT_COMPONENT_LIB_FILE: ${{ inputs.custom_streamlit_component_lib_file }} + run: | + find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do + ( + cd "${line}"; + npm install "${STREAMLIT_COMPONENT_LIB_FILE}" + ) + done + + - name: Install node dependencies + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: ./dev.py all-npm-install + + - name: Build frontend code + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: ./dev.py all-npm-build + + - name: Build wheel packages + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: | + find examples template template-reactless -maxdepth 3 -name '__init__.py' |\ + xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/'; + + ./dev.py all-python-build-package + + - name: Set outputs + working-directory: ${{ github.action_path }}/../../../ + shell: bash + id: final_step + run: | + output_dir="$(readlink -e dist)" + echo "output_directory=${output_dir}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/build_streamlit_component_library/action.yaml b/.github/actions/build_streamlit_component_library/action.yaml new file mode 100644 index 00000000..3b32f500 --- /dev/null +++ b/.github/actions/build_streamlit_component_library/action.yaml @@ -0,0 +1,54 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + working_directory: + required: false + description: "" + default: "streamlit" + +outputs: + output_file: + description: '' + value: ${{ steps.final_step.outputs.output_file }} + +runs: + using: "composite" + steps: + - name: Check prerequisite + shell: bash + run: | + if command -v node > /dev/null + then + echo "Node is required to use this action" + exit 1 + fi + + - name: Checkout streamlit/streamlit + uses: actions/checkout@v3 + with: + persist-credentials: false + repository: streamlit/streamlit + ref: develop + path: ${{ inputs.working_directory }} + + - name: Install node dependencies for streamlit-component-lib + working-directory: ${{ inputs.working_directory }}/component-lib/ + shell: bash + run: | + npm install -g yarn + yarn install + + - name: Build streamlit-component-lib package + working-directory: ${{ inputs.working_directory }}/component-lib/ + shell: bash + run: yarn run build && npm pack + + - name: Link develop version of streamlit-component-lib + if: inputs.custom_streamlit_component_lib_file != '' + working-directory: ${{ inputs.working_directory }}/component-lib/ + id: final_step + shell: bash + run: | + component_lib_tar_gz=$(find "./" -maxdepth 1 -name 'streamlit-component-lib-*.tgz') + component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}") + echo "output_file=${component_lib_tar_gz}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 97bc60b3..31ad649f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,69 +57,31 @@ jobs: run: ./dev.py e2e-utils-check working-directory: ./component-template/ - - name: Checkout streamlit/streamlit + - name: Build Streamlit Component Library + uses: ./component-template/.github/actions/build_streamlit_component_library if: matrix.component_lib_version == 'develop' - uses: actions/checkout@v3 - with: - persist-credentials: false - repository: streamlit/streamlit - ref: develop - path: streamlit - - - name: Install node dependencies for streamlit-component-lib - if: matrix.component_lib_version == 'develop' - working-directory: ./streamlit/component-lib/ - run: | - npm install -g yarn - yarn install - - - name: Build streamlit-component-lib package - if: matrix.component_lib_version == 'develop' - working-directory: ./streamlit/component-lib/ - run: yarn run build && npm pack - - - name: Link develop version of streamlit-component-lib - if: matrix.component_lib_version == 'develop' - working-directory: ./component-template/ - env: - COMPONENT_LIB_DIR: ${{ github.workspace }}/streamlit/component-lib/ - run: | - component_lib_tar_gz=$(find "${COMPONENT_LIB_DIR}" -maxdepth 1 -name 'streamlit-component-lib-*.tgz') - component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}") - - find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do - ( - cd "${line}"; - npm install "${component_lib_tar_gz}" - ) - done - - - name: Install node dependencies - working-directory: ./component-template/ - run: ./dev.py all-npm-install - - - name: Build frontend code - working-directory: ./component-template/ - run: ./dev.py all-npm-build + id: streamlit_component_library - - name: Build wheel packages - working-directory: ./component-template/ - run: | - find examples template template-reactless -maxdepth 3 -name '__init__.py' |\ - xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/'; - - ./dev.py all-python-build-package + - name: Build components wheels + uses: ./component-template/.github/actions/build_component_wheels + id: component_wheels + with: + custom_streamlit_component_lib_file: >- + ${{ matrix.component_lib_version && steps.streamlit_component_library.outputs.output_file || '' }} - name: Upload wheel packages uses: actions/upload-artifact@v3 with: name: all-wheel - path: ./component-template/dist/*.whl + path: ${{ steps.component_wheels.outputs.output_directory }}/*.whl if-no-files-found: error - name: Run E2E tests if: matrix.node_version == '19.x' uses: ./component-template/.github/actions/run_e2e + with: + python_version: ${{ env.PYTHON_VERSION }} + streamlit_version: ${{ env.STREAMLIT_VERSION }} build-cookiecutter: runs-on: ubuntu-latest From ae66de20d042f7a0f82613e5552d6a3732e83e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 00:02:57 +0200 Subject: [PATCH 07/19] fixup! Modularize workflows --- .github/actions/build_streamlit_component_library/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_streamlit_component_library/action.yaml b/.github/actions/build_streamlit_component_library/action.yaml index 3b32f500..a440a56d 100644 --- a/.github/actions/build_streamlit_component_library/action.yaml +++ b/.github/actions/build_streamlit_component_library/action.yaml @@ -17,7 +17,7 @@ runs: - name: Check prerequisite shell: bash run: | - if command -v node > /dev/null + if ! command -v node > /dev/null then echo "Node is required to use this action" exit 1 From 10beb5312ca70e9da22ffb4e65db1a9acac845ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 00:51:12 +0200 Subject: [PATCH 08/19] Support development streamlit versions --- .github/actions/run_e2e/action.yaml | 26 ++++++- .github/workflows/ci.yaml | 81 ++++++++++++++++++- .gitignore | 1 + Dockerfile | 21 ++++- dev.py | 116 ++++++++++++++++++---------- 5 files changed, 196 insertions(+), 49 deletions(-) diff --git a/.github/actions/run_e2e/action.yaml b/.github/actions/run_e2e/action.yaml index 3024ff96..cb4dc78e 100644 --- a/.github/actions/run_e2e/action.yaml +++ b/.github/actions/run_e2e/action.yaml @@ -5,26 +5,44 @@ inputs: required: true default: '3.8' streamlit_version: - required: true - default: '1.25.0' - streamlit_path: + required: false + default: '' + streamlit_wheel_file: + required: false default: '' runs: using: "composite" steps: + - name: Check prerequisite + run: | + One and exactly one input is required: streamlit_version, streamlit_wheel_file + exit 1 + if: inputs.streamlit_version == '' && inputs.streamlit_wheel_file == '' + shell: bash + - name: Build docker images run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" working-directory: ${{ github.action_path }}/../../../ + if: inputs.streamlit_version != '' shell: bash env: STREAMLIT_VERSION: ${{ inputs.streamlit_version }} PYTHON_VERSION: ${{ inputs.python_version }} + - name: Build docker images + run: ./dev.py e2e-build-images "--streamlit-wheel_file=${{ env.STREAMLIT_WHEEL_FILE }}" "--python-version=${{ env.PYTHON_VERSION }}" + working-directory: ${{ github.action_path }}/../../../ + if: inputs.streamlit_wheel_file != '' + shell: bash + env: + STREAMLIT_WHEEL_FILE: ${{ inputs.streamlit_wheel_file }} + PYTHON_VERSION: ${{ inputs.python_version }} + - name: Run e2e tests run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" shell: bash working-directory: ${{ github.action_path }}/../../../ env: - STREAMLIT_VERSION: ${{ inputs.streamlit_version }} + STREAMLIT_VERSION: ${{ inputs.streamlit_wheel_file == '' && inputs.streamlit_version || 'custom' }} PYTHON_VERSION: ${{ inputs.python_version }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 31ad649f..5fd56e2f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,7 +67,7 @@ jobs: id: component_wheels with: custom_streamlit_component_lib_file: >- - ${{ matrix.component_lib_version && steps.streamlit_component_library.outputs.output_file || '' }} + ${{ matrix.component_lib_version == 'develop' && steps.streamlit_component_library.outputs.output_file || '' }} - name: Upload wheel packages uses: actions/upload-artifact@v3 @@ -83,6 +83,85 @@ jobs: python_version: ${{ env.PYTHON_VERSION }} streamlit_version: ${{ env.STREAMLIT_VERSION }} + test-custom-streamlit-wheel: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node_version: + # # For details, see: https://nodejs.dev/en/about/releases/ + # # Maintenance LTS. End Of Life: 2023-09-11 + # - 16.x + # # Active LTS. End Of Life: 2025-04-30 + # - 18.x + # Current version + - 19.x + component_lib_version: + - current +# - develop + + env: + NODE_VERSION: ${{ matrix.node_version }} + PYTHON_VERSION: 3.8 # Oldest version supported by Streamlit + COMPONENT_LIB_VERSION: ${{ matrix.component_lib_version }} + + name: Examples + Templates / node_version=${{ matrix.node_version }} / component_lib_version=${{ matrix.component_lib_version }} + + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + path: component-template + + - name: Use Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Check dependencies for examples + run: ./dev.py examples-check-deps + working-directory: ./component-template/ + + - name: Check e2e utils files + run: ./dev.py e2e-utils-check + working-directory: ./component-template/ + + - name: Build Streamlit Component Library + uses: ./component-template/.github/actions/build_streamlit_component_library + if: matrix.component_lib_version == 'develop' + id: streamlit_component_library + + - name: Build components wheels + uses: ./component-template/.github/actions/build_component_wheels + id: component_wheels + with: + custom_streamlit_component_lib_file: >- + ${{ matrix.component_lib_version == 'develop' && steps.streamlit_component_library.outputs.output_file || '' }} + + - name: Upload wheel packages + uses: actions/upload-artifact@v3 + with: + name: all-wheel + path: ${{ steps.component_wheels.outputs.output_directory }}/*.whl + if-no-files-found: error + + - name: Download specific streamlit version + id: download_streamlit + run: | + cd $(mktemp -d) + pip install --no-deps "streamlit==1.23.0" + whl_file=$(find . -type f) + whl_file="$(readlink -e "${whl_file}")" + echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT + + - name: Run E2E tests + if: matrix.node_version == '19.x' + uses: ./component-template/.github/actions/run_e2e + with: + python_version: ${{ env.PYTHON_VERSION }} + streamlit_wheel_file: ${{ steps.download_streamlit.outputs.whl_file }} + build-cookiecutter: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6e42962b..8a7d7a25 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ template-reactless/**/package-lock.json # VSCode ######################################################################## .vscode/ +buildcontext/ diff --git a/Dockerfile b/Dockerfile index b2d99893..f3804419 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.4 ARG PYTHON_VERSION -FROM python:${PYTHON_VERSION}-slim-bullseye +FROM python:${PYTHON_VERSION}-slim-bullseye as e2e_base SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] @@ -15,13 +15,15 @@ RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version ARG PLAYWRIGHT_VERSION="1.36.0" ENV PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION} -RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps +# RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps ENV PYTHONUNBUFFERED=1 ENV PIP_ROOT_USER_ACTION=ignore RUN mkdir /component WORKDIR /component +FROM e2e_base AS e2e_pip + # Install streamlit and components ARG STREAMLIT_VERSION="latest" ENV E2E_STREAMLIT_VERSION=${STREAMLIT_VERSION} @@ -30,7 +32,7 @@ RUN <<"EOF" if [[ "${E2E_STREAMLIT_VERSION}" == "latest" ]]; then pip install --no-cache-dir "streamlit" elif [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then - pip uninstall --yes streamlit + pip uninstall --yes "streamlit" pip install --no-cache-dir "streamlit-nightly" else pip install --no-cache-dir "streamlit==${E2E_STREAMLIT_VERSION}" @@ -44,4 +46,17 @@ if [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then else echo "${installed_streamlit_version}" | grep -v 'dev' fi +EOF + +FROM e2e_base AS e2e_whl + +ARG STREAMLIT_WHEEL_PATH + +RUN --mount=type=bind,source=./buildcontext,target=/buildcontext <<"EOF" +pip uninstall --yes "streamlit" "streamlit-nightly" +find /buildcontext/ -name '*.whl'| xargs -t pip install --no-cache-dir + +# Coherence check +installed_streamlit_version=$(python -c "import streamlit; print(streamlit.__version__)") +echo "Installed Streamlit version: ${installed_streamlit_version}" EOF \ No newline at end of file diff --git a/dev.py b/dev.py index 10df75d7..321ae7d2 100755 --- a/dev.py +++ b/dev.py @@ -58,20 +58,35 @@ def cmd_e2e_build_images(args): e2e_dir = next(project_dir.glob("**/e2e/"), None) if e2e_dir and os.listdir(e2e_dir): # Define the image tag for the docker image + streamlit_version = args.streamlit_version if args.streamlit_wheel_file else 'custom' image_tag = ( - f"component-template:py-{args.python_version}-st-{args.streamlit_version}-component-{project_dir.parts[-1]}" + f"component-template:py-{args.python_version}-st-{streamlit_version}-component-{project_dir.parts[-1]}" ) + # Build the docker image with specified build arguments - run_verbose( - [ - "docker", - "build", - ".", + docker_args = [ + "docker", + "build", + ".", + f"--build-arg=PYTHON_VERSION={args.python_version}", + f"--tag={image_tag}", + "--progress=plain", + ] + if args.streamlit_wheel_file: + buildcontext_path = THIS_DIRECTORY / "buildcontext" + shutil.rmtree(buildcontext_path, ignore_errors=True) + buildcontext_path.mkdir() + shutil.copy(args.streamlit_wheel_file, buildcontext_path) + docker_args.extend([ + f"--target=e2e_whl", + ]) + else: + docker_args.extend([ f"--build-arg=STREAMLIT_VERSION={args.streamlit_version}", - f"--build-arg=PYTHON_VERSION={args.python_version}", - f"--tag={image_tag}", - "--progress=plain", - ], + f"--target=e2e_pip", + ]) + run_verbose( + docker_args, env={**os.environ, "DOCKER_BUILDKIT": "1"}, ) @@ -285,36 +300,55 @@ def cmd_update_templates(args): shutil.copytree(output_template, cookiecutter_variant.repo_directory) print() - -COMMANDS = { - "all-npm-install": cmd_all_npm_install, - "all-npm-build": cmd_all_npm_build, - "all-python-build-package": cmd_all_python_build_package, - "examples-check-deps": cmd_example_check_deps, - "templates-check-not-modified": cmd_check_templates_using_cookiecutter, - "templates-update": cmd_update_templates, - "e2e-utils-check": cmd_check_test_utils, - "e2e-build-images": cmd_e2e_build_images, - "e2e-run-tests": cmd_e2e_run, - "docker-images-cleanup": cmd_docker_images_cleanup -} - ARG_STREAMLIT_VERSION = ("--streamlit-version", "latest", "Streamlit version for which tests will be run.") +ARG_STREAMLIT_WHEEL_FILE = ("--streamlit-wheel-file", "", "") ARG_PYTHON_VERSION = ("--python-version", os.environ.get("PYTHON_VERSION", "3.11.4"), "Python version for which tests will be run.") -ARGUMENTS = { - "e2e-build-images": [ - ARG_STREAMLIT_VERSION, - ARG_PYTHON_VERSION - ], - "e2e-run-tests": [ - ARG_STREAMLIT_VERSION, - ARG_PYTHON_VERSION - ], - "docker-images-cleanup": [ - (*ARG_STREAMLIT_VERSION[:2], f"Streamlit version used to create the Docker resources"), - (*ARG_PYTHON_VERSION[:2], f"Python version used to create the Docker resources") - ] +COMMANDS = { + "all-npm-install": { + "fn": cmd_all_npm_install} + , + "all-npm-build": { + "fn": cmd_all_npm_build + }, + "all-python-build-package": { + "fn": cmd_all_python_build_package + }, + "examples-check-deps": { + "fn": cmd_example_check_deps + }, + "templates-check-not-modified": { + "fn": cmd_check_templates_using_cookiecutter + }, + "templates-update": { + "fn": cmd_update_templates + }, + "e2e-utils-check": { + "fn": cmd_check_test_utils + }, + "e2e-build-images": { + "fn": cmd_e2e_build_images, + "arguments": [ + ARG_STREAMLIT_VERSION, + ARG_STREAMLIT_WHEEL_FILE, + ARG_PYTHON_VERSION, + ] + }, + "e2e-run-tests": { + "fn": cmd_e2e_run, + "arguments": [ + ARG_STREAMLIT_VERSION, + ARG_PYTHON_VERSION, + ] + }, + "docker-images-cleanup": { + "fn": cmd_docker_images_cleanup, + "arguments": [ + (*ARG_STREAMLIT_VERSION[:2], f"Streamlit version used to create the Docker resources"), + (*ARG_STREAMLIT_WHEEL_FILE[:2], f""), + (*ARG_PYTHON_VERSION[:2], f"Python version used to create the Docker resources") + ] + } } @@ -323,12 +357,12 @@ def get_parser(): parser = argparse.ArgumentParser(prog=__file__, description=__doc__) subparsers = parser.add_subparsers(dest="subcommand", metavar="COMMAND") subparsers.required = True - for command_name, command_fn in COMMANDS.items(): + for command_name, command_info in COMMANDS.items(): + command_fn = command_info['fn'] subparser = subparsers.add_parser(command_name, help=command_fn.__doc__) - if command_name in ARGUMENTS: - for arg_name, arg_default, arg_help in ARGUMENTS[command_name]: - subparser.add_argument(arg_name, default=arg_default, help=arg_help) + for arg_name, arg_default, arg_help in command_info.get('arguments', []): + subparser.add_argument(arg_name, default=arg_default, help=arg_help) subparser.set_defaults(func=command_fn) From 020b7f96b557f8deea1d240f16aec0071e25d898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 00:54:35 +0200 Subject: [PATCH 09/19] fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5fd56e2f..929befb6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -151,7 +151,8 @@ jobs: run: | cd $(mktemp -d) pip install --no-deps "streamlit==1.23.0" - whl_file=$(find . -type f) + set -x + whl_file="$(find . -type f)" whl_file="$(readlink -e "${whl_file}")" echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT From 1e63063241cfbfc18342341be9fb6d7611206435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 00:56:56 +0200 Subject: [PATCH 10/19] fixup! fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 929befb6..cd6fd42e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,9 +149,9 @@ jobs: - name: Download specific streamlit version id: download_streamlit run: | - cd $(mktemp -d) + set +x + cd "$(mktemp -d)" pip install --no-deps "streamlit==1.23.0" - set -x whl_file="$(find . -type f)" whl_file="$(readlink -e "${whl_file}")" echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT From c9648ca0c9f75373ab669d723356f4cebc7c21de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:01:34 +0200 Subject: [PATCH 11/19] fixup! fixup! fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd6fd42e..0de9a824 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,7 +149,7 @@ jobs: - name: Download specific streamlit version id: download_streamlit run: | - set +x + set -x cd "$(mktemp -d)" pip install --no-deps "streamlit==1.23.0" whl_file="$(find . -type f)" From 57fdc5c4963fcd23adf7b6308b815a2a6a32e46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:02:25 +0200 Subject: [PATCH 12/19] fixup! fixup! fixup! fixup! Support development streamlit versions --- dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev.py b/dev.py index 321ae7d2..137b63f8 100755 --- a/dev.py +++ b/dev.py @@ -58,7 +58,7 @@ def cmd_e2e_build_images(args): e2e_dir = next(project_dir.glob("**/e2e/"), None) if e2e_dir and os.listdir(e2e_dir): # Define the image tag for the docker image - streamlit_version = args.streamlit_version if args.streamlit_wheel_file else 'custom' + streamlit_version = args.streamlit_version if not args.streamlit_wheel_file else 'custom' image_tag = ( f"component-template:py-{args.python_version}-st-{streamlit_version}-component-{project_dir.parts[-1]}" ) From 6b867b73b0a4b758ceb9c2bf5e6b31417c3512ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:06:26 +0200 Subject: [PATCH 13/19] fixup! fixup! fixup! fixup! fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0de9a824..ecdb4768 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,8 +152,7 @@ jobs: set -x cd "$(mktemp -d)" pip install --no-deps "streamlit==1.23.0" - whl_file="$(find . -type f)" - whl_file="$(readlink -e "${whl_file}")" + whl_file="$(readlink -e "$(find . -type f)")" echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT - name: Run E2E tests From f7f329427f92eb426b990da8039cd3315628f979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:11:31 +0200 Subject: [PATCH 14/19] fixup! fixup! fixup! fixup! fixup! fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ecdb4768..2bf58ac8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,6 +152,7 @@ jobs: set -x cd "$(mktemp -d)" pip install --no-deps "streamlit==1.23.0" + ls *.whl whl_file="$(readlink -e "$(find . -type f)")" echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT From 8ae97470f99e11af8e9d2eb77eaa99143d564cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:17:57 +0200 Subject: [PATCH 15/19] fixup! fixup! fixup! fixup! fixup! fixup! fixup! Support development streamlit versions --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2bf58ac8..cb3e9571 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -151,8 +151,7 @@ jobs: run: | set -x cd "$(mktemp -d)" - pip install --no-deps "streamlit==1.23.0" - ls *.whl + pip download --no-deps "streamlit==1.23.0" whl_file="$(readlink -e "$(find . -type f)")" echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT From bf42ed03ab02f12b4201c9cba7264dac0636032c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:21:11 +0200 Subject: [PATCH 16/19] fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Support development streamlit versions --- .github/actions/run_e2e/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run_e2e/action.yaml b/.github/actions/run_e2e/action.yaml index cb4dc78e..7451332c 100644 --- a/.github/actions/run_e2e/action.yaml +++ b/.github/actions/run_e2e/action.yaml @@ -31,7 +31,7 @@ runs: PYTHON_VERSION: ${{ inputs.python_version }} - name: Build docker images - run: ./dev.py e2e-build-images "--streamlit-wheel_file=${{ env.STREAMLIT_WHEEL_FILE }}" "--python-version=${{ env.PYTHON_VERSION }}" + run: ./dev.py e2e-build-images "--streamlit-wheel-file=${{ env.STREAMLIT_WHEEL_FILE }}" "--python-version=${{ env.PYTHON_VERSION }}" working-directory: ${{ github.action_path }}/../../../ if: inputs.streamlit_wheel_file != '' shell: bash From 8a52bfe1d5380cf86a20f70269665b51ef0aa659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 26 Aug 2023 01:28:01 +0200 Subject: [PATCH 17/19] Enable all tests --- .github/workflows/ci.yaml | 10 +++++----- Dockerfile | 2 +- dev.py | 3 --- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb3e9571..f7345203 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,11 +16,11 @@ jobs: fail-fast: false matrix: node_version: -# # For details, see: https://nodejs.dev/en/about/releases/ -# # Maintenance LTS. End Of Life: 2023-09-11 -# - 16.x -# # Active LTS. End Of Life: 2025-04-30 -# - 18.x + # For details, see: https://nodejs.dev/en/about/releases/ + # Maintenance LTS. End Of Life: 2023-09-11 + - 16.x + # Active LTS. End Of Life: 2025-04-30 + - 18.x # Current version - 19.x component_lib_version: diff --git a/Dockerfile b/Dockerfile index f3804419..16ca380d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version ARG PLAYWRIGHT_VERSION="1.36.0" ENV PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION} -# RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps +RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps ENV PYTHONUNBUFFERED=1 ENV PIP_ROOT_USER_ACTION=ignore diff --git a/dev.py b/dev.py index 137b63f8..fb67eafd 100755 --- a/dev.py +++ b/dev.py @@ -23,9 +23,6 @@ THIS_DIRECTORY / "template", THIS_DIRECTORY / "template-reactless", ] -# TODO(kbregula): Enable tests for examples -EXAMPLE_DIRECTORIES = [] - # Utilities function def run_verbose(cmd_args, *args, **kwargs): From d6a343cb9bca2eddd3eec5ba617c669009da1d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Mon, 28 Aug 2023 09:29:07 +0200 Subject: [PATCH 18/19] Update dev.py --- dev.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dev.py b/dev.py index fb67eafd..71953ef7 100755 --- a/dev.py +++ b/dev.py @@ -107,15 +107,7 @@ def cmd_e2e_run(args): image_tag, "/bin/sh", "-c", # Run a shell command inside the container "find /component/dist/ -name '*.whl' | xargs -I {} echo '{}[devel]' | xargs pip install && " # Install whl package and dev dependencies - f"playwright install webkit chromium firefox --with-deps && " # Install browsers - f"pytest", # Run pytest - "-s", - "--browser", "webkit", - "--browser", "chromium", - "--browser", "firefox", - "--reruns", "5", - "--capture=no", - "--setup-show" + f"pytest -S --browser webkit --browser chromium --browser firefox --reruns 5 --capture=no --setup-show", # Run pytest ]) From de2c13fa725df4d7a637eab29479f28dfd99a667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Mon, 28 Aug 2023 09:37:29 +0200 Subject: [PATCH 19/19] Update dev.py --- dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev.py b/dev.py index 71953ef7..4ddeed88 100755 --- a/dev.py +++ b/dev.py @@ -107,7 +107,7 @@ def cmd_e2e_run(args): image_tag, "/bin/sh", "-c", # Run a shell command inside the container "find /component/dist/ -name '*.whl' | xargs -I {} echo '{}[devel]' | xargs pip install && " # Install whl package and dev dependencies - f"pytest -S --browser webkit --browser chromium --browser firefox --reruns 5 --capture=no --setup-show", # Run pytest + f"pytest -s --browser webkit --browser chromium --browser firefox --reruns 5 --capture=no --setup-show", # Run pytest ])