diff --git a/.github/actions/build_component_wheels/action.yaml b/.github/actions/build_component_wheels/action.yaml new file mode 100644 index 00000000..aab89a18 --- /dev/null +++ b/.github/actions/build_component_wheels/action.yaml @@ -0,0 +1,56 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + custom_streamlit_component_lib_file: + required: false + description: '' + default: '' + +outputs: + output_directory: + description: '' + value: ${{ steps.final_step.outputs.output_directory }} + +runs: + using: "composite" + steps: + - name: Link develop version of streamlit-component-lib + if: inputs.custom_streamlit_component_lib_file != '' + working-directory: ${{ github.action_path }}/../../../ + shell: bash + env: + STREAMLIT_COMPONENT_LIB_FILE: ${{ inputs.custom_streamlit_component_lib_file }} + run: | + find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do + ( + cd "${line}"; + npm install "${STREAMLIT_COMPONENT_LIB_FILE}" + ) + done + + - name: Install node dependencies + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: ./dev.py all-npm-install + + - name: Build frontend code + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: ./dev.py all-npm-build + + - name: Build wheel packages + working-directory: ${{ github.action_path }}/../../../ + shell: bash + run: | + find examples template template-reactless -maxdepth 3 -name '__init__.py' |\ + xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/'; + + ./dev.py all-python-build-package + + - name: Set outputs + working-directory: ${{ github.action_path }}/../../../ + shell: bash + id: final_step + run: | + output_dir="$(readlink -e dist)" + echo "output_directory=${output_dir}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/build_streamlit_component_library/action.yaml b/.github/actions/build_streamlit_component_library/action.yaml new file mode 100644 index 00000000..a440a56d --- /dev/null +++ b/.github/actions/build_streamlit_component_library/action.yaml @@ -0,0 +1,54 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + working_directory: + required: false + description: "" + default: "streamlit" + +outputs: + output_file: + description: '' + value: ${{ steps.final_step.outputs.output_file }} + +runs: + using: "composite" + steps: + - name: Check prerequisite + shell: bash + run: | + if ! command -v node > /dev/null + then + echo "Node is required to use this action" + exit 1 + fi + + - name: Checkout streamlit/streamlit + uses: actions/checkout@v3 + with: + persist-credentials: false + repository: streamlit/streamlit + ref: develop + path: ${{ inputs.working_directory }} + + - name: Install node dependencies for streamlit-component-lib + working-directory: ${{ inputs.working_directory }}/component-lib/ + shell: bash + run: | + npm install -g yarn + yarn install + + - name: Build streamlit-component-lib package + working-directory: ${{ inputs.working_directory }}/component-lib/ + shell: bash + run: yarn run build && npm pack + + - name: Link develop version of streamlit-component-lib + if: inputs.custom_streamlit_component_lib_file != '' + working-directory: ${{ inputs.working_directory }}/component-lib/ + id: final_step + shell: bash + run: | + component_lib_tar_gz=$(find "./" -maxdepth 1 -name 'streamlit-component-lib-*.tgz') + component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}") + echo "output_file=${component_lib_tar_gz}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/run_e2e/action.yaml b/.github/actions/run_e2e/action.yaml new file mode 100644 index 00000000..7451332c --- /dev/null +++ b/.github/actions/run_e2e/action.yaml @@ -0,0 +1,48 @@ +name: 'Hello World' +description: 'Greet someone' +inputs: + python_version: + required: true + default: '3.8' + streamlit_version: + required: false + default: '' + streamlit_wheel_file: + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Check prerequisite + run: | + One and exactly one input is required: streamlit_version, streamlit_wheel_file + exit 1 + if: inputs.streamlit_version == '' && inputs.streamlit_wheel_file == '' + shell: bash + + - name: Build docker images + run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + working-directory: ${{ github.action_path }}/../../../ + if: inputs.streamlit_version != '' + shell: bash + env: + STREAMLIT_VERSION: ${{ inputs.streamlit_version }} + PYTHON_VERSION: ${{ inputs.python_version }} + + - name: Build docker images + run: ./dev.py e2e-build-images "--streamlit-wheel-file=${{ env.STREAMLIT_WHEEL_FILE }}" "--python-version=${{ env.PYTHON_VERSION }}" + working-directory: ${{ github.action_path }}/../../../ + if: inputs.streamlit_wheel_file != '' + shell: bash + env: + STREAMLIT_WHEEL_FILE: ${{ inputs.streamlit_wheel_file }} + PYTHON_VERSION: ${{ inputs.python_version }} + + - name: Run e2e tests + run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + shell: bash + working-directory: ${{ github.action_path }}/../../../ + env: + STREAMLIT_VERSION: ${{ inputs.streamlit_wheel_file == '' && inputs.streamlit_version || 'custom' }} + PYTHON_VERSION: ${{ inputs.python_version }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4931eab0..f7345203 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,7 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false + path: component-template - name: Use Node.js ${{ env.NODE_VERSION }} uses: actions/setup-node@v3 @@ -50,77 +51,116 @@ jobs: - name: Check dependencies for examples run: ./dev.py examples-check-deps + working-directory: ./component-template/ - name: Check e2e utils files run: ./dev.py e2e-utils-check + working-directory: ./component-template/ - - name: Checkout streamlit/streamlit + - name: Build Streamlit Component Library + uses: ./component-template/.github/actions/build_streamlit_component_library if: matrix.component_lib_version == 'develop' - uses: actions/checkout@v3 + id: streamlit_component_library + + - name: Build components wheels + uses: ./component-template/.github/actions/build_component_wheels + id: component_wheels with: - persist-credentials: false - repository: streamlit/streamlit - ref: develop - path: streamlit + custom_streamlit_component_lib_file: >- + ${{ matrix.component_lib_version == 'develop' && steps.streamlit_component_library.outputs.output_file || '' }} - - name: Install node dependencies for streamlit-component-lib - if: matrix.component_lib_version == 'develop' - working-directory: ./streamlit/component-lib/ - run: | - npm install -g yarn - yarn install + - name: Upload wheel packages + uses: actions/upload-artifact@v3 + with: + name: all-wheel + path: ${{ steps.component_wheels.outputs.output_directory }}/*.whl + if-no-files-found: error - - name: Build streamlit-component-lib package - if: matrix.component_lib_version == 'develop' - working-directory: ./streamlit/component-lib/ - run: yarn run build && npm pack + - name: Run E2E tests + if: matrix.node_version == '19.x' + uses: ./component-template/.github/actions/run_e2e + with: + python_version: ${{ env.PYTHON_VERSION }} + streamlit_version: ${{ env.STREAMLIT_VERSION }} - - name: Link develop version of streamlit-component-lib - if: matrix.component_lib_version == 'develop' - env: - COMPONENT_LIB_DIR: ${{ github.workspace }}/streamlit/component-lib/ - run: | - component_lib_tar_gz=$(find "${COMPONENT_LIB_DIR}" -maxdepth 1 -name 'streamlit-component-lib-*.tgz') - component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}") + test-custom-streamlit-wheel: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node_version: + # # For details, see: https://nodejs.dev/en/about/releases/ + # # Maintenance LTS. End Of Life: 2023-09-11 + # - 16.x + # # Active LTS. End Of Life: 2025-04-30 + # - 18.x + # Current version + - 19.x + component_lib_version: + - current +# - develop - find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do - ( - cd "${line}"; - npm install "${component_lib_tar_gz}" - ) - done + env: + NODE_VERSION: ${{ matrix.node_version }} + PYTHON_VERSION: 3.8 # Oldest version supported by Streamlit + COMPONENT_LIB_VERSION: ${{ matrix.component_lib_version }} - - name: Install node dependencies - run: ./dev.py all-npm-install + name: Examples + Templates / node_version=${{ matrix.node_version }} / component_lib_version=${{ matrix.component_lib_version }} - - name: Build frontend code - run: ./dev.py all-npm-build + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + path: component-template - - name: Build wheel packages - run: | - find examples template template-reactless -maxdepth 3 -name '__init__.py' |\ - xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/'; + - name: Use Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} - ./dev.py all-python-build-package + - name: Check dependencies for examples + run: ./dev.py examples-check-deps + working-directory: ./component-template/ - - name: Upload wheel packages - uses: actions/upload-artifact@v3 - with: - name: all-wheel - path: dist/*.whl - if-no-files-found: error + - name: Check e2e utils files + run: ./dev.py e2e-utils-check + working-directory: ./component-template/ - - name: Set up Docker Buildx - if: matrix.node_version == '19.x' - uses: docker/setup-buildx-action@7703e82fbced3d0c9eec08dff4429c023a5fd9a9 # v2.9.1 + - name: Build Streamlit Component Library + uses: ./component-template/.github/actions/build_streamlit_component_library + if: matrix.component_lib_version == 'develop' + id: streamlit_component_library - - name: Build docker images - if: matrix.node_version == '19.x' - run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + - name: Build components wheels + uses: ./component-template/.github/actions/build_component_wheels + id: component_wheels + with: + custom_streamlit_component_lib_file: >- + ${{ matrix.component_lib_version == 'develop' && steps.streamlit_component_library.outputs.output_file || '' }} - - name: Run e2e tests - if: matrix.node_version == '19.x' - run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}" + - name: Upload wheel packages + uses: actions/upload-artifact@v3 + with: + name: all-wheel + path: ${{ steps.component_wheels.outputs.output_directory }}/*.whl + if-no-files-found: error + + - name: Download specific streamlit version + id: download_streamlit + run: | + set -x + cd "$(mktemp -d)" + pip download --no-deps "streamlit==1.23.0" + whl_file="$(readlink -e "$(find . -type f)")" + echo "whl_file=${whl_file}" >> $GITHUB_OUTPUT + + - name: Run E2E tests + if: matrix.node_version == '19.x' + uses: ./component-template/.github/actions/run_e2e + with: + python_version: ${{ env.PYTHON_VERSION }} + streamlit_wheel_file: ${{ steps.download_streamlit.outputs.whl_file }} build-cookiecutter: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6e42962b..8a7d7a25 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ template-reactless/**/package-lock.json # VSCode ######################################################################## .vscode/ +buildcontext/ diff --git a/Dockerfile b/Dockerfile index 73fed010..16ca380d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.4 -ARG PYTHON_VERSION="3.11.4" -FROM python:${PYTHON_VERSION}-slim-bullseye +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-bullseye as e2e_base SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] @@ -11,11 +11,19 @@ ENV PIP_VERSION=${PIP_VERSION} RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version +# Setup Playwright +ARG PLAYWRIGHT_VERSION="1.36.0" +ENV PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION} + +RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps + ENV PYTHONUNBUFFERED=1 ENV PIP_ROOT_USER_ACTION=ignore RUN mkdir /component WORKDIR /component +FROM e2e_base AS e2e_pip + # Install streamlit and components ARG STREAMLIT_VERSION="latest" ENV E2E_STREAMLIT_VERSION=${STREAMLIT_VERSION} @@ -24,7 +32,7 @@ RUN <<"EOF" if [[ "${E2E_STREAMLIT_VERSION}" == "latest" ]]; then pip install --no-cache-dir "streamlit" elif [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then - pip uninstall --yes streamlit + pip uninstall --yes "streamlit" pip install --no-cache-dir "streamlit-nightly" else pip install --no-cache-dir "streamlit==${E2E_STREAMLIT_VERSION}" @@ -38,4 +46,17 @@ if [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then else echo "${installed_streamlit_version}" | grep -v 'dev' fi +EOF + +FROM e2e_base AS e2e_whl + +ARG STREAMLIT_WHEEL_PATH + +RUN --mount=type=bind,source=./buildcontext,target=/buildcontext <<"EOF" +pip uninstall --yes "streamlit" "streamlit-nightly" +find /buildcontext/ -name '*.whl'| xargs -t pip install --no-cache-dir + +# Coherence check +installed_streamlit_version=$(python -c "import streamlit; print(streamlit.__version__)") +echo "Installed Streamlit version: ${installed_streamlit_version}" EOF \ No newline at end of file diff --git a/dev.py b/dev.py index b97a529e..4ddeed88 100755 --- a/dev.py +++ b/dev.py @@ -24,7 +24,6 @@ THIS_DIRECTORY / "template-reactless", ] - # Utilities function def run_verbose(cmd_args, *args, **kwargs): kwargs.setdefault("check", True) @@ -56,20 +55,35 @@ def cmd_e2e_build_images(args): e2e_dir = next(project_dir.glob("**/e2e/"), None) if e2e_dir and os.listdir(e2e_dir): # Define the image tag for the docker image + streamlit_version = args.streamlit_version if not args.streamlit_wheel_file else 'custom' image_tag = ( - f"component-template:py-{args.python_version}-st-{args.streamlit_version}-component-{project_dir.parts[-1]}" + f"component-template:py-{args.python_version}-st-{streamlit_version}-component-{project_dir.parts[-1]}" ) + # Build the docker image with specified build arguments - run_verbose( - [ - "docker", - "build", - ".", + docker_args = [ + "docker", + "build", + ".", + f"--build-arg=PYTHON_VERSION={args.python_version}", + f"--tag={image_tag}", + "--progress=plain", + ] + if args.streamlit_wheel_file: + buildcontext_path = THIS_DIRECTORY / "buildcontext" + shutil.rmtree(buildcontext_path, ignore_errors=True) + buildcontext_path.mkdir() + shutil.copy(args.streamlit_wheel_file, buildcontext_path) + docker_args.extend([ + f"--target=e2e_whl", + ]) + else: + docker_args.extend([ f"--build-arg=STREAMLIT_VERSION={args.streamlit_version}", - f"--build-arg=PYTHON_VERSION={args.python_version}", - f"--tag={image_tag}", - "--progress=plain", - ], + f"--target=e2e_pip", + ]) + run_verbose( + docker_args, env={**os.environ, "DOCKER_BUILDKIT": "1"}, ) @@ -93,15 +107,7 @@ def cmd_e2e_run(args): image_tag, "/bin/sh", "-c", # Run a shell command inside the container "find /component/dist/ -name '*.whl' | xargs -I {} echo '{}[devel]' | xargs pip install && " # Install whl package and dev dependencies - f"playwright install webkit chromium firefox --with-deps && " # Install browsers - f"pytest", # Run pytest - "-s", - "--browser", "webkit", - "--browser", "chromium", - "--browser", "firefox", - "--reruns", "5", - "--capture=no", - "--setup-show" + f"pytest -s --browser webkit --browser chromium --browser firefox --reruns 5 --capture=no --setup-show", # Run pytest ]) @@ -283,36 +289,55 @@ def cmd_update_templates(args): shutil.copytree(output_template, cookiecutter_variant.repo_directory) print() - -COMMANDS = { - "all-npm-install": cmd_all_npm_install, - "all-npm-build": cmd_all_npm_build, - "all-python-build-package": cmd_all_python_build_package, - "examples-check-deps": cmd_example_check_deps, - "templates-check-not-modified": cmd_check_templates_using_cookiecutter, - "templates-update": cmd_update_templates, - "e2e-utils-check": cmd_check_test_utils, - "e2e-build-images": cmd_e2e_build_images, - "e2e-run-tests": cmd_e2e_run, - "docker-images-cleanup": cmd_docker_images_cleanup -} - ARG_STREAMLIT_VERSION = ("--streamlit-version", "latest", "Streamlit version for which tests will be run.") +ARG_STREAMLIT_WHEEL_FILE = ("--streamlit-wheel-file", "", "") ARG_PYTHON_VERSION = ("--python-version", os.environ.get("PYTHON_VERSION", "3.11.4"), "Python version for which tests will be run.") -ARGUMENTS = { - "e2e-build-images": [ - ARG_STREAMLIT_VERSION, - ARG_PYTHON_VERSION - ], - "e2e-run-tests": [ - ARG_STREAMLIT_VERSION, - ARG_PYTHON_VERSION - ], - "docker-images-cleanup": [ - (*ARG_STREAMLIT_VERSION[:2], f"Streamlit version used to create the Docker resources"), - (*ARG_PYTHON_VERSION[:2], f"Python version used to create the Docker resources") - ] +COMMANDS = { + "all-npm-install": { + "fn": cmd_all_npm_install} + , + "all-npm-build": { + "fn": cmd_all_npm_build + }, + "all-python-build-package": { + "fn": cmd_all_python_build_package + }, + "examples-check-deps": { + "fn": cmd_example_check_deps + }, + "templates-check-not-modified": { + "fn": cmd_check_templates_using_cookiecutter + }, + "templates-update": { + "fn": cmd_update_templates + }, + "e2e-utils-check": { + "fn": cmd_check_test_utils + }, + "e2e-build-images": { + "fn": cmd_e2e_build_images, + "arguments": [ + ARG_STREAMLIT_VERSION, + ARG_STREAMLIT_WHEEL_FILE, + ARG_PYTHON_VERSION, + ] + }, + "e2e-run-tests": { + "fn": cmd_e2e_run, + "arguments": [ + ARG_STREAMLIT_VERSION, + ARG_PYTHON_VERSION, + ] + }, + "docker-images-cleanup": { + "fn": cmd_docker_images_cleanup, + "arguments": [ + (*ARG_STREAMLIT_VERSION[:2], f"Streamlit version used to create the Docker resources"), + (*ARG_STREAMLIT_WHEEL_FILE[:2], f""), + (*ARG_PYTHON_VERSION[:2], f"Python version used to create the Docker resources") + ] + } } @@ -321,12 +346,12 @@ def get_parser(): parser = argparse.ArgumentParser(prog=__file__, description=__doc__) subparsers = parser.add_subparsers(dest="subcommand", metavar="COMMAND") subparsers.required = True - for command_name, command_fn in COMMANDS.items(): + for command_name, command_info in COMMANDS.items(): + command_fn = command_info['fn'] subparser = subparsers.add_parser(command_name, help=command_fn.__doc__) - if command_name in ARGUMENTS: - for arg_name, arg_default, arg_help in ARGUMENTS[command_name]: - subparser.add_argument(arg_name, default=arg_default, help=arg_help) + for arg_name, arg_default, arg_help in command_info.get('arguments', []): + subparser.add_argument(arg_name, default=arg_default, help=arg_help) subparser.set_defaults(func=command_fn)