Skip to content

Commit

Permalink
Cleanup workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
Fokko committed Nov 22, 2023
1 parent 2247ae7 commit 949e279
Show file tree
Hide file tree
Showing 7 changed files with 13 additions and 35 deletions.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
38 changes: 5 additions & 33 deletions .github/workflows/Rest.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Rest
on: [push, pull_request, repository_dispatch]
on: [pull_request, repository_dispatch]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
Expand All @@ -11,41 +11,25 @@ jobs:
rest:
name: Test against Rest Catalog
runs-on: ubuntu-latest
container: ${{ matrix.container }}
strategy:
matrix:
# Add commits/tags to build against other DuckDB versions
duckdb_version: [ '<submodule_version>' ]
arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4']
vcpkg_version: [ '2023.04.15' ]
include:
- arch: 'linux_amd64_gcc4'
container: 'quay.io/pypa/manylinux2014_x86_64'
vcpkg_triplet: 'x64-linux'
- arch: 'linux_amd64'
container: 'ubuntu:18.04'
vcpkg_triplet: 'x64-linux'
- arch: 'linux_arm64'
container: 'ubuntu:18.04'
vcpkg_triplet: 'arm64-linux'
container: 'ubuntu:20.04'
env:
VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
VCPKG_TARGET_TRIPLET: 'x64-linux'
GEN: Ninja
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake

steps:
- name: Install required ubuntu packages
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
run: |
apt-get update -y -qq
apt-get install -y -qq software-properties-common
add-apt-repository ppa:git-core/ppa
apt-get update -y -qq
apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
apt-get install -y -qq tar pkg-config
curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
- name: Install Git 2.18.5
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
run: |
wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
tar xvf v2.18.5.tar.gz
Expand All @@ -59,19 +43,11 @@ jobs:
fetch-depth: 0
submodules: 'true'

- name: Checkout DuckDB to version
if: ${{ matrix.duckdb_version != '<submodule_version>'}}
run: |
cd duckdb
git checkout ${{ matrix.duckdb_version }}
- name: Setup ManyLinux2014
if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
run: |
./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias
- name: Setup Ubuntu
if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
uses: ./duckdb/.github/actions/ubuntu_18_setup
with:
aarch64_cross_compile: 1
Expand All @@ -81,18 +57,14 @@ jobs:
with:
vcpkgGitCommitId: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1

# Build extension
- name: Build extension
env:
GEN: ninja
STATIC_LIBCPP: 1
CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}
CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}
run: |
make release
- name: Start Rest Catalog
if: ${{ matrix.arch == 'linux_amd64_gcc4' || matrix.arch == 'linux_amd64' }}
working-directory: scripts/
run: |
./start-rest-catalog.sh
4 changes: 2 additions & 2 deletions scripts/provision.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@
)
.config("spark.sql.catalog.demo", "org.apache.iceberg.spark.SparkCatalog")
.config("spark.sql.catalog.demo.type", "rest")
.config("spark.sql.catalog.demo.uri", "http://localhost:8181")
.config("spark.sql.catalog.demo.uri", "http://127.0.0.1:8181")
.config("spark.sql.catalog.demo.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
.config("spark.sql.catalog.demo.warehouse", "s3://warehouse/wh/")
.config("spark.sql.catalog.demo.s3.endpoint", "http://localhost:9000")
.config("spark.sql.catalog.demo.s3.endpoint", "http://127.0.0.1:9000")
.config("spark.sql.catalog.demo.s3.path-style-access", "true")
.config("spark.sql.defaultCatalog", "demo")
.config("spark.sql.catalogImplementation", "in-memory")
Expand Down
6 changes: 6 additions & 0 deletions scripts/start-rest-catalog.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ docker-compose rm -f
docker-compose up -d
docker-compose logs -f mc

sleep 10

docker-compose logs rest

docker-compose ps

pip3 install -r requirements.txt

python3 provision.py
Expand Down

0 comments on commit 949e279

Please sign in to comment.