From a7ba60ef137bae67cfdc51f5e424c9f9f6bf5836 Mon Sep 17 00:00:00 2001 From: Varsha U N Date: Sun, 1 Dec 2024 15:35:34 +0530 Subject: [PATCH] Co-authored-by: Ayan Sinha Mahapatra Signed-off-by: Varsha U N Signed-off-by: Varsha U N Co-authored-by: Ayan Sinha Mahapatra Signed-off-by: Varsha U N Update dockerfile.py Signed-off-by: Varsha U N --- src/packagedcode/__init__.py | 4 +- .../{dockerfileLABEL.py => dockerfile.py} | 51 +++++------ ...ct_toml.cpython-312-pytest-8.3.3.pyc.26520 | Bin 0 -> 2086 bytes .../data/docker/containerfile-expected.json | 13 +++ .../data/docker/psql-expected.json | 10 +++ .../packagedcode/data/docker/psql.dockerfile | 5 ++ .../data/docker/test-dockerfile-expected.json | 18 ++++ .../data/docker/test.containerfile | 80 ++++++++++++++++++ .../packagedcode/data/docker/test.dockerfile | 36 ++++++++ tests/packagedcode/test_dockerfile.py | 60 +++++++++++++ 10 files changed, 243 insertions(+), 34 deletions(-) rename src/packagedcode/{dockerfileLABEL.py => dockerfile.py} (51%) create mode 100644 tests/packagedcode/__pycache__/test_parse_pyproject_toml.cpython-312-pytest-8.3.3.pyc.26520 create mode 100644 tests/packagedcode/data/docker/containerfile-expected.json create mode 100644 tests/packagedcode/data/docker/psql-expected.json create mode 100644 tests/packagedcode/data/docker/psql.dockerfile create mode 100644 tests/packagedcode/data/docker/test-dockerfile-expected.json create mode 100644 tests/packagedcode/data/docker/test.containerfile create mode 100644 tests/packagedcode/data/docker/test.dockerfile create mode 100644 tests/packagedcode/test_dockerfile.py diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index dc9cde4589..69fc770bec 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -20,7 +20,7 @@ from packagedcode import debian from packagedcode import debian_copyright from packagedcode import distro -from packagedcode import dockerfileLABEL +from packagedcode import dockerfile from packagedcode import conda from packagedcode import conan from packagedcode import cocoapods @@ -98,7 +98,7 @@ debian.DebianSourcePackageTarballHandler, distro.EtcOsReleaseHandler, - dockerfileLABEL.DockerfileHandler, + dockerfile.DockerfileHandler, freebsd.CompactManifestHandler, diff --git a/src/packagedcode/dockerfileLABEL.py b/src/packagedcode/dockerfile.py similarity index 51% rename from src/packagedcode/dockerfileLABEL.py rename to src/packagedcode/dockerfile.py index b9e46dddc2..cc97bdc147 100644 --- a/src/packagedcode/dockerfileLABEL.py +++ b/src/packagedcode/dockerfile.py @@ -14,29 +14,33 @@ from dockerfile_parse import DockerfileParser from packagedcode import models from packagedcode import utils +import fnmatch class DockerfileHandler(models.DatafileHandler): - datasource_id = 'dockerfile' - default_package_type = 'docker-image' - path_patterns = ('Dockerfile', 'containerfile', '*.dockerfile') - description = 'Dockerfile (OCI) metadata handler' - documentation_url = 'https://docs.docker.com/engine/reference/builder/' - + datasource_id = 'dockerfile_oci_labels' + + @classmethod + def is_datafile(cls, path): + patterns = ['Dockerfile', 'containerfile', '*.dockerfile'] + filename=os.path.basename(path) + for pattern in patterns: + if fnmatch.fnmatch(filename, pattern): + return True + return False + @classmethod def parse(cls, location, package_only=False): """ Parse a Dockerfile and yield one or more PackageData objects with OCI labels and metadata. """ labels = cls.extract_oci_labels_from_dockerfile(location) - - package_data = { 'datasource_id': cls.datasource_id, 'type': cls.default_package_type, - 'name': labels.get('name', 'unknown'), - 'version': labels.get('version', 'unknown'), - 'license_expression': labels.get('license', 'unknown'), + 'name': labels.get('name', 'None'), + 'version': labels.get('version', 'None'), + 'license_expression': labels.get('license', 'None'), 'labels': labels, } @@ -45,28 +49,11 @@ def parse(cls, location, package_only=False): @classmethod def extract_oci_labels_from_dockerfile(cls, dockerfile_path): """ - Extract OCI labels from the Dockerfile using container-inspector. + Extract OCI labels from the Dockerfile using DockerfileParser. """ labels = {} parser = DockerfileParser() - parser.parse(dockerfile_path) - labels = parser.labels + with open(dockerfile_path, 'r') as dockerfile: + parser.content = dockerfile.read() + labels = parser.labels return labels - - @classmethod - def assemble(cls, package_data, resource, codebase, package_adder): - """ - Assemble a Package from the parsed Dockerfile data. - """ - if package_data.purl: - package = models.Package.from_package_data(package_data=package_data, datafile_path=resource.path) - - - package.populate_license_fields() - - yield package - - - package_adder(package.package_uid, resource, codebase) - - yield resource diff --git a/tests/packagedcode/__pycache__/test_parse_pyproject_toml.cpython-312-pytest-8.3.3.pyc.26520 b/tests/packagedcode/__pycache__/test_parse_pyproject_toml.cpython-312-pytest-8.3.3.pyc.26520 new file mode 100644 index 0000000000000000000000000000000000000000..1eb645b2da6cf3b0de46e3c067534f642a59b285 GIT binary patch literal 2086 zcmdT_&2Jk;6rb4-+v~*na6?E@ft8w;_)w=UEr+V9KwznnP~ienElaDZXB|6P@2+Mh zO>M2!3VO%^Au0|n3Wyv*`4f=1^~9x8Z$qUbIB^R~rAR*UX6=cs0)hh^7<=cF+Ues}Ft1EF7(q6g^dIJgYrKB9=?09A1+jkZ}gFlt*>8xK60Y9{q$ ztJzfMsyS-hMOCMROXjaC(6xFRZa8H@n(!Qa59dI1+(XppVSzO$LDbwEa=3_khEV8q ztXq0Dl8Lc5+}ShXeE8|T7>&im&&5=O|8EPpu6k zwGz95shyaK)iCyk>}f_6G;V!LuSRljxV(w@3uEzTMhLx1Or1L$M;5hzBuB^)(+ti2 zYYeP^%-{$K46-!$rP0km6*FTQ&W-r$6?GDG2hn_D?VzuYBG8F00O(|rY2h?VR=lWp zjKy2fwvkM-ny;20L0|Xm4myd>A&$48x341%zt8X|?k)2=+QNV0F$bPf;aayUd@lHM zOzInC&8PK<`pc9EvK)8fmYP_!?$^Z`5w(M5E=WiTqhJ@!DD-01x=DoZ4R?>=Wd-jx z|GU^{{97^5@@aXUN1?>5g!{(IN(pz1rw zXf>VhzVSX`+^4R>xis>*8%Dxy6H#xvqUpPOX9_6i2k1*~cOqXIk^yBVe}!=FGm##C z0S<@Q7fJLds}=EtlawPV73ISFp>#%3!J zt&q!15CMYA1&=qQ&48v*Z!L&!kiZK`+n0{VNyAqWAX+10K;Ih>YV$QY^;XKQ6@?YX zA|{V%2;>%!kO=UUa>IJXL^NelXYDDraMNJ38m+)*(Ef<2_mG@W90!M-8z_T=UmsU4 z@*H5@;pmgMFV#MPI`P_vQ1vFM@j3}r_d%yc!A48`(U5jiZM;)!Y2lzw9C}c{o9m-|_ zCXd5ysN$CgO#w|+X8smy3FyxwlgtinY#R13^(A4GYPS*b^n$)cKnRSlz|_xOZbx)8 m@UO7fKvkbiuHKLPCdT+FLU+-}6F(yOI|tS@zG|@3;QSqlyYlP+ literal 0 HcmV?d00001 diff --git a/tests/packagedcode/data/docker/containerfile-expected.json b/tests/packagedcode/data/docker/containerfile-expected.json new file mode 100644 index 0000000000..92fe52b842 --- /dev/null +++ b/tests/packagedcode/data/docker/containerfile-expected.json @@ -0,0 +1,13 @@ +[ + { + "datasource_id": "dockerfile_oci_labels", + "type": "default", + "name": "Unknown", + "version": "Unknown", + "license_expression": "GPL-2.0-only AND BSD-2-Clause", + "labels": { + "source": "https://github.com/kubernetes-sigs/blixt", + "licenses": "GPL-2.0-only,BSD-2-Clause" + } + } +] diff --git a/tests/packagedcode/data/docker/psql-expected.json b/tests/packagedcode/data/docker/psql-expected.json new file mode 100644 index 0000000000..d9f8bb4c5c --- /dev/null +++ b/tests/packagedcode/data/docker/psql-expected.json @@ -0,0 +1,10 @@ +[ + { + "datasource_id": "dockerfile_oci_labels", + "type": "default", + "license_expression": "MIT", + "labels": { + "source": "https://github.com/kreneskyp/ix" + } + } +] diff --git a/tests/packagedcode/data/docker/psql.dockerfile b/tests/packagedcode/data/docker/psql.dockerfile new file mode 100644 index 0000000000..cbdd9e54eb --- /dev/null +++ b/tests/packagedcode/data/docker/psql.dockerfile @@ -0,0 +1,5 @@ +FROM postgres:15.3 +LABEL org.opencontainers.image.source https://github.com/kreneskyp/ix + +RUN apt update -y && \ + apt install -y postgresql-15-pgvector \ diff --git a/tests/packagedcode/data/docker/test-dockerfile-expected.json b/tests/packagedcode/data/docker/test-dockerfile-expected.json new file mode 100644 index 0000000000..2a843acee2 --- /dev/null +++ b/tests/packagedcode/data/docker/test-dockerfile-expected.json @@ -0,0 +1,18 @@ +[ + { + "datasource_id": "dockerfile_oci_labels", + "type": "default", + "name": "Kanboard", + "version": "1.2.42", + "license_expression": "MIT", + "labels": { + "source": "https://github.com/kanboard/kanboard", + "title": "Kanboard", + "description": "Kanboard is project management software that focuses on the Kanban methodology", + "vendor": "Kanboard", + "licenses": "MIT", + "url": "https://kanboard.org", + "documentation": "https://docs.kanboard.org" + } + } +] diff --git a/tests/packagedcode/data/docker/test.containerfile b/tests/packagedcode/data/docker/test.containerfile new file mode 100644 index 0000000000..2145e4476d --- /dev/null +++ b/tests/packagedcode/data/docker/test.containerfile @@ -0,0 +1,80 @@ +#Copied from https://github.com/kubernetes-sigs/blixt/blob + + +FROM rust:1.79-slim-bookworm as builder + +ARG TARGETARCH +ARG LLVM_VERSION=19 + +RUN apt-get update +RUN apt-get install --yes \ + build-essential \ + protobuf-compiler \ + pkg-config \ + musl-tools \ + clang \ + wget + +RUN apt install --yes lsb-release software-properties-common gnupg +RUN wget -O /tmp/llvm.sh https://apt.llvm.org/llvm.sh +RUN chmod +x /tmp/llvm.sh +RUN /bin/sh -c "/tmp/llvm.sh ${LLVM_VERSION} all" + +RUN rustup default stable +RUN rustup install nightly +RUN rustup component add rust-src --toolchain nightly +RUN --mount=type=cache,target=/root/.cargo/registry \ + cargo install bpf-linker + +WORKDIR /workspace +# Docker uses the amd64/arm64 convention while Rust uses the x86_64/aarch64 convention. +# Since Dockerfile doesn't support conditional variables (sigh), write the arch in Rust's +# convention to a file for later usage. +RUN if [ "$TARGETARCH" = "amd64" ]; \ + then echo "x86_64" >> arch; \ + else echo "aarch64" >> arch; \ + fi +RUN rustup target add $(eval cat arch)-unknown-linux-musl + +COPY dataplane dataplane +COPY tools/udp-test-server tools/udp-test-server +COPY xtask xtask +COPY Cargo.toml Cargo.toml +COPY Cargo.lock Cargo.lock +COPY .cargo .cargo + +# We need to tell bpf-linker where it can find LLVM's shared library file. +# Ref: https://github.com/aya-rs/rustc-llvm-proxy/blob/cbcb3c6/src/lib.rs#L48 +ENV LD_LIBRARY_PATH="/usr/lib/llvm-$LLVM_VERSION/lib" +ENV CC_aarch64_unknown_linux_musl="/usr/lib/llvm-$LLVM_VERSION/bin/clang" +ENV AR_aarch64_unknown_linux_musl="/usr/lib/llvm-$LLVM_VERSION/bin/llvm-ar" +ENV CC_x86_64_unknown_linux_musl="/usr/lib/llvm-$LLVM_VERSION/bin/clang" +ENV AR_x86_64_unknown_linux_musl="/usr/lib/llvm-$LLVM_VERSION/bin/llvm-ar" +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" + +RUN --mount=type=cache,target=/workspace/target/ \ + --mount=type=cache,target=/root/.cargo/registry \ + cargo xtask build-ebpf --release +RUN --mount=type=cache,target=/workspace/target/ \ + --mount=type=cache,target=/root/.cargo/registry \ + RUSTFLAGS=-Ctarget-feature=+crt-static cargo build \ + --workspace \ + --exclude ebpf \ + --release \ + --target=$(eval cat arch)-unknown-linux-musl +RUN --mount=type=cache,target=/workspace/target/ \ + cp /workspace/target/$(eval cat arch)-unknown-linux-musl/release/loader /workspace/dataplane-release + +FROM alpine + +LABEL org.opencontainers.image.source=https://github.com/kubernetes-sigs/blixt +LABEL org.opencontainers.image.licenses=GPL-2.0-only,BSD-2-Clause + +WORKDIR /opt/blixt/ + +COPY --from=builder /workspace/dataplane-release /opt/blixt/dataplane + +COPY dataplane/LICENSE.GPL-2.0 /opt/blixt/LICENSE.GPL-2.0 +COPY dataplane/LICENSE.BSD-2-Clause /opt/blixt/LICENSE.BSD-2-Clause + +ENTRYPOINT ["/opt/blixt/dataplane"] \ No newline at end of file diff --git a/tests/packagedcode/data/docker/test.dockerfile b/tests/packagedcode/data/docker/test.dockerfile new file mode 100644 index 0000000000..bb5efa2459 --- /dev/null +++ b/tests/packagedcode/data/docker/test.dockerfile @@ -0,0 +1,36 @@ +#Copied from https://github.com/kanboard/kanboard + +FROM alpine:3.21 + +LABEL org.opencontainers.image.source https://github.com/kanboard/kanboard +LABEL org.opencontainers.image.title=Kanboard +LABEL org.opencontainers.image.description="Kanboard is project management software that focuses on the Kanban methodology" +LABEL org.opencontainers.image.vendor=Kanboard +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.url=https://kanboard.org +LABEL org.opencontainers.image.documentation=https://docs.kanboard.org + +VOLUME /var/www/app/data +VOLUME /var/www/app/plugins +VOLUME /etc/nginx/ssl + +EXPOSE 80 443 + +ARG VERSION + +RUN apk --no-cache --update add \ + tzdata openssl unzip nginx bash ca-certificates s6 curl ssmtp mailx php83 php83-phar php83-curl \ + php83-fpm php83-json php83-zlib php83-xml php83-dom php83-ctype php83-opcache php83-zip php83-iconv \ + php83-pdo php83-pdo_mysql php83-pdo_sqlite php83-pdo_pgsql php83-mbstring php83-session php83-bcmath \ + php83-gd php83-openssl php83-sockets php83-posix php83-ldap php83-simplexml php83-xmlwriter && \ + rm -rf /var/www/localhost && \ + rm -f /etc/php83/php-fpm.d/www.conf && \ + ln -sf /usr/bin/php83 /usr/bin/php + +ADD . /var/www/app +ADD docker/ / + +RUN rm -rf /var/www/app/docker && echo $VERSION > /var/www/app/app/version.txt + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD [] \ No newline at end of file diff --git a/tests/packagedcode/test_dockerfile.py b/tests/packagedcode/test_dockerfile.py new file mode 100644 index 0000000000..34353f056e --- /dev/null +++ b/tests/packagedcode/test_dockerfile.py @@ -0,0 +1,60 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from packagedcode import dockerfile +import pytest +import os.path +import json +from pathlib import Path +from packagedcode.dockerfile import DockerfileHandler + +class TestDockerfileHandler: + + def get_test_loc(self, path): + return Path(os.path.join(os.path.dirname(__file__), 'data')) + + def load_expected(self, expected_file): + with open(expected_file) as f: + return json.load(f) + + def test_is_datafile(self): + dockerfiles = [ + 'test.dockerfile', + 'test.containerfile', + 'psql.dockerfile' + ] + for dockerfile in dockerfiles: + test_file = self.get_test_loc(f'data/docker/{dockerfile}') + assert DockerfileHandler.is_datafile(str(test_file)) + + def test_parse_dockerfile(self): + test_files = [ + ('test.dockerfile', 'test-dockerfile-expected.json'), + ('test.containerfile', 'containerfile-expected.json'), + ('psql.dockerfile', 'psql-expected.json') + ] + for dockerfile, expected in test_files: + test_file = self.get_test_loc(f'data/docker/{dockerfile}') + expected_loc = self.get_test_loc(f'data/docker/{expected}') + packages = list(DockerfileHandler.parse(str(test_file))) + expected_packages = self.load_expected(expected_loc) + assert packages == expected_packages + + def test_extract_oci_labels_from_dockerfile(self, mocker): + dockerfiles = [ + 'test.dockerfile', + 'test.containerfile', + 'psql.dockerfile' + ] + for dockerfile in dockerfiles: + dockerfile_path = self.get_test_loc(f'data/docker/{dockerfile}') + labels = DockerfileHandler.extract_oci_labels_from_dockerfile(str(dockerfile_path)) + expected_loc = self.get_test_loc(f'data/docker/{dockerfile.replace(".dockerfile", "-expected.json").replace(".containerfile", "-expected.json")}') + expected_labels = self.load_expected(expected_loc)[0]['labels'] + assert labels == expected_labels