From 362299d38cdb845d1bd40f55d2c1b2612f8edb20 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 10:19:05 -0400 Subject: [PATCH 01/11] avoid loading pkg until we have to --- src/LazyArtifacts.jl | 58 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index b783276..a8b6455 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -9,7 +9,61 @@ using Artifacts: Artifacts, export artifact_exists, artifact_path, artifact_meta, artifact_hash, select_downloadable_artifacts, find_artifacts_toml, @artifact_str -# define a function for satisfying lazy Artifact downloads -using Pkg.Artifacts: ensure_artifact_installed +using Base.BinaryPlatforms: AbstractPlatform, HostPlatform +using Base: SHA1 + +""" + ensure_artifact_installed(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing, + verbose::Bool = false, + quiet_download::Bool = false, + io::IO=stderr) + +Ensures an artifact is installed, downloading it via the download information stored in +`artifacts_toml` if necessary. Throws an error if unable to install. +""" +function ensure_artifact_installed(name::String, artifacts_toml::String; + platform::AbstractPlatform=HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing, + verbose::Bool=false, + quiet_download::Bool=false, + io::IO=stderr) + meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform) + if meta === nothing + error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") + end + + return ensure_artifact_installed(name, meta, artifacts_toml; + platform, verbose, quiet_download, io) +end + +function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String; + platform::AbstractPlatform=HostPlatform(), + verbose::Bool=false, + quiet_download::Bool=false, + io::IO=stderr) + + hash = SHA1(meta["git-tree-sha1"]) + if !artifact_exists(hash) + # loading Pkg is a bit slow, so we only do it if we need to + # and do it in a subprocess to avoid precompilation complexity + return read(`$(Base.julia_cmd()) -E ' + Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); + Pkg.try_artifact_download_sources( + $(repr(name)), + $(repr(hash)), + $(repr(meta)), + $(repr(artifacts_toml)); + $(repr(platform)), + $(repr(verbose)), + $(repr(quiet_download)), + $(repr(io)) + )' + `, String) + else + return artifact_path(hash) + end +end end From e80fa25849f232d0b7cdb931e9f90da735011498 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 10:37:25 -0400 Subject: [PATCH 02/11] tryfix --- .gitignore | 2 ++ src/LazyArtifacts.jl | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d1d02a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +Manifest.toml diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index a8b6455..334c29e 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -48,19 +48,20 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str if !artifact_exists(hash) # loading Pkg is a bit slow, so we only do it if we need to # and do it in a subprocess to avoid precompilation complexity - return read(`$(Base.julia_cmd()) -E ' - Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); - Pkg.try_artifact_download_sources( - $(repr(name)), - $(repr(hash)), - $(repr(meta)), - $(repr(artifacts_toml)); - $(repr(platform)), - $(repr(verbose)), - $(repr(quiet_download)), - $(repr(io)) - )' - `, String) + cmd = run(pipeline(`$(Base.julia_cmd()) -E ' + Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); + Pkg.try_artifact_download_sources( + $(repr(name)), + $(repr(hash)), + $(repr(meta)), + $(repr(artifacts_toml)); + platform = $(repr(platform)), + verbose = $(repr(verbose)), + quiet_download = $(repr(quiet_download)), + io = $(repr(io)) + )'`, + stderr=stderr)) + return read(cmd, String) else return artifact_path(hash) end From e3fd69f296993d393f297d3c6801310a6c3b2bc5 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 12:49:17 -0400 Subject: [PATCH 03/11] fixes --- src/LazyArtifacts.jl | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 334c29e..41268c7 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -48,20 +48,22 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str if !artifact_exists(hash) # loading Pkg is a bit slow, so we only do it if we need to # and do it in a subprocess to avoid precompilation complexity - cmd = run(pipeline(`$(Base.julia_cmd()) -E ' - Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); - Pkg.try_artifact_download_sources( - $(repr(name)), - $(repr(hash)), - $(repr(meta)), - $(repr(artifacts_toml)); - platform = $(repr(platform)), - verbose = $(repr(verbose)), - quiet_download = $(repr(quiet_download)), - io = $(repr(io)) - )'`, - stderr=stderr)) - return read(cmd, String) + code = """ + Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); + ret = Pkg.Artifacts.try_artifact_download_sources( + $(repr(name)), + Base.$(repr(hash)), + $(repr(meta)), + $(repr(artifacts_toml)); + platform = Base.BinaryPlatforms.$(repr(platform)), + verbose = $(repr(verbose)), + quiet_download = $(repr(quiet_download)), + io = stderr + ) + println(stdout, ret) + """ + ret = String(readchomp(pipeline(`$(Base.julia_cmd()) -e $code`, stderr=io))) + return ret else return artifact_path(hash) end From 80dd849df74dd9d2ee17b40e032715db3818de68 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 12:52:32 -0400 Subject: [PATCH 04/11] don't cache artifacts because we need to test downloading --- .github/workflows/ci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 301d9e8..c7f7a52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,17 +26,6 @@ jobs: with: version: ${{ matrix.julia-version }} arch: ${{ matrix.julia-arch }} - - name: Cache artifacts - uses: actions/cache@v2 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-runtest@latest - uses: julia-actions/julia-uploadcodecov@v0.1 From 2a6b4e4af46a48e229827bc52282d8fc871ebdc9 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 12:55:55 -0400 Subject: [PATCH 05/11] update CI --- .github/workflows/ci.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7f7a52..85d1f05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,21 +14,25 @@ jobs: strategy: fail-fast: false matrix: - julia-version: - - "nightly" - os: - - ubuntu-latest - julia-arch: - - x64 + include: + - os: ubuntu-latest + julia-arch: x64 + julia-version: 'nightly' + - os: windows-latest + julia-arch: x64 + julia-version: 'nightly' + - os: macos-latest + julia-arch: aarch64 + julia-version: 'nightly' steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.julia-version }} arch: ${{ matrix.julia-arch }} - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-runtest@latest - - uses: julia-actions/julia-uploadcodecov@v0.1 - continue-on-error: true - - uses: julia-actions/julia-uploadcoveralls@v0.1 - continue-on-error: true + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v2 + with: + files: lcov.info From 3aeb8bea1281c2d7fbcccc78368c7ca74fa345fa Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 13:10:48 -0400 Subject: [PATCH 06/11] fix getting output back from subprocess --- src/LazyArtifacts.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 41268c7..4f494fc 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -50,7 +50,7 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str # and do it in a subprocess to avoid precompilation complexity code = """ Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); - ret = Pkg.Artifacts.try_artifact_download_sources( + Pkg.Artifacts.try_artifact_download_sources( $(repr(name)), Base.$(repr(hash)), $(repr(meta)), @@ -60,10 +60,9 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str quiet_download = $(repr(quiet_download)), io = stderr ) - println(stdout, ret) """ - ret = String(readchomp(pipeline(`$(Base.julia_cmd()) -e $code`, stderr=io))) - return ret + out = readchomp(pipeline(`$(Base.julia_cmd()) -E $code`, stderr=io)) + return Meta.parse(out) else return artifact_path(hash) end From 9999ed5cc326929a644cecac51a264227b05a397 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 13:22:42 -0400 Subject: [PATCH 07/11] add comment --- src/LazyArtifacts.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 4f494fc..94e9f18 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -12,6 +12,11 @@ export artifact_exists, artifact_path, artifact_meta, artifact_hash, using Base.BinaryPlatforms: AbstractPlatform, HostPlatform using Base: SHA1 +# We are mimicking Pkg.Artifacts.ensure_artifact_installed here so that we can +# check if the artifact is already installed before loading Pkg. +# Then if we need to load Pkg we do it in a subprocess to avoid precompilation +# complexity. + """ ensure_artifact_installed(name::String, artifacts_toml::String; platform::AbstractPlatform = HostPlatform(), From 80b12abefb068da68cf71b8428a1c6f51b8a8fed Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 10 Sep 2024 13:44:24 -0400 Subject: [PATCH 08/11] add a precompile --- src/LazyArtifacts.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 94e9f18..0424c14 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -73,4 +73,8 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str end end +if Base.generating_output() + precompile(Tuple{typeof(Artifacts._artifact_str), Module, String, Base.SubString{String}, String, Base.Dict{String, Any}, Base.SHA1, Base.BinaryPlatforms.Platform, Base.Val{LazyArtifacts}}) +end + end From 83e3328911307d16053cc2194917c011c1fe3410 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 11 Sep 2024 13:43:22 -0400 Subject: [PATCH 09/11] only use a subprocess if generating output (precompiling) --- src/LazyArtifacts.jl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 0424c14..b42e55e 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -50,11 +50,13 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str io::IO=stderr) hash = SHA1(meta["git-tree-sha1"]) + pkg_pkgid = Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg") if !artifact_exists(hash) # loading Pkg is a bit slow, so we only do it if we need to - # and do it in a subprocess to avoid precompilation complexity - code = """ - Pkg = Base.require_stdlib(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")); + if Base.generating_output() + # if precompiling load and use Pkg in a subprocess to avoid precompilation complexity + code = """ + Pkg = Base.require_stdlib($pkg_pkgid); Pkg.Artifacts.try_artifact_download_sources( $(repr(name)), Base.$(repr(hash)), @@ -65,9 +67,13 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str quiet_download = $(repr(quiet_download)), io = stderr ) - """ - out = readchomp(pipeline(`$(Base.julia_cmd()) -E $code`, stderr=io)) - return Meta.parse(out) + """ + out = readchomp(pipeline(`$(Base.julia_cmd()) -E $code`, stderr=io)) + return Meta.parse(out) + else + Pkg = Base.require_stdlib(pkg_pkgid); + return Pkg.Artifacts.try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io) + end else return artifact_path(hash) end From 6cde8c2489909f46666c52ced6a1903676b82eaf Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 11 Sep 2024 13:45:44 -0400 Subject: [PATCH 10/11] add invokelatest --- src/LazyArtifacts.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index b42e55e..9de45a8 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -72,7 +72,8 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str return Meta.parse(out) else Pkg = Base.require_stdlib(pkg_pkgid); - return Pkg.Artifacts.try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io) + return @invokelatest Pkg.Artifacts.try_artifact_download_sources(name, hash, meta, artifacts_toml; + platform, verbose, quiet_download, io) end else return artifact_path(hash) From ca58bba37eb761ba7fde415a2b0bee48dc392e7d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 11 Sep 2024 14:03:41 -0400 Subject: [PATCH 11/11] improve comments --- src/LazyArtifacts.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/LazyArtifacts.jl b/src/LazyArtifacts.jl index 9de45a8..eb9c5f7 100644 --- a/src/LazyArtifacts.jl +++ b/src/LazyArtifacts.jl @@ -13,9 +13,7 @@ using Base.BinaryPlatforms: AbstractPlatform, HostPlatform using Base: SHA1 # We are mimicking Pkg.Artifacts.ensure_artifact_installed here so that we can -# check if the artifact is already installed before loading Pkg. -# Then if we need to load Pkg we do it in a subprocess to avoid precompilation -# complexity. +# check if the artifact is already installed before loading Pkg, then load if needed. """ ensure_artifact_installed(name::String, artifacts_toml::String; @@ -54,7 +52,7 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str if !artifact_exists(hash) # loading Pkg is a bit slow, so we only do it if we need to if Base.generating_output() - # if precompiling load and use Pkg in a subprocess to avoid precompilation complexity + # if precompiling, load and use Pkg in a subprocess to avoid precompilation complexity code = """ Pkg = Base.require_stdlib($pkg_pkgid); Pkg.Artifacts.try_artifact_download_sources(