diff --git a/.changeset/great-eggs-unite.md b/.changeset/great-eggs-unite.md new file mode 100644 index 0000000..f21fb6a --- /dev/null +++ b/.changeset/great-eggs-unite.md @@ -0,0 +1,5 @@ +--- +"@zk-kit/artifacts": minor +--- + +download artifacts from https://snark-artifacts.pse.dev diff --git a/packages/artifacts/src/download/download.ts b/packages/artifacts/src/download/download.ts index 9c0da57..03c1fdd 100644 --- a/packages/artifacts/src/download/download.ts +++ b/packages/artifacts/src/download/download.ts @@ -1,18 +1,9 @@ import { createWriteStream, existsSync } from 'node:fs' import { mkdir } from 'node:fs/promises' import { dirname } from 'node:path' -import type { Urls } from './urls.ts' -async function fetchRetry(urls: string[]): Promise> { - const [url] = urls - if (!url) throw new Error('No urls to try') - return fetch(url).catch(() => fetchRetry(urls.slice(1))) -} - -export async function download(urls: Urls | string[] | string, outputPath: string) { - const { body, ok, statusText, url } = Array.isArray(urls) - ? await fetchRetry(urls as string[]) - : await fetch(urls as string) +export async function download(url: string, outputPath: string) { + const { body, ok, statusText } = await fetch(url) if (!ok) throw new Error(`Failed to fetch ${url}: ${statusText}`) if (!body) throw new Error('Failed to get response body') @@ -42,7 +33,7 @@ export async function download(urls: Urls | string[] | string, outputPath: strin } } -export async function maybeDownload(urls: Urls | string[] | string, outputPath: string) { - if (!existsSync(outputPath)) await download(urls, outputPath) +export async function maybeDownload(url: string, outputPath: string) { + if (!existsSync(outputPath)) await download(url, outputPath) return outputPath } diff --git a/packages/artifacts/src/download/index.browser.ts b/packages/artifacts/src/download/index.browser.ts index 12a1722..c7574e6 100644 --- a/packages/artifacts/src/download/index.browser.ts +++ b/packages/artifacts/src/download/index.browser.ts @@ -1,15 +1,26 @@ -import type { SnarkArtifacts } from './types' -import { getSnarkArtifactUrls } from './urls' +import { type Project, projects } from '../projects' +import type { SnarkArtifacts, Version } from './types' +import { getBaseUrl } from './urls' -// TODO: retry for browser? -// beisdes, is caching already handled by circom/snarkjs? export default async function maybeGetSnarkArtifacts( - ...pars: Parameters + project: Project, + options: { + parameters?: (bigint | number | string)[] + version?: Version + cdnUrl?: string + } = {}, ): Promise { - const { wasms, zkeys } = await getSnarkArtifactUrls(...pars) + if (!projects.includes(project)) + throw new Error(`Project '${project}' is not supported`) + + options.version ??= 'latest' + const url = await getBaseUrl(project, options.version) + const parameters = options.parameters + ? `-${options.parameters.join('-')}` + : '' return { - wasm: wasms[0], - zkey: zkeys[0], + wasm: `${url}${parameters}.wasm`, + zkey: `${url}${parameters}.zkey`, } } diff --git a/packages/artifacts/src/download/index.node.ts b/packages/artifacts/src/download/index.node.ts index ba28861..fa10cd6 100644 --- a/packages/artifacts/src/download/index.node.ts +++ b/packages/artifacts/src/download/index.node.ts @@ -1,10 +1,9 @@ import { tmpdir } from 'node:os' import { maybeDownload } from './download.ts' +import _maybeGetSnarkArtifacts from './index.browser.ts' import type { SnarkArtifacts } from './types' -import { getSnarkArtifactUrls } from './urls' -// https://unpkg.com/@zk-kit/poseidon-artifacts@latest/poseidon.wasm -> @zk/poseidon-artifacts@latest/poseidon.wasm -const extractEndPath = (url: string) => url.substring(url.indexOf('@zk')) +const extractEndPath = (url: string) => url.split('pse.dev/')[1] /** * Downloads SNARK artifacts (`wasm` and `zkey`) files if not already present in OS tmp folder. @@ -18,17 +17,17 @@ const extractEndPath = (url: string) => url.substring(url.indexOf('@zk')) * @returns {@link SnarkArtifacts} */ export default async function maybeGetSnarkArtifacts( - ...pars: Parameters + ...pars: Parameters ): Promise { - const { wasms, zkeys } = await getSnarkArtifactUrls( + const urls = await _maybeGetSnarkArtifacts( ...pars, ) - const outputPath = `${tmpdir()}/${extractEndPath(wasms[0])}` + const outputPath = `${tmpdir()}/snark-artifacts/${extractEndPath(urls.wasm)}` const [wasm, zkey] = await Promise.all([ - maybeDownload(wasms, outputPath), - maybeDownload(zkeys, outputPath.replace(/.wasm$/, '.zkey')), + maybeDownload(urls.wasm, outputPath), + maybeDownload(urls.zkey, outputPath.replace(/.wasm$/, '.zkey')), ]) return { diff --git a/packages/artifacts/src/download/types.ts b/packages/artifacts/src/download/types.ts index e48b334..1493623 100644 --- a/packages/artifacts/src/download/types.ts +++ b/packages/artifacts/src/download/types.ts @@ -5,10 +5,6 @@ */ export type SnarkArtifacts = Record<'wasm' | 'zkey', string> -// Recursively build an array type of length L with elements of type T. -export type ArrayOf = A['length'] extends L ? A - : ArrayOf - type Digit = `${number}` type PreRelease = 'alpha' | 'beta' diff --git a/packages/artifacts/src/download/urls.ts b/packages/artifacts/src/download/urls.ts index 9489efc..87a236c 100644 --- a/packages/artifacts/src/download/urls.ts +++ b/packages/artifacts/src/download/urls.ts @@ -1,7 +1,7 @@ -import { type Project, projects } from '../projects' -import type { ArrayOf, Version } from './types' +import type { Project } from '../projects' +import type { Version } from './types' -export type Urls = ArrayOf +const BASE_URL = 'https://snark-artifacts.pse.dev' export async function getAvailableVersions(project: Project) { const res = await fetch(`https://registry.npmjs.org/@zk-kit/${project}-artifacts`) @@ -15,35 +15,7 @@ async function isVersionAvailableOrThrow(project: Project, version: Version) { throw new Error(`Version '${version}' is not available for project '${project}'`) } -export async function getBaseUrls(project: Project, version: Version): Promise { +export async function getBaseUrl(project: Project, version: Version): Promise { await isVersionAvailableOrThrow(project, version) - return [ - `https://unpkg.com/@zk-kit/${project}-artifacts@${version}/${project}`, - `https://raw.githubusercontent.com/privacy-scaling-explorations/snark-artifacts/@zk-kit/${project}-artifacts@${version}/packages/${project}/${project}`, - `https://cdn.jsdelivr.net/npm/@zk-kit/${project}-artifacts@${version}/${project}`, - ] -} - -export async function getSnarkArtifactUrls( - project: Project, - options: { - parameters?: (bigint | number | string)[] - version?: Version - cdnUrl?: string - } = {}, -) { - if (!projects.includes(project)) - throw new Error(`Project '${project}' is not supported`) - - options.version ??= 'latest' - const urls = await getBaseUrls(project, options.version) - - const parameters = options.parameters - ? `-${options.parameters.join('-')}` - : '' - - return { - wasms: urls.map((url) => `${url}${parameters}.wasm`) as unknown as Urls, - zkeys: urls.map(url => `${url}${parameters}.zkey`) as unknown as Urls, - } + return `${BASE_URL}/${project}/${version}/${project}` } diff --git a/packages/artifacts/test/download.test.ts b/packages/artifacts/test/download.test.ts index 8284a78..38885a7 100644 --- a/packages/artifacts/test/download.test.ts +++ b/packages/artifacts/test/download.test.ts @@ -2,10 +2,10 @@ import fs from 'node:fs' import fsPromises from 'node:fs/promises' import maybeGetSnarkArtifactsBrowser from '../src/download/index.browser' import maybeGetSnarkArtifacts from '../src/download/index.node' -import { getAvailableVersions, getSnarkArtifactUrls } from '../src/download/urls' +import { getAvailableVersions } from '../src/download/urls' import { Project } from '../src/projects' -const version = '1.0.0' +const version = '1.0.0-beta.1' describe('getAvailableVersions', () => { it('Should return available versions', async () => { @@ -15,189 +15,205 @@ describe('getAvailableVersions', () => { }, 20_000) }) -describe('getSnarkArtifactUrls', () => { - it('Should return valid urls', async () => { - const { wasms, zkeys } = await getSnarkArtifactUrls(Project.POSEIDON, { - parameters: ['2'], - version, +describe('maybeGetSnarkArtifacts', () => { + describe('browser', () => { + it('Should return valid urls', async () => { + const { wasm, zkey } = await maybeGetSnarkArtifactsBrowser( + Project.POSEIDON, + { + parameters: ['2'], + version, + }, + ) + + await expect(fetch(wasm)).resolves.toHaveProperty('ok', true) + await expect(fetch(zkey)).resolves.toHaveProperty('ok', true) + }, 20_000) + + it('should throw if the project is not supported', async () => { + await expect( + maybeGetSnarkArtifactsBrowser('project' as Project, { + parameters: ['2'], + version: 'latest', + }), + ).rejects.toThrow("Project 'project' is not supported") }) - for (const url of wasms) - await expect(fetch(url)).resolves.toHaveProperty('ok', true) - for (const url of zkeys) - await expect(fetch(url)).resolves.toHaveProperty('ok', true) - }, 20_000) - - it('should throw if the project is not supported', async () => { - await expect( - getSnarkArtifactUrls('project' as Project, { - parameters: ['2'], - version: 'latest', - }), - ).rejects.toThrow("Project 'project' is not supported") - }) - - it('should throw if the version is not available', async () => { - await expect( - getSnarkArtifactUrls(Project.POSEIDON, { - parameters: ['2'], - version: '0.1.0-beta', - }), - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Version '0.1.0-beta' is not available for project 'poseidon'"`, - ) - }) -}) - -describe('MaybeGetSnarkArtifacts', () => { - let fetchSpy: jest.SpyInstance - let mkdirSpy: jest.SpyInstance - let createWriteStreamSpy: jest.SpyInstance - let existsSyncSpy: jest.SpyInstance - - beforeEach(() => { - // @ts-expect-error non exhaustive mock of fetch - fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValueOnce({ - json: async () => ({ versions: { [version]: {} } }), + it('should throw if the version is not available', async () => { + await expect( + maybeGetSnarkArtifactsBrowser(Project.POSEIDON, { + parameters: ['2'], + version: '0.1.0-beta', + }), + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Version '0.1.0-beta' is not available for project 'poseidon'"`, + ) }) - createWriteStreamSpy = jest.spyOn(fs, 'createWriteStream') - existsSyncSpy = jest.spyOn(fs, 'existsSync') - mkdirSpy = jest.spyOn(fsPromises, 'mkdir') - mkdirSpy.mockResolvedValue(undefined) - }) - - afterEach(() => { - jest.restoreAllMocks() - }) - - it('Should throw an error if the project is not supported', async () => { - await expect( - maybeGetSnarkArtifacts('project' as Project, { - parameters: ['2'], - version: 'latest', - }), - ).rejects.toThrow("Project 'project' is not supported") - await expect( - maybeGetSnarkArtifactsBrowser('project' as Project), - ).rejects.toThrow("Project 'project' is not supported") - }) - - it('Should throw on fetch errors', async () => { - existsSyncSpy.mockReturnValue(false) - fetchSpy.mockResolvedValueOnce({ - ok: false, - statusText: 'TEST', - url: 'https://test.com', + it('Should return artifact file paths with parameters', async () => { + const { wasm, zkey } = await maybeGetSnarkArtifactsBrowser( + Project.POSEIDON, + { + parameters: ['2'], + }, + ) + + expect(wasm).toMatchInlineSnapshot( + `"https://snark-artifacts.pse.dev/poseidon/latest/poseidon-2.wasm"`, + ) + expect(zkey).toMatchInlineSnapshot( + `"https://snark-artifacts.pse.dev/poseidon/latest/poseidon-2.zkey"`, + ) }) - await expect( - maybeGetSnarkArtifacts(Project.POSEIDON, { - parameters: ['2'], - version: 'latest', - }), - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Failed to fetch https://test.com: TEST"`, - ) + it('Should return artifact files paths without parameters', async () => { + const { wasm, zkey } = await maybeGetSnarkArtifactsBrowser( + Project.SEMAPHORE, + ) + + expect(wasm).toMatchInlineSnapshot( + `"https://snark-artifacts.pse.dev/semaphore/latest/semaphore.wasm"`, + ) + expect(zkey).toMatchInlineSnapshot( + `"https://snark-artifacts.pse.dev/semaphore/latest/semaphore.zkey"`, + ) + }) }) - it('Should throw if missing body', async () => { - existsSyncSpy.mockReturnValue(false) - fetchSpy.mockResolvedValueOnce({ - ok: true, - statusText: 'OK', + describe('node', () => { + let fetchSpy: jest.SpyInstance + let mkdirSpy: jest.SpyInstance + let createWriteStreamSpy: jest.SpyInstance + let existsSyncSpy: jest.SpyInstance + + beforeEach(() => { + // @ts-expect-error non exhaustive mock of fetch + fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValueOnce({ + json: async () => ({ versions: { [version]: {} } }), + }) + createWriteStreamSpy = jest.spyOn(fs, 'createWriteStream') + existsSyncSpy = jest.spyOn(fs, 'existsSync') + mkdirSpy = jest.spyOn(fsPromises, 'mkdir') + mkdirSpy.mockResolvedValue(undefined) }) - await expect( - maybeGetSnarkArtifacts(Project.POSEIDON, { - parameters: ['2'], - }), - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Failed to get response body"`, - ) - }) + afterEach(() => { + jest.restoreAllMocks() + }) - it('Should throw on stream error', async () => { - existsSyncSpy.mockReturnValue(false) - const mockResponseStream = { - body: { - getReader: jest.fn(() => ({ - read: jest.fn().mockRejectedValueOnce(new Error('TEST STREAM ERROR')), - })), - }, - ok: true, - statusText: 'OK', - } - fetchSpy.mockResolvedValue(mockResponseStream) - createWriteStreamSpy.mockReturnValue({ - close: jest.fn(), - end: jest.fn(), - write: jest.fn(), + it('Should throw an error if the project is not supported', async () => { + await expect( + maybeGetSnarkArtifacts('project' as Project, { + parameters: ['2'], + version: 'latest', + }), + ).rejects.toThrow("Project 'project' is not supported") + + await expect( + maybeGetSnarkArtifactsBrowser('project' as Project), + ).rejects.toThrow("Project 'project' is not supported") }) - await expect( - maybeGetSnarkArtifacts(Project.POSEIDON, { - parameters: ['2'], - }), - ).rejects.toThrowErrorMatchingInlineSnapshot(`"TEST STREAM ERROR"`) - }) + it('Should throw on fetch errors', async () => { + existsSyncSpy.mockReturnValue(false) + fetchSpy.mockResolvedValueOnce({ + ok: false, + statusText: 'TEST', + url: 'https://test.com', + }) + + await expect( + maybeGetSnarkArtifacts(Project.POSEIDON, { + parameters: ['2'], + version: 'latest', + }), + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to fetch https://snark-artifacts.pse.dev/poseidon/latest/poseidon-2.wasm: TEST"`, + ) + }) - it("Should download files only if don't exist yet", async () => { - existsSyncSpy.mockReturnValue(true) + it('Should throw if missing body', async () => { + existsSyncSpy.mockReturnValue(false) + fetchSpy.mockResolvedValueOnce({ + ok: true, + statusText: 'OK', + }) + + await expect( + maybeGetSnarkArtifacts(Project.POSEIDON, { + parameters: ['2'], + }), + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to get response body"`, + ) + }) - await maybeGetSnarkArtifacts(Project.POSEIDON, { parameters: ['2'] }) + it('Should throw on stream error', async () => { + existsSyncSpy.mockReturnValue(false) + const mockResponseStream = { + body: { + getReader: jest.fn(() => ({ + read: jest + .fn() + .mockRejectedValueOnce(new Error('TEST STREAM ERROR')), + })), + }, + ok: true, + statusText: 'OK', + } + fetchSpy.mockResolvedValue(mockResponseStream) + createWriteStreamSpy.mockReturnValue({ + close: jest.fn(), + end: jest.fn(), + write: jest.fn(), + }) + + await expect( + maybeGetSnarkArtifacts(Project.POSEIDON, { + parameters: ['2'], + }), + ).rejects.toThrowErrorMatchingInlineSnapshot(`"TEST STREAM ERROR"`) + }) - expect(global.fetch).toHaveBeenCalledTimes(1) - expect(global.fetch).toHaveBeenLastCalledWith('https://registry.npmjs.org/@zk-kit/poseidon-artifacts') - }) + it("Should download files only if don't exist yet", async () => { + existsSyncSpy.mockReturnValue(true) - it('Should return artifact file paths in node environment', async () => { - mkdirSpy.mockRestore() - existsSyncSpy.mockReturnValue(false) + await maybeGetSnarkArtifacts(Project.POSEIDON, { parameters: ['2'] }) - const { wasm, zkey } = await maybeGetSnarkArtifacts(Project.POSEIDON, { - parameters: ['2'], + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(global.fetch).toHaveBeenLastCalledWith( + 'https://registry.npmjs.org/@zk-kit/poseidon-artifacts', + ) }) - expect(wasm).toMatchInlineSnapshot( - `"/tmp/@zk-kit/poseidon-artifacts@latest/poseidon-2.wasm"`, - ) - expect(zkey).toMatchInlineSnapshot( - `"/tmp/@zk-kit/poseidon-artifacts@latest/poseidon-2.zkey"`, - ) - - expect(fetchSpy).toHaveBeenCalledTimes(3) - expect(fetchSpy).toHaveBeenNthCalledWith(1, 'https://registry.npmjs.org/@zk-kit/poseidon-artifacts') - expect(fetchSpy).toHaveBeenNthCalledWith(2, 'https://unpkg.com/@zk-kit/poseidon-artifacts@latest/poseidon-2.wasm') - expect(fetchSpy).toHaveBeenNthCalledWith(3, 'https://unpkg.com/@zk-kit/poseidon-artifacts@latest/poseidon-2.zkey') - }, 25_000) - - it('Should return artifact file paths with parameters in browser environment', async () => { - const { wasm, zkey } = await maybeGetSnarkArtifactsBrowser( - Project.POSEIDON, - { - parameters: ['2'], - }, - ) - - expect(wasm).toMatchInlineSnapshot( - `"https://unpkg.com/@zk-kit/poseidon-artifacts@latest/poseidon-2.wasm"`, - ) - expect(zkey).toMatchInlineSnapshot( - `"https://unpkg.com/@zk-kit/poseidon-artifacts@latest/poseidon-2.zkey"`, - ) - }) + it('Should return artifact file paths in node environment', async () => { + mkdirSpy.mockRestore() + existsSyncSpy.mockReturnValue(false) - it('Should return artifact files paths without parameters in browser environment', async () => { - const { wasm, zkey } = await maybeGetSnarkArtifactsBrowser( - Project.SEMAPHORE, - ) - - expect(wasm).toMatchInlineSnapshot( - `"https://unpkg.com/@zk-kit/semaphore-artifacts@latest/semaphore.wasm"`, - ) - expect(zkey).toMatchInlineSnapshot( - `"https://unpkg.com/@zk-kit/semaphore-artifacts@latest/semaphore.zkey"`, - ) + const { wasm, zkey } = await maybeGetSnarkArtifacts(Project.POSEIDON, { + parameters: ['2'], + }) + + expect(wasm).toMatchInlineSnapshot( + `"/tmp/snark-artifacts/poseidon/latest/poseidon-2.wasm"`, + ) + expect(zkey).toMatchInlineSnapshot( + `"/tmp/snark-artifacts/poseidon/latest/poseidon-2.zkey"`, + ) + + expect(fetchSpy).toHaveBeenCalledTimes(3) + expect(fetchSpy).toHaveBeenNthCalledWith( + 1, + 'https://registry.npmjs.org/@zk-kit/poseidon-artifacts', + ) + expect(fetchSpy).toHaveBeenNthCalledWith( + 2, + 'https://snark-artifacts.pse.dev/poseidon/latest/poseidon-2.wasm', + ) + expect(fetchSpy).toHaveBeenNthCalledWith( + 3, + 'https://snark-artifacts.pse.dev/poseidon/latest/poseidon-2.zkey', + ) + }, 25_000) }) }) diff --git a/packages/cli/test/integration.test.ts b/packages/cli/test/integration.test.ts index 13974bf..a73fd63 100644 --- a/packages/cli/test/integration.test.ts +++ b/packages/cli/test/integration.test.ts @@ -82,6 +82,7 @@ Commands: 1.0.0-beta poseidon 1.0.0 + 1.0.0-beta.1 semaphore 1.0.0 4.0.0-beta.9