diff --git a/package.json b/package.json index d63719a..80df341 100644 --- a/package.json +++ b/package.json @@ -34,17 +34,15 @@ "license": "MIT", "dependencies": { "@ensdomains/content-hash": "^3.0.0", - "@ipld/car": "^5.2.4", - "@ipld/dag-cbor": "^9.0.6", "@ipld/unixfs": "^2.1.2", "@stauro/filebase-upload": "^0.0.7", "@stauro/piggybank": "^0.0.5", "ascii-bar": "^1.0.3", "cac": "^6.7.14", + "cborg": "^4.0.5", "colorette": "^2.0.20", "globby": "^14.0.0", "multiformats": "^12.1.3", - "semver": "^7.5.4", "table": "^6.8.1", "viem": "^1.19.9" }, @@ -52,7 +50,6 @@ "@eslint/js": "^8.54.0", "@stylistic/eslint-plugin": "^1.4.1", "@types/node": "^20.10.0", - "@types/semver": "^7.5.6", "@typescript-eslint/parser": "^6.13.1", "eslint": "^8.54.0", "globals": "^13.23.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2262fc0..3fa1cc1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,6 @@ dependencies: '@ensdomains/content-hash': specifier: ^3.0.0 version: 3.0.0 - '@ipld/car': - specifier: ^5.2.4 - version: 5.2.4 - '@ipld/dag-cbor': - specifier: ^9.0.6 - version: 9.0.6 '@ipld/unixfs': specifier: ^2.1.2 version: 2.1.2 @@ -29,6 +23,9 @@ dependencies: cac: specifier: ^6.7.14 version: 6.7.14 + cborg: + specifier: ^4.0.5 + version: 4.0.5 colorette: specifier: ^2.0.20 version: 2.0.20 @@ -38,9 +35,6 @@ dependencies: multiformats: specifier: ^12.1.3 version: 12.1.3 - semver: - specifier: ^7.5.4 - version: 7.5.4 table: specifier: ^6.8.1 version: 6.8.1 @@ -58,9 +52,6 @@ devDependencies: '@types/node': specifier: ^20.10.0 version: 20.10.0 - '@types/semver': - specifier: ^7.5.6 - version: 7.5.6 '@typescript-eslint/parser': specifier: ^6.13.1 version: 6.13.1(eslint@8.54.0)(typescript@5.3.2) @@ -246,24 +237,6 @@ packages: resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} dev: true - /@ipld/car@5.2.4: - resolution: {integrity: sha512-YoVXE/o5HLXKi/Oqh9Nhcn423sdn9brRFKnbUid68/1D332/XINcoyCTvBluFcCw/9IeiTx+sEAV+onXZ/A4eA==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - dependencies: - '@ipld/dag-cbor': 9.0.6 - cborg: 4.0.5 - multiformats: 12.1.3 - varint: 6.0.0 - dev: false - - /@ipld/dag-cbor@9.0.6: - resolution: {integrity: sha512-3kNab5xMppgWw6DVYx2BzmFq8t7I56AGWfp5kaU1fIPkwHVpBRglJJTYsGtbVluCi/s/q97HZM3bC+aDW4sxbQ==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - dependencies: - cborg: 4.0.5 - multiformats: 12.1.3 - dev: false - /@ipld/dag-pb@4.0.6: resolution: {integrity: sha512-wOij3jfDKZsb9yjhQeHp+TQy0pu1vmUkGv324xciFFZ7xGbDfAGTQW03lSA5aJ/7HBBNYgjEE0nvHmNW1Qjfag==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -3073,10 +3046,6 @@ packages: spdx-expression-parse: 3.0.1 dev: true - /varint@6.0.0: - resolution: {integrity: sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==} - dev: false - /viem@1.19.9(typescript@5.3.2): resolution: {integrity: sha512-Sf9U2x4jU0S/FALqYypcspWOGene0NZyD470oUripNhE0Ta6uOE/OgE4toTDVfRxov8qw0JFinr/wPGxYE3+HQ==} peerDependencies: diff --git a/src/ipfs-car/car.ts b/src/ipfs-car/car.ts index b274a7a..2173091 100644 --- a/src/ipfs-car/car.ts +++ b/src/ipfs-car/car.ts @@ -1,8 +1,8 @@ import * as varint from './varint.js' -import { encode as cborEncode } from '@ipld/dag-cbor' import { UnknownLink } from 'multiformats' import { Block } from '@ipld/unixfs' import { TransformStream } from 'node:stream/web' +import { cborEncode } from './dag-cbor.js' function encodeHeader(roots: UnknownLink[]) { const headerBytes = cborEncode({ version: 1, roots }) diff --git a/src/ipfs-car/dag-cbor.ts b/src/ipfs-car/dag-cbor.ts new file mode 100644 index 0000000..12b5fbb --- /dev/null +++ b/src/ipfs-car/dag-cbor.ts @@ -0,0 +1,43 @@ +import { CID } from 'multiformats/cid' +import { Token, Type, encode, decode } from 'cborg' + +const CID_CBOR_TAG = 42 + +function cidEncoder(obj: any): Token[] | null { + if (obj.asCID !== obj && obj['/'] !== obj.bytes) { + return null + } + const cid = CID.asCID(obj) + if (!cid) { + return null + } + const bytes = new Uint8Array(cid.bytes.byteLength + 1) + bytes.set(cid.bytes, 1) + return [ + new Token(Type.tag, CID_CBOR_TAG), + new Token(Type.bytes, bytes), + ] +} + +function cidDecoder(bytes: Uint8Array) { + if (bytes[0] !== 0) { + throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00') + } + return CID.decode(bytes.subarray(1)) // ignore leading 0x00 +} + +export const cborEncode = (data: unknown) => encode(data, { + float64: true, + typeEncoders: { + Object: cidEncoder, + }, +}) + +export const cborDecode = (data: Uint8Array) => decode(data, { + allowIndefinite: false, + coerceUndefinedToNull: true, + strict: true, + useMaps: false, + rejectDuplicateMapKeys: true, + tags: { [CID_CBOR_TAG]: cidDecoder }, +}) diff --git a/src/ipfs-car/varint.ts b/src/ipfs-car/varint.ts index 424d31c..90f7851 100644 --- a/src/ipfs-car/varint.ts +++ b/src/ipfs-car/varint.ts @@ -27,3 +27,32 @@ export function encode(num: number, out: number[] = [], offset: number = 0): num return out } + +interface ReadResult { + bytes: number +} + +export function decode(buf: Uint8Array, offset = 0): [number, number] { + let res = 0 + let shift = 0 + let counter = offset + let b: number + const l = buf.length + let bytes = 0 + + do { + if (counter >= l || shift > 49) { + bytes = 0 + throw new RangeError('Could not decode varint') + } + b = buf[counter++] + res += shift < 28 + ? (b & REST) << shift + : (b & REST) * Math.pow(2, shift) + shift += 7 + } while (b >= MSB) + + bytes = counter - offset + + return [res, bytes] +} diff --git a/src/utils/car-writer.ts b/src/utils/car-writer.ts new file mode 100644 index 0000000..9c7864b --- /dev/null +++ b/src/utils/car-writer.ts @@ -0,0 +1,157 @@ +import { CID } from 'multiformats/cid' +import { FileHandle } from 'node:fs/promises' +import { read, write } from 'node:fs' +import { promisify } from 'node:util' +import { cborDecode, cborEncode } from '../ipfs-car/dag-cbor.js' +import * as varint from '../ipfs-car/varint.js' + +const fsread = promisify(read) +const fswrite = promisify(write) + +interface Seekable { + seek(length: number): void +} + +interface BytesReader extends Seekable { + upTo(length: number): Promise + + exactly(length: number, seek?: boolean): Promise + + pos: number +} + +function decodeVarint(bytes: Uint8Array, seeker: Seekable) { + if (!bytes.length) { + throw new Error('Unexpected end of data') + } + const [i, len] = varint.decode(bytes) + seeker.seek(len) + return i +} + +async function readHeader(reader: BytesReader) { + const length = decodeVarint(await reader.upTo(8), reader) + if (length === 0) { + throw new Error('Invalid CAR header (zero length)') + } + const header = await reader.exactly(length, true) + const block = cborDecode(header) + + if (!Array.isArray(block.roots)) { + throw new Error('Invalid CAR header format') + } + return block +} + +function chunkReader(readChunk: () => Promise): BytesReader { + let pos = 0 + let have = 0 + let offset = 0 + let currentChunk = new Uint8Array(0) + + const read = async (length: number) => { + have = currentChunk.length - offset + const bufa = [currentChunk.subarray(offset)] + while (have < length) { + const chunk = await readChunk() + if (chunk == null) { + break + } + if (have < 0) { + if (chunk.length > have) { + bufa.push(chunk.subarray(-have)) + } + } + else { + bufa.push(chunk) + } + have += chunk.length + } + currentChunk = new Uint8Array(bufa.reduce((p, c) => p + c.length, 0)) + let off = 0 + for (const b of bufa) { + currentChunk.set(b, off) + off += b.length + } + offset = 0 + } + + return { + async upTo(length: number) { + if (currentChunk.length - offset < length) { + await read(length) + } + return currentChunk.subarray(offset, offset + Math.min(currentChunk.length - offset, length)) + }, + + async exactly(length: number, seek = false) { + if (currentChunk.length - offset < length) { + await read(length) + } + if (currentChunk.length - offset < length) { + throw new Error('Unexpected end of data') + } + const out = currentChunk.subarray(offset, offset + length) + if (seek) { + pos += length + offset += length + } + return out + }, + + seek(length: number) { + pos += length + offset += length + }, + + get pos() { + return pos + }, + } +} + +function createHeader(roots: CID[]) { + const headerBytes = cborEncode({ version: 1, roots }) + const varintBytes = varint.encode(headerBytes.length) + const header = new Uint8Array(varintBytes.length + headerBytes.length) + header.set(varintBytes, 0) + header.set(headerBytes, varintBytes.length) + return header +} + +export async function updateRootsInFile(fd: FileHandle, roots: CID[]) { + const chunkSize = 256 + let bytes: Uint8Array + let offset = 0 + + let readChunk: () => Promise + + if (typeof fd === 'number') { + readChunk = async () => (await fsread(fd, bytes, 0, chunkSize, offset)).bytesRead + } + else if (typeof fd === 'object' && typeof fd.read === 'function') { + readChunk = async () => (await fd.read(bytes, 0, chunkSize, offset)).bytesRead + } + else { + throw new TypeError('Bad fd') + } + + const fdReader = chunkReader(async () => { + bytes = new Uint8Array(chunkSize) + const read = await readChunk() + offset += read + return read < chunkSize ? bytes.subarray(0, read) : bytes + }) + + await readHeader(fdReader) + const newHeader = createHeader(roots) + if (fdReader.pos !== newHeader.length) { + throw new Error(`old header is ${fdReader.pos} bytes, new header is ${newHeader.length} bytes`) + } + if (typeof fd === 'number') { + await fswrite(fd, newHeader, 0, newHeader.length, 0) + } + else if (typeof fd === 'object' && typeof fd.read === 'function') { + await fd.write(newHeader, 0, newHeader.length, 0) + } +} diff --git a/src/utils/ipfs.ts b/src/utils/ipfs.ts index 0c0656d..ed1f093 100644 --- a/src/utils/ipfs.ts +++ b/src/utils/ipfs.ts @@ -2,12 +2,12 @@ import { tmpdir } from 'node:os' import { readFile, open } from 'node:fs/promises' import { createWriteStream } from 'node:fs' import { CID } from 'multiformats/cid' -import { CarWriter } from '@ipld/car/writer' import type { FileEntry } from '../types.js' import { TransformStream } from 'node:stream/web' import { createDirectoryEncoderStream, CAREncoderStream } from '../ipfs-car/index.js' import { Block } from '@ipld/unixfs' import { Writable } from 'node:stream' +import { updateRootsInFile } from './car-writer.js' const tmp = tmpdir() @@ -32,7 +32,7 @@ export const packCAR = async (files: FileEntry[], name: string, dir = tmp) => { .pipeTo(Writable.toWeb(createWriteStream(output))) const fd = await open(output, 'r+') - await CarWriter.updateRootsInFile(fd, [rootCID!]) + await updateRootsInFile(fd, [rootCID!]) await fd.close() const file = await readFile(output)