Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support idempotency on create export in the CLI. #585

Merged
merged 4 commits into from
Jan 29, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@
"build:sea": "node ./sea/build.cjs",
"format": "prettier -w --log-level silent .",
"format:check": "prettier -c .",
"prepare": "husky"
"prepare": "husky",
"pr-check": "npm run format:check && npm run lint && npm run test"
},
"husky": {
"hooks": {
Expand Down
16 changes: 16 additions & 0 deletions src/commands/export/create.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ async function createS3Export(argv) {
maxWait,
quiet,
destination,
idempotency,
} = argv;
const logger = container.resolve("logger");
const { createExport } = container.resolve("accountAPI");
Expand All @@ -37,6 +38,7 @@ async function createS3Export(argv) {
collections,
destination: destinationInput,
format,
idempotency,
});

if (wait && !createdExport.is_terminal) {
Expand Down Expand Up @@ -64,6 +66,10 @@ const sharedExamples = [
"$0 export create s3 --bucket doc-example-bucket --path exports/my_db",
"You can also specify the S3 location using --bucket and --path options rather than --destination.",
],
[
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --idempotency f47ac10b-58cc-4372-a567-0e02b2c3d479",
"Set an idempotency key. Avoids reprocessing successful requests with the same key for 24 hours",
],
[
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --json",
"Output the full JSON of the export request.",
Expand Down Expand Up @@ -117,6 +123,13 @@ function buildCreateS3ExportCommand(yargs) {
default: "simple",
group: "API:",
},
idempotency: {
type: "string",
required: false,
description:
"Idempotency key. Avoids reprocessing successful requests with the same key for 24 hours.",
group: "API:",
},
})
.options(WAIT_OPTIONS)
.check((argv) => {
Expand All @@ -141,6 +154,9 @@ function buildCreateS3ExportCommand(yargs) {
"Either --destination or both --bucket and --path are required to create an export.",
);
}
if (argv.idempotency?.trim() === "") {
throw new ValidationError("--idempotency can't be an empty string.");
}
return true;
})
.example(sharedExamples);
Expand Down
8 changes: 7 additions & 1 deletion src/lib/account-api.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,7 @@ async function createKey({ path, role, ttl, name }) {
* @param {string} params.destination.s3.bucket - The name of the S3 bucket to export to.
* @param {string} params.destination.s3.path - The key prefix to export to.
* @param {string} params.format - The format for the export.
* @param {string | undefined} params.idempotency - The idempotency key, if any, to use in the request
* @returns {Promise<Object>} - A promise that resolves when the export is created.
* @throws {AuthorizationError | AuthenticationError | CommandError | Error} If the response is not OK
*/
Expand All @@ -421,12 +422,14 @@ async function createExport({
destination,
format,
collections = undefined,
idempotency,
}) {
const url = toResource({ endpoint: "/exports", version: API_VERSIONS.v2 });
const response = await fetchWithAccountKey(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
...(idempotency !== undefined && { "Idempotency-Key": idempotency }),
},
body: JSON.stringify({
database: standardizeRegion(database),
Expand All @@ -437,7 +440,10 @@ async function createExport({
});

const data = await responseHandler(response);
return data.response;
const idempotentReplayed =
response.headers.get("Idempotent-Replayed") === "true";
// eslint-disable-next-line camelcase
return { ...data.response, idempotent_replayed: idempotentReplayed };
}

/**
Expand Down
44 changes: 44 additions & 0 deletions test/commands/export/create.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ describe("export create s3", () => {
database,
destination: expectedDestination,
format: "simple",
idempotent_replayed: false,
});
createExport.resolves(stubbedResponse);

Expand All @@ -88,15 +89,48 @@ destination:
uri: s3://test-bucket/test/key
created_at: 2025-01-02T22:59:51
updated_at: 2025-01-02T22:59:51
idempotent_replayed: false
`);
expect(createExport).to.have.been.calledWith({
database,
collections: [],
destination: expectedDestArgs,
format: "simple",
idempotency: undefined,
});
});

it(`handles idempotent replay with idempotent_replayed: true ${description}`, async () => {
const database = "us-std/example";
const stubbedResponse = createExportStub({
database,
destination: expectedDestination,
format: "simple",
idempotent_replayed: true,
});
createExport.resolves(stubbedResponse);

await run(
`export create s3 --database '${database}' ${args} --idempotency XYZ`,
container,
);
await stdout.waitForWritten();

expect(stdout.getWritten()).to.equal(`id: test-export-id
state: Pending
database: us-std/example
format: simple
destination:
s3:
bucket: test-bucket
path: /test/key
uri: s3://test-bucket/test/key
created_at: 2025-01-02T22:59:51
updated_at: 2025-01-02T22:59:51
idempotent_replayed: true
`);
});

it(`outputs the full response with --json ${description}`, async () => {
const database = "us-std/example";
const stubbedResponse = createExportStub({
Expand Down Expand Up @@ -166,6 +200,16 @@ updated_at: 2025-01-02T22:59:51
expectedError:
"Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.",
},
{
description: "an empty string is given as the --idempotency input",
args: "--destination s3://test-bucket/test/key --idempotency ",
expectedError: "--idempotency can't be an empty string.",
},
{
description: "an blank string is given as the --idempotency input",
args: "--destination s3://test-bucket/test/key --idempotency ' '",
expectedError: "--idempotency can't be an empty string.",
},
];

invalidScenarios.forEach(({ description, args, expectedError }) => {
Expand Down
3 changes: 2 additions & 1 deletion test/helpers.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ import sinon from "sinon";

// small helper for sinon to wrap your return value
// in the shape fetch would return it from the network
export function f(returnValue, status) {
export function f(returnValue, status, headers) {
return new Response(JSON.stringify(returnValue), {
status: status || 200,
headers: {
"Content-type": "application/json",
...headers,
},
});
}
Expand Down
83 changes: 82 additions & 1 deletion test/lib/account-api/account-api.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,88 @@ describe("accountAPI", () => {
}),
}),
);
expect(data).to.deep.equal(testExport);
expect(data).to.deep.equal({
...testExport,
idempotent_replayed: false,
});
});

it("should call the endpoint and handle idempotent replay", async () => {
fetch
.withArgs(
sinon.match({ href: "https://account.fauna.com/v2/exports" }),
sinon.match({ method: "POST" }),
)
.resolves(
f({ response: testExport }, 201, { "Idempotent-Replayed": "true" }),
);

const data = await accountAPI.createExport({
database: "us/demo",
format: "simple",
destination: {
s3: {
bucket: "test-bucket",
path: "test/key",
},
},
idempotency: "Foo",
});

expect(fetch).to.have.been.calledWith(
sinon.match({ href: "https://account.fauna.com/v2/exports" }),
sinon.match({
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: "Bearer some-account-key",
"Idempotency-Key": "Foo",
},
body: JSON.stringify({
database: "us-std/demo",
destination: {
s3: {
bucket: "test-bucket",
path: "test/key",
},
},
format: "simple",
}),
}),
);
expect(data).to.deep.equal({
...testExport,
idempotent_replayed: true,
});
});

it("should handle non-idempotent replay", async () => {
fetch
.withArgs(
sinon.match({ href: "https://account.fauna.com/v2/exports" }),
sinon.match({ method: "POST" }),
)
.resolves(
f({ response: testExport }, 201, {
"Idempotent-Replayed": "false",
}),
);

const data = await accountAPI.createExport({
database: "us/demo",
format: "simple",
destination: {
s3: {
bucket: "test-bucket",
path: "test/key",
},
},
});

expect(data).to.deep.equal({
...testExport,
idempotent_replayed: false,
});
});
});
});
Expand Down