Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/prevent zombie containers #53

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
5 changes: 5 additions & 0 deletions cli/src/cmd/deploy.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ import { getIExecDebug } from '../utils/iexec.js';
import { goToProjectRoot } from '../cli-helpers/goToProjectRoot.js';
import * as color from '../cli-helpers/color.js';
import { hintBox } from '../cli-helpers/box.js';
import { createAbortSignal } from '../utils/abortController.js';

export async function deploy() {
const spinner = getSpinner();
const abortSignal = createAbortSignal();
abortSignal.addEventListener('abort', async () => {
throw Error('Execution aborted');
});
try {
await goToProjectRoot({ spinner });
const dockerhubUsername = await askForDockerhubUsername({ spinner });
Expand Down
144 changes: 90 additions & 54 deletions cli/src/execDocker/docker.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import Docker from 'dockerode';
import os from 'os';
import { createAbortSignal } from '../utils/abortController.js';

const docker = new Docker();

Expand Down Expand Up @@ -31,71 +32,88 @@ export async function dockerBuild({
if (osType === 'Darwin' && isForTest) {
platform = 'linux/arm64';
}

const abortSignal = createAbortSignal();
// Perform the Docker build operation
const buildImageStream = await docker.buildImage(buildArgs, {
t: tag,
platform,
pull: true, // docker store does not support multi platform image, this can cause issues when switching build target platform, pulling ensures the right image is used
abortSignal,
});

const imageId = await new Promise((resolve, reject) => {
docker.modem.followProgress(buildImageStream, onFinished, onProgress);

function onFinished(err, output) {
/**
* expected output format for image id
* ```
* {
* aux: {
* ID: 'sha256:e994101ce877e9b42f31f1508e11bbeb8fa5096a1fb2d0c650a6a26797b1906b'
* }
* },
* ```
*/
const builtImageId = output?.find((row) => row?.aux?.ID)?.aux?.ID;
let imageId = null;

/**
* 3 kind of error possible, we want to catch each of them:
* - stream error
* - build error
* - no image id (should not happen)
*
* expected output format for build error
* ```
* {
* errorDetail: {
* code: 1,
* message: "The command '/bin/sh -c npm ci' returned a non-zero code: 1"
* },
* error: "The command '/bin/sh -c npm ci' returned a non-zero code: 1"
* }
* ```
*/
const errorOrErrorMessage =
err || // stream error
output.find((row) => row?.error)?.error || // build error message
(!builtImageId && 'Failed to retrieve generated image ID'); // no image id -> error message
try {
const imageId = await new Promise((resolve, reject) => {
// Handle abort signal
if (abortSignal) {
abortSignal.addEventListener('abort', () => {
buildImageStream.destroy();
reject(new Error('Docker build aborted'));
});
}

if (errorOrErrorMessage) {
const error =
errorOrErrorMessage instanceof Error
? errorOrErrorMessage
: Error(errorOrErrorMessage);
reject(error);
} else {
resolve(builtImageId);
docker.modem.followProgress(buildImageStream, onFinished, onProgress);

function onFinished(err, output) {
/**
* expected output format for image id
* ```
* {
* aux: {
* ID: 'sha256:e994101ce877e9b42f31f1508e11bbeb8fa5096a1fb2d0c650a6a26797b1906b'
* }
* },
* ```
*/
const builtImageId = output?.find((row) => row?.aux?.ID)?.aux?.ID;

/**
* 3 kind of error possible, we want to catch each of them:
* - stream error
* - build error
* - no image id (should not happen)
*
* expected output format for build error
* ```
* {
* errorDetail: {
* code: 1,
* message: "The command '/bin/sh -c npm ci' returned a non-zero code: 1"
* },
* error: "The command '/bin/sh -c npm ci' returned a non-zero code: 1"
* }
* ```
*/
const errorOrErrorMessage =
err || // stream error
output.find((row) => row?.error)?.error || // build error message
(!builtImageId && 'Failed to retrieve generated image ID'); // no image id -> error message

if (errorOrErrorMessage) {
const error =
errorOrErrorMessage instanceof Error
? errorOrErrorMessage
: Error(errorOrErrorMessage);
reject(error);
} else {
resolve(builtImageId);
}
}
}

function onProgress(event) {
if (event?.stream) {
progressCallback(event.stream);
function onProgress(event) {
if (event?.stream) {
progressCallback(event.stream);
}
}
});
return imageId;
} catch (error) {
if (imageId) {
await docker.getImage(imageId).remove();
}
});

return imageId;
throw error;
}
}

// Function to push a Docker image
Expand All @@ -109,15 +127,24 @@ export async function pushDockerImage({
throw new Error('Missing DockerHub credentials.');
}
const dockerImage = docker.getImage(tag);
const abortSignal = createAbortSignal();

const imagePushStream = await dockerImage.push({
authconfig: {
username: dockerhubUsername,
password: dockerhubAccessToken,
},
abortSignal,
});

await new Promise((resolve, reject) => {
// Handle abort signal
if (abortSignal) {
abortSignal.addEventListener('abort', () => {
imagePushStream.destroy();
reject(new Error('Docker push aborted'));
});
}

docker.modem.followProgress(imagePushStream, onFinished, onProgress);

function onFinished(err, output) {
Expand Down Expand Up @@ -166,6 +193,7 @@ export async function runDockerContainer({
memory = undefined,
logsCallback = () => {},
}) {
const abortSignal = createAbortSignal();
const container = await docker.createContainer({
Image: image,
Cmd: cmd,
Expand All @@ -175,10 +203,18 @@ export async function runDockerContainer({
Memory: memory,
},
Env: env,
abortSignal,
});

// Handle abort signal
if (abortSignal) {
abortSignal.addEventListener('abort', async () => {
await container.kill();
logsCallback('Container execution aborted');
});
}

// Start the container
// TODO we should handle abort signal to stop the container and avoid containers running after command is interrupted
await container.start();

// get the logs stream
Expand Down
10 changes: 10 additions & 0 deletions cli/src/utils/abortController.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
export function createAbortSignal() {
const abortController = new AbortController();
const { signal: signalAbort } = abortController;
const handleAbort = () => {
abortController.abort();
process.off('SIGINT', handleAbort);
};
process.on('SIGINT', handleAbort);
return signalAbort;
}