diff --git a/dev-packages/cli/package.json b/dev-packages/cli/package.json index 4a7d0c73d9579..177f5b7e41097 100644 --- a/dev-packages/cli/package.json +++ b/dev-packages/cli/package.json @@ -40,6 +40,7 @@ "@types/tar": "^4.0.3", "chai": "^4.2.0", "colors": "^1.4.0", + "decompress": "^4.2.1", "https-proxy-agent": "^5.0.0", "mkdirp": "^0.5.0", "mocha": "^7.0.0", @@ -47,9 +48,7 @@ "proxy-from-env": "^1.1.0", "puppeteer": "^2.0.0", "puppeteer-to-istanbul": "^1.2.2", - "tar": "^4.0.0", "temp": "^0.9.1", - "unzip-stream": "^0.3.0", "yargs": "^11.1.0" }, "devDependencies": { diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 1ce96ee0cf55a..d83c12f0b029b 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -19,13 +19,13 @@ import fetch, { Response, RequestInit } from 'node-fetch'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { getProxyForUrl } from 'proxy-from-env'; -import * as fs from 'fs'; +import { promises as fs, createWriteStream } from 'fs'; import * as mkdirp from 'mkdirp'; import * as path from 'path'; import * as process from 'process'; import * as stream from 'stream'; -import * as tar from 'tar'; -import * as zlib from 'zlib'; +import * as decompress from 'decompress'; +import * as temp from 'temp'; import { green, red } from 'colors/safe'; @@ -33,7 +33,7 @@ import { promisify } from 'util'; const mkdirpAsPromised = promisify(mkdirp); const pipelineAsPromised = promisify(stream.pipeline); -const unzip = require('unzip-stream'); +temp.track(); /** * Available options when downloading. @@ -69,92 +69,93 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = console.log(red('error: missing mandatory \'theiaPlugins\' property.')); return; } + try { + await Promise.all(Object.keys(pck.theiaPlugins).map( + plugin => downloadPluginAsync(failures, plugin, pck.theiaPlugins[plugin], pluginsDir, packed) + )); + } finally { + temp.cleanupSync(); + } + failures.forEach(console.error); +} - await Promise.all(Object.keys(pck.theiaPlugins).map(async plugin => { - if (!plugin) { - return; - } - const pluginUrl = pck.theiaPlugins[plugin]; - - let fileExt: string; - if (pluginUrl.endsWith('tar.gz')) { - fileExt = '.tar.gz'; - } else if (pluginUrl.endsWith('vsix')) { - fileExt = '.vsix'; - } else { - console.error(red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`)); - return; - } +/** + * Downloads a plugin, will make multiple attempts before actually failing. + * + * @param failures reference to an array storing all failures + * @param plugin plugin short name + * @param pluginUrl url to download the plugin at + * @param pluginsDir where to download the plugin in + * @param packed whether to decompress or not + */ +async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean): Promise { + if (!plugin) { + return; + } + let fileExt: string; + if (pluginUrl.endsWith('tar.gz')) { + fileExt = '.tar.gz'; + } else if (pluginUrl.endsWith('vsix')) { + fileExt = '.vsix'; + } else { + console.error(red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`)); + return; + } + const targetPath = path.join(process.cwd(), pluginsDir, `${plugin}${packed === true ? fileExt : ''}`); + // Skip plugins which have previously been downloaded. + if (await isDownloaded(targetPath)) { + console.warn('- ' + plugin + ': already downloaded - skipping'); + return; + } - const targetPath = path.join(process.cwd(), pluginsDir, `${plugin}${packed === true ? fileExt : ''}`); + const maxAttempts = 5; + const retryDelay = 2000; - // Skip plugins which have previously been downloaded. - if (isDownloaded(targetPath)) { - console.warn('- ' + plugin + ': already downloaded - skipping'); - return; - } + let attempts: number; + let lastError: Error | undefined; + let response: Response | undefined; - const maxAttempts = 5; - const retryDelay = 2000; - - let attempts: number; - let lastError: Error | undefined; - let response: Response | undefined; - - for (attempts = 0; attempts < maxAttempts; attempts++) { - if (attempts > 0) { - await new Promise(resolve => setTimeout(resolve, retryDelay)); - } - lastError = undefined; - try { - response = await xfetch(pluginUrl); - } catch (error) { - lastError = error; - continue; - } - const retry = response.status === 439 || response.status >= 500; - if (!retry) { - break; - } - } - if (lastError) { - failures.push(red(`x ${plugin}: failed to download, last error:\n ${lastError}`)); - return; + for (attempts = 0; attempts < maxAttempts; attempts++) { + if (attempts > 0) { + await new Promise(resolve => setTimeout(resolve, retryDelay)); } - if (typeof response === 'undefined') { - failures.push(red(`x ${plugin}: failed to download (unknown reason)`)); - return; + lastError = undefined; + try { + response = await xfetch(pluginUrl); + } catch (error) { + lastError = error; + continue; } - if (response.status !== 200) { - failures.push(red(`x ${plugin}: failed to download with: ${response.status} ${response.statusText}`)); - return; + const retry = response.status === 439 || response.status >= 500; + if (!retry) { + break; } + } + if (lastError) { + failures.push(red(`x ${plugin}: failed to download, last error:\n ${lastError}`)); + return; + } + if (typeof response === 'undefined') { + failures.push(red(`x ${plugin}: failed to download (unknown reason)`)); + return; + } + if (response.status !== 200) { + failures.push(red(`x ${plugin}: failed to download with: ${response.status} ${response.statusText}`)); + return; + } - if (fileExt === '.tar.gz') { - // Decompress .tar.gz files. - await mkdirpAsPromised(targetPath); - const gunzip = zlib.createGunzip({ - finishFlush: zlib.Z_SYNC_FLUSH, - flush: zlib.Z_SYNC_FLUSH - }); - const untar = tar.x({ cwd: targetPath }); - await pipelineAsPromised(response.body, gunzip, untar); - } else { - if (packed === true) { - // Download .vsix without decompressing. - const file = fs.createWriteStream(targetPath); - await pipelineAsPromised(response.body, file); - } else { - // Decompress .vsix. - await pipelineAsPromised(response.body, unzip.Extract({ path: targetPath })); - } - } + if (fileExt === '.vsix' && packed === true) { + // Download .vsix without decompressing. + const file = createWriteStream(targetPath); + await pipelineAsPromised(response.body, file); + } else { + await mkdirpAsPromised(targetPath); + const tempFile = temp.createWriteStream('theia-plugin-download'); + await pipelineAsPromised(response.body, tempFile); + await decompress(tempFile.path, targetPath); + } - console.warn(green(`+ ${plugin}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`)); - })); - failures.forEach(failure => { - console.log(failure); - }); + console.warn(green(`+ ${plugin}: downloaded successfully ${attempts > 1 ? `(after ${attempts} attempts)` : ''}`)); } /** @@ -163,8 +164,8 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = * * @returns `true` if the resource is already downloaded, else `false`. */ -function isDownloaded(filePath: string): boolean { - return fs.existsSync(filePath); +async function isDownloaded(filePath: string): Promise { + return fs.stat(filePath).then(() => true, () => false); } /** diff --git a/yarn.lock b/yarn.lock index 4b8fb3e1462e4..adba1e52b9307 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4860,6 +4860,20 @@ decompress@4.2.0: pify "^2.3.0" strip-dirs "^2.0.0" +decompress@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/decompress/-/decompress-4.2.1.tgz#007f55cc6a62c055afa37c07eb6a4ee1b773f118" + integrity sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ== + dependencies: + decompress-tar "^4.0.0" + decompress-tarbz2 "^4.0.0" + decompress-targz "^4.0.0" + decompress-unzip "^4.0.1" + graceful-fs "^4.1.10" + make-dir "^1.0.0" + pify "^2.3.0" + strip-dirs "^2.0.0" + dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"