Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add worker_threads optimization for setup #90

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

add worker_threads optimization for setup #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f3e4754
add worker_threads optimization for setup
devland Jun 25, 2024
7e8228f
add worker exit event handler and remove unused toSkip logic
devland Jun 26, 2024
2389a87
remove debug logs
devland Jun 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 83 additions & 39 deletions cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const fs = require('fs');
const logic = require('./logic.js');
const config = require('./config.js');
const utils = require('./assets/utils/cli.js');
const { Worker } = require('worker_threads');
let marked = require('marked');
const markedTerminal = require('marked-terminal');
marked.setOptions({
Expand Down Expand Up @@ -172,51 +173,94 @@ const cli = {
}
},
// computes & installs dependencies for h5p library
setup: async function(library, version, download) {
const isUrl = ['http', 'git@'].includes(library.slice(0, 4)) ? true : false;
const url = library;
const missingOptionals = {};
try {
if (isUrl) {
const entry = await this.register(url);
library = logic.machineToShort(Object.keys(entry)[0]);
setup: function(library, version, download) {
return new Promise(async (resolve, reject) => {
const isUrl = ['http', 'git@'].includes(library.slice(0, 4)) ? true : false;
const url = library;
const missingOptionals = {};
const benchStart = new Date();
let preparing = true;
const toDo = {
pending: 0,
done: 0
}
let toSkip = [];
const action = parseInt(download) ? 'download' : 'clone';
const latest = version ? false : true;
let result = await logic.computeDependencies(library, 'view', 1, version);
for (let item in result) {
// setup editor dependencies for every view dependency
if (!result[item].id) {
handleMissingOptionals(missingOptionals, result, item);
}
else {
toSkip = await logic.getWithDependencies(action, item, 'edit', 1, latest, toSkip);
const handleWorkerDone = () => {
toDo.done++;
console.log(`> ${toDo.done}/${toDo.pending} done`);
if (!preparing && toDo.done === toDo.pending) {
console.log(`> done setting up ${library} (${new Date() - benchStart} ms)`);
resolve();
}
}
result = await logic.computeDependencies(library, 'edit', 1, version);
for (let item in result) {
if (!result[item].id) {
handleMissingOptionals(missingOptionals, result, item);
}
const runWorker = (data) => {
console.log('<<< worker started');
const worker = new Worker(`${require.main.path}/logic.js` , { workerData : data });
worker.on('message', (result) => {
console.log('<<< worker done');
});
worker.on('error', (error) => {
console.log('<<< worker error');
console.error(error);
});
worker.on('exit', () => {
console.log('<<< worker exit');
handleWorkerDone();
});
}
toSkip = [];
console.log(`> ${action} ${library} library "view" dependencies into "${config.folders.libraries}" folder`);
toSkip = await logic.getWithDependencies(action, library, 'view', 1, latest, toSkip);
console.log(`> ${action} ${library} library "edit" dependencies into "${config.folders.libraries}" folder`);
toSkip = await logic.getWithDependencies(action, library, 'edit', 1, latest, toSkip);
if (Object.keys(missingOptionals).length) {
console.log('!!! missing optional libraries');
for (let item in missingOptionals) {
console.log(`${item} (${missingOptionals[item].optional ? 'optional' : 'required'}) required by ${missingOptionals[item].parent}`);
try {
if (isUrl) {
const entry = await this.register(url);
library = logic.machineToShort(Object.keys(entry)[0]);
}
const action = parseInt(download) ? 'download' : 'clone';
const latest = version ? false : true;
const viewDeps = await logic.computeDependencies(library, 'view', 1, version);
const editDeps = await logic.computeDependencies(library, 'edit', 1, version);
for (let item in viewDeps) {
// setup editor dependencies for every view dependency
if (!viewDeps[item].id) {
handleMissingOptionals(missingOptionals, viewDeps, item);
}
else {
toDo.pending++;
runWorker({
function: 'getWithDependencies',
arguments: [action, item, 'edit', 1, latest]
});
}
}
for (let item in editDeps) {
if (!editDeps[item].id) {
handleMissingOptionals(missingOptionals, editDeps, item);
}
}
console.log(`> ${action} ${library} library "view" dependencies into "${config.folders.libraries}" folder`);
toDo.pending++;
runWorker({
function: 'getWithDependencies',
arguments: [action, library, 'view', 1, latest]
});
console.log(`> ${action} ${library} library "edit" dependencies into "${config.folders.libraries}" folder`);
toDo.pending++;
runWorker({
function: 'getWithDependencies',
arguments: [action, library, 'edit', 1, latest]
});
if (Object.keys(missingOptionals).length) {
console.log('!!! missing optional libraries');
for (let item in missingOptionals) {
console.log(`${item} (${missingOptionals[item].optional ? 'optional' : 'required'}) required by ${missingOptionals[item].parent}`);
}
}
console.log('<<< done preparing');
preparing = false;
}
console.log(`> done setting up ${library}`);
}
catch (error) {
console.log('> error');
console.log(error);
}
catch (error) {
console.log('> error');
console.log(error);
reject(error);
}
});
},
// updates local library registry entry
register: async (input) => {
Expand Down
40 changes: 25 additions & 15 deletions logic.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ const getFileList = (folder) => {
}
module.exports = {
// imports content type from zip archive file in the .h5p format
import: (folder, archive) => {
import: function (folder, archive) {
const target = `${config.folders.temp}/${folder}`;
new admZip(archive).extractAllTo(target);
fs.renameSync(`${target}/content`, `content/${folder}`);
Expand All @@ -115,7 +115,7 @@ module.exports = {
return folder;
},
// creates zip archive export file in the .h5p format
export: (library, folder) => {
export: function (library, folder) {
const libsFile = `${config.folders.cache}/${library}.json`;
const editLibsFile = `${config.folders.cache}/${library}_edit.json`;
const target = `${config.folders.temp}/${folder}`;
Expand Down Expand Up @@ -151,7 +151,7 @@ module.exports = {
},
/* retrieves list of h5p librarie
ignoreCache - if true cache file is overwritten with online data */
getRegistry: async (ignoreCache) => {
getRegistry: async function (ignoreCache) {
const registryFile = `${config.folders.cache}/${config.registry}`;
let list;
if (!ignoreCache && fs.existsSync(registryFile)) {
Expand Down Expand Up @@ -192,7 +192,7 @@ module.exports = {
saveToCache - if true list is saved to cache folder
version - optional version to compute; defaults to 'master'
folder - optional local library folder to use instead of git repo; use "" to ignore */
computeDependencies: async (library, mode, saveToCache, version, folder) => {
computeDependencies: async function (library, mode, saveToCache, version, folder) {
console.log(`> ${library} deps ${mode}`);
version = version || 'master';
let level = -1;
Expand Down Expand Up @@ -369,7 +369,7 @@ module.exports = {
return output;
},
// list tags for library using git
tags: (org, repo, mainBranch = 'master') => {
tags: function (org, repo, mainBranch = 'master') {
const library = getRepoFile(fromTemplate(config.urls.library.clone, { org, repo }), 'library.json', mainBranch, true);
const label = `${repo}_${mainBranch}`;
const folder = `${config.folders.temp}/${label}`;
Expand All @@ -394,7 +394,7 @@ module.exports = {
return output;
},
// download & unzip repository
download: async (org, repo, version, target) => {
download: async function (org, repo, version, target) {
const blob = (await superAgent.get(fromTemplate(config.urls.library.zip, { org, repo, version })))._body;
const zipFile = `${config.folders.temp}/temp.zip`;
fs.writeFileSync(zipFile, blob);
Expand All @@ -403,15 +403,15 @@ module.exports = {
fs.renameSync(`${config.folders.libraries}/${repo}-master`, target);
},
// clone repository using git
clone: (org, repo, branch, target) => {
clone: function (org, repo, branch, target) {
return execSync(`git clone ${fromTemplate(config.urls.library.clone, {org, repo})} ${target} --branch ${branch}`, { cwd: config.folders.libraries }).toString();
},
/* clones/downloads dependencies to libraries folder using git and runs relevant npm commands
mode - 'view' or 'edit' to fetch non-editor or editor libraries
useCache - if true cached dependency list is used
latest - if true master branch libraries are used; otherwise the versions found in the cached deps list are used
toSkip - optional array of libraries to skip; after a library is parsed by the function it's auto-added to the array so it's skipped for efficiency */
getWithDependencies: async (action, library, mode, useCache, latest, toSkip = []) => {
getWithDependencies: async function (action, library, mode, useCache, latest, toSkip = []) {
let list;
const doneFile = `${config.folders.cache}/${library}${mode == 'edit' ? '_edit' : ''}.json`;
if (useCache && fs.existsSync(doneFile)) {
Expand All @@ -422,10 +422,10 @@ module.exports = {
list = await module.exports.computeDependencies(library, mode, 1);
}
for (let item in list) {
if (toSkip.indexOf(item) != -1) {
if (toSkip?.indexOf(item) != -1) {
continue;
}
toSkip.push(item);
toSkip?.push(item);
if (!list[item].id) {
if (list[item].optional) {
console.log(`> skipping optional unregistered ${item} library`);
Expand Down Expand Up @@ -475,7 +475,7 @@ module.exports = {
},
/* checks if dependency lists are cached and dependencies are installed for a given library;
returns a report with boolean statuses; the overall status is reflected under the "ok" attribute;*/
verifySetup: async (library) => {
verifySetup: async function (library) {
const registry = await module.exports.getRegistry();
const viewList = `${config.folders.cache}/${library}.json`;
const editList = `${config.folders.cache}/${library}_edit.json`;
Expand Down Expand Up @@ -509,7 +509,7 @@ module.exports = {
return output;
},
// generates h5p.json file with info describing the library in the specified folder
generateInfo: (folder, library) => {
generateInfo: function (folder, library) {
const target = `content/${folder}`;
const lib = JSON.parse(fs.readFileSync(`${config.folders.cache}/${library}.json`, 'utf-8'))[library];
const viewDepsFile = `${config.folders.cache}/${library}.json`;
Expand Down Expand Up @@ -545,7 +545,7 @@ module.exports = {
fs.writeFileSync(`${target}/h5p.json`, JSON.stringify(info));
},
// upgrades content via current main library upgrades.js scripts
upgrade: (folder, library) => {
upgrade: function (folder, library) {
const lib = JSON.parse(fs.readFileSync(`${config.folders.cache}/${library}.json`, 'utf-8'))[library];
const info = JSON.parse(fs.readFileSync(`content/${folder}/h5p.json`, 'utf-8'));
const extraAttrs = [
Expand Down Expand Up @@ -606,11 +606,11 @@ module.exports = {
fs.writeFileSync(contentFile, JSON.stringify(content));
module.exports.generateInfo(folder, library);
},
machineToShort: (machineName) => {
machineToShort: function (machineName) {
machineName = machineName.replace('H5PEditor', 'H5P-Editor');
return machineName.replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase().replace('.', '-');
},
registryEntryFromRepoUrl: function(gitUrl) {
registryEntryFromRepoUrl: function (gitUrl) {
let { host, org, repoName } = parseGitUrl(gitUrl);
const list = getRepoFile(gitUrl, 'library.json', 'master', true);
const shortName = this.machineToShort(list.machineName);
Expand Down Expand Up @@ -701,3 +701,13 @@ const parseSemanticLibraries = (entries) => {
}
return output;
}

const { workerData, parentPort } = require('worker_threads');
if (parentPort && workerData) {
console.log('<<< worker working...');
const run = async () => {
const result = await module.exports[workerData.function].apply(null, workerData.arguments);
parentPort.postMessage(result);
}
run();
}