Skip to content

Commit

Permalink
wip on dmno build / flattening
Browse files Browse the repository at this point in the history
  • Loading branch information
theoephraim committed Aug 8, 2024
1 parent cddac7f commit 6126b87
Show file tree
Hide file tree
Showing 29 changed files with 679 additions and 344 deletions.
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ tmp-package-registry

changesets-summary.json

# DMNO files ###
# dmno files ###
# local cache for resolved values
**/.dmno/cache.json
# encryption key used for cache
Expand All @@ -28,4 +28,5 @@ changesets-summary.json
**/.dmno/.icon-cache
# local config overrides
**/.dmno/.env.local

# built/flattened config
**/.dmno-built
10 changes: 2 additions & 8 deletions example-repo/.dmno/config.mts
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,17 @@ import { OnePasswordDmnoPlugin, OnePasswordTypes } from '@dmno/1password-plugin'
import { EncryptedVaultDmnoPlugin, EncryptedVaultTypes } from '@dmno/encrypted-vault-plugin';



const OnePassSecretsProd = new OnePasswordDmnoPlugin('1pass/prod', {
token: configPath('OP_TOKEN'),
envItemLink: 'https://start.1password.com/open/i?a=I3GUA2KU6BD3FBHA47QNBIVEV4&v=ut2dftalm3ugmxc6klavms6tfq&i=n4wmgfq77mydg5lebtroa3ykvm&h=dmnoinc.1password.com',
// token: InjectPluginInputByType,
// token: 'asdf',
});
const OnePassSecretsDev = new OnePasswordDmnoPlugin('1pass', {
const OnePassSecretsDev = new OnePasswordDmnoPlugin('1pass/dev', {
token: configPath('OP_TOKEN'),
envItemLink: 'https://start.1password.com/open/i?a=I3GUA2KU6BD3FBHA47QNBIVEV4&v=ut2dftalm3ugmxc6klavms6tfq&i=n4wmgfq77mydg5lebtroa3ykvm&h=dmnoinc.1password.com',
// token: InjectPluginInputByType,
// token: 'asdf',
envItemLink: 'https://start.1password.com/open/i?a=I3GUA2KU6BD3FBHA47QNBIVEV4&v=ut2dftalm3ugmxc6klavms6tfq&i=4u4klfhpldobgdxrcjwb2bqsta&h=dmnoinc.1password.com',
});


const ProdVault = new EncryptedVaultDmnoPlugin('vault/prod', {
key: configPath('DMNO_VAULT_KEY'),
name: 'prod',
Expand Down Expand Up @@ -58,8 +54,6 @@ export default defineDmnoService({
}),
},



DMNO_VAULT_KEY: {
extends: EncryptedVaultTypes.encryptionKey,
// required: true
Expand Down
4 changes: 4 additions & 0 deletions example-repo/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,7 @@
**/.dmno/.typegen
# iconify cache used in generated types
**/.dmno/.icon-cache
# built/flattened config
**/.dmno-built
# local config overrides
**/.dmno/.env.local
6 changes: 5 additions & 1 deletion example-repo/packages/api/.dmno/config.mts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { defineDmnoService, DmnoBaseTypes, NodeEnvType, configPath, dmnoFormula,
import { OnePasswordDmnoPlugin } from '@dmno/1password-plugin';
import { EncryptedVaultDmnoPlugin } from '@dmno/encrypted-vault-plugin';

const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass');
const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass/prod');
const VaultPlugin = EncryptedVaultDmnoPlugin.injectInstance('vault/prod');

export default defineDmnoService({
Expand All @@ -11,6 +11,7 @@ export default defineDmnoService({
pick: [
'NODE_ENV',
'DMNO_ENV',
'ROOT_ONLY',
],
schema: {
OP_ITEM_1: {
Expand All @@ -27,6 +28,9 @@ export default defineDmnoService({
allowedDomains: ['*']
},
},
// REQUIRED_ITEM: {
// required: true,
// },

STRIPE_SECRET_KEY: {
value: 'fake-stripe-secret-key',
Expand Down
1 change: 1 addition & 0 deletions example-repo/packages/api/src/routes/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export type CustomRouter = Router<CustomAppState, CustomAppContext>;
router.get('/', async (ctx) => {
ctx.body = {
systemStatus: 'nope',
pickedFromRoot: DMNO_CONFIG.ROOT_ONLY,
envCheck: DMNO_CONFIG.API_ONLY || 'env-var-not-loaded',
dmnoTest: DMNO_CONFIG.PORT,
public: DMNO_PUBLIC_CONFIG.PUBLIC_EXAMPLE,
Expand Down
4 changes: 2 additions & 2 deletions example-repo/packages/astro-web/.dmno/config.mts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { DmnoBaseTypes, DmnoDataType, DmnoDataTypeFactoryFn, ExtractSettingsSchema, cacheFunctionResult, createDmnoDataType, defineDmnoService, dmnoFormula, switchByDmnoEnv, switchByNodeEnv, } from 'dmno';
import { DmnoBaseTypes, defineDmnoService } from 'dmno';
import { OnePasswordDmnoPlugin, OnePasswordTypes } from '@dmno/1password-plugin';

const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass');
const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass/prod');

export default defineDmnoService({
name: 'astroweb',
Expand Down
2 changes: 1 addition & 1 deletion example-repo/packages/group1/.dmno/config.mts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { DmnoBaseTypes, defineDmnoService } from 'dmno';
import { OnePasswordDmnoPlugin } from '@dmno/1password-plugin';

const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass');
const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass/prod');

export default defineDmnoService({
name: 'group1',
Expand Down
4 changes: 2 additions & 2 deletions example-repo/packages/nextjs-web/.dmno/config.mts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { DmnoBaseTypes, DmnoDataType, DmnoDataTypeFactoryFn, ExtractSettingsSchema, cacheFunctionResult, createDmnoDataType, defineDmnoService, dmnoFormula, switchByDmnoEnv, switchByNodeEnv, } from 'dmno';
import { defineDmnoService } from 'dmno';
import { OnePasswordDmnoPlugin } from '@dmno/1password-plugin';

const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass');
const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass/prod');

export default defineDmnoService({
name: 'nextweb',
Expand Down
10 changes: 5 additions & 5 deletions example-repo/packages/webapp/.dmno/config.mts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { DmnoBaseTypes, DmnoDataType, DmnoDataTypeFactoryFn, ExtractSettingsSchema, cacheFunctionResult, createDmnoDataType, defineDmnoService, dmnoFormula, switchByDmnoEnv, switchByNodeEnv, } from 'dmno';
import { OnePasswordDmnoPlugin } from '@dmno/1password-plugin';

const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass');
const OnePassBackend = OnePasswordDmnoPlugin.injectInstance('1pass/prod');

const customUrlType = createDmnoDataType({
typeLabel: 'my-custom-url',
Expand Down Expand Up @@ -41,10 +41,10 @@ export default defineDmnoService({
}
],
schema: {
// OP_ITEM_1: {
// sensitive: true,
// value: OnePassBackend.item(),
// },
OP_ITEM_1: {
sensitive: true,
value: OnePassBackend.item(),
},

// EX1: {
// value: (ctx) => DMNO_CONFIG.BOOLEAN_EXAMPLE,
Expand Down
10 changes: 5 additions & 5 deletions example-repo/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 8 additions & 3 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@
"@dagrejs/graphlib": "^2.2.2",
"@inquirer/core": "^8.0.1",
"@inquirer/prompts": "^5.0.1",
"@rollup/plugin-commonjs": "^26.0.1",
"@rollup/plugin-node-resolve": "^15.2.3",
"@rollup/plugin-sucrase": "^5.0.2",
"acorn": "^8.11.3",
"acorn-typescript": "^1.4.13",
"acorn-walk": "^8.3.3",
Expand All @@ -127,11 +130,13 @@
"outdent": "^0.8.0",
"picomatch": "^3.0.1",
"read-yaml-file": "^2.1.0",
"rollup": "^4.20.0",
"svgo": "^3.2.0",
"tslib": "^2.6.3",
"typescript": "^5.4.5",
"validate-npm-package-name": "^5.0.0",
"vite": "^5.2.10",
"vite-node": "^1.5.0",
"validate-npm-package-name": "^5.0.1",
"vite": "^5.3.5",
"vite-node": "^2.0.5",
"which": "^4.0.0"
}
}
2 changes: 2 additions & 0 deletions packages/core/src/cli/cli-executable.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import { PluginCommand } from './commands/plugin.command';
import { InitCommand } from './commands/init.command';
import { ClearCacheCommand } from './commands/clear-cache.command';
import { PrintEnvCommand } from './commands/printenv.command';
import { BuildCommand } from './commands/build.command';



Expand All @@ -43,6 +44,7 @@ program.addCommand(InitCommand);
program.addCommand(ClearCacheCommand);
program.addCommand(PluginCommand);
program.addCommand(PrintEnvCommand);
program.addCommand(BuildCommand);



Expand Down
135 changes: 135 additions & 0 deletions packages/core/src/cli/commands/build.command.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import {
copyFile, rm, cp, writeFile,
} from 'fs/promises';
import { exec } from 'child_process';
import { rollup, RollupBuild } from 'rollup';
import { nodeResolve } from '@rollup/plugin-node-resolve';
import commonjsRollupPlugin from '@rollup/plugin-commonjs';
// import typescriptRollupPlugin from '@rollup/plugin-typescript';
import sucraseRollupPlugin from '@rollup/plugin-sucrase';

import kleur from 'kleur';
import _ from 'lodash-es';
import { DmnoCommand } from '../lib/dmno-command';

import { addServiceSelection } from '../lib/selection-helpers';
import { getCliRunCtx } from '../lib/cli-ctx';
import { addCacheFlags } from '../lib/cache-helpers';
import { addWatchMode } from '../lib/watch-mode-helpers';
import { checkForSchemaErrors } from '../lib/check-errors-helpers';
import { CliExitError } from '../lib/cli-error';
import { DmnoBuildInfo } from '../../config-loader/find-services';

const program = new DmnoCommand('build')
.summary('build compiled dmno config')
.description('Builds DMNO config files into minimal code appropriate for deployed contexts without access to all dependencies')
.example('dmno build', 'Build the config using current detected service')
.example('dmno build --service service1', 'Build dmno config files for use in service1');

addWatchMode(program); // must be first
addCacheFlags(program);
addServiceSelection(program);

program.action(async (opts: {
// these args should be handled already by the helpers
// service?: string,
// watch?: boolean,
// skipCache?: boolean,
// clearCache?: boolean,

format?: string,
public?: boolean,
showAll?: boolean,
}, thisCommand) => {
const ctx = getCliRunCtx();

if (opts.format) ctx.expectingOutput = true;

if (!ctx.selectedService) return; // error message already handled

ctx.log(`\nResolving config for service ${kleur.magenta(ctx.selectedService.serviceName)}\n`);

const workspace = ctx.workspace!;
const service = ctx.selectedService;
checkForSchemaErrors(workspace);

if (!ctx.configLoader.workspaceInfo.isMonorepo) {
throw new CliExitError('`dmno build` only works in monorepos');
}

const buildDirPath = `${service.path}/.dmno-built`;
// delete existing build folder
try {
await rm(buildDirPath, { recursive: true });
} catch (err) {}

if (!ctx.workspace) throw new Error('workspace not loaded');

// TODO: do we want to write a top level json file with metadata?
const buildMetadata: DmnoBuildInfo = {
isMonorepo: ctx.configLoader.workspaceInfo.isMonorepo,
rootService: ctx.workspace.rootService.serviceName,
selectedService: ctx.selectedService.serviceName,
};

const buildMeta = ctx.workspace.getServiceMetaForBuild(service.serviceName);



for (const workspaceService of ctx.workspace?.allServices || []) {
// skip any unrelated services according to the DAG (related via parent or pick)
if (
workspaceService !== ctx.selectedService
&& !buildMeta.requiredServices.includes(workspaceService.serviceName)
) continue;

const configFilePath = `${workspaceService.path}/.dmno/config.mts`;
const buildPackageDirPath = `${buildDirPath}/${workspaceService.serviceName.replaceAll('/', '__')}`;
console.log('Bundling dmno config file for service ', workspaceService.serviceName);
console.log('> config file:', configFilePath);
console.log('> bundled config package dir:', buildPackageDirPath);

let bundle: RollupBuild | undefined;
bundle = await rollup({
input: configFilePath,
external: ['dmno', /@dmno\/.*/],
plugins: [
commonjsRollupPlugin(),
nodeResolve({
preferBuiltins: true, // not sure about this - but false and undefined show warnings
}),

// typescriptRollupPlugin({
// include: ['**/*.{mts,ts}'],
// }),

// alternative to @rollup/plugin-typescript which does faster builds without typechecking
sucraseRollupPlugin({
include: ['**/*.{mts,ts}'],
exclude: ['node_modules/**'],
transforms: ['typescript'],
}),
],
});

// write built JS config file
await bundle.write({
file: `${buildPackageDirPath}/.dmno/config.js`,
});
await bundle?.close();


// copy vault file(s)
// TODO: will need to figure out how to generalize this
// we could copy everything that is not git-ignored?
// we could let plugins specify a list of patterns to copy
await exec(`cp ${workspaceService.path}/.dmno/*.vault.json ${buildPackageDirPath}/.dmno`);

// copy package.json file
await copyFile(`${workspaceService.path}/package.json`, `${buildPackageDirPath}/package.json`);
}

await writeFile(`${buildDirPath}/dmno-build-info.json`, JSON.stringify(buildMetadata, null, 2), 'utf8');
});

export const BuildCommand = program;
5 changes: 0 additions & 5 deletions packages/core/src/cli/commands/clear-cache.command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,7 @@ import kleur from 'kleur';
import _ from 'lodash-es';
import { outdent } from 'outdent';
import { DmnoCommand } from '../lib/dmno-command';
import { formatError, formattedValue, getItemSummary } from '../lib/formatting';
import { getCliRunCtx } from '../lib/cli-ctx';
import { ConfigServer } from '../../config-loader/config-server';
import { addCacheFlags } from '../lib/cache-helpers';
import { addServiceSelection } from '../lib/selection-helpers';
import { fallingDmnoLoader, fallingDmnosAnimation } from '../lib/loaders';
import { pathExists } from '../../config-loader/find-services';

const program = new DmnoCommand('clear-cache')
Expand Down
5 changes: 2 additions & 3 deletions packages/core/src/cli/commands/dev.command.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import kleur from 'kleur';
import _ from 'lodash-es';
import { outdent } from 'outdent';
import gradient from 'gradient-string';
import { DmnoCommand } from '../lib/dmno-command';
import { formatError, formattedValue, getItemSummary } from '../lib/formatting';
import { getItemSummary } from '../lib/formatting';
import { getCliRunCtx } from '../lib/cli-ctx';
import { ConfigServer } from '../../config-loader/config-server';
import { addCacheFlags } from '../lib/cache-helpers';
import { addServiceSelection } from '../lib/selection-helpers';
import { DMNO_DEV_BANNER, fallingDmnoLoader, fallingDmnosAnimation } from '../lib/loaders';
import { DMNO_DEV_BANNER, fallingDmnosAnimation } from '../lib/loaders';

const TERMINAL_COLS = Math.floor(process.stdout.columns * 0.75);

Expand Down
Loading

0 comments on commit 6126b87

Please sign in to comment.