feat(decorators): validate decorators against model (#916)

* feat(decorators): validate decorators against model

Signed-off-by: Dan Selman <[email protected]>

packages/concerto-core/api.txt

@@ -2,7 +2,7 @@ class AstModelManager extends BaseModelManager {
    + void constructor(object?) 
 }
 class BaseModelManager {
-   + void constructor(object?,boolean?,Object?,boolean?,boolean?,boolean?,boolean?,processFile?) 
+   + void constructor(object?,boolean?,Object?,boolean?,boolean?,boolean?,boolean?,object?,string?,string?,processFile?) 
    + boolean isModelManager() 
    + boolean isStrict() 
    + boolean isAliasedTypeEnabled() 
@@ -16,6 +16,7 @@ class BaseModelManager {
    + void validateModelFiles() 
    + Promise updateExternalModels(Object?,FileDownloader?) throws IllegalModelException
    + void writeModelsToFileSystem(string,Object?,boolean) 
+   + object getDecoratorValidation() 
    + Object[] getModels(Object?,boolean) 
    + void clearModelFiles() 
    + ModelFile getModelFile(string) 
@@ -35,7 +36,7 @@ class BaseModelManager {
    + boolean derivesFrom(string,string) 
    + object resolveMetaModel(object) 
    + void fromAst(ast) 
-   + void getAst(boolean?) 
+   + void getAst(boolean?,boolean?) 
    + BaseModelManager filter(FilterFunction) 
 }
 class Concerto {
@@ -57,7 +58,7 @@ class Concerto {
    + object setCurrentTime() 
 class DecoratorManager {
    + ModelManager validate(decoratorCommandSet,ModelFile[]) throws Error
-   + ModelManager decorateModels(ModelManager,decoratorCommandSet,object?,boolean?,boolean?,boolean?,boolean?) 
+   + ModelManager decorateModels(ModelManager,decoratorCommandSet,object?,boolean?,boolean?,boolean?,boolean?,boolean?) 
    + ExtractDecoratorsResult extractDecorators(ModelManager,object,boolean,string) 
    + ExtractDecoratorsResult extractVocabularies(ModelManager,object,boolean,string) 
    + ExtractDecoratorsResult extractNonVocabDecorators(ModelManager,object,boolean,string) 
@@ -69,6 +70,7 @@ class DecoratorManager {
 }
    + string[] intersect() 
    + boolean isUnversionedNamespaceEqual() 
+   + object getDecoratorModel() 
 class Factory {
    + string newId() 
    + void constructor(ModelManager) 

packages/concerto-core/changelog.txt

@@ -24,6 +24,10 @@
 # Note that the latest public API is documented using JSDocs and is available in api.txt.
 #
 
+Version 3.19.2 {56bc38dc7305eee0a06f08a8cf639910} 2024-10-02
+- validateDecorators option added to ModelManager
+- update DecoratorManager to support validated decorators
+
 Version 3.17.5 {9bd69f9522c14a99a085f077e12ac4b2} 2024-08-29
 - importAliasing added to ModelManager parameters
 

packages/concerto-core/lib/basemodelmanager.js

@@ -27,6 +27,7 @@ const ModelUtil = require('./modelutil');
 const Serializer = require('./serializer');
 const TypeNotFoundException = require('./typenotfoundexception');
 const { getRootModel } = require('./rootmodel');
+const { getDecoratorModel } = require('./decoratormodel');
 const MetamodelException = require('./metamodelexception');
 
 // Types needed for TypeScript generation.
@@ -57,6 +58,16 @@ const defaultProcessFile = (name, data) => {
     };
 };
 
+// default decorator validation configuration
+const DEFAULT_DECORATOR_VALIDATION = {
+    missingDecorator: undefined, // 'error' | 'warn' (see Logger.levels)...,
+    invalidDecorator: undefined, // 'error' | 'warn' ...
+};
+
+// these namespaces are internal and excluded by default by getModelFiles
+// and ignored by fromAst
+const EXCLUDE_NS = ['[email protected]', 'concerto', '[email protected]'];
+
 /**
  * Manages the Concerto model files.
  *
@@ -83,6 +94,9 @@ class BaseModelManager {
      * @param {boolean} [options.addMetamodel] - When true, the Concerto metamodel is added to the model manager
      * @param {boolean} [options.enableMapType] - When true, the Concerto Map Type feature is enabled
      * @param {boolean} [options.importAliasing] - When true, the Concerto Aliasing feature is enabled
+     * @param {object} [options.decoratorValidation] - the decorator validation configuration
+     * @param {string} [options.decoratorValidation.missingDecorator] - the validation log level for missingDecorator decorators: off, warning, error
+     * @param {string} [options.decoratorValidation.invalidDecorator] - the validation log level for invalidDecorator decorators: off, warning, error
      * @param {*} [processFile] - how to obtain a concerto AST from an input to the model manager
      */
     constructor(options, processFile) {
@@ -93,12 +107,15 @@ class BaseModelManager {
         this.decoratorFactories = [];
         this.strict = !!options?.strict;
         this.options = options;
+        this.addDecoratorModel();
         this.addRootModel();
+        this.decoratorValidation = options?.decoratorValidation ? options?.decoratorValidation : DEFAULT_DECORATOR_VALIDATION;
 
         // TODO Remove on release of MapType
         // Supports both env var and property based flag
         this.enableMapType = !!options?.enableMapType;
         this.importAliasing = process?.env?.IMPORT_ALIASING === 'true' || !!options?.importAliasing;
+
         // Cache a copy of the Metamodel ModelFile for use when validating the structure of ModelFiles later.
         this.metamodelModelFile = new ModelFile(this, MetaModelUtil.metaModelAst, undefined, MetaModelNamespace);
 
@@ -155,6 +172,16 @@ class BaseModelManager {
         }
     }
 
+    /**
+     * Adds decorator types
+     * @private
+     */
+    addDecoratorModel() {
+        const {decoratorModelAst, decoratorModelCto, decoratorModelFile} = getDecoratorModel();
+        const m = new ModelFile(this, decoratorModelAst, decoratorModelCto, decoratorModelFile);
+        this.addModelFile(m, decoratorModelCto, decoratorModelFile, true);
+    }
+
     /**
      * Visitor design pattern
      * @param {Object} visitor - the visitor
@@ -469,6 +496,14 @@ class BaseModelManager {
         ModelWriter.writeModelsToFileSystem(this.getModelFiles(), path, options);
     }
 
+    /**
+     * Returns the status of the decorator validation options
+     * @returns {object} returns an object that indicates the log levels for defined and undefined decorators
+     */
+    getDecoratorValidation() {
+        return this.decoratorValidation;
+    }
+
     /**
      * Get the array of model file instances
      * @param {Boolean} [includeConcertoNamespace] - whether to include the concerto namespace
@@ -482,7 +517,7 @@ class BaseModelManager {
 
         for (let n = 0; n < keys.length; n++) {
             const ns = keys[n];
-            if(includeConcertoNamespace || (ns !== '[email protected]' && ns !== 'concerto')) {
+            if(includeConcertoNamespace || (!EXCLUDE_NS.includes(ns))) {
                 result.push(this.modelFiles[ns]);
             }
         }
@@ -561,6 +596,7 @@ class BaseModelManager {
      */
     clearModelFiles() {
         this.modelFiles = {};
+        this.addDecoratorModel();
         this.addRootModel();
     }
 
@@ -751,7 +787,7 @@ class BaseModelManager {
      * @return {object} the resolved metamodel
      */
     resolveMetaModel(metaModel) {
-        const priorModels = this.getAst();
+        const priorModels = this.getAst(false, true);
         return MetaModelUtil.resolveLocalNames(priorModels, metaModel);
     }
 
@@ -762,23 +798,26 @@ class BaseModelManager {
     fromAst(ast) {
         this.clearModelFiles();
         ast.models.forEach( model => {
-            const modelFile = new ModelFile( this, model );
-            this.addModelFile( modelFile, null, null, true );
+            if(!EXCLUDE_NS.includes(model.namespace)) { // excludes the internal namespaces, already added
+                const modelFile = new ModelFile( this, model );
+                this.addModelFile( modelFile, null, null, true );
+            }
         });
         this.validateModelFiles();
     }
 
     /**
      * Get the full ast (metamodel instances) for a modelmanager
      * @param {boolean} [resolve] - whether to resolve names
+     * @param {boolean} [includeConcertoNamespaces] - whether to include the concerto namespaces
      * @returns {*} the metamodel
      */
-    getAst(resolve) {
+    getAst(resolve,includeConcertoNamespaces) {
         const result = {
             $class: `${MetaModelNamespace}.Models`,
             models: [],
         };
-        const modelFiles = this.getModelFiles();
+        const modelFiles = this.getModelFiles(includeConcertoNamespaces);
         modelFiles.forEach((thisModelFile) => {
             let metaModel = thisModelFile.getAst();
             if (resolve) {

packages/concerto-core/lib/decoratormanager.js

@@ -31,10 +31,10 @@ if (global === undefined) {
 }
 /* eslint-enable no-unused-vars */
 
-const DCS_VERSION = '0.3.0';
+const DCS_VERSION = '0.4.0';
 
 const DCS_MODEL = `concerto version "^3.0.0"
-namespace org.accordproject.decoratorcommands@0.3.0
+namespace org.accordproject.decoratorcommands@0.4.0
 
 import [email protected]
 
@@ -83,6 +83,7 @@ concept Command {
     o CommandTarget target
     o Decorator decorator
     o CommandType type
+    o String decoratorNamespace optional
 }
 
 /**
@@ -325,7 +326,7 @@ class DecoratorManager {
             validationModelManager.addModelFiles(modelManager.getModelFiles());
             validationModelManager.addCTOModel(
                 DCS_MODEL,
-                'decoratorcommands@0.3.0.cto'
+                'decoratorcommands@0.4.0.cto'
             );
             const factory = new Factory(validationModelManager);
             const serializer = new Serializer(factory, validationModelManager);
@@ -365,17 +366,39 @@ class DecoratorManager {
      * @param {boolean} [options.validateCommands] - validate the decorator command set targets. Note that
      * the validate option must also be true
      * @param {boolean} [options.migrate] - migrate the decoratorCommandSet $class to match the dcs model version
+     * @param {boolean} [options.defaultNamespace] - the default namespace to use for decorator commands that include a decorator without a namespace
      * @param {boolean} [options.enableDcsNamespaceTarget] - flag to control applying namespace targeted decorators on top of the namespace instead of all declarations in that namespace
      * @returns {ModelManager} a new model manager with the decorations applied
      */
     static decorateModels(modelManager, decoratorCommandSet, options) {
 
         this.migrateAndValidate(modelManager, decoratorCommandSet, options?.migrate, options?.validate, options?.validateCommands);
 
+        // we create synthetic imports for all decorator declarations
+        // along with any of their type reference arguments
+        const decoratorImports = decoratorCommandSet.commands.map(command => {
+            return [{
+                $class: `${MetaModelNamespace}.ImportType`,
+                name: command.decorator.name,
+                namespace: command.decorator.namespace ? command.decorator.namespace : options?.defaultNamespace
+            }].concat(command.decorator.arguments ? command.decorator.arguments?.filter(a => a.type)
+                .map(a => {
+                    return {
+                        $class: `${MetaModelNamespace}.ImportType`,
+                        name: a.type.name,
+                        namespace: a.type.namespace ? a.type.namespace : options?.defaultNamespace
+                    };
+                })
+                : []);
+        }).flat().filter(i => i.namespace);
         const { namespaceCommandsMap, declarationCommandsMap, propertyCommandsMap, mapElementCommandsMap, typeCommandsMap }  = this.getDecoratorMaps(decoratorCommandSet);
-        const ast = modelManager.getAst(true);
+        const ast = modelManager.getAst(true, true);
         const decoratedAst = JSON.parse(JSON.stringify(ast));
         decoratedAst.models.forEach((model) => {
+            // remove the imports for types defined in this namespace
+            const neededImports = decoratorImports.filter(i => i.namespace !== model.namespace);
+            // add the imports for decorators, in case they get added below
+            model.imports = model.imports ? model.imports.concat(neededImports) : neededImports;
             model.declarations.forEach((decl) => {
                 const declarationDecoratorCommandSets = [];
                 const { name: declarationName, $class: $classForDeclaration } = decl;
@@ -421,8 +444,12 @@ class DecoratorManager {
 
             });
         });
+
         const enableMapType = modelManager?.enableMapType ? true : false;
-        const newModelManager = new ModelManager({ enableMapType });
+        const newModelManager = new ModelManager({
+            strict: modelManager.isStrict(),
+            enableMapType,
+            decoratorValidation: modelManager.getDecoratorValidation()});
         newModelManager.fromAst(decoratedAst);
         return newModelManager;
     }
@@ -447,7 +474,7 @@ class DecoratorManager {
             locale:'en',
             ...options
         };
-        const sourceAst = modelManager.getAst(true);
+        const sourceAst = modelManager.getAst(true, true);
         const decoratorExtrator = new DecoratorExtractor(options.removeDecoratorsFromModel, options.locale, DCS_VERSION, sourceAst, DecoratorExtractor.Action.EXTRACT_ALL);
         const collectionResp = decoratorExtrator.extract();
         return {
@@ -470,7 +497,7 @@ class DecoratorManager {
             locale:'en',
             ...options
         };
-        const sourceAst = modelManager.getAst(true);
+        const sourceAst = modelManager.getAst(true, true);
         const decoratorExtrator = new DecoratorExtractor(options.removeDecoratorsFromModel, options.locale, DCS_VERSION, sourceAst, DecoratorExtractor.Action.EXTRACT_VOCAB);
         const collectionResp = decoratorExtrator.extract();
         return {

packages/concerto-core/lib/decoratormodel.js

@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+/** @type unknown */
+const decoratorModelAst = require('./decoratormodel.json');
+
+/**
+ * Gets the decorator 'concerto.decorator' model
+ * @returns {object} decoratorModelFile, decoratorModelCto and decoratorModelAst
+ */
+function getDecoratorModel() {
+    const decoratorModelFile = 'concerto_decorator_1.0.0.cto';
+    const decoratorModelCto = `namespace [email protected]
+    abstract concept Decorator {}
+    concept DotNetNamespace extends Decorator {
+       o String namespace
+    }`;
+    const ast = JSON.parse(JSON.stringify(decoratorModelAst));
+    return { decoratorModelFile, decoratorModelCto, decoratorModelAst: ast };
+}
+
+module.exports = { getDecoratorModel };

packages/concerto-core/lib/decoratormodel.json

@@ -0,0 +1,31 @@
+{
+    "$class": "[email protected]",
+    "decorators": [],
+    "namespace": "[email protected]",
+    "imports": [],
+    "declarations": [
+        {
+            "$class": "[email protected]",
+            "name": "Decorator",
+            "isAbstract": true,
+            "properties": []
+        },
+        {
+            "$class": "[email protected]",
+            "name": "DotNetNamespace",
+            "superType": {
+                "$class": "[email protected]",
+                "name": "Decorator"
+            },
+            "isAbstract": false,
+            "properties": [
+                {
+                    "$class": "[email protected]",
+                    "name": "namespace",
+                    "isArray": false,
+                    "isOptional": false
+                }
+            ]
+        }
+    ]
+}