Resource: Support policy set definition grouping (Azure#11503)

* Support policy set definition grouping

* Re-record managedapp test in 2019_03_01 profile

* Record policy insights tests with new policy api-version

azure-cli.pyproj

@@ -1324,6 +1324,9 @@
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_param_def.json" />
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_rule.json" />
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_set.json" />
+    <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_set_grouping.json" />
+    <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_groups_def.json" />
+    <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\sample_policy_groups_def2.json" />
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\simple_deploy.json" />
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\simple_deploy_parameters.json" />
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\latest\test-object.json" />

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -134,7 +134,7 @@ def default_api_version(self):
         ResourceType.MGMT_RESOURCE_FEATURES: '2015-12-01',
         ResourceType.MGMT_RESOURCE_LINKS: '2016-09-01',
         ResourceType.MGMT_RESOURCE_LOCKS: '2016-09-01',
-        ResourceType.MGMT_RESOURCE_POLICY: '2019-06-01',
+        ResourceType.MGMT_RESOURCE_POLICY: '2019-09-01',
         ResourceType.MGMT_RESOURCE_RESOURCES: '2019-07-01',
         ResourceType.MGMT_RESOURCE_SUBSCRIPTIONS: '2016-06-01',
         ResourceType.MGMT_NETWORK_DNS: '2018-05-01',

src/azure-cli-core/setup.py

@@ -69,7 +69,7 @@
     'requests~=2.20',
     'six~=1.12',
     'wheel==0.30.0',
-    'azure-mgmt-resource~=4.0',
+    'azure-mgmt-resource~=6.0',
 ]
 
 TESTS_REQUIRE = [

src/azure-cli-core/tox.ini

@@ -6,6 +6,6 @@ skip_missing_interpreters = True
 deps = pytest
        mock
        pip
-       azure-mgmt-resource~=4.0.0
+       azure-mgmt-resource~=6.0.0
        -e ../azure-cli-telemetry
 commands = pytest

src/azure-cli/HISTORY.rst

@@ -13,6 +13,10 @@ Release History
 * Fix issue #11217: webapp: az webapp config ssl upload should support slot parameter
 * Fix issue #10965: Error: Name cannot be empty. Allow remove by ip_address and subnet
 
+**ARM**
+
+* Update azure-mgmt-resource package to use 6.0.0
+
 **Compute**
 
 * vmss create/update: Add --scale-in-policy, which decides which virtual machines are chosen for removal when a VMSS is scaled-in.
@@ -47,6 +51,11 @@ Release History
 * Add back edge builds for pip install
 * Add Ubuntu eoan package
 
+**Policy**
+
+* Support for Policy API version 2019-09-01.
+* az policy set-definition: Support grouping within policy set definitions with `--definition-groups` parameter
+
 **Storage**
 
 * GA Release Large File Shares property for storage account create and update command

src/azure-cli/azure/cli/command_modules/policyinsights/tests/latest/test_policyinsights_scenario.py

@@ -327,7 +327,7 @@ def test_policy_insights_remediation_management_group(self):
     @AllowLargeResponse()
     def test_policy_insights_remediation_complete(self):
         self.kwargs.update({
-            'pan': 'cd7ac64c77ec441dbff7af7c',
+            'pan': '2a47116300b347c599c4c4d3',
             'rg': 'az-cli-policy-insights-test',
             'rn': self.create_random_name('azurecli-test-remediation', 40)
         })

src/azure-cli/azure/cli/command_modules/resource/_help.py

@@ -884,7 +884,7 @@
 parameters:
   - name: --definitions
     type: string
-    short-summary: Policy definitions in JSON format, or a path to a file containing JSON rules.
+    short-summary: Policy definitions in JSON format, or a path to a file or URI containing JSON rules.
   - name: --management-group
     type: string
     short-summary: Name of management group the new policy set definition can be assigned in.
@@ -896,16 +896,34 @@
     text: |
         az policy set-definition create -n readOnlyStorage --definitions '[
                 {
-                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy"
+                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy",
+                    "parameters": { "storageSku": { "value": "[parameters(\\"requiredSku\\")]" } }
                 }
+            ]' \\
+            --params '{ "requiredSku": { "type": "String" } }'
+  - name: Create a policy set definition with parameters.
+    text: |
+        az policy set-definition create -n readOnlyStorage --definitions '[
+                { "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy" }
             ]'
-  - name: Create a policy set definition to be used by a subscription.
+  - name: Create a policy set definition in a subscription.
     text: |
         az policy set-definition create -n readOnlyStorage --subscription '0b1f6471-1bf0-4dda-aec3-111122223333' --definitions '[
+                { "policyDefinitionId": "/subscriptions/0b1f6471-1bf0-4dda-aec3-111122223333/providers/Microsoft.Authorization/policyDefinitions/storagePolicy" }
+            ]'
+  - name: Create a policy set definition with policy definition groups.
+    text: |
+        az policy set-definition create -n computeRequirements --definitions '[
+                {
+                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy",
+                    "groupNames": [ "CostSaving", "Organizational" ]
+                },
                 {
-                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy"
+                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/tagPolicy",
+                    "groupNames": [ "Organizational" ]
                 }
-            ]'
+            ]' \\
+            --definition-groups '[{ "name": "CostSaving" }, { "name": "Organizational" } ]'
 """
 
 helps['policy set-definition delete'] = """
@@ -935,14 +953,24 @@
 type: command
 short-summary: Update a policy set definition.
 examples:
-  - name: Update a policy set definition. (autogenerated)
+  - name: Update a policy set definition.
     text: |-
         az policy set-definition update --definitions '[
+                { "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy" }
+            ]' --name MyPolicySetDefinition
+  - name: Update the groups and definitions within a policy set definition.
+    text: |
+        az policy set-definition update -n computeRequirements --definitions '[
+                {
+                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy",
+                    "groupNames": [ "CostSaving", "Organizational" ]
+                },
                 {
-                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/storagePolicy"
+                    "policyDefinitionId": "/subscriptions/mySubId/providers/Microsoft.Authorization/policyDefinitions/tagPolicy",
+                    "groupNames": [ "Organizational" ]
                 }
-            ]' --name MyPolicySetDefinition
-    crafted: true
+            ]' \\
+            --definition-groups '[{ "name": "CostSaving" }, { "name": "Organizational" } ]'
 """
 
 helps['provider'] = """

src/azure-cli/azure/cli/command_modules/resource/_params.py

@@ -156,6 +156,7 @@ def load_arguments(self, _):
         c.argument('description', help='Description of policy set definition.')
         c.argument('params', help='JSON formatted string or a path to a file or uri with parameter definitions.', type=file_type, completer=FilesCompleter())
         c.argument('definitions', help='JSON formatted string or a path to a file or uri containing definitions.', type=file_type, completer=FilesCompleter())
+        c.argument('definition_groups', min_api='2019-09-01', help='JSON formatted string or a path to a file or uri containing policy definition groups. Groups are used to organize policy definitions within a policy set.', type=file_type, completer=FilesCompleter())
         c.argument('management_group', arg_type=management_group_name_type)
         c.argument('subscription', arg_type=subscription_type)
         c.ignore('_subscription')  # disable global subscription

src/azure-cli/azure/cli/command_modules/resource/custom.py

@@ -1439,14 +1439,15 @@ def create_policy_definition(cmd, name, rules=None, params=None, display_name=No
 
 
 def create_policy_setdefinition(cmd, name, definitions, params=None, display_name=None, description=None,
-                                subscription=None, management_group=None):
+                                subscription=None, management_group=None, definition_groups=None):
     definitions = _load_file_string_or_uri(definitions, 'definitions')
     params = _load_file_string_or_uri(params, 'params', False)
+    definition_groups = _load_file_string_or_uri(definition_groups, 'definition_groups', False)
 
     policy_client = _resource_policy_client_factory(cmd.cli_ctx)
     PolicySetDefinition = cmd.get_models('PolicySetDefinition')
     parameters = PolicySetDefinition(policy_definitions=definitions, parameters=params, description=description,
-                                     display_name=display_name)
+                                     display_name=display_name, policy_definition_groups=definition_groups)
     if cmd.supported_api_version(min_api='2018-03-01'):
         enforce_mutually_exclusive(subscription, management_group)
         if management_group:
@@ -1554,10 +1555,11 @@ def update_policy_definition(cmd, policy_definition_name, rules=None, params=Non
 
 def update_policy_setdefinition(cmd, policy_set_definition_name, definitions=None, params=None,
                                 display_name=None, description=None,
-                                subscription=None, management_group=None):
+                                subscription=None, management_group=None, definition_groups=None):
 
     definitions = _load_file_string_or_uri(definitions, 'definitions', False)
     params = _load_file_string_or_uri(params, 'params', False)
+    definition_groups = _load_file_string_or_uri(definition_groups, 'definition_groups', False)
 
     policy_client = _resource_policy_client_factory(cmd.cli_ctx)
     definition = _get_custom_or_builtin_policy(cmd, policy_client, policy_set_definition_name, subscription, management_group, True)
@@ -1567,7 +1569,8 @@ def update_policy_setdefinition(cmd, policy_set_definition_name, definitions=Non
         policy_definitions=definitions if definitions is not None else definition.policy_definitions,
         description=description if description is not None else definition.description,
         display_name=display_name if display_name is not None else definition.display_name,
-        parameters=params if params is not None else definition.parameters)
+        parameters=params if params is not None else definition.parameters,
+        policy_definition_groups=definition_groups if definition_groups is not None else definition.policy_definition_groups)
     if cmd.supported_api_version(min_api='2018-03-01'):
         enforce_mutually_exclusive(subscription, management_group)
         if management_group: