From 6931561b137d70a84f5b00b1be41331f69b6dddb Mon Sep 17 00:00:00 2001 From: Kedar Vijay Kulkarni Date: Mon, 23 Sep 2019 16:59:42 -0400 Subject: [PATCH] initial commit with vault+sprout integration via dynaconf --- sprout/appliances/models.py | 2 +- sprout/appliances/utils/__init__.py | 0 sprout/appliances/utils/providers.py | 71 +++++++++++++++ sprout/appliances/views.py | 2 +- sprout/requirements.txt | 1 + sprout/vault/__init__.py | 0 sprout/vault/approle_setup.sh | 29 ++++++ sprout/vault/cfme-qe-infra-approle-token.json | 10 +++ sprout/vault/cfme-qe-infra-ro-policy.json | 90 +++++++++++++++++++ sprout/vault/vault.py | 53 +++++++++++ 10 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 sprout/appliances/utils/__init__.py create mode 100644 sprout/appliances/utils/providers.py create mode 100644 sprout/vault/__init__.py create mode 100755 sprout/vault/approle_setup.sh create mode 100644 sprout/vault/cfme-qe-infra-approle-token.json create mode 100644 sprout/vault/cfme-qe-infra-ro-policy.json create mode 100644 sprout/vault/vault.py diff --git a/sprout/appliances/models.py b/sprout/appliances/models.py index 20bda55592..b0034065b6 100644 --- a/sprout/appliances/models.py +++ b/sprout/appliances/models.py @@ -34,7 +34,7 @@ from cfme.utils.appliance import Appliance as CFMEAppliance, IPAppliance from cfme.utils.bz import Bugzilla from cfme.utils.conf import cfme_data -from cfme.utils.providers import get_mgmt +from sprout.appliances.utils.providers import get_mgmt from cfme.utils.timeutil import nice_seconds from cfme.utils.version import Version diff --git a/sprout/appliances/utils/__init__.py b/sprout/appliances/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sprout/appliances/utils/providers.py b/sprout/appliances/utils/providers.py new file mode 100644 index 0000000000..4ebc19f6a4 --- /dev/null +++ b/sprout/appliances/utils/providers.py @@ -0,0 +1,71 @@ +from collections import Mapping + +import six + +from cfme.common.provider import all_types +from cfme.exceptions import UnknownProviderType +from cfme.utils import conf +from cfme.utils.log import logger +from sprout.vault.vault import settings + +providers_data = conf.cfme_data.get("management_systems", {}) + +PROVIDER_MGMT_CACHE = {} + + +def get_class_from_type(prov_type): + try: + return all_types()[prov_type] + except KeyError: + raise UnknownProviderType("Unknown provider type: {}!".format(prov_type)) + + +def get_mgmt(provider_key, providers=None, credentials=None): + """ Provides a ``wrapanapi`` object, based on the request. + + Args: + provider_key: The name of a provider, as supplied in the yaml configuration files. + You can also use the dictionary if you want to pass the provider data directly. + providers: A set of data in the same format as the ``management_systems`` section in the + configuration yamls. If ``None`` then the configuration is loaded from the default + locations. Expects a dict. + credentials: A set of credentials in the same format as the ``credentials`` yamls files. + If ``None`` then credentials are loaded from the vault using dynaconf. Expects a dict. + Return: A provider instance of the appropriate ``wrapanapi.WrapanapiAPIBase`` + subclass + """ + if providers is None: + providers = providers_data + # provider_key can also be provider_data for some reason + # TODO rename the parameter; might break things + if isinstance(provider_key, Mapping): + provider_data = provider_key + provider_key = provider_data['name'] + else: + provider_data = providers[provider_key] + + if credentials is None: + # create env matching provider_keys in vault to hold credentials + with settings.using_env(provider_key): + credentials = {key.lower(): val for key, val in settings.as_dict().items() + if 'VAULT' not in key} + + # Munge together provider dict and creds, + # Let the provider do whatever they need with them + provider_kwargs = provider_data.copy() + provider_kwargs.update(credentials) + + if not provider_kwargs.get('username') and provider_kwargs.get('principal'): + provider_kwargs['username'] = provider_kwargs['principal'] + provider_kwargs['password'] = provider_kwargs['secret'] + + if isinstance(provider_key, six.string_types): + provider_kwargs['provider_key'] = provider_key + provider_kwargs['logger'] = logger + + if provider_key not in PROVIDER_MGMT_CACHE: + mgmt_instance = get_class_from_type(provider_data['type']).mgmt_class(**provider_kwargs) + PROVIDER_MGMT_CACHE[provider_key] = mgmt_instance + else: + logger.debug("returning cached mgmt class for '%s'", provider_key) + return PROVIDER_MGMT_CACHE[provider_key] diff --git a/sprout/appliances/views.py b/sprout/appliances/views.py index e80f572d9c..54c0e0b39e 100644 --- a/sprout/appliances/views.py +++ b/sprout/appliances/views.py @@ -26,7 +26,7 @@ from sprout.log import create_logger from cfme.utils.bz import Bugzilla -from cfme.utils.providers import get_mgmt +from sprout.appliances.utils.providers import get_mgmt from cfme.utils.version import Version from wrapanapi import Openshift diff --git a/sprout/requirements.txt b/sprout/requirements.txt index a3c8f270f7..7c59a30833 100644 --- a/sprout/requirements.txt +++ b/sprout/requirements.txt @@ -4,6 +4,7 @@ Django<1.10 celery[redis]<3.2 django-object-actions django-ipware +dynaconf[vault] flower gunicorn pika diff --git a/sprout/vault/__init__.py b/sprout/vault/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sprout/vault/approle_setup.sh b/sprout/vault/approle_setup.sh new file mode 100755 index 0000000000..95228f688f --- /dev/null +++ b/sprout/vault/approle_setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This file briefly describes the steps and can execute those as well - to create Approle Token. +# make sure to have exported VAULT_ADDR=https://addr:port +# and VAULT_SKIP_VERIFY=true to disable ssl verification + +echo 'login with kerberos - make sure you are admin by reading listed policies' +vault login -method=ldap -tls-skip-verify=true username= + +# ======================================================================================= +# NOTE: we do not need to run this every time, but only when you need a new AppRole Token. +echo 'write policy' +vault policy write cfme-qe-infra-ro-policy cfme-qe-infra-ro-policy.json + +echo 'enable AppRole auth' +vault auth enable approle + +echo 'create an AppRole called' +vault write auth/approle/role/cfme-qe-infra secret_id_ttl=10m secret_id_num_uses=0 token_num_uses=20 token_ttl=30m token_max_ttl=60m policies=cfme-qe-infra-ro-policy + +echo 'Creating a Limited-Use Token' +vault policy write cfme-qe-infra-approle-token cfme-qe-infra-approle-token.json +vault token create -policy=cfme-qe-infra-approle-token +# ======================================================================================= + +echo 'Set following env variable.' +echo 'export VAULT_ENABLED_FOR_DYNACONF=true' +echo 'export VAULT_URL_FOR_DYNACONF=https://infra-assets.cfme2.lab.eng.rdu2.redhat.com:8201' +echo 'export VAULT_APPROLE_TOKEN=' +echo 'export VAULT_VERIFY_FOR_DYNACONF=false' diff --git a/sprout/vault/cfme-qe-infra-approle-token.json b/sprout/vault/cfme-qe-infra-approle-token.json new file mode 100644 index 0000000000..86c5311df0 --- /dev/null +++ b/sprout/vault/cfme-qe-infra-approle-token.json @@ -0,0 +1,10 @@ +{ + "path": { + "auth/approle/role/cfme-qe-infra/role-id": { + "capabilities": ["read"] + }, + "auth/approle/role/cfme-qe-infra/secret-id": { + "capabilities": ["create", "update"] + } + } +} diff --git a/sprout/vault/cfme-qe-infra-ro-policy.json b/sprout/vault/cfme-qe-infra-ro-policy.json new file mode 100644 index 0000000000..5c38641c4f --- /dev/null +++ b/sprout/vault/cfme-qe-infra-ro-policy.json @@ -0,0 +1,90 @@ +{ + "path": { + "secret/*": { + "capabilities": [ + "list" + ] + }, + "secret/cfme-qe-sprout/*": { + "capabilities": [ + "read", + "list" + ] + }, + "secret/data/cfme-qe-sprout/*": { + "capabilities": [ + "read", + "list" + ] + }, + "auth/token/lookup-self": { + "capabilities": [ + "read" + ] + }, + "auth/token/renew-self": { + "capabilities": [ + "update" + ] + }, + "auth/token/revoke-self": { + "capabilities": [ + "update" + ] + }, + "sys/capabilities-self": { + "capabilities": [ + "update" + ] + }, + "sys/renew": { + "capabilities": [ + "update" + ] + }, + "sys/leases/renew": { + "capabilities": [ + "update" + ] + }, + "sys/leases/lookup": { + "capabilities": [ + "update" + ] + }, + "cubbyhole/*": { + "capabilities": [ + "create", + "read", + "update", + "delete", + "list" + ] + }, + "sys/wrapping/wrap": { + "capabilities": [ + "update" + ] + }, + "sys/wrapping/lookup": { + "capabilities": [ + "update" + ] + }, + "sys/wrapping/unwrap": { + "capabilities": [ + "update" + ] + }, + "sys/mounts": { + "capabilities": [ + "read" + ] + }, + "sys/auth": { + "capabilities": [ + "read" + ] + } + } +} diff --git a/sprout/vault/vault.py b/sprout/vault/vault.py new file mode 100644 index 0000000000..276ac2cd1f --- /dev/null +++ b/sprout/vault/vault.py @@ -0,0 +1,53 @@ +import os + +from dynaconf import LazySettings +from hvac import Client + +VAULT_APPROLE = 'cfme-qe-infra' +extra_dynaconf_args = {} + +# The process to authenticate is to basically create AppRole Token and use that to authenticate +# with vault and every time you authenticate also renew the token. That token will allow you +# to create role_id and secret_id which together lets you authenticate to AppRole +vault_approle_token = os.environ.get("VAULT_APPROLE_TOKEN", None) +vault_url = os.environ.get("VAULT_URL_FOR_DYNACONF", None) + + +def _login_and_renew_token(url, token): + """Log into Vault, renew the token, and return the Vault client""" + vault = Client(url=url, token=token, verify=False) + if not vault.is_authenticated(): + return None + # Renew the token so that it's valid for another 7 days + vault.renew_token() + return vault + + +def _get_approle_ids(url, token): + vault = _login_and_renew_token(url, token) + if not vault: + return None + role_id = vault.get_role_id(VAULT_APPROLE) + secret_id = vault.create_role_secret_id(VAULT_APPROLE).get("data", {}).get("secret_id") + return {"role_id": role_id, "secret_id": secret_id} + + +if vault_approle_token and vault_url: + # Generate secret id + vault_approle_ids = _get_approle_ids(vault_url, vault_approle_token) + if not vault_approle_ids: + raise Exception(f"Cannot auth with Vault with AppRole token '{vault_approle_token}'") + extra_dynaconf_args.update( + { + "VAULT_ROLE_ID": vault_approle_ids["role_id"], + "VAULT_ROLE_ID_FOR_DYNACONF": vault_approle_ids["role_id"], # Jenkins vault + "VAULT_SECRET_ID": vault_approle_ids["secret_id"], + "VAULT_SECRET_ID_FOR_DYNACONF": vault_approle_ids["secret_id"], # Jenkins vault + } + ) +settings = LazySettings( + VAULT_PATH_FOR_DYNACONF="cfme-qe-sprout", + VAULT_VERIFY_FOR_DYNACONF=False, + VAULT_ENABLED_FOR_DYNACONF=True, + **extra_dynaconf_args +)