Adding first working commit. This commit creates workspaces, pushes v…

…ars, and creates host project

.gitignore

@@ -1,4 +1,3 @@
 .idea/
 .terraform/
-common.tfvars
-tfe-init.tfvars
+credentials.json

infra/.gitignore

@@ -0,0 +1,2 @@
+org.auto.tfvars
+backend.hcl

infra/backend.hcl.example

@@ -0,0 +1,4 @@
+organization = "Your terraform cloud org"
+workspaces {
+  name = "desired workspace"
+}

infra/backend.tf

@@ -0,0 +1,3 @@
+terraform {
+  backend "remote" {}
+}

infra/main.tf

@@ -0,0 +1,41 @@
+resource "google_folder" "org" {
+  count = var.parent_folder == "" ? 1 : 0
+  display_name = upper(var.environment)
+  parent = "organizations/${var.org_id}"
+}
+
+resource "google_folder" "folder" {
+  count = var.parent_folder != "" ? 1 : 0
+  display_name = upper(var.environment)
+  parent = "folders/${var.parent_folder}"
+}
+
+resource "local_file" "creds" {
+  filename = "${path.module}/creds.json"
+  sensitive_content = var.credentials
+}
+
+module "host-vpc-project" {
+  source = "[email protected]:ops-guru/terraform-google-project-factory.git?ref=8837b4b6d825be74b1351165671a9e88202af0cd"
+  name = "host-vpc-${var.environment}"
+  random_project_id = true
+  org_id = var.org_id
+  billing_account = var.billing_account_id
+  folder_id = var.parent_folder == "" ? google_folder.org[0].id : google_folder.folder[0].id
+  credentials_path  = local_file.creds.filename
+  activate_apis = [
+    "appengine.googleapis.com",
+    "cloudapis.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
+    "compute.googleapis.com",
+    "iam.googleapis.com",
+    "serviceusage.googleapis.com",
+    "storage-api.googleapis.com",
+    "storage-component.googleapis.com",
+    "container.googleapis.com",
+    "sqladmin.googleapis.com",
+    "sourcerepo.googleapis.com",
+    "servicenetworking.googleapis.com",
+  ]
+  default_service_account = "keep"
+}

infra/org.auto.tfvars.example

@@ -0,0 +1,2 @@
+org_id = "Your organization id"
+billing_account_id = "Billing Account Id"

infra/outputs.tf

@@ -0,0 +1,4 @@
+output "host_project_id" {
+  description = "Project ID"
+  value       = module.host-vpc-project.project_id
+}

infra/provider.tf

@@ -0,0 +1,9 @@
+provider "google" {
+  version = "~> 2.14.0"
+  credentials = var.credentials
+}
+
+provider "google-beta" {
+  version = "~> 2.14.0"
+  credentials = var.credentials
+}

infra/variables.tf

@@ -0,0 +1,20 @@
+variable "credentials" {
+  description = "Path to service account credentials"
+}
+
+variable "environment" {
+  description = "Environment. Valid values are dev, stg, and prd"
+}
+
+variable "org_id" {
+  description = "Organization ID"
+}
+
+variable "billing_account_id" {
+  description = "Billing Account ID"
+}
+
+variable "parent_folder" {
+  description = "Parent Folder"
+  default = ""
+}

workspaces/.gitignore

@@ -1,2 +1 @@
-*.auto.tfvars
-terraform.tfstate*
+test*

workspaces/create-workspaces.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import hcl
+import json
+import os
+import argparse
+import requests
+import yaml
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Create workspaces')
+    parser.add_argument('--workspaces', default='workspaces.yaml', help='Yaml file with workspaces\' definitions')
+    parser.add_argument('--org', required=True, default='gcp-landing-zone', help='Terraform cloud organization')
+    parser.add_argument('--oauth_token_id', default=None, help='Terraform vcs oauth token id. Set this variable if'
+                                                               'you are setting vcs_repo for your workspace in '
+                                                               'workspaces.yaml')
+    args = parser.parse_args()
+    return args
+
+
+def get_token():
+    if os.environ.get('TF_TOKEN'):
+        return os.environ.get('TF_TOKEN')
+    if os.path.isfile(os.path.expanduser('~/.terraformrc')):
+        with open(os.path.expanduser('~/.terraformrc'), 'r') as fp:
+            obj = hcl.load(fp)
+            try:
+                return obj['credentials']["app.terraform.io"]["token"]
+            except KeyError:
+                pass
+    else:
+        error_message = 'You need to define terraform cloud API token either through TF_TOKEN variable or ~/.terraformrc. ' \
+                        'See: https://www.terraform.io/docs/commands/cli-config.html#available-settings'
+        raise Exception(error_message)
+
+
+def create_workspace(name, org, token, working_directory='', oauth_token_id=None, vcs_repo={}):
+    url = 'https://app.terraform.io/api/v2/organizations/{}/workspaces'.format(org)
+    if oauth_token_id and "identifier" in vcs_repo:
+        vcs_repo["oauth-token-id"] = oauth_token_id
+    if oauth_token_id in vcs_repo and not oauth_token_id:
+        message = 'Workspace {} has vcs-repo.identifier defined but no oauth-token-id.'.format(name)
+        raise Exception(message)
+    body = {
+        "data": {
+            "attributes": {
+                "name": name,
+                "working-directory": working_directory,
+                "vcs-repo": vcs_repo
+            },
+            "type": "workspaces"
+        }
+    }
+
+    session = requests.session()
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/vnd.api+json'
+    }
+
+    res = session.get(url + '/' + name, headers=headers)
+    if res.status_code != 200:
+        res = session.post(url, headers=headers, data=json.dumps(body))
+    elif res.status_code == 200:
+        res = session.patch(url + '/' + name, headers=headers, data=json.dumps(body))
+    if res.status_code not in [200, 201]:
+        raise Exception('Status code: {}. Exception: {}'.format(res.status_code, res.content))
+
+
+def load_workspaces(workspaces):
+    with open(workspaces, 'r') as workspaces:
+        return yaml.load(workspaces)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    token = get_token()
+    workspaces = load_workspaces(args.workspaces)
+    org = args.org
+    oauth_token_id = args.oauth_token_id
+
+    for workspace in workspaces:
+        print('Creating/Modifying {}'.format(workspace['name']))
+        working_directory = ''
+        vcs_repo = {}
+        if 'working_directory' in workspace:
+            working_directory = workspace['working_directory']
+        if 'vcs_repo' in workspace:
+            vcs_repo = workspace['vcs_repo']
+        create_workspace(workspace['name'], org, token, working_directory=working_directory,
+                         oauth_token_id=oauth_token_id, vcs_repo=vcs_repo)

workspaces/infra-dev.hcl

@@ -0,0 +1,4 @@
+workspace = "infra-dev"
+credentials = "../credentials.json"
+environment = "dev"
+parent_folder = 331329180875

workspaces/pushvars.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import hcl
+import json
+import os
+import argparse
+import requests
+import yaml
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Add variables to workspace')
+    parser.add_argument('varfiles', nargs='+', help='HCL files that hold variables')
+    parser.add_argument('--org', default='gcp-landing-zone', required=True, help='terraform cloud secrets file')
+    args = parser.parse_args()
+    return args
+
+
+def get_token():
+    if os.environ.get('TF_TOKEN'):
+        return os.environ.get('TF_TOKEN')
+    if os.path.isfile(os.path.expanduser('~/.terraformrc')):
+        with open(os.path.expanduser('~/.terraformrc'), 'r') as fp:
+            obj = hcl.load(fp)
+            try:
+                return obj['credentials']["app.terraform.io"]["token"]
+            except KeyError:
+                pass
+    else:
+        error_message = 'You need to define terraform cloud API token either through TF_TOKEN variable or ~/.terraformrc. ' \
+                        'See: https://www.terraform.io/docs/commands/cli-config.html#available-settings'
+        raise Exception(error_message)
+
+
+def load_variables_file(variables_file):
+    with open(variables_file, 'r') as fp:
+        obj = hcl.load(fp)
+    return obj
+
+
+def get_workspace_id(workspace, org, token):
+    url = 'https://app.terraform.io/api/v2/organizations/{}/workspaces/{}'.format(org, workspace)
+    session = requests.session()
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/vnd.api+json'
+    }
+    res = session.get(url, headers=headers)
+    if res.status_code != 200:
+        message = 'workspace {} does not exists'.format(workspace)
+        raise Exception(message)
+    response = json.loads(res.content)
+    return response['data']['id']
+
+
+def get_variables_for_workspace(org, workspace, token):
+    url = 'https://app.terraform.io/api/v2/vars'
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/vnd.api+json'
+    }
+    params = {'filter[organization][name]': org, 'filter[workspace][name]': workspace}
+    session = requests.Session()
+    res_get_var = session.get(url, headers=headers, params=params)
+    if res_get_var.status_code != 200:
+        message = 'workspace={} in org={} doesn\'t exists. Error code={}. Error response={} '\
+            .format(org, workspace, res_get_var.status_code, res_get_var.text)
+        raise Exception(message)
+    variables = {}
+    for var in json.loads(res_get_var.content)['data']:
+        variables[var['attributes']['key']] = var['id']
+    return variables
+
+
+def set_variable(key, value, current_variables, workspace_id, token, sensitive=False):
+    url = 'https://app.terraform.io/api/v2/vars'
+    if isinstance(value, str):
+        hcl_type = False
+    else:
+        hcl_type = True
+        # Converting to hcl
+        value = json.dumps(value, indent=2).replace('": ', '" = ')
+    if 'credentials' in key.lower() or 'password' in key.lower():
+        sensitive=True
+    if key == 'credentials':
+        with open(value, 'r') as cred_file:
+            value = cred_file.read()
+    body = {
+        "data": {
+            "type": "vars",
+            "attributes": {
+                "key": key,
+                "value": value,
+                "category": "terraform",
+                "hcl": hcl_type,
+                "sensitive": sensitive
+            },
+            "relationships": {
+                "workspace": {
+                    "data": {
+                        "id": workspace_id,
+                        "type": "workspaces"
+                    }
+                }
+            }
+        }
+    }
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/vnd.api+json'
+    }
+
+    session = requests.Session()
+    if key in current_variables:
+        res = session.patch(url + '/' + current_variables[key], headers=headers, data=json.dumps(body))
+    else:
+        res = session.post(url, headers=headers, data=json.dumps(body))
+    if res.status_code not in [200, 201]:
+        message = 'Status code: {}, {}'.format(res.status_code, res.text)
+        raise Exception(message)
+
+
+def get_workspace(variables):
+    if 'workspace' in variables:
+        return variables['workspace']
+    raise Exception('You have to define workspace variable in your variables file')
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    token = get_token()
+    org = args.org
+
+    for variable_file in args.varfiles:
+        variables = load_variables_file(variable_file)
+        workspace = get_workspace(variables)
+        workspace_id = get_workspace_id(workspace, org, token)
+        current_variables = get_variables_for_workspace(org, workspace, token)
+        for key, value in variables.items():
+            if key == 'workspace':
+                continue
+            print('Setting {}'.format(key))
+            set_variable(key, value, current_variables, workspace_id, token)

workspaces/workspaces.yaml

@@ -0,0 +1,4 @@
+- name: infra-dev
+  #vcs_repo:
+  #  identifier: "ops-guru/gcp-terraform-cloud"
+  #  ingress-submodules: true