Skip to content

Commit

Permalink
Merge pull request #6 from komminarlab/tk/improvements
Browse files Browse the repository at this point in the history
made few improvements
  • Loading branch information
thulasirajkomminar authored Dec 19, 2023
2 parents a68829a + a9a8947 commit e329f20
Show file tree
Hide file tree
Showing 7 changed files with 231 additions and 86 deletions.
156 changes: 116 additions & 40 deletions .github/workflows/terraform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,64 +4,140 @@ name: Terraform
on:
pull_request:

permissions:
contents: write
pull-requests: write

env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
terraform-fmt:
fmt-lint-validate:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@master
- name: Checkout code
uses: actions/checkout@v4

- name: Setup Terraform
uses: hashicorp/setup-terraform@v2

- name: Setup Terraform Linters
uses: terraform-linters/setup-tflint@v4
with:
github_token: ${{ env.GITHUB_TOKEN }}

- name: Terraform Format
uses: hashicorp/terraform-github-actions@master
id: fmt
run: terraform fmt -check -recursive

- name: Terraform Init
id: init
run: terraform init

- name: Terraform Validate
id: validate
run: terraform validate -no-color

- name: Terraform Lint
id: lint
run: tflint --no-color --recursive --format compact

- uses: actions/github-script@v6
if: github.event_name == 'pull_request' || always()
with:
tf_actions_version: latest
tf_actions_subcommand: fmt
tf_actions_comment: true
github-token: ${{ env.GITHUB_TOKEN }}
script: |
// 1. Retrieve existing bot comments for the PR
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
})
const botComment = comments.find(comment => {
return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style')
})
terraform-validate:
// 2. Prepare format of the comment
const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
#### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
#### Terraform Lint 📖\`${{ steps.lint.outcome }}\`
#### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
<details><summary>Validation Output</summary>
\`\`\`\n
${{ steps.validate.outputs.stdout }}
\`\`\`
</details>`;
// 3. If we have a comment, update it, otherwise create a new one
if (botComment) {
github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body: output
})
} else {
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: output
})
}
tfsec:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@master
- name: Terraform Init
uses: hashicorp/terraform-github-actions@master
- name: Checkout code
uses: actions/checkout@v4

- name: Terraform security scan
uses: aquasecurity/[email protected]
with:
tf_actions_version: latest
tf_actions_subcommand: init
tf_actions_comment: true
- name: Terraform Validate
uses: hashicorp/terraform-github-actions@master
env:
AWS_DEFAULT_REGION: eu-west-1
github_token: ${{ env.GITHUB_TOKEN }}
soft_fail: false

- name: Terraform pr commenter
uses: aquasecurity/[email protected]
with:
tf_actions_version: latest
tf_actions_subcommand: validate
tf_actions_comment: true
github_token: ${{ env.GITHUB_TOKEN }}
tfsec_args: --concise-output --force-all-dirs

terraform-docs:
checkov:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.ref }}
- name: Update module usage docs and push any changes back to PR branch
uses: Dirrk/[email protected]
uses: actions/checkout@v4

- name: Run Checkov
uses: bridgecrewio/[email protected]
with:
tf_docs_args: '--sort-inputs-by-required'
tf_docs_git_commit_message: 'terraform-docs: Update module usage'
tf_docs_git_push: 'true'
tf_docs_output_file: README.md
tf_docs_output_method: inject
tf_docs_find_dir: .
container_user: 1000
directory: "/"
download_external_modules: false
framework: terraform
output_format: sarif
quiet: true
skip_check: "CKV_TF_1,CKV_AWS_108,CKV_AWS_109,CKV_AWS_111,CKV_AWS_356"
soft_fail: false

tfsec:
name: tfsec
docs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v2
- name: Terraform security scan
uses: triat/[email protected]
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}

- name: Render terraform docs inside the README.md and push changes back to PR branch
uses: terraform-docs/[email protected]
with:
args: --sort-by required
git-commit-message: "docs(readme): update module usage"
git-push: true
output-file: README.md
output-method: inject
working-dir: .
continue-on-error: true # added this to prevent a PR from a remote fork failing the workflow
62 changes: 62 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,65 @@ Terraform module to create and manage a AWS Glue job
| trigger\_id | The Glue trigger name |

<!--- END_TF_DOCS --->

<!-- BEGIN_TF_DOCS -->
## Requirements

| Name | Version |
|------|---------|
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3 |
| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.62.0 |

## Providers

| Name | Version |
|------|---------|
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.62.0 |

## Modules

No modules.

## Resources

| Name | Type |
|------|------|
| [aws_glue_job.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_job) | resource |
| [aws_glue_trigger.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_trigger) | resource |
| [aws_iam_role.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy_attachment.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_policy_document.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |

## Inputs

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_name"></a> [name](#input\_name) | The name of the Glue job | `string` | n/a | yes |
| <a name="input_script_location"></a> [script\_location](#input\_script\_location) | The S3 path to the script that is executed by the job | `string` | n/a | yes |
| <a name="input_tags"></a> [tags](#input\_tags) | A mapping of tags to assign to all resources | `map(string)` | n/a | yes |
| <a name="input_command_name"></a> [command\_name](#input\_command\_name) | The name of the job command. Defaults to glueetl | `string` | `"glueetl"` | no |
| <a name="input_connections"></a> [connections](#input\_connections) | A list with connections for this job | `list(string)` | `[]` | no |
| <a name="input_default_arguments"></a> [default\_arguments](#input\_default\_arguments) | A map with default arguments for the job | `map(string)` | `{}` | no |
| <a name="input_glue_version"></a> [glue\_version](#input\_glue\_version) | The Glue version to use | `string` | `"4.0"` | no |
| <a name="input_max_capacity"></a> [max\_capacity](#input\_max\_capacity) | The maximum number of data processing units that can be allocated | `number` | `null` | no |
| <a name="input_max_retries"></a> [max\_retries](#input\_max\_retries) | The maximum number of times to retry the failing job | `number` | `0` | no |
| <a name="input_number_of_workers"></a> [number\_of\_workers](#input\_number\_of\_workers) | The number of workers that are allocated when the job runs | `string` | `null` | no |
| <a name="input_python_version"></a> [python\_version](#input\_python\_version) | The Python version (2 or 3) being used to execute a Python shell job | `string` | `"3"` | no |
| <a name="input_role_arn"></a> [role\_arn](#input\_role\_arn) | An optional Glue execution role | `string` | `null` | no |
| <a name="input_role_policy"></a> [role\_policy](#input\_role\_policy) | A valid Glue IAM policy JSON document | `string` | `null` | no |
| <a name="input_schedule"></a> [schedule](#input\_schedule) | A cron expression used to specify the schedule for the glue trigger | `string` | `null` | no |
| <a name="input_schedule_active"></a> [schedule\_active](#input\_schedule\_active) | Whether the glue trigger should be active | `bool` | `true` | no |
| <a name="input_security_configuration"></a> [security\_configuration](#input\_security\_configuration) | The name of the Security Configuration to be associated with the job | `string` | `null` | no |
| <a name="input_trigger_type"></a> [trigger\_type](#input\_trigger\_type) | The type ('ON\_DEMAND' or 'SCHEDULED') of the trigger | `string` | `null` | no |
| <a name="input_worker_type"></a> [worker\_type](#input\_worker\_type) | The type ('Standard' or 'G.1X' or 'G.2X') of predefined worker that is allocated when the job runs | `string` | `null` | no |

## Outputs

| Name | Description |
|------|-------------|
| <a name="output_arn"></a> [arn](#output\_arn) | The Glue job ARN |
| <a name="output_id"></a> [id](#output\_id) | The Glue job name |
| <a name="output_trigger_arn"></a> [trigger\_arn](#output\_trigger\_arn) | The Glue trigger ARN |
| <a name="output_trigger_id"></a> [trigger\_id](#output\_trigger\_id) | The Glue trigger name |
<!-- END_TF_DOCS -->
2 changes: 1 addition & 1 deletion examples/example.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module "example_glue_job" {
source = "github.com/komminar/terraform-aws-glue-job?ref=v0.1.0"
source = "github.com/komminarlab/terraform-aws-glue-job?ref=v1.0.0"
name = "example-glue-job"
max_retries = 1
number_of_workers = 2
Expand Down
29 changes: 29 additions & 0 deletions iam.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
data "aws_iam_policy_document" "default" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["glue.amazonaws.com"]
}
}
}

resource "aws_iam_role" "default" {
count = var.role_arn == null ? 1 : 0
name = "GlueExecutionRole-${var.name}"
assume_role_policy = data.aws_iam_policy_document.default.json
tags = var.tags
}

resource "aws_iam_role_policy" "default" {
count = var.role_arn == null && var.role_policy != null ? 1 : 0
name = "GlueExecutionRole-${var.name}"
role = aws_iam_role.default[0].id
policy = var.role_policy
}

resource "aws_iam_role_policy_attachment" "default" {
count = var.role_arn == null ? 1 : 0
role = aws_iam_role.default[0].id
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
}
53 changes: 11 additions & 42 deletions main.tf
Original file line number Diff line number Diff line change
@@ -1,46 +1,15 @@
data "aws_iam_policy_document" "default" {
statement {
actions = [
"sts:AssumeRole"
]
principals {
type = "Service"
identifiers = ["glue.amazonaws.com"]
}
}
}

resource "aws_iam_role" "default" {
count = var.role_arn == null ? 1 : 0
name = "GlueRole-${var.name}"
assume_role_policy = data.aws_iam_policy_document.default.json
tags = var.tags
}

resource "aws_iam_role_policy" "default" {
count = var.role_arn == null && var.role_policy != null ? 1 : 0
name = "GlueRole-${var.name}"
role = aws_iam_role.default[0].id
policy = var.role_policy
}

resource "aws_iam_role_policy_attachment" "default" {
count = var.role_arn == null ? 1 : 0
role = aws_iam_role.default[0].id
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
}

resource "aws_glue_job" "default" {
name = var.name
connections = var.connections
default_arguments = var.default_arguments
glue_version = var.glue_version
max_capacity = var.max_capacity
max_retries = var.max_retries
number_of_workers = var.number_of_workers
role_arn = var.role_arn != null ? var.role_arn : aws_iam_role.default[0].arn
worker_type = var.worker_type
tags = var.tags
name = var.name
connections = var.connections
default_arguments = var.default_arguments
glue_version = var.glue_version
max_capacity = var.max_capacity
max_retries = var.max_retries
number_of_workers = var.number_of_workers
role_arn = var.role_arn != null ? var.role_arn : aws_iam_role.default[0].arn
security_configuration = var.security_configuration
worker_type = var.worker_type
tags = var.tags

command {
name = var.command_name
Expand Down
8 changes: 7 additions & 1 deletion variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ variable "default_arguments" {

variable "glue_version" {
type = string
default = "2.0"
default = "4.0"
description = "The Glue version to use"
}

Expand Down Expand Up @@ -74,6 +74,12 @@ variable "schedule" {
description = "A cron expression used to specify the schedule for the glue trigger"
}

variable "security_configuration" {
type = string
default = null
description = "The name of the Security Configuration to be associated with the job"
}

variable "schedule_active" {
type = bool
default = true
Expand Down
7 changes: 5 additions & 2 deletions versions.tf
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
terraform {
required_version = ">= 0.13.0"
required_version = ">= 1.3"

required_providers {
aws = ">= 3.10"
aws = {
source = "hashicorp/aws"
version = ">= 4.62.0"
}
}
}

0 comments on commit e329f20

Please sign in to comment.