diff --git a/sfn-glue-terraform/README.md b/sfn-glue-terraform/README.md index dbbe486eef..79a2c95051 100644 --- a/sfn-glue-terraform/README.md +++ b/sfn-glue-terraform/README.md @@ -1,6 +1,6 @@ -# AWS Step Functions to start a AWS Glue Job Through a Cloudwatch event rule +# AWS Step Functions to start a AWS Glue Job Through a CloudWatch event rule -The Terraform template deploys a AWS Step Function, a AWS Glue Job, a Cloudwatch Event Rule, a Amazon S3 bucket and the minimum IAM resources required to run the application. +The Terraform template deploys a AWS Step Functions state machine, a AWS Glue Job, a CloudWatch Event Rule, a Amazon S3 bucket and the minimum IAM resources required to run the application. ## Architecture ![Alt](./resources/architecture.png) @@ -8,13 +8,13 @@ The Terraform template deploys a AWS Step Function, a AWS Glue Job, a Cloudwatch This pattern demonstrates the use of Terraform modules and deploys the below resources: * Amazon S3 bucket and load the sample Python script as an object * Sample AWS Glue Job which executes the script in the S3 bucket -* AWS Step Function to invoke the AWS Glue Job synchronously. The Function will wait until the Job is completed -* Cloudwatch Event Rule which is configured to start the AWS Step Function evey 10 minutes +* AWS Step Functions to invoke the AWS Glue Job synchronously. The Function will wait until the Job is completed +* CloudWatch Event Rule which is configured to start the AWS Step Functions evey 10 minutes ## How it works -The AWS Cloudwatch rule is configured to start a Step Function execution every 10 minutes. The Step function then invokes a AWS Glue Job with some default arguments and a test message. +The AWS CloudWatch rule is configured to start a Step Functions execution every 10 minutes. The Step function then invokes a AWS Glue Job with some default arguments and a test message. The Arguments to the AWS Glue Job, the Python script and the CloudWatch event rule can be modified as per requirement. @@ -54,7 +54,17 @@ Important: this application uses various AWS services and there are costs associ ## Testing -After deployment, go to the cloudwatch logs to check the event details. +After deployment, the Amazon EventBridge rule automatically triggers the AWS Step Functions state machine every 10 minutes. Wait for at least 10 minutes, then verify that the execution completed successfully. + +1. Verify the AWS Step Functions state machine execution `status` is `SUCCEEDED`. + ```sh + aws stepfunctions list-executions --state-machine-arn $(terraform output -raw aws_step_function_arn) + ``` +1. Verify the AWS Glue job run `JobRunState` is `SUCCEEDED`. + ```sh + aws glue get-job-runs --job-name sample-glue-job-terraform + ``` + ## Cleanup 1. Change directory to the pattern directory: @@ -63,7 +73,7 @@ After deployment, go to the cloudwatch logs to check the event details. ``` 1. Delete all created resources ```bash - terraform apply -destroy + terraform apply --destroy ``` 1. During the prompts: * Enter yes @@ -71,7 +81,7 @@ After deployment, go to the cloudwatch logs to check the event details. ```bash terraform show ``` - ``` + ---- Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/sfn-glue-terraform/main.tf b/sfn-glue-terraform/main.tf index 072fb8ee6a..6cc7327a18 100644 --- a/sfn-glue-terraform/main.tf +++ b/sfn-glue-terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 3.27" + version = "~> 6.0" } } @@ -55,7 +55,7 @@ output "aws_cloudwatch_event_rule_arn" { } output "aws_step_function_arn" { - value = module.aws_sfn_state_machine.stf_role_arn + value = module.aws_sfn_state_machine.stf_arn } output "aws_glue_job_arn" { diff --git a/sfn-glue-terraform/modules/terraform-amazon-s3/main.tf b/sfn-glue-terraform/modules/terraform-amazon-s3/main.tf index ba38567ba6..a11c195803 100644 --- a/sfn-glue-terraform/modules/terraform-amazon-s3/main.tf +++ b/sfn-glue-terraform/modules/terraform-amazon-s3/main.tf @@ -12,18 +12,18 @@ resource "aws_s3_bucket" "s3_sample_glue_bucket" { resource "aws_s3_bucket_public_access_block" "s3_glue_bucket_block_public_access" { bucket = aws_s3_bucket.s3_sample_glue_bucket.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true + block_public_acls = true + block_public_policy = true + ignore_public_acls = true restrict_public_buckets = true } # upload the AWS Glue script to the bucket -resource "aws_s3_bucket_object" "glue_script_object" { +resource "aws_s3_object" "glue_script_object" { bucket = aws_s3_bucket.s3_sample_glue_bucket.bucket key = "glue_script.py" source = "./resources/glue_script.py" - acl = "private" + acl = "private" } ## IAM Resources @@ -39,5 +39,5 @@ output "bucket_arn" { } output "glue_script_name" { - value = aws_s3_bucket_object.glue_script_object.key + value = aws_s3_object.glue_script_object.key } \ No newline at end of file diff --git a/sfn-glue-terraform/modules/terraform-aws-cloudwatch/main.tf b/sfn-glue-terraform/modules/terraform-aws-cloudwatch/main.tf index f87aad99c9..eb7bdbb21d 100644 --- a/sfn-glue-terraform/modules/terraform-aws-cloudwatch/main.tf +++ b/sfn-glue-terraform/modules/terraform-aws-cloudwatch/main.tf @@ -1,5 +1,5 @@ -locals{ - schedule_expression= "rate(10 minutes)" +locals { + schedule_expression = "rate(10 minutes)" } # Variables @@ -30,8 +30,8 @@ resource "aws_iam_role" "allow_cloudwatch_to_execute_role" { } resource "aws_iam_role_policy" "state_execution" { - name = "state_execution_policy" - role = aws_iam_role.allow_cloudwatch_to_execute_role.id + name = "state_execution_policy" + role = aws_iam_role.allow_cloudwatch_to_execute_role.id policy = <