From 660aa9b1c5657c0e7d282a038ab1e32941a289ea Mon Sep 17 00:00:00 2001 From: kakakakakku Date: Mon, 6 Jul 2026 23:24:30 +0900 Subject: [PATCH 1/5] sfn-glue-sync-cdk: Update aws-cdk-lib --- sfn-glue-sync-cdk/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sfn-glue-sync-cdk/requirements.txt b/sfn-glue-sync-cdk/requirements.txt index 3a672bfa6f..81bf648456 100644 --- a/sfn-glue-sync-cdk/requirements.txt +++ b/sfn-glue-sync-cdk/requirements.txt @@ -1,2 +1,2 @@ -aws-cdk-lib==2.13.0 +aws-cdk-lib==2.260.0 constructs>=10.0.0,<11.0.0 From 0db99747988ef294dae9aa1b5fef7375ca634f0e Mon Sep 17 00:00:00 2001 From: kakakakakku Date: Mon, 6 Jul 2026 23:30:23 +0900 Subject: [PATCH 2/5] sfn-glue-sync-cdk: Replace deprecated definition with definitionBody and taskTimeout --- sfn-glue-sync-cdk/app.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sfn-glue-sync-cdk/app.py b/sfn-glue-sync-cdk/app.py index 951ec08774..9fe6985b87 100644 --- a/sfn-glue-sync-cdk/app.py +++ b/sfn-glue-sync-cdk/app.py @@ -20,15 +20,14 @@ class SfnGlueCdkStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) - # Glue job execution IAM Role + # Glue job execution IAM Role glue_job_role = iam.Role( self, 'Glue-Job-Role', assumed_by=iam.ServicePrincipal('glue.amazonaws.com'), managed_policies = [iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSGlueServiceRole')] ) - - + S3_BUCKET_NAME = "MyCdkGlueJobBucket" # S3 Bucket to host glue scripts @@ -36,7 +35,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: auto_delete_objects=True, block_public_access=s3.BlockPublicAccess.BLOCK_ALL) # asset to sync local scripts folder with S3 bucket - asset = s3deploy.Source.asset("./resources/glue-scripts") + asset = s3deploy.Source.asset("./resources/glue-scripts") # Sync local scripts with S3 bucket s3deploy.BucketDeployment(self, "DeployGlueJobScripts", @@ -66,15 +65,15 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: arguments=sfn.TaskInput.from_object({ "--message": sfn.JsonPath.string_at("$.message") }), - timeout=Duration.minutes(6), + task_timeout=sfn.Timeout.duration(Duration.minutes(6)), notify_delay_after= Duration.minutes(6) ) - # State Function defination + # State Function definition definition = glue_task state_machine = sfn.StateMachine( self, "GlueJobStateMachine", - definition=definition, + definition_body=sfn.DefinitionBody.from_chainable(definition), timeout=Duration.minutes(10) ) From 928c76cea1424bb27a1faf270aedc1d02b688be2 Mon Sep 17 00:00:00 2001 From: kakakakakku Date: Mon, 6 Jul 2026 23:36:56 +0900 Subject: [PATCH 3/5] sfn-glue-sync-cdk: Replace deprecated Python shell 3.6 with 3.9 --- sfn-glue-sync-cdk/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sfn-glue-sync-cdk/app.py b/sfn-glue-sync-cdk/app.py index 9fe6985b87..9dcb685498 100644 --- a/sfn-glue-sync-cdk/app.py +++ b/sfn-glue-sync-cdk/app.py @@ -54,7 +54,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: role=glue_job_role.role_arn, command=glue.CfnJob.JobCommandProperty( name='pythonshell', - python_version='3', + python_version='3.9', script_location=scriptLocation )) From d77187510d97dee1547261a3b0030d39f5aa8be6 Mon Sep 17 00:00:00 2001 From: kakakakakku Date: Mon, 6 Jul 2026 23:42:14 +0900 Subject: [PATCH 4/5] sfn-glue-sync-cdk: Add note about Python shell version in README --- sfn-glue-sync-cdk/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sfn-glue-sync-cdk/README.md b/sfn-glue-sync-cdk/README.md index 4df267d7a2..74c086b1c2 100644 --- a/sfn-glue-sync-cdk/README.md +++ b/sfn-glue-sync-cdk/README.md @@ -59,6 +59,9 @@ You can call the `StartJobRun` API from a Task state with Task `GlueStartJobRun` defined in the `app.py` triggers a glue job and wait for job completion before transitioning to next step. Python-shell glue job is defined with `glue.CfnJob` in `app.py` which triggers a python script `hello.py` + +> [!NOTE] +> This pattern uses a Python shell job (Python 3.9), which is the latest Python version available for Python shell jobs. If you need a newer Python version, consider other options described in [Migrate from AWS Glue Python shell jobs](https://docs.aws.amazon.com/glue/latest/dg/pyshell-migration.html). ## Steps for Testing @@ -210,4 +213,4 @@ aws stepfunctions describe-execution --execution-arn "{executionArn}" ---- Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -SPDX-License-Identifier: MIT-0 \ No newline at end of file +SPDX-License-Identifier: MIT-0 From 3f8289837d7361e3ac24c0790f460cd4b0e9f227 Mon Sep 17 00:00:00 2001 From: kakakakakku Date: Tue, 7 Jul 2026 00:04:25 +0900 Subject: [PATCH 5/5] sfn-glue-sync-cdk: Update README.md --- sfn-glue-sync-cdk/README.md | 74 ++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/sfn-glue-sync-cdk/README.md b/sfn-glue-sync-cdk/README.md index 74c086b1c2..0ddecbee55 100644 --- a/sfn-glue-sync-cdk/README.md +++ b/sfn-glue-sync-cdk/README.md @@ -1,8 +1,6 @@ # AWS Step Functions workflow to integrate with AWS Glue Job using CDK. -This CDK application deploys a Step Functions workflow, that takes in a payload and trigger a AWS Glue job synchronously. In this pattern, the state machine does wait for Glue job to finish. The application contains the minimum IAM resources required to run the workflow and Glue job. - - +This CDK application deploys a Step Functions workflow, that takes in a payload and trigger a AWS Glue job synchronously. In this pattern, the state machine does wait for Glue job to finish. The application contains the minimum IAM resources required to run the workflow and Glue job. Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/sfn-glue-sync-cdk @@ -13,7 +11,7 @@ Important: This application uses various AWS services and there are costs associ * [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. * [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured * [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -* [AWS Cloud Development Kit](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) (AWS CDK >= 1.124.0) Installed +* [AWS Cloud Development Kit](https://docs.aws.amazon.com/cdk/v2/guide/getting-started.html) installed ## Deployment Instructions @@ -52,7 +50,6 @@ Important: This application uses various AWS services and there are costs associ The CDK stack deploys the resources and the IAM permissions required to run the application. This stack will deploy S3 bucket to store glue job scripts, python-shell glue job and state function which executes 'glue:startJobRun.sync' step. - Step Functions supports AWS Glue through the service integration pattern. You can call the `StartJobRun` API from a Task state with [Run a Job (.sync)](https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-sync) integration pattern. @@ -62,7 +59,7 @@ Python-shell glue job is defined with `glue.CfnJob` in `app.py` which triggers a > [!NOTE] > This pattern uses a Python shell job (Python 3.9), which is the latest Python version available for Python shell jobs. If you need a newer Python version, consider other options described in [Migrate from AWS Glue Python shell jobs](https://docs.aws.amazon.com/glue/latest/dg/pyshell-migration.html). - + ## Steps for Testing * Start the Step Function execution with the sample event payload. @@ -82,30 +79,33 @@ aws stepfunctions start-execution --state-machine-arn "{StateMachineArn}" --inpu ```json { - "executionArn": "arn:aws:states:us-east-1:<*AccountNumber*>:execution:GlueJobStateMachineE759615B-t1nYjCro8h5F:36bd81e5-787a-4e34-ac6a-1f21064802ff", - "startDate": "2021-11-17T22:37:02.301000-06:00" + "executionArn": "arn:aws:states:us-east-1:000000000000:execution:GlueJobStateMachineE759615B-at44lw4ZXLFV:132bb420-2b55-4fa4-9085-63a0e226e2d7", + "startDate": "2026-07-06T23:55:20.379000+09:00" } ``` Note the `executionArn` from the above output and run the following CLI command to get the status of the execution. ```bash -aws stepfunctions describe-execution --execution-arn "{executionArn}" +aws stepfunctions describe-execution --execution-arn "{executionArn}" ``` ### Get execution status output: ```json { - "executionArn": "arn:aws:states:us-east-1:<*AccountNumber*>:execution:GlueJobStateMachineE759615B-t1nYjCro8h5F:36bd81e5-787a-4e34-ac6a-1f21064802ff", - "stateMachineArn": "arn:aws:states:us-east-1:<*AccountNumber*>:stateMachine:GlueJobStateMachineE759615B-t1nYjCro8h5F", - "name": "36bd81e5-787a-4e34-ac6a-1f21064802ff", + "executionArn": "arn:aws:states:us-east-1:000000000000:execution:GlueJobStateMachineE759615B-at44lw4ZXLFV:132bb420-2b55-4fa4-9085-63a0e226e2d7", + "stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:GlueJobStateMachineE759615B-at44lw4ZXLFV", + "name": "132bb420-2b55-4fa4-9085-63a0e226e2d7", "status": "RUNNING", - "startDate": "2021-11-17T22:37:02.301000-06:00", + "startDate": "2026-07-06T23:55:20.379000+09:00", "input": "{\"message\": \"Hello from sfn glue step\"}", "inputDetails": { "included": true - } + }, + "redriveCount": 0, + "redriveStatus": "NOT_REDRIVABLE", + "redriveStatusReason": "Execution is RUNNING and cannot be redriven" } ``` @@ -121,24 +121,28 @@ aws glue get-job-runs --job-name "cdk-test-glue-python-job" { "JobRuns": [ { - "Id": "jr_c14390561406a75616f270734b658e17f7175895f3fc3c289279b708840e5584", + "Id": "jr_3d888acaf38939653452dcfe972966c87b6b325c6d44d91f629406d03c79c3f4", "Attempt": 0, "JobName": "cdk-test-glue-python-job", - "StartedOn": "2021-11-17T22:37:02.488000-06:00", - "LastModifiedOn": "2021-11-17T22:37:06.008000-06:00", - "JobRunState": "RUNNING", + "JobMode": "SCRIPT", + "JobRunQueuingEnabled": false, + "StartedOn": "2026-07-06T23:55:20.582000+09:00", + "LastModifiedOn": "2026-07-06T23:56:38.460000+09:00", + "CompletedOn": "2026-07-06T23:56:38.460000+09:00", + "JobRunState": "SUCCEEDED", "Arguments": { "--message": "Hello from sfn glue step" }, "PredecessorRuns": [], "AllocatedCapacity": 0, - "ExecutionTime": 45, + "ExecutionTime": 71, "Timeout": 6, "MaxCapacity": 0.0625, "LogGroupName": "/aws-glue/python-jobs", "NotificationProperty": { "NotifyDelayAfter": 6 - } + }, + "GlueVersion": "5.1" } ] } @@ -154,44 +158,46 @@ aws logs tail "/aws-glue/python-jobs/output" ### Log output: ```log -2021-11-18T04:40:18.556000+00:00 jr_c14390561406a75616f270734b658e17f7175895f3fc3c289279b708840e5584 python-shell glue job message - Hello from sfn glue step - +2026-07-06T14:56:28.170000+00:00 jr_3d888acaf38939653452dcfe972966c87b6b325c6d44d91f629406d03c79c3f4 python-shell glue job message - Hello from sfn glue step ``` -Re-run the following CLI command to get the latest status of the staefunction execution and confirm the `status` is `SUCCEEDED`. +Re-run the following CLI command to get the latest status of the staefunction execution and confirm the `status` is `SUCCEEDED`. :information_source: It might take some time for the statefunction execution to complete after glue job is completed. ```bash -aws stepfunctions describe-execution --execution-arn "{executionArn}" +aws stepfunctions describe-execution --execution-arn "{executionArn}" ``` ### Get execution status output: ```bash { - "executionArn": "arn:aws:states:us-east-1:<*AccountNumber*>:execution:GlueJobStateMachineE759615B-t1nYjCro8h5F:36bd81e5-787a-4e34-ac6a-1f21064802ff", - "stateMachineArn": "arn:aws:states:us-east-1:<*AccountNumber*>:stateMachine:GlueJobStateMachineE759615B-t1nYjCro8h5F", - "name": "36bd81e5-787a-4e34-ac6a-1f21064802ff", + "executionArn": "arn:aws:states:us-east-1:000000000000:execution:GlueJobStateMachineE759615B-at44lw4ZXLFV:132bb420-2b55-4fa4-9085-63a0e226e2d7", + "stateMachineArn": "arn:aws:states:us-east-1:000000000000:stateMachine:GlueJobStateMachineE759615B-at44lw4ZXLFV", + "name": "132bb420-2b55-4fa4-9085-63a0e226e2d7", "status": "SUCCEEDED", - "startDate": "2021-11-17T22:37:02.301000-06:00", - "stopDate": "2021-11-17T22:41:08.340000-06:00", + "startDate": "2026-07-06T23:55:20.379000+09:00", + "stopDate": "2026-07-06T23:57:24.983000+09:00", "input": "{\"message\": \"Hello from sfn glue step\"}", "inputDetails": { "included": true }, - "output": "{\"AllocatedCapacity\":0,\"Arguments\":{\"--message\":\"Hello from sfn glue step\"},\"Attempt\":0,\"CompletedOn\":1637210428964,\"ExecutionTime\":197,\"Id\":\"jr_c14390561406a75616f270734b658e17f7175895f3fc3c289279b708840e5584\",\"JobName\":\"cdk-test-glue-python-job\",\"JobRunState\":\"SUCCEEDED\",\"LastModifiedOn\":1637210428964,\"LogGroupName\":\"/aws-glue/python-jobs\",\"MaxCapacity\":0.0625,\"NotificationProperty\":{\"NotifyDelayAfter\":6},\"PredecessorRuns\":[],\"StartedOn\":1637210222488,\"Timeout\":6}", + "output": "{\"AllocatedCapacity\":0,\"Arguments\":{\"--message\":\"Hello from sfn glue step\"},\"Attempt\":0,\"CompletedOn\":1783349798460,\"ExecutionTime\":71,\"GlueVersion\":\"5.1\",\"Id\":\"jr_3d888acaf38939653452dcfe972966c87b6b325c6d44d91f629406d03c79c3f4\",\"JobMode\":\"SCRIPT\",\"JobName\":\"cdk-test-glue-python-job\",\"JobRunState\":\"SUCCEEDED\",\"LastModifiedOn\":1783349798460,\"LogGroupName\":\"/aws-glue/python-jobs\",\"MaxCapacity\":0.0625,\"NotificationProperty\":{\"NotifyDelayAfter\":6},\"PredecessorRuns\":[],\"StartedOn\":1783349720582,\"Timeout\":6}", "outputDetails": { "included": true - } + }, + "redriveCount": 0, + "redriveStatus": "NOT_REDRIVABLE", + "redriveStatusReason": "Execution is SUCCEEDED and cannot be redriven" } ``` ## Cleanup - -1. Run the given command to delete the resources that were created. It might take some time for the CloudFormation stack to get deleted. + +1. Run the given command to delete the resources that were created. It might take some time for the CloudFormation stack to get deleted. - :warning: This will delete S3 bucket defind as part of this CDK stack(S3 bucket name is provided in the CDK stack outputs)! + :warning: This will delete S3 bucket defined as part of this CDK stack(S3 bucket name is provided in the CDK stack outputs)! ```bash cdk destroy