Job class
Provides a Glue Job resource.
> Glue functionality, such as monitoring and logging of jobs, is typically managed with the defaultArguments argument. See the Special Parameters Used by AWS Glue topic in the Glue developer guide for additional information.
Example Usage
Python Glue Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
// IAM role for Glue jobs
const glueJobRole = new aws.iam.Role("glue_job_role", {
name: "glue-job-role",
assumeRolePolicy: JSON.stringify({
Version: "2012-10-17",
Statement: [{
Action: "sts:AssumeRole",
Effect: "Allow",
Principal: {
Service: "glue.amazonaws.com",
},
}],
}),
});
const etlJob = new aws.glue.Job("etl_job", {
command: {
scriptLocation: `s3://${glueScripts.bucket}/jobs/etl_job.py`,
name: "glueetl",
pythonVersion: "3",
},
notificationProperty: {
notifyDelayAfter: 3,
},
executionProperty: {
maxConcurrentRuns: 1,
},
name: "example-etl-job",
description: "An example Glue ETL job",
roleArn: glueJobRole.arn,
glueVersion: "5.0",
maxRetries: 0,
timeout: 2880,
numberOfWorkers: 2,
workerType: "G.1X",
connections: [example.name],
executionClass: "STANDARD",
defaultArguments: {
"--job-language": "python",
"--continuous-log-logGroup": "/aws-glue/jobs",
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
"--enable-auto-scaling": "true",
},
tags: {
ManagedBy: "AWS",
},
});
const glueEtlScript = new aws.s3.BucketObjectv2("glue_etl_script", {
bucket: glueScripts.id,
key: "jobs/etl_job.py",
source: new pulumi.asset.FileAsset("jobs/etl_job.py"),
});
import pulumi
import json
import pulumi_aws as aws
# IAM role for Glue jobs
glue_job_role = aws.iam.Role("glue_job_role",
name="glue-job-role",
assume_role_policy=json.dumps({
"Version": "2012-10-17",
"Statement": [{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "glue.amazonaws.com",
},
}],
}))
etl_job = aws.glue.Job("etl_job",
command={
"script_location": f"s3://{glue_scripts['bucket']}/jobs/etl_job.py",
"name": "glueetl",
"python_version": "3",
},
notification_property={
"notify_delay_after": 3,
},
execution_property={
"max_concurrent_runs": 1,
},
name="example-etl-job",
description="An example Glue ETL job",
role_arn=glue_job_role.arn,
glue_version="5.0",
max_retries=0,
timeout=2880,
number_of_workers=2,
worker_type="G.1X",
connections=[example["name"]],
execution_class="STANDARD",
default_arguments={
"--job-language": "python",
"--continuous-log-logGroup": "/aws-glue/jobs",
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
"--enable-auto-scaling": "true",
},
tags={
"ManagedBy": "AWS",
})
glue_etl_script = aws.s3.BucketObjectv2("glue_etl_script",
bucket=glue_scripts["id"],
key="jobs/etl_job.py",
source=pulumi.FileAsset("jobs/etl_job.py"))
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
// IAM role for Glue jobs
var glueJobRole = new Aws.Iam.Role("glue_job_role", new()
{
Name = "glue-job-role",
AssumeRolePolicy = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["Version"] = "2012-10-17",
["Statement"] = new[]
{
new Dictionary<string, object?>
{
["Action"] = "sts:AssumeRole",
["Effect"] = "Allow",
["Principal"] = new Dictionary<string, object?>
{
["Service"] = "glue.amazonaws.com",
},
},
},
}),
});
var etlJob = new Aws.Glue.Job("etl_job", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = $"s3://{glueScripts.Bucket}/jobs/etl_job.py",
Name = "glueetl",
PythonVersion = "3",
},
NotificationProperty = new Aws.Glue.Inputs.JobNotificationPropertyArgs
{
NotifyDelayAfter = 3,
},
ExecutionProperty = new Aws.Glue.Inputs.JobExecutionPropertyArgs
{
MaxConcurrentRuns = 1,
},
Name = "example-etl-job",
Description = "An example Glue ETL job",
RoleArn = glueJobRole.Arn,
GlueVersion = "5.0",
MaxRetries = 0,
Timeout = 2880,
NumberOfWorkers = 2,
WorkerType = "G.1X",
Connections = new[]
{
example.Name,
},
ExecutionClass = "STANDARD",
DefaultArguments =
{
{ "--job-language", "python" },
{ "--continuous-log-logGroup", "/aws-glue/jobs" },
{ "--enable-continuous-cloudwatch-log", "true" },
{ "--enable-continuous-log-filter", "true" },
{ "--enable-metrics", "" },
{ "--enable-auto-scaling", "true" },
},
Tags =
{
{ "ManagedBy", "AWS" },
},
});
var glueEtlScript = new Aws.S3.BucketObjectv2("glue_etl_script", new()
{
Bucket = glueScripts.Id,
Key = "jobs/etl_job.py",
Source = new FileAsset("jobs/etl_job.py"),
});
});
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/s3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
tmpJSON0, err := json.Marshal(map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
map[string]interface{}{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": map[string]string{
"Service": "glue.amazonaws.com",
},
},
},
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
// IAM role for Glue jobs
glueJobRole, err := iam.NewRole(ctx, "glue_job_role", &iam.RoleArgs{
Name: pulumi.String("glue-job-role"),
AssumeRolePolicy: pulumi.String(json0),
})
if err != nil {
return err
}
_, err = glue.NewJob(ctx, "etl_job", &glue.JobArgs{
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.Sprintf("s3://%v/jobs/etl_job.py", glueScripts.Bucket),
Name: pulumi.String("glueetl"),
PythonVersion: pulumi.String("3"),
},
NotificationProperty: &glue.JobNotificationPropertyArgs{
NotifyDelayAfter: pulumi.Int(3),
},
ExecutionProperty: &glue.JobExecutionPropertyArgs{
MaxConcurrentRuns: pulumi.Int(1),
},
Name: pulumi.String("example-etl-job"),
Description: pulumi.String("An example Glue ETL job"),
RoleArn: glueJobRole.Arn,
GlueVersion: pulumi.String("5.0"),
MaxRetries: pulumi.Int(0),
Timeout: pulumi.Int(2880),
NumberOfWorkers: pulumi.Int(2),
WorkerType: pulumi.String("G.1X"),
Connections: pulumi.StringArray{
example.Name,
},
ExecutionClass: pulumi.String("STANDARD"),
DefaultArguments: pulumi.StringMap{
"--job-language": pulumi.String("python"),
"--continuous-log-logGroup": pulumi.String("/aws-glue/jobs"),
"--enable-continuous-cloudwatch-log": pulumi.String("true"),
"--enable-continuous-log-filter": pulumi.String("true"),
"--enable-metrics": pulumi.String(""),
"--enable-auto-scaling": pulumi.String("true"),
},
Tags: pulumi.StringMap{
"ManagedBy": pulumi.String("AWS"),
},
})
if err != nil {
return err
}
_, err = s3.NewBucketObjectv2(ctx, "glue_etl_script", &s3.BucketObjectv2Args{
Bucket: pulumi.Any(glueScripts.Id),
Key: pulumi.String("jobs/etl_job.py"),
Source: pulumi.NewFileAsset("jobs/etl_job.py"),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_job" "etl_job" {
command = {
script_location ="s3://${glueScripts.bucket}/jobs/etl_job.py"
name = "glueetl"
python_version = "3"
}
notification_property = {
notify_delay_after = 3
}
# delay in minutes
execution_property = {
max_concurrent_runs = 1
}
name = "example-etl-job"
description = "An example Glue ETL job"
role_arn = aws_iam_role.glue_job_role.arn
glue_version = "5.0"
max_retries = 0
timeout = 2880
number_of_workers = 2
worker_type = "G.1X"
connections = [example.name]
execution_class = "STANDARD"
default_arguments = {
"--job-language" = "python"
"--continuous-log-logGroup" = "/aws-glue/jobs"
"--enable-continuous-cloudwatch-log" = "true"
"--enable-continuous-log-filter" = "true"
"--enable-metrics" = ""
"--enable-auto-scaling" = "true"
}
tags = {
"ManagedBy" = "AWS"
}
}
# IAM role for Glue jobs
resource "aws_iam_role" "glue_job_role" {
name = "glue-job-role"
assume_role_policy = jsonencode({
"Version" = "2012-10-17"
"Statement" = [{
"Action" = "sts:AssumeRole"
"Effect" = "Allow"
"Principal" = {
"Service" = "glue.amazonaws.com"
}
}]
})
}
resource "aws_s3_bucketobjectv2" "glue_etl_script" {
bucket = glueScripts.id
key = "jobs/etl_job.py"
source = fileAsset("jobs/etl_job.py")
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.iam.Role;
import com.pulumi.aws.iam.RoleArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import com.pulumi.aws.glue.inputs.JobNotificationPropertyArgs;
import com.pulumi.aws.glue.inputs.JobExecutionPropertyArgs;
import com.pulumi.aws.s3.BucketObjectv2;
import com.pulumi.aws.s3.BucketObjectv2Args;
import com.pulumi.asset.FileAsset;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
// IAM role for Glue jobs
var glueJobRole = new Role("glueJobRole", RoleArgs.builder()
.name("glue-job-role")
.assumeRolePolicy(serializeJson(
jsonObject(
jsonProperty("Version", "2012-10-17"),
jsonProperty("Statement", jsonArray(jsonObject(
jsonProperty("Action", "sts:AssumeRole"),
jsonProperty("Effect", "Allow"),
jsonProperty("Principal", jsonObject(
jsonProperty("Service", "glue.amazonaws.com")
))
)))
)))
.build());
var etlJob = new Job("etlJob", JobArgs.builder()
.command(JobCommandArgs.builder()
.scriptLocation(String.format("s3://%s/jobs/etl_job.py", glueScripts.bucket()))
.name("glueetl")
.pythonVersion("3")
.build())
.notificationProperty(JobNotificationPropertyArgs.builder()
.notifyDelayAfter(3)
.build())
.executionProperty(JobExecutionPropertyArgs.builder()
.maxConcurrentRuns(1)
.build())
.name("example-etl-job")
.description("An example Glue ETL job")
.roleArn(glueJobRole.arn())
.glueVersion("5.0")
.maxRetries(0)
.timeout(2880)
.numberOfWorkers(2)
.workerType("G.1X")
.connections(example.name())
.executionClass("STANDARD")
.defaultArguments(Map.ofEntries(
Map.entry("--job-language", "python"),
Map.entry("--continuous-log-logGroup", "/aws-glue/jobs"),
Map.entry("--enable-continuous-cloudwatch-log", "true"),
Map.entry("--enable-continuous-log-filter", "true"),
Map.entry("--enable-metrics", ""),
Map.entry("--enable-auto-scaling", "true")
))
.tags(Map.of("ManagedBy", "AWS"))
.build());
var glueEtlScript = new BucketObjectv2("glueEtlScript", BucketObjectv2Args.builder()
.bucket(glueScripts.id())
.key("jobs/etl_job.py")
.source(new FileAsset("jobs/etl_job.py"))
.build());
}
}
resources:
etlJob:
type: aws:glue:Job
name: etl_job
properties:
command:
scriptLocation: s3://${glueScripts.bucket}/jobs/etl_job.py
name: glueetl
pythonVersion: '3'
notificationProperty:
notifyDelayAfter: 3
executionProperty:
maxConcurrentRuns: 1
name: example-etl-job
description: An example Glue ETL job
roleArn: ${glueJobRole.arn}
glueVersion: '5.0'
maxRetries: 0
timeout: 2880
numberOfWorkers: 2
workerType: G.1X
connections:
- ${example.name}
executionClass: STANDARD
defaultArguments:
--job-language: python
--continuous-log-logGroup: /aws-glue/jobs
--enable-continuous-cloudwatch-log: 'true'
--enable-continuous-log-filter: 'true'
--enable-metrics: ""
--enable-auto-scaling: 'true'
tags:
ManagedBy: AWS
# IAM role for Glue jobs
glueJobRole:
type: aws:iam:Role
name: glue_job_role
properties:
name: glue-job-role
assumeRolePolicy:
fn::toJSON:
Version: 2012-10-17
Statement:
- Action: sts:AssumeRole
Effect: Allow
Principal:
Service: glue.amazonaws.com
glueEtlScript:
type: aws:s3:BucketObjectv2
name: glue_etl_script
properties:
bucket: ${glueScripts.id}
key: jobs/etl_job.py
source:
fn::fileAsset: jobs/etl_job.py
Pythonshell Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
// IAM role for Glue jobs
const glueJobRole = new aws.iam.Role("glue_job_role", {
name: "glue-job-role",
assumeRolePolicy: JSON.stringify({
Version: "2012-10-17",
Statement: [{
Action: "sts:AssumeRole",
Effect: "Allow",
Principal: {
Service: "glue.amazonaws.com",
},
}],
}),
});
const pythonShellJob = new aws.glue.Job("python_shell_job", {
command: {
scriptLocation: `s3://${glueScripts.bucket}/jobs/shell_job.py`,
name: "pythonshell",
pythonVersion: "3.9",
},
executionProperty: {
maxConcurrentRuns: 1,
},
name: "example-python-shell-job",
description: "An example Python shell job",
roleArn: glueJobRole.arn,
maxCapacity: 0.0625,
maxRetries: 0,
timeout: 2880,
connections: [example.name],
defaultArguments: {
"--job-language": "python",
"--continuous-log-logGroup": "/aws-glue/jobs",
"--enable-continuous-cloudwatch-log": "true",
"library-set": "analytics",
},
tags: {
ManagedBy: "AWS",
},
});
const pythonShellScript = new aws.s3.BucketObjectv2("python_shell_script", {
bucket: glueScripts.id,
key: "jobs/shell_job.py",
source: new pulumi.asset.FileAsset("jobs/shell_job.py"),
});
import pulumi
import json
import pulumi_aws as aws
# IAM role for Glue jobs
glue_job_role = aws.iam.Role("glue_job_role",
name="glue-job-role",
assume_role_policy=json.dumps({
"Version": "2012-10-17",
"Statement": [{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "glue.amazonaws.com",
},
}],
}))
python_shell_job = aws.glue.Job("python_shell_job",
command={
"script_location": f"s3://{glue_scripts['bucket']}/jobs/shell_job.py",
"name": "pythonshell",
"python_version": "3.9",
},
execution_property={
"max_concurrent_runs": 1,
},
name="example-python-shell-job",
description="An example Python shell job",
role_arn=glue_job_role.arn,
max_capacity=0.0625,
max_retries=0,
timeout=2880,
connections=[example["name"]],
default_arguments={
"--job-language": "python",
"--continuous-log-logGroup": "/aws-glue/jobs",
"--enable-continuous-cloudwatch-log": "true",
"library-set": "analytics",
},
tags={
"ManagedBy": "AWS",
})
python_shell_script = aws.s3.BucketObjectv2("python_shell_script",
bucket=glue_scripts["id"],
key="jobs/shell_job.py",
source=pulumi.FileAsset("jobs/shell_job.py"))
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
// IAM role for Glue jobs
var glueJobRole = new Aws.Iam.Role("glue_job_role", new()
{
Name = "glue-job-role",
AssumeRolePolicy = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["Version"] = "2012-10-17",
["Statement"] = new[]
{
new Dictionary<string, object?>
{
["Action"] = "sts:AssumeRole",
["Effect"] = "Allow",
["Principal"] = new Dictionary<string, object?>
{
["Service"] = "glue.amazonaws.com",
},
},
},
}),
});
var pythonShellJob = new Aws.Glue.Job("python_shell_job", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = $"s3://{glueScripts.Bucket}/jobs/shell_job.py",
Name = "pythonshell",
PythonVersion = "3.9",
},
ExecutionProperty = new Aws.Glue.Inputs.JobExecutionPropertyArgs
{
MaxConcurrentRuns = 1,
},
Name = "example-python-shell-job",
Description = "An example Python shell job",
RoleArn = glueJobRole.Arn,
MaxCapacity = 0.0625,
MaxRetries = 0,
Timeout = 2880,
Connections = new[]
{
example.Name,
},
DefaultArguments =
{
{ "--job-language", "python" },
{ "--continuous-log-logGroup", "/aws-glue/jobs" },
{ "--enable-continuous-cloudwatch-log", "true" },
{ "library-set", "analytics" },
},
Tags =
{
{ "ManagedBy", "AWS" },
},
});
var pythonShellScript = new Aws.S3.BucketObjectv2("python_shell_script", new()
{
Bucket = glueScripts.Id,
Key = "jobs/shell_job.py",
Source = new FileAsset("jobs/shell_job.py"),
});
});
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/iam"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/s3"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
tmpJSON0, err := json.Marshal(map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
map[string]interface{}{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": map[string]string{
"Service": "glue.amazonaws.com",
},
},
},
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
// IAM role for Glue jobs
glueJobRole, err := iam.NewRole(ctx, "glue_job_role", &iam.RoleArgs{
Name: pulumi.String("glue-job-role"),
AssumeRolePolicy: pulumi.String(json0),
})
if err != nil {
return err
}
_, err = glue.NewJob(ctx, "python_shell_job", &glue.JobArgs{
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.Sprintf("s3://%v/jobs/shell_job.py", glueScripts.Bucket),
Name: pulumi.String("pythonshell"),
PythonVersion: pulumi.String("3.9"),
},
ExecutionProperty: &glue.JobExecutionPropertyArgs{
MaxConcurrentRuns: pulumi.Int(1),
},
Name: pulumi.String("example-python-shell-job"),
Description: pulumi.String("An example Python shell job"),
RoleArn: glueJobRole.Arn,
MaxCapacity: pulumi.Float64(0.0625),
MaxRetries: pulumi.Int(0),
Timeout: pulumi.Int(2880),
Connections: pulumi.StringArray{
example.Name,
},
DefaultArguments: pulumi.StringMap{
"--job-language": pulumi.String("python"),
"--continuous-log-logGroup": pulumi.String("/aws-glue/jobs"),
"--enable-continuous-cloudwatch-log": pulumi.String("true"),
"library-set": pulumi.String("analytics"),
},
Tags: pulumi.StringMap{
"ManagedBy": pulumi.String("AWS"),
},
})
if err != nil {
return err
}
_, err = s3.NewBucketObjectv2(ctx, "python_shell_script", &s3.BucketObjectv2Args{
Bucket: pulumi.Any(glueScripts.Id),
Key: pulumi.String("jobs/shell_job.py"),
Source: pulumi.NewFileAsset("jobs/shell_job.py"),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_job" "python_shell_job" {
command = {
script_location ="s3://${glueScripts.bucket}/jobs/shell_job.py"
name = "pythonshell"
python_version = "3.9"
}
execution_property = {
max_concurrent_runs = 1
}
name = "example-python-shell-job"
description = "An example Python shell job"
role_arn = aws_iam_role.glue_job_role.arn
max_capacity = "0.0625"
max_retries = 0
timeout = 2880
connections = [example.name]
default_arguments = {
"--job-language" = "python"
"--continuous-log-logGroup" = "/aws-glue/jobs"
"--enable-continuous-cloudwatch-log" = "true"
"library-set" = "analytics"
}
tags = {
"ManagedBy" = "AWS"
}
}
# IAM role for Glue jobs
resource "aws_iam_role" "glue_job_role" {
name = "glue-job-role"
assume_role_policy = jsonencode({
"Version" = "2012-10-17"
"Statement" = [{
"Action" = "sts:AssumeRole"
"Effect" = "Allow"
"Principal" = {
"Service" = "glue.amazonaws.com"
}
}]
})
}
resource "aws_s3_bucketobjectv2" "python_shell_script" {
bucket = glueScripts.id
key = "jobs/shell_job.py"
source = fileAsset("jobs/shell_job.py")
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.iam.Role;
import com.pulumi.aws.iam.RoleArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import com.pulumi.aws.glue.inputs.JobExecutionPropertyArgs;
import com.pulumi.aws.s3.BucketObjectv2;
import com.pulumi.aws.s3.BucketObjectv2Args;
import com.pulumi.asset.FileAsset;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
// IAM role for Glue jobs
var glueJobRole = new Role("glueJobRole", RoleArgs.builder()
.name("glue-job-role")
.assumeRolePolicy(serializeJson(
jsonObject(
jsonProperty("Version", "2012-10-17"),
jsonProperty("Statement", jsonArray(jsonObject(
jsonProperty("Action", "sts:AssumeRole"),
jsonProperty("Effect", "Allow"),
jsonProperty("Principal", jsonObject(
jsonProperty("Service", "glue.amazonaws.com")
))
)))
)))
.build());
var pythonShellJob = new Job("pythonShellJob", JobArgs.builder()
.command(JobCommandArgs.builder()
.scriptLocation(String.format("s3://%s/jobs/shell_job.py", glueScripts.bucket()))
.name("pythonshell")
.pythonVersion("3.9")
.build())
.executionProperty(JobExecutionPropertyArgs.builder()
.maxConcurrentRuns(1)
.build())
.name("example-python-shell-job")
.description("An example Python shell job")
.roleArn(glueJobRole.arn())
.maxCapacity(0.0625)
.maxRetries(0)
.timeout(2880)
.connections(example.name())
.defaultArguments(Map.ofEntries(
Map.entry("--job-language", "python"),
Map.entry("--continuous-log-logGroup", "/aws-glue/jobs"),
Map.entry("--enable-continuous-cloudwatch-log", "true"),
Map.entry("library-set", "analytics")
))
.tags(Map.of("ManagedBy", "AWS"))
.build());
var pythonShellScript = new BucketObjectv2("pythonShellScript", BucketObjectv2Args.builder()
.bucket(glueScripts.id())
.key("jobs/shell_job.py")
.source(new FileAsset("jobs/shell_job.py"))
.build());
}
}
resources:
pythonShellJob:
type: aws:glue:Job
name: python_shell_job
properties:
command:
scriptLocation: s3://${glueScripts.bucket}/jobs/shell_job.py
name: pythonshell
pythonVersion: '3.9'
executionProperty:
maxConcurrentRuns: 1
name: example-python-shell-job
description: An example Python shell job
roleArn: ${glueJobRole.arn}
maxCapacity: '0.0625'
maxRetries: 0
timeout: 2880
connections:
- ${example.name}
defaultArguments:
--job-language: python
--continuous-log-logGroup: /aws-glue/jobs
--enable-continuous-cloudwatch-log: 'true'
library-set: analytics
tags:
ManagedBy: AWS
# IAM role for Glue jobs
glueJobRole:
type: aws:iam:Role
name: glue_job_role
properties:
name: glue-job-role
assumeRolePolicy:
fn::toJSON:
Version: 2012-10-17
Statement:
- Action: sts:AssumeRole
Effect: Allow
Principal:
Service: glue.amazonaws.com
pythonShellScript:
type: aws:s3:BucketObjectv2
name: python_shell_script
properties:
bucket: ${glueScripts.id}
key: jobs/shell_job.py
source:
fn::fileAsset: jobs/shell_job.py
Ray Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
command: {
name: "glueray",
pythonVersion: "3.9",
runtime: "Ray2.4",
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.py`,
},
name: "example",
roleArn: exampleAwsIamRole.arn,
glueVersion: "4.0",
workerType: "Z.2X",
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
command={
"name": "glueray",
"python_version": "3.9",
"runtime": "Ray2.4",
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.py",
},
name="example",
role_arn=example_aws_iam_role["arn"],
glue_version="4.0",
worker_type="Z.2X")
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
Name = "glueray",
PythonVersion = "3.9",
Runtime = "Ray2.4",
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.py",
},
Name = "example",
RoleArn = exampleAwsIamRole.Arn,
GlueVersion = "4.0",
WorkerType = "Z.2X",
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Command: &glue.JobCommandArgs{
Name: pulumi.String("glueray"),
PythonVersion: pulumi.String("3.9"),
Runtime: pulumi.String("Ray2.4"),
ScriptLocation: pulumi.Sprintf("s3://%v/example.py", exampleAwsS3Bucket.Bucket),
},
Name: pulumi.String("example"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
GlueVersion: pulumi.String("4.0"),
WorkerType: pulumi.String("Z.2X"),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_job" "example" {
command = {
name = "glueray"
python_version = "3.9"
runtime = "Ray2.4"
script_location ="s3://${exampleAwsS3Bucket.bucket}/example.py"
}
name = "example"
role_arn = exampleAwsIamRole.arn
glue_version = "4.0"
worker_type = "Z.2X"
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.command(JobCommandArgs.builder()
.name("glueray")
.pythonVersion("3.9")
.runtime("Ray2.4")
.scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
.build())
.name("example")
.roleArn(exampleAwsIamRole.arn())
.glueVersion("4.0")
.workerType("Z.2X")
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
command:
name: glueray
pythonVersion: '3.9'
runtime: Ray2.4
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.py
name: example
roleArn: ${exampleAwsIamRole.arn}
glueVersion: '4.0'
workerType: Z.2X
Scala Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
command: {
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.scala`,
},
name: "example",
roleArn: exampleAwsIamRole.arn,
defaultArguments: {
"--job-language": "scala",
},
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
command={
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.scala",
},
name="example",
role_arn=example_aws_iam_role["arn"],
default_arguments={
"--job-language": "scala",
})
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.scala",
},
Name = "example",
RoleArn = exampleAwsIamRole.Arn,
DefaultArguments =
{
{ "--job-language", "scala" },
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Command: &glue.JobCommandArgs{
ScriptLocation: pulumi.Sprintf("s3://%v/example.scala", exampleAwsS3Bucket.Bucket),
},
Name: pulumi.String("example"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
DefaultArguments: pulumi.StringMap{
"--job-language": pulumi.String("scala"),
},
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_job" "example" {
command = {
script_location ="s3://${exampleAwsS3Bucket.bucket}/example.scala"
}
name = "example"
role_arn = exampleAwsIamRole.arn
default_arguments = {
"--job-language" = "scala"
}
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.command(JobCommandArgs.builder()
.scriptLocation(String.format("s3://%s/example.scala", exampleAwsS3Bucket.bucket()))
.build())
.name("example")
.roleArn(exampleAwsIamRole.arn())
.defaultArguments(Map.of("--job-language", "scala"))
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
command:
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.scala
name: example
roleArn: ${exampleAwsIamRole.arn}
defaultArguments:
--job-language: scala
Streaming Job
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Job("example", {
command: {
name: "gluestreaming",
scriptLocation: `s3://${exampleAwsS3Bucket.bucket}/example.script`,
},
name: "example streaming job",
roleArn: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Job("example",
command={
"name": "gluestreaming",
"script_location": f"s3://{example_aws_s3_bucket['bucket']}/example.script",
},
name="example streaming job",
role_arn=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Job("example", new()
{
Command = new Aws.Glue.Inputs.JobCommandArgs
{
Name = "gluestreaming",
ScriptLocation = $"s3://{exampleAwsS3Bucket.Bucket}/example.script",
},
Name = "example streaming job",
RoleArn = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewJob(ctx, "example", &glue.JobArgs{
Command: &glue.JobCommandArgs{
Name: pulumi.String("gluestreaming"),
ScriptLocation: pulumi.Sprintf("s3://%v/example.script", exampleAwsS3Bucket.Bucket),
},
Name: pulumi.String("example streaming job"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_job" "example" {
command = {
name = "gluestreaming"
script_location ="s3://${exampleAwsS3Bucket.bucket}/example.script"
}
name = "example streaming job"
role_arn = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobCommandArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Job("example", JobArgs.builder()
.command(JobCommandArgs.builder()
.name("gluestreaming")
.scriptLocation(String.format("s3://%s/example.script", exampleAwsS3Bucket.bucket()))
.build())
.name("example streaming job")
.roleArn(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:glue:Job
properties:
command:
name: gluestreaming
scriptLocation: s3://${exampleAwsS3Bucket.bucket}/example.script
name: example streaming job
roleArn: ${exampleAwsIamRole.arn}
Enabling CloudWatch Logs and Metrics
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.cloudwatch.LogGroup("example", {
name: "example",
retentionInDays: 14,
});
const exampleJob = new aws.glue.Job("example", {defaultArguments: {
"--continuous-log-logGroup": example.name,
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
}});
import pulumi
import pulumi_aws as aws
example = aws.cloudwatch.LogGroup("example",
name="example",
retention_in_days=14)
example_job = aws.glue.Job("example", default_arguments={
"--continuous-log-logGroup": example.name,
"--enable-continuous-cloudwatch-log": "true",
"--enable-continuous-log-filter": "true",
"--enable-metrics": "",
})
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.CloudWatch.LogGroup("example", new()
{
Name = "example",
RetentionInDays = 14,
});
var exampleJob = new Aws.Glue.Job("example", new()
{
DefaultArguments =
{
{ "--continuous-log-logGroup", example.Name },
{ "--enable-continuous-cloudwatch-log", "true" },
{ "--enable-continuous-log-filter", "true" },
{ "--enable-metrics", "" },
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/cloudwatch"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := cloudwatch.NewLogGroup(ctx, "example", &cloudwatch.LogGroupArgs{
Name: pulumi.String("example"),
RetentionInDays: pulumi.Int(14),
})
if err != nil {
return err
}
_, err = glue.NewJob(ctx, "example", &glue.JobArgs{
DefaultArguments: pulumi.StringMap{
"--continuous-log-logGroup": example.Name,
"--enable-continuous-cloudwatch-log": pulumi.String("true"),
"--enable-continuous-log-filter": pulumi.String("true"),
"--enable-metrics": pulumi.String(""),
},
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_cloudwatch_loggroup" "example" {
name = "example"
retention_in_days = 14
}
resource "aws_glue_job" "example" {
default_arguments = {
"--continuous-log-logGroup" = aws_cloudwatch_loggroup.example.name
"--enable-continuous-cloudwatch-log" = "true"
"--enable-continuous-log-filter" = "true"
"--enable-metrics" = ""
}
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.cloudwatch.LogGroup;
import com.pulumi.aws.cloudwatch.LogGroupArgs;
import com.pulumi.aws.glue.Job;
import com.pulumi.aws.glue.JobArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new LogGroup("example", LogGroupArgs.builder()
.name("example")
.retentionInDays(14)
.build());
var exampleJob = new Job("exampleJob", JobArgs.builder()
.defaultArguments(Map.ofEntries(
Map.entry("--continuous-log-logGroup", example.name()),
Map.entry("--enable-continuous-cloudwatch-log", "true"),
Map.entry("--enable-continuous-log-filter", "true"),
Map.entry("--enable-metrics", "")
))
.build());
}
}
resources:
example:
type: aws:cloudwatch:LogGroup
properties:
name: example
retentionInDays: 14
exampleJob:
type: aws:glue:Job
name: example
properties:
defaultArguments:
--continuous-log-logGroup: ${example.name}
--enable-continuous-cloudwatch-log: 'true'
--enable-continuous-log-filter: 'true'
--enable-metrics: ""
Import
Identity Schema
Required
name(String) Name of the Glue Job.
Optional
accountId(String) AWS account ID.region(String) Region where this resource is managed.
Using pulumi import, import Glue Jobs using name. For example:
$ pulumi import aws:glue/job:Job example example
Constructors
- Job(String name, {JobArgs? args, CustomResourceOptions? options})
-
Creates a new Job.
nameThe Pulumi resource name.argsArguments used to configure this Job. The set of arguments for Job.optionsResource options controlling this resource's behavior. - Job.reference(String urn)
- Creates a typed reference to an existing Job resource.
Properties
-
arn
↔ Output<
String> -
ARN of Glue Job
latefinal
-
childResources
→ Set<
Resource> -
finalinherited
-
command
↔ Output<
JobCommand> -
The command of the job. Defined below.
latefinal
-
completionSources
↔ Map<
String, IOutputCompletionSource> -
latefinalinherited
-
connections
↔ Output<
List< String> ?> -
The list of connections used for this job.
latefinal
-
defaultArguments
↔ Output<
Map< String, String> ?> -
The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the Calling AWS Glue APIs in Python topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the Special Parameters Used by AWS Glue topic in the developer guide.
latefinal
-
description
↔ Output<
String?> -
Description of the job.
latefinal
-
executionClass
↔ Output<
String?> -
Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value:
FLEX,STANDARD.latefinal -
executionProperty
↔ Output<
JobExecutionProperty> -
Execution property of the job. Defined below.
latefinal
-
glueVersion
↔ Output<
String> -
The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the AWS Glue Release Notes.
latefinal
- hashCode → int
-
The hash code for this object.
no setterinherited
-
id
↔ Output<
String> -
getter/setter pairinherited
- isCustom → bool
-
Returns whether this resource is provider-managed.
no setterinherited
- isProtected → bool
-
Returns whether this resource is protected from deletion.
no setterinherited
- isRemote → bool
-
Whether this resource is registered as remote.
no setterinherited
- isResourceReference → bool
-
Whether this instance represents a resource value returned over RPC.
finalinherited
-
jobMode
↔ Output<
String> -
Describes how a job was created. Valid values are
SCRIPT,NOTEBOOKandVISUAL.latefinal -
jobRunQueuingEnabled
↔ Output<
bool?> -
Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
latefinal
-
maintenanceWindow
↔ Output<
String?> -
Specifies the day of the week and hour for the maintenance window for streaming jobs.
latefinal
-
maxCapacity
↔ Output<
double> -
The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
Requiredwhenpythonshellis set, accept either0.0625or1.0. UsenumberOfWorkersandworkerTypearguments instead withglueVersion2.0and above.latefinal -
maxRetries
↔ Output<
int?> -
The maximum number of times to retry this job if it fails.
latefinal
-
name
↔ Output<
String> -
The name you assign to this job. It must be unique in your account.
latefinal
-
nonOverridableArguments
↔ Output<
Map< String, String> ?> -
Non-overridable arguments for this job, specified as name-value pairs.
latefinal
-
notificationProperty
↔ Output<
JobNotificationProperty> -
Notification property of the job. Defined below.
latefinal
-
numberOfWorkers
↔ Output<
int> -
The number of workers of a defined workerType that are allocated when a job runs.
latefinal
-
region
↔ Output<
String> -
Region where this resource will be managed. Defaults to the Region set in the provider configuration.
latefinal
-
resourceTransforms
→ List<
ResourceTransform> -
Inherited/explicit async transforms.
no setterinherited
-
roleArn
↔ Output<
String> -
The ARN of the IAM role associated with this job.
latefinal
- runtimeType → Type
-
A representation of the runtime type of the object.
no setterinherited
-
securityConfiguration
↔ Output<
String?> -
The name of the Security Configuration to be associated with the job.
latefinal
-
sourceControlDetails
↔ Output<
JobSourceControlDetails?> -
The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository. Defined below.
latefinal
-
Key-value map of resource tags. If configured with a provider
defaultTagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.latefinal -
A map of tags assigned to the resource, including those inherited from the provider
defaultTagsconfiguration block.latefinal -
timeout
↔ Output<
int> -
The job timeout in minutes. The default is 2880 minutes (48 hours) for
glueetlandpythonshelljobs, and 0 (unlimited) forgluestreamingjobs. Leave this attribute argumnet unconfigured forgluerayjobs.latefinal -
transformations
→ List<
ResourceTransformation> -
Inherited/explicit legacy transformations.
no setterinherited
-
urn
↔ Output<
String> -
latefinalinherited
-
workerType
↔ Output<
String> -
The type of predefined worker that is allocated when a job runs. Valid values:
Standard,G.1X,G.2X,G.025X,G.4X,G.8X,G.12X,G.16X,R.1X,R.2X,R.4X,R.8X,Z.2X(Ray jobs). See the AWS documentation for details.latefinal
Methods
-
failId(
Object error) → void -
Completes this resource ID with an error when registration fails.
inherited
-
failOutputs(
Object error) → void -
Completes all output properties with
error.inherited -
failUrn(
Object error) → void -
Completes this resource URN with an error when registration fails.
inherited
-
getProvider(
String moduleMember) → ProviderResource? -
Returns provider for
moduleMember's package, if configured.inherited -
getResourceName(
) → String -
Returns this resource's logical name.
inherited
-
getResourceType(
) → String -
Returns this resource's Pulumi type token.
inherited
-
noSuchMethod(
Invocation invocation) → dynamic -
Invoked when a nonexistent method or property is accessed.
inherited
-
registerOutput<
T> (String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output< T> -
Registers a dynamic output property for this resource.
inherited
-
resolveId(
String? value, {required bool isKnown}) → void -
Resolves the provider-assigned ID for this resource.
inherited
-
resolveOutputs(
Struct outputs) → void -
Resolves all output properties from a monitor response payload.
inherited
-
resolveUrn(
String value) → void -
Resolves this resource's URN once assigned by the engine.
inherited
-
serializeProperties(
Map< String, dynamic> properties) → Future<Struct> -
Serializes resource properties for RPC transmission.
inherited
-
toString(
) → String -
A string representation of this object.
inherited
Operators
-
operator ==(
Object other) → bool -
The equality operator.
inherited