Cluster class
Provides an Elastic MapReduce Cluster, a web service that makes it easy to process large amounts of data efficiently. See Amazon Elastic MapReduce Documentation for more information.
To configure Instance Groups for task nodes, see the aws.emr.InstanceGroup resource.
Example Usage
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const cluster = new aws.emr.Cluster("cluster", {
ec2Attributes: {
subnetId: main.id,
emrManagedMasterSecurityGroup: sg.id,
emrManagedSlaveSecurityGroup: sg.id,
instanceProfile: emrProfile.arn,
},
masterInstanceGroup: {
instanceType: "m4.large",
},
coreInstanceGroup: {
ebsConfigs: [{
size: 40,
type: "gp2",
volumesPerInstance: 1,
}],
instanceType: "c4.large",
instanceCount: 1,
bidPrice: "0.30",
autoscalingPolicy: `{
\\"Constraints\\": {
\\"MinCapacity\\": 1,
\\"MaxCapacity\\": 2
},
\\"Rules\\": [
{
\\"Name\\": \\"ScaleOutMemoryPercentage\\",
\\"Description\\": \\"Scale out if YARNMemoryAvailablePercentage is less than 15\\",
\\"Action\\": {
\\"SimpleScalingPolicyConfiguration\\": {
\\"AdjustmentType\\": \\"CHANGE_IN_CAPACITY\\",
\\"ScalingAdjustment\\": 1,
\\"CoolDown\\": 300
}
},
\\"Trigger\\": {
\\"CloudWatchAlarmDefinition\\": {
\\"ComparisonOperator\\": \\"LESS_THAN\\",
\\"EvaluationPeriods\\": 1,
\\"MetricName\\": \\"YARNMemoryAvailablePercentage\\",
\\"Namespace\\": \\"AWS/ElasticMapReduce\\",
\\"Period\\": 300,
\\"Statistic\\": \\"AVERAGE\\",
\\"Threshold\\": 15.0,
\\"Unit\\": \\"PERCENT\\"
}
}
}
]
}
`,
},
bootstrapActions: [{
path: "s3://elasticmapreduce/bootstrap-actions/run-if",
name: "runif",
args: [
"instance.isMaster=true",
"echo running on master node",
],
}],
name: "emr-test-arn",
releaseLabel: "emr-4.6.0",
applications: ["Spark"],
additionalInfo: `{
\\"instanceAwsClientConfiguration\\": {
\\"proxyPort\\": 8099,
\\"proxyHost\\": \\"myproxy.example.com\\"
}
}
`,
terminationProtection: false,
keepJobFlowAliveWhenNoSteps: true,
ebsRootVolumeSize: 100,
tags: {
role: "rolename",
env: "env",
},
configurationsJson: ` [
{
\\"Classification\\": \\"hadoop-env\\",
\\"Configurations\\": [
{
\\"Classification\\": \\"export\\",
\\"Properties\\": {
\\"JAVA_HOME\\": \\"/usr/lib/jvm/java-1.8.0\\"
}
}
],
\\"Properties\\": {}
},
{
\\"Classification\\": \\"spark-env\\",
\\"Configurations\\": [
{
\\"Classification\\": \\"export\\",
\\"Properties\\": {
\\"JAVA_HOME\\": \\"/usr/lib/jvm/java-1.8.0\\"
}
}
],
\\"Properties\\": {}
}
]
`,
serviceRole: iamEmrServiceRole.arn,
});
import pulumi
import pulumi_aws as aws
cluster = aws.emr.Cluster("cluster",
ec2_attributes={
"subnet_id": main["id"],
"emr_managed_master_security_group": sg["id"],
"emr_managed_slave_security_group": sg["id"],
"instance_profile": emr_profile["arn"],
},
master_instance_group={
"instance_type": "m4.large",
},
core_instance_group={
"ebs_configs": [{
"size": 40,
"type": "gp2",
"volumes_per_instance": 1,
}],
"instance_type": "c4.large",
"instance_count": 1,
"bid_price": "0.30",
"autoscaling_policy": """{
\"Constraints\": {
\"MinCapacity\": 1,
\"MaxCapacity\": 2
},
\"Rules\": [
{
\"Name\": \"ScaleOutMemoryPercentage\",
\"Description\": \"Scale out if YARNMemoryAvailablePercentage is less than 15\",
\"Action\": {
\"SimpleScalingPolicyConfiguration\": {
\"AdjustmentType\": \"CHANGE_IN_CAPACITY\",
\"ScalingAdjustment\": 1,
\"CoolDown\": 300
}
},
\"Trigger\": {
\"CloudWatchAlarmDefinition\": {
\"ComparisonOperator\": \"LESS_THAN\",
\"EvaluationPeriods\": 1,
\"MetricName\": \"YARNMemoryAvailablePercentage\",
\"Namespace\": \"AWS/ElasticMapReduce\",
\"Period\": 300,
\"Statistic\": \"AVERAGE\",
\"Threshold\": 15.0,
\"Unit\": \"PERCENT\"
}
}
}
]
}
""",
},
bootstrap_actions=[{
"path": "s3://elasticmapreduce/bootstrap-actions/run-if",
"name": "runif",
"args": [
"instance.isMaster=true",
"echo running on master node",
],
}],
name="emr-test-arn",
release_label="emr-4.6.0",
applications=["Spark"],
additional_info="""{
\"instanceAwsClientConfiguration\": {
\"proxyPort\": 8099,
\"proxyHost\": \"myproxy.example.com\"
}
}
""",
termination_protection=False,
keep_job_flow_alive_when_no_steps=True,
ebs_root_volume_size=100,
tags={
"role": "rolename",
"env": "env",
},
configurations_json=""" [
{
\"Classification\": \"hadoop-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
},
{
\"Classification\": \"spark-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
}
]
""",
service_role=iam_emr_service_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var cluster = new Aws.Emr.Cluster("cluster", new()
{
Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
{
SubnetId = main.Id,
EmrManagedMasterSecurityGroup = sg.Id,
EmrManagedSlaveSecurityGroup = sg.Id,
InstanceProfile = emrProfile.Arn,
},
MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
{
InstanceType = "m4.large",
},
CoreInstanceGroup = new Aws.Emr.Inputs.ClusterCoreInstanceGroupArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceGroupEbsConfigArgs
{
Size = 40,
Type = "gp2",
VolumesPerInstance = 1,
},
},
InstanceType = "c4.large",
InstanceCount = 1,
BidPrice = "0.30",
AutoscalingPolicy = @"{
\""Constraints\"": {
\""MinCapacity\"": 1,
\""MaxCapacity\"": 2
},
\""Rules\"": [
{
\""Name\"": \""ScaleOutMemoryPercentage\"",
\""Description\"": \""Scale out if YARNMemoryAvailablePercentage is less than 15\"",
\""Action\"": {
\""SimpleScalingPolicyConfiguration\"": {
\""AdjustmentType\"": \""CHANGE_IN_CAPACITY\"",
\""ScalingAdjustment\"": 1,
\""CoolDown\"": 300
}
},
\""Trigger\"": {
\""CloudWatchAlarmDefinition\"": {
\""ComparisonOperator\"": \""LESS_THAN\"",
\""EvaluationPeriods\"": 1,
\""MetricName\"": \""YARNMemoryAvailablePercentage\"",
\""Namespace\"": \""AWS/ElasticMapReduce\"",
\""Period\"": 300,
\""Statistic\"": \""AVERAGE\"",
\""Threshold\"": 15.0,
\""Unit\"": \""PERCENT\""
}
}
}
]
}
",
},
BootstrapActions = new[]
{
new Aws.Emr.Inputs.ClusterBootstrapActionArgs
{
Path = "s3://elasticmapreduce/bootstrap-actions/run-if",
Name = "runif",
Args = new[]
{
"instance.isMaster=true",
"echo running on master node",
},
},
},
Name = "emr-test-arn",
ReleaseLabel = "emr-4.6.0",
Applications = new[]
{
"Spark",
},
AdditionalInfo = @"{
\""instanceAwsClientConfiguration\"": {
\""proxyPort\"": 8099,
\""proxyHost\"": \""myproxy.example.com\""
}
}
",
TerminationProtection = false,
KeepJobFlowAliveWhenNoSteps = true,
EbsRootVolumeSize = 100,
Tags =
{
{ "role", "rolename" },
{ "env", "env" },
},
ConfigurationsJson = @" [
{
\""Classification\"": \""hadoop-env\"",
\""Configurations\"": [
{
\""Classification\"": \""export\"",
\""Properties\"": {
\""JAVA_HOME\"": \""/usr/lib/jvm/java-1.8.0\""
}
}
],
\""Properties\"": {}
},
{
\""Classification\"": \""spark-env\"",
\""Configurations\"": [
{
\""Classification\"": \""export\"",
\""Properties\"": {
\""JAVA_HOME\"": \""/usr/lib/jvm/java-1.8.0\""
}
}
],
\""Properties\"": {}
}
]
",
ServiceRole = iamEmrServiceRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := emr.NewCluster(ctx, "cluster", &emr.ClusterArgs{
Ec2Attributes: &emr.ClusterEc2AttributesArgs{
SubnetId: pulumi.Any(main.Id),
EmrManagedMasterSecurityGroup: pulumi.Any(sg.Id),
EmrManagedSlaveSecurityGroup: pulumi.Any(sg.Id),
InstanceProfile: pulumi.Any(emrProfile.Arn),
},
MasterInstanceGroup: &emr.ClusterMasterInstanceGroupArgs{
InstanceType: pulumi.String("m4.large"),
},
CoreInstanceGroup: &emr.ClusterCoreInstanceGroupArgs{
EbsConfigs: emr.ClusterCoreInstanceGroupEbsConfigArray{
&emr.ClusterCoreInstanceGroupEbsConfigArgs{
Size: pulumi.Int(40),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
InstanceType: pulumi.String("c4.large"),
InstanceCount: pulumi.Int(1),
BidPrice: pulumi.String("0.30"),
AutoscalingPolicy: pulumi.String(`{
\"Constraints\": {
\"MinCapacity\": 1,
\"MaxCapacity\": 2
},
\"Rules\": [
{
\"Name\": \"ScaleOutMemoryPercentage\",
\"Description\": \"Scale out if YARNMemoryAvailablePercentage is less than 15\",
\"Action\": {
\"SimpleScalingPolicyConfiguration\": {
\"AdjustmentType\": \"CHANGE_IN_CAPACITY\",
\"ScalingAdjustment\": 1,
\"CoolDown\": 300
}
},
\"Trigger\": {
\"CloudWatchAlarmDefinition\": {
\"ComparisonOperator\": \"LESS_THAN\",
\"EvaluationPeriods\": 1,
\"MetricName\": \"YARNMemoryAvailablePercentage\",
\"Namespace\": \"AWS/ElasticMapReduce\",
\"Period\": 300,
\"Statistic\": \"AVERAGE\",
\"Threshold\": 15.0,
\"Unit\": \"PERCENT\"
}
}
}
]
}
`),
},
BootstrapActions: emr.ClusterBootstrapActionArray{
&emr.ClusterBootstrapActionArgs{
Path: pulumi.String("s3://elasticmapreduce/bootstrap-actions/run-if"),
Name: pulumi.String("runif"),
Args: pulumi.StringArray{
pulumi.String("instance.isMaster=true"),
pulumi.String("echo running on master node"),
},
},
},
Name: pulumi.String("emr-test-arn"),
ReleaseLabel: pulumi.String("emr-4.6.0"),
Applications: pulumi.StringArray{
pulumi.String("Spark"),
},
AdditionalInfo: pulumi.String(`{
\"instanceAwsClientConfiguration\": {
\"proxyPort\": 8099,
\"proxyHost\": \"myproxy.example.com\"
}
}
`),
TerminationProtection: pulumi.Bool(false),
KeepJobFlowAliveWhenNoSteps: pulumi.Bool(true),
EbsRootVolumeSize: pulumi.Int(100),
Tags: pulumi.StringMap{
"role": pulumi.String("rolename"),
"env": pulumi.String("env"),
},
ConfigurationsJson: pulumi.String(` [
{
\"Classification\": \"hadoop-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
},
{
\"Classification\": \"spark-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
}
]
`),
ServiceRole: pulumi.Any(iamEmrServiceRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_emr_cluster" "cluster" {
ec2_attributes = {
subnet_id = main.id
emr_managed_master_security_group = sg.id
emr_managed_slave_security_group = sg.id
instance_profile = emrProfile.arn
}
master_instance_group = {
instance_type = "m4.large"
}
core_instance_group = {
ebs_configs = [{
"size" = "40"
"type" = "gp2"
"volumesPerInstance" = 1
}]
instance_type = "c4.large"
instance_count = 1
bid_price = "0.30"
autoscaling_policy = "{\n\\\"Constraints\\\": {\n \\\"MinCapacity\\\": 1,\n \\\"MaxCapacity\\\": 2\n},\n\\\"Rules\\\": [\n {\n \\\"Name\\\": \\\"ScaleOutMemoryPercentage\\\",\n \\\"Description\\\": \\\"Scale out if YARNMemoryAvailablePercentage is less than 15\\\",\n \\\"Action\\\": {\n \\\"SimpleScalingPolicyConfiguration\\\": {\n \\\"AdjustmentType\\\": \\\"CHANGE_IN_CAPACITY\\\",\n \\\"ScalingAdjustment\\\": 1,\n \\\"CoolDown\\\": 300\n }\n },\n \\\"Trigger\\\": {\n \\\"CloudWatchAlarmDefinition\\\": {\n \\\"ComparisonOperator\\\": \\\"LESS_THAN\\\",\n \\\"EvaluationPeriods\\\": 1,\n \\\"MetricName\\\": \\\"YARNMemoryAvailablePercentage\\\",\n \\\"Namespace\\\": \\\"AWS/ElasticMapReduce\\\",\n \\\"Period\\\": 300,\n \\\"Statistic\\\": \\\"AVERAGE\\\",\n \\\"Threshold\\\": 15.0,\n \\\"Unit\\\": \\\"PERCENT\\\"\n }\n }\n }\n]\n}\n"
}
bootstrap_actions {
path = "s3://elasticmapreduce/bootstrap-actions/run-if"
name = "runif"
args = ["instance.isMaster=true", "echo running on master node"]
}
name = "emr-test-arn"
release_label = "emr-4.6.0"
applications = ["Spark"]
additional_info = "{\n \\\"instanceAwsClientConfiguration\\\": {\n \\\"proxyPort\\\": 8099,\n \\\"proxyHost\\\": \\\"myproxy.example.com\\\"\n }\n}\n"
termination_protection = false
keep_job_flow_alive_when_no_steps = true
ebs_root_volume_size = 100
tags = {
"role" = "rolename"
"env" = "env"
}
configurations_json = " [\n {\n \\\"Classification\\\": \\\"hadoop-env\\\",\n \\\"Configurations\\\": [\n {\n \\\"Classification\\\": \\\"export\\\",\n \\\"Properties\\\": {\n \\\"JAVA_HOME\\\": \\\"/usr/lib/jvm/java-1.8.0\\\"\n }\n }\n ],\n \\\"Properties\\\": {}\n },\n {\n \\\"Classification\\\": \\\"spark-env\\\",\n \\\"Configurations\\\": [\n {\n \\\"Classification\\\": \\\"export\\\",\n \\\"Properties\\\": {\n \\\"JAVA_HOME\\\": \\\"/usr/lib/jvm/java-1.8.0\\\"\n }\n }\n ],\n \\\"Properties\\\": {}\n }\n ]\n"
service_role = iamEmrServiceRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupEbsConfigArgs;
import com.pulumi.aws.emr.inputs.ClusterBootstrapActionArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var cluster = new Cluster("cluster", ClusterArgs.builder()
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(main.id())
.emrManagedMasterSecurityGroup(sg.id())
.emrManagedSlaveSecurityGroup(sg.id())
.instanceProfile(emrProfile.arn())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceType("m4.large")
.build())
.coreInstanceGroup(ClusterCoreInstanceGroupArgs.builder()
.ebsConfigs(ClusterCoreInstanceGroupEbsConfigArgs.builder()
.size(40)
.type("gp2")
.volumesPerInstance(1)
.build())
.instanceType("c4.large")
.instanceCount(1)
.bidPrice("0.30")
.autoscalingPolicy("""
{
\"Constraints\": {
\"MinCapacity\": 1,
\"MaxCapacity\": 2
},
\"Rules\": [
{
\"Name\": \"ScaleOutMemoryPercentage\",
\"Description\": \"Scale out if YARNMemoryAvailablePercentage is less than 15\",
\"Action\": {
\"SimpleScalingPolicyConfiguration\": {
\"AdjustmentType\": \"CHANGE_IN_CAPACITY\",
\"ScalingAdjustment\": 1,
\"CoolDown\": 300
}
},
\"Trigger\": {
\"CloudWatchAlarmDefinition\": {
\"ComparisonOperator\": \"LESS_THAN\",
\"EvaluationPeriods\": 1,
\"MetricName\": \"YARNMemoryAvailablePercentage\",
\"Namespace\": \"AWS/ElasticMapReduce\",
\"Period\": 300,
\"Statistic\": \"AVERAGE\",
\"Threshold\": 15.0,
\"Unit\": \"PERCENT\"
}
}
}
]
}
""")
.build())
.bootstrapActions(ClusterBootstrapActionArgs.builder()
.path("s3://elasticmapreduce/bootstrap-actions/run-if")
.name("runif")
.args(
"instance.isMaster=true",
"echo running on master node")
.build())
.name("emr-test-arn")
.releaseLabel("emr-4.6.0")
.applications("Spark")
.additionalInfo("""
{
\"instanceAwsClientConfiguration\": {
\"proxyPort\": 8099,
\"proxyHost\": \"myproxy.example.com\"
}
}
""")
.terminationProtection(false)
.keepJobFlowAliveWhenNoSteps(true)
.ebsRootVolumeSize(100)
.tags(Map.ofEntries(
Map.entry("role", "rolename"),
Map.entry("env", "env")
))
.configurationsJson("""
[
{
\"Classification\": \"hadoop-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
},
{
\"Classification\": \"spark-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
}
]
""")
.serviceRole(iamEmrServiceRole.arn())
.build());
}
}
resources:
cluster:
type: aws:emr:Cluster
properties:
ec2Attributes:
subnetId: ${main.id}
emrManagedMasterSecurityGroup: ${sg.id}
emrManagedSlaveSecurityGroup: ${sg.id}
instanceProfile: ${emrProfile.arn}
masterInstanceGroup:
instanceType: m4.large
coreInstanceGroup:
ebsConfigs:
- size: '40'
type: gp2
volumesPerInstance: 1
instanceType: c4.large
instanceCount: 1
bidPrice: '0.30'
autoscalingPolicy: |
{
\"Constraints\": {
\"MinCapacity\": 1,
\"MaxCapacity\": 2
},
\"Rules\": [
{
\"Name\": \"ScaleOutMemoryPercentage\",
\"Description\": \"Scale out if YARNMemoryAvailablePercentage is less than 15\",
\"Action\": {
\"SimpleScalingPolicyConfiguration\": {
\"AdjustmentType\": \"CHANGE_IN_CAPACITY\",
\"ScalingAdjustment\": 1,
\"CoolDown\": 300
}
},
\"Trigger\": {
\"CloudWatchAlarmDefinition\": {
\"ComparisonOperator\": \"LESS_THAN\",
\"EvaluationPeriods\": 1,
\"MetricName\": \"YARNMemoryAvailablePercentage\",
\"Namespace\": \"AWS/ElasticMapReduce\",
\"Period\": 300,
\"Statistic\": \"AVERAGE\",
\"Threshold\": 15.0,
\"Unit\": \"PERCENT\"
}
}
}
]
}
bootstrapActions:
- path: s3://elasticmapreduce/bootstrap-actions/run-if
name: runif
args:
- instance.isMaster=true
- echo running on master node
name: emr-test-arn
releaseLabel: emr-4.6.0
applications:
- Spark
additionalInfo: |
{
\"instanceAwsClientConfiguration\": {
\"proxyPort\": 8099,
\"proxyHost\": \"myproxy.example.com\"
}
}
terminationProtection: false
keepJobFlowAliveWhenNoSteps: true
ebsRootVolumeSize: 100
tags:
role: rolename
env: env
configurationsJson: |2
[
{
\"Classification\": \"hadoop-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
},
{
\"Classification\": \"spark-env\",
\"Configurations\": [
{
\"Classification\": \"export\",
\"Properties\": {
\"JAVA_HOME\": \"/usr/lib/jvm/java-1.8.0\"
}
}
],
\"Properties\": {}
}
]
serviceRole: ${iamEmrServiceRole.arn}
The aws.emr.Cluster resource typically requires two IAM roles, one for the EMR Cluster to use as a service role, and another is assigned to every EC2 instance in a cluster and each application process that runs on a cluster assumes this role for permissions to interact with other AWS services. An additional role, the Auto Scaling role, is required if your cluster uses automatic scaling in Amazon EMR.
The default AWS managed EMR service role is called EMR_DefaultRole with Amazon managed policy AmazonEMRServicePolicy_v2 attached. The name of default instance profile role is EMR_EC2_DefaultRole with default managed policy AmazonElasticMapReduceforEC2Role attached, but it is on the path to deprecation and will not be replaced with another default managed policy. You'll need to create and specify an instance profile to replace the deprecated role and default policy. See the Configure IAM service roles for Amazon EMR guide for more information on these IAM roles. There is also a fully-bootable example Pulumi configuration at the bottom of this page.
Instance Fleet
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.emr.Cluster("example", {
masterInstanceFleet: {
instanceTypeConfigs: [{
instanceType: "m4.xlarge",
}],
targetOnDemandCapacity: 1,
},
coreInstanceFleet: {
launchSpecifications: {
spotSpecifications: [{
allocationStrategy: "capacity-optimized",
blockDurationMinutes: 0,
timeoutAction: "SWITCH_TO_ON_DEMAND",
timeoutDurationMinutes: 10,
}],
},
instanceTypeConfigs: [
{
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
bidPriceAsPercentageOfOnDemandPrice: 80,
instanceType: "m3.xlarge",
weightedCapacity: 1,
},
{
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
bidPriceAsPercentageOfOnDemandPrice: 100,
instanceType: "m4.xlarge",
weightedCapacity: 1,
},
{
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
bidPriceAsPercentageOfOnDemandPrice: 100,
instanceType: "m4.2xlarge",
weightedCapacity: 2,
},
],
name: "core fleet",
targetOnDemandCapacity: 2,
targetSpotCapacity: 2,
},
});
const task = new aws.emr.InstanceFleet("task", {
launchSpecifications: {
spotSpecifications: [{
allocationStrategy: "capacity-optimized",
blockDurationMinutes: 0,
timeoutAction: "TERMINATE_CLUSTER",
timeoutDurationMinutes: 10,
}],
},
instanceTypeConfigs: [
{
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
bidPriceAsPercentageOfOnDemandPrice: 100,
instanceType: "m4.xlarge",
weightedCapacity: 1,
},
{
ebsConfigs: [{
size: 100,
type: "gp2",
volumesPerInstance: 1,
}],
bidPriceAsPercentageOfOnDemandPrice: 100,
instanceType: "m4.2xlarge",
weightedCapacity: 2,
},
],
clusterId: example.id,
name: "task fleet",
targetOnDemandCapacity: 1,
targetSpotCapacity: 1,
});
import pulumi
import pulumi_aws as aws
example = aws.emr.Cluster("example",
master_instance_fleet={
"instance_type_configs": [{
"instance_type": "m4.xlarge",
}],
"target_on_demand_capacity": 1,
},
core_instance_fleet={
"launch_specifications": {
"spot_specifications": [{
"allocation_strategy": "capacity-optimized",
"block_duration_minutes": 0,
"timeout_action": "SWITCH_TO_ON_DEMAND",
"timeout_duration_minutes": 10,
}],
},
"instance_type_configs": [
{
"ebs_configs": [{
"size": 100,
"type": "gp2",
"volumes_per_instance": 1,
}],
"bid_price_as_percentage_of_on_demand_price": float(80),
"instance_type": "m3.xlarge",
"weighted_capacity": 1,
},
{
"ebs_configs": [{
"size": 100,
"type": "gp2",
"volumes_per_instance": 1,
}],
"bid_price_as_percentage_of_on_demand_price": float(100),
"instance_type": "m4.xlarge",
"weighted_capacity": 1,
},
{
"ebs_configs": [{
"size": 100,
"type": "gp2",
"volumes_per_instance": 1,
}],
"bid_price_as_percentage_of_on_demand_price": float(100),
"instance_type": "m4.2xlarge",
"weighted_capacity": 2,
},
],
"name": "core fleet",
"target_on_demand_capacity": 2,
"target_spot_capacity": 2,
})
task = aws.emr.InstanceFleet("task",
launch_specifications={
"spot_specifications": [{
"allocation_strategy": "capacity-optimized",
"block_duration_minutes": 0,
"timeout_action": "TERMINATE_CLUSTER",
"timeout_duration_minutes": 10,
}],
},
instance_type_configs=[
{
"ebs_configs": [{
"size": 100,
"type": "gp2",
"volumes_per_instance": 1,
}],
"bid_price_as_percentage_of_on_demand_price": float(100),
"instance_type": "m4.xlarge",
"weighted_capacity": 1,
},
{
"ebs_configs": [{
"size": 100,
"type": "gp2",
"volumes_per_instance": 1,
}],
"bid_price_as_percentage_of_on_demand_price": float(100),
"instance_type": "m4.2xlarge",
"weighted_capacity": 2,
},
],
cluster_id=example.id,
name="task fleet",
target_on_demand_capacity=1,
target_spot_capacity=1)
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Emr.Cluster("example", new()
{
MasterInstanceFleet = new Aws.Emr.Inputs.ClusterMasterInstanceFleetArgs
{
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.ClusterMasterInstanceFleetInstanceTypeConfigArgs
{
InstanceType = "m4.xlarge",
},
},
TargetOnDemandCapacity = 1,
},
CoreInstanceFleet = new Aws.Emr.Inputs.ClusterCoreInstanceFleetArgs
{
LaunchSpecifications = new Aws.Emr.Inputs.ClusterCoreInstanceFleetLaunchSpecificationsArgs
{
SpotSpecifications = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs
{
AllocationStrategy = "capacity-optimized",
BlockDurationMinutes = 0,
TimeoutAction = "SWITCH_TO_ON_DEMAND",
TimeoutDurationMinutes = 10,
},
},
},
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPriceAsPercentageOfOnDemandPrice = 80,
InstanceType = "m3.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPriceAsPercentageOfOnDemandPrice = 100,
InstanceType = "m4.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPriceAsPercentageOfOnDemandPrice = 100,
InstanceType = "m4.2xlarge",
WeightedCapacity = 2,
},
},
Name = "core fleet",
TargetOnDemandCapacity = 2,
TargetSpotCapacity = 2,
},
});
var task = new Aws.Emr.InstanceFleet("task", new()
{
LaunchSpecifications = new Aws.Emr.Inputs.InstanceFleetLaunchSpecificationsArgs
{
SpotSpecifications = new[]
{
new Aws.Emr.Inputs.InstanceFleetLaunchSpecificationsSpotSpecificationArgs
{
AllocationStrategy = "capacity-optimized",
BlockDurationMinutes = 0,
TimeoutAction = "TERMINATE_CLUSTER",
TimeoutDurationMinutes = 10,
},
},
},
InstanceTypeConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPriceAsPercentageOfOnDemandPrice = 100,
InstanceType = "m4.xlarge",
WeightedCapacity = 1,
},
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigArgs
{
EbsConfigs = new[]
{
new Aws.Emr.Inputs.InstanceFleetInstanceTypeConfigEbsConfigArgs
{
Size = 100,
Type = "gp2",
VolumesPerInstance = 1,
},
},
BidPriceAsPercentageOfOnDemandPrice = 100,
InstanceType = "m4.2xlarge",
WeightedCapacity = 2,
},
},
ClusterId = example.Id,
Name = "task fleet",
TargetOnDemandCapacity = 1,
TargetSpotCapacity = 1,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
example, err := emr.NewCluster(ctx, "example", &emr.ClusterArgs{
MasterInstanceFleet: &emr.ClusterMasterInstanceFleetArgs{
InstanceTypeConfigs: emr.ClusterMasterInstanceFleetInstanceTypeConfigArray{
&emr.ClusterMasterInstanceFleetInstanceTypeConfigArgs{
InstanceType: pulumi.String("m4.xlarge"),
},
},
TargetOnDemandCapacity: pulumi.Int(1),
},
CoreInstanceFleet: &emr.ClusterCoreInstanceFleetArgs{
LaunchSpecifications: &emr.ClusterCoreInstanceFleetLaunchSpecificationsArgs{
SpotSpecifications: emr.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArray{
&emr.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs{
AllocationStrategy: pulumi.String("capacity-optimized"),
BlockDurationMinutes: pulumi.Int(0),
TimeoutAction: pulumi.String("SWITCH_TO_ON_DEMAND"),
TimeoutDurationMinutes: pulumi.Int(10),
},
},
},
InstanceTypeConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(80),
InstanceType: pulumi.String("m3.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
InstanceType: pulumi.String("m4.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.ClusterCoreInstanceFleetInstanceTypeConfigArgs{
EbsConfigs: emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
InstanceType: pulumi.String("m4.2xlarge"),
WeightedCapacity: pulumi.Int(2),
},
},
Name: pulumi.String("core fleet"),
TargetOnDemandCapacity: pulumi.Int(2),
TargetSpotCapacity: pulumi.Int(2),
},
})
if err != nil {
return err
}
_, err = emr.NewInstanceFleet(ctx, "task", &emr.InstanceFleetArgs{
LaunchSpecifications: &emr.InstanceFleetLaunchSpecificationsArgs{
SpotSpecifications: emr.InstanceFleetLaunchSpecificationsSpotSpecificationArray{
&emr.InstanceFleetLaunchSpecificationsSpotSpecificationArgs{
AllocationStrategy: pulumi.String("capacity-optimized"),
BlockDurationMinutes: pulumi.Int(0),
TimeoutAction: pulumi.String("TERMINATE_CLUSTER"),
TimeoutDurationMinutes: pulumi.Int(10),
},
},
},
InstanceTypeConfigs: emr.InstanceFleetInstanceTypeConfigArray{
&emr.InstanceFleetInstanceTypeConfigArgs{
EbsConfigs: emr.InstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.InstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
InstanceType: pulumi.String("m4.xlarge"),
WeightedCapacity: pulumi.Int(1),
},
&emr.InstanceFleetInstanceTypeConfigArgs{
EbsConfigs: emr.InstanceFleetInstanceTypeConfigEbsConfigArray{
&emr.InstanceFleetInstanceTypeConfigEbsConfigArgs{
Size: pulumi.Int(100),
Type: pulumi.String("gp2"),
VolumesPerInstance: pulumi.Int(1),
},
},
BidPriceAsPercentageOfOnDemandPrice: pulumi.Float64(100),
InstanceType: pulumi.String("m4.2xlarge"),
WeightedCapacity: pulumi.Int(2),
},
},
ClusterId: example.ID().ToIDOutput().ToStringOutput(),
Name: pulumi.String("task fleet"),
TargetOnDemandCapacity: pulumi.Int(1),
TargetSpotCapacity: pulumi.Int(1),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_emr_cluster" "example" {
master_instance_fleet = {
instance_type_configs = [{
"instanceType" = "m4.xlarge"
}]
target_on_demand_capacity = 1
}
core_instance_fleet = {
launch_specifications = {
spot_specifications = [{
"allocationStrategy" = "capacity-optimized"
"blockDurationMinutes" = 0
"timeoutAction" = "SWITCH_TO_ON_DEMAND"
"timeoutDurationMinutes" = 10
}]
}
instance_type_configs = [{
"ebsConfigs" = [{
"size" = 100
"type" = "gp2"
"volumesPerInstance" = 1
}]
"bidPriceAsPercentageOfOnDemandPrice" = 80
"instanceType" = "m3.xlarge"
"weightedCapacity" = 1
}, {
"ebsConfigs" = [{
"size" = 100
"type" = "gp2"
"volumesPerInstance" = 1
}]
"bidPriceAsPercentageOfOnDemandPrice" = 100
"instanceType" = "m4.xlarge"
"weightedCapacity" = 1
}, {
"ebsConfigs" = [{
"size" = 100
"type" = "gp2"
"volumesPerInstance" = 1
}]
"bidPriceAsPercentageOfOnDemandPrice" = 100
"instanceType" = "m4.2xlarge"
"weightedCapacity" = 2
}]
name = "core fleet"
target_on_demand_capacity = 2
target_spot_capacity = 2
}
}
resource "aws_emr_instancefleet" "task" {
launch_specifications = {
spot_specifications = [{
"allocationStrategy" = "capacity-optimized"
"blockDurationMinutes" = 0
"timeoutAction" = "TERMINATE_CLUSTER"
"timeoutDurationMinutes" = 10
}]
}
instance_type_configs {
ebs_configs {
size = 100
type = "gp2"
volumes_per_instance = 1
}
bid_price_as_percentage_of_on_demand_price = 100
instance_type = "m4.xlarge"
weighted_capacity = 1
}
instance_type_configs {
ebs_configs {
size = 100
type = "gp2"
volumes_per_instance = 1
}
bid_price_as_percentage_of_on_demand_price = 100
instance_type = "m4.2xlarge"
weighted_capacity = 2
}
cluster_id = aws_emr_cluster.example.id
name = "task fleet"
target_on_demand_capacity = 1
target_spot_capacity = 1
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceFleetInstanceTypeConfigArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetLaunchSpecificationsArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetInstanceTypeConfigArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs;
import com.pulumi.aws.emr.InstanceFleet;
import com.pulumi.aws.emr.InstanceFleetArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetLaunchSpecificationsArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetLaunchSpecificationsSpotSpecificationArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetInstanceTypeConfigArgs;
import com.pulumi.aws.emr.inputs.InstanceFleetInstanceTypeConfigEbsConfigArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.masterInstanceFleet(ClusterMasterInstanceFleetArgs.builder()
.instanceTypeConfigs(ClusterMasterInstanceFleetInstanceTypeConfigArgs.builder()
.instanceType("m4.xlarge")
.build())
.targetOnDemandCapacity(1)
.build())
.coreInstanceFleet(ClusterCoreInstanceFleetArgs.builder()
.launchSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(ClusterCoreInstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("SWITCH_TO_ON_DEMAND")
.timeoutDurationMinutes(10)
.build())
.build())
.instanceTypeConfigs(
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPriceAsPercentageOfOnDemandPrice(80.0)
.instanceType("m3.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPriceAsPercentageOfOnDemandPrice(100.0)
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
ClusterCoreInstanceFleetInstanceTypeConfigArgs.builder()
.ebsConfigs(ClusterCoreInstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPriceAsPercentageOfOnDemandPrice(100.0)
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.name("core fleet")
.targetOnDemandCapacity(2)
.targetSpotCapacity(2)
.build())
.build());
var task = new InstanceFleet("task", InstanceFleetArgs.builder()
.launchSpecifications(InstanceFleetLaunchSpecificationsArgs.builder()
.spotSpecifications(InstanceFleetLaunchSpecificationsSpotSpecificationArgs.builder()
.allocationStrategy("capacity-optimized")
.blockDurationMinutes(0)
.timeoutAction("TERMINATE_CLUSTER")
.timeoutDurationMinutes(10)
.build())
.build())
.instanceTypeConfigs(
InstanceFleetInstanceTypeConfigArgs.builder()
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPriceAsPercentageOfOnDemandPrice(100.0)
.instanceType("m4.xlarge")
.weightedCapacity(1)
.build(),
InstanceFleetInstanceTypeConfigArgs.builder()
.ebsConfigs(InstanceFleetInstanceTypeConfigEbsConfigArgs.builder()
.size(100)
.type("gp2")
.volumesPerInstance(1)
.build())
.bidPriceAsPercentageOfOnDemandPrice(100.0)
.instanceType("m4.2xlarge")
.weightedCapacity(2)
.build())
.clusterId(example.id())
.name("task fleet")
.targetOnDemandCapacity(1)
.targetSpotCapacity(1)
.build());
}
}
resources:
example:
type: aws:emr:Cluster
properties:
masterInstanceFleet:
instanceTypeConfigs:
- instanceType: m4.xlarge
targetOnDemandCapacity: 1
coreInstanceFleet:
launchSpecifications:
spotSpecifications:
- allocationStrategy: capacity-optimized
blockDurationMinutes: 0
timeoutAction: SWITCH_TO_ON_DEMAND
timeoutDurationMinutes: 10
instanceTypeConfigs:
- ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
bidPriceAsPercentageOfOnDemandPrice: 80
instanceType: m3.xlarge
weightedCapacity: 1
- ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
bidPriceAsPercentageOfOnDemandPrice: 100
instanceType: m4.xlarge
weightedCapacity: 1
- ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
bidPriceAsPercentageOfOnDemandPrice: 100
instanceType: m4.2xlarge
weightedCapacity: 2
name: core fleet
targetOnDemandCapacity: 2
targetSpotCapacity: 2
task:
type: aws:emr:InstanceFleet
properties:
launchSpecifications:
spotSpecifications:
- allocationStrategy: capacity-optimized
blockDurationMinutes: 0
timeoutAction: TERMINATE_CLUSTER
timeoutDurationMinutes: 10
instanceTypeConfigs:
- ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
bidPriceAsPercentageOfOnDemandPrice: 100
instanceType: m4.xlarge
weightedCapacity: 1
- ebsConfigs:
- size: 100
type: gp2
volumesPerInstance: 1
bidPriceAsPercentageOfOnDemandPrice: 100
instanceType: m4.2xlarge
weightedCapacity: 2
clusterId: ${example.id}
name: task fleet
targetOnDemandCapacity: 1
targetSpotCapacity: 1
Enable Debug Logging
Debug logging in EMR is implemented as a step. It is highly recommended that you utilize the resource options configuration with ignoreChanges if other steps are being managed outside of this provider.
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.emr.Cluster("example", {steps: [{
hadoopJarStep: {
jar: "command-runner.jar",
args: ["state-pusher-script"],
},
actionOnFailure: "TERMINATE_CLUSTER",
name: "Setup Hadoop Debugging",
}]}, {
ignoreChanges: ["steps"],
});
import pulumi
import pulumi_aws as aws
example = aws.emr.Cluster("example", steps=[{
"hadoop_jar_step": {
"jar": "command-runner.jar",
"args": ["state-pusher-script"],
},
"action_on_failure": "TERMINATE_CLUSTER",
"name": "Setup Hadoop Debugging",
}],
opts = pulumi.ResourceOptions(ignore_changes=["steps"]))
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Emr.Cluster("example", new()
{
Steps = new[]
{
new Aws.Emr.Inputs.ClusterStepArgs
{
HadoopJarStep = new Aws.Emr.Inputs.ClusterStepHadoopJarStepArgs
{
Jar = "command-runner.jar",
Args = new[]
{
"state-pusher-script",
},
},
ActionOnFailure = "TERMINATE_CLUSTER",
Name = "Setup Hadoop Debugging",
},
},
}, new CustomResourceOptions
{
IgnoreChanges =
{
"steps",
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := emr.NewCluster(ctx, "example", &emr.ClusterArgs{
Steps: emr.ClusterStepArray{
&emr.ClusterStepArgs{
HadoopJarStep: &emr.ClusterStepHadoopJarStepArgs{
Jar: pulumi.String("command-runner.jar"),
Args: pulumi.StringArray{
pulumi.String("state-pusher-script"),
},
},
ActionOnFailure: pulumi.String("TERMINATE_CLUSTER"),
Name: pulumi.String("Setup Hadoop Debugging"),
},
},
}, pulumi.IgnoreChanges([]string{
"steps",
}))
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_emr_cluster" "example" {
lifecycle {
ignore_changes = [steps]
}
steps {
hadoop_jar_step = {
jar = "command-runner.jar"
args = ["state-pusher-script"]
}
action_on_failure = "TERMINATE_CLUSTER"
name = "Setup Hadoop Debugging"
}
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterStepArgs;
import com.pulumi.aws.emr.inputs.ClusterStepHadoopJarStepArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.builder()
.steps(ClusterStepArgs.builder()
.hadoopJarStep(ClusterStepHadoopJarStepArgs.builder()
.jar("command-runner.jar")
.args("state-pusher-script")
.build())
.actionOnFailure("TERMINATE_CLUSTER")
.name("Setup Hadoop Debugging")
.build())
.build(), CustomResourceOptions.builder()
.ignoreChanges("steps")
.build());
}
}
resources:
example:
type: aws:emr:Cluster
properties:
steps:
- hadoopJarStep:
jar: command-runner.jar
args:
- state-pusher-script
actionOnFailure: TERMINATE_CLUSTER
name: Setup Hadoop Debugging
options:
ignoreChanges:
- steps
Multiple Node Master Instance Group
Available in EMR version 5.23.0 and later, an EMR Cluster can be launched with three master nodes for high availability. Additional information about this functionality and its requirements can be found in the EMR Management Guide.
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
const example = new aws.ec2.Subnet("example", {mapPublicIpOnLaunch: true});
const exampleCluster = new aws.emr.Cluster("example", {
ec2Attributes: {
subnetId: example.id,
},
masterInstanceGroup: {
instanceCount: 3,
},
coreInstanceGroup: {},
releaseLabel: "emr-5.24.1",
terminationProtection: true,
});
import pulumi
import pulumi_aws as aws
# This configuration is for illustrative purposes and highlights
# only relevant configurations for working with this functionality.
# Map public IP on launch must be enabled for public (Internet accessible) subnets
example = aws.ec2.Subnet("example", map_public_ip_on_launch=True)
example_cluster = aws.emr.Cluster("example",
ec2_attributes={
"subnet_id": example.id,
},
master_instance_group={
"instance_count": 3,
},
core_instance_group={},
release_label="emr-5.24.1",
termination_protection=True)
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
var example = new Aws.Ec2.Subnet("example", new()
{
MapPublicIpOnLaunch = true,
});
var exampleCluster = new Aws.Emr.Cluster("example", new()
{
Ec2Attributes = new Aws.Emr.Inputs.ClusterEc2AttributesArgs
{
SubnetId = example.Id,
},
MasterInstanceGroup = new Aws.Emr.Inputs.ClusterMasterInstanceGroupArgs
{
InstanceCount = 3,
},
CoreInstanceGroup = null,
ReleaseLabel = "emr-5.24.1",
TerminationProtection = true,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/ec2"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
example, err := ec2.NewSubnet(ctx, "example", &ec2.SubnetArgs{
MapPublicIpOnLaunch: pulumi.Bool(true),
})
if err != nil {
return err
}
_, err = emr.NewCluster(ctx, "example", &emr.ClusterArgs{
Ec2Attributes: &emr.ClusterEc2AttributesArgs{
SubnetId: example.ID().ToIDOutput().ToStringOutput(),
},
MasterInstanceGroup: &emr.ClusterMasterInstanceGroupArgs{
InstanceCount: pulumi.Int(3),
},
CoreInstanceGroup: &emr.ClusterCoreInstanceGroupArgs{},
ReleaseLabel: pulumi.String("emr-5.24.1"),
TerminationProtection: pulumi.Bool(true),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
# This configuration is for illustrative purposes and highlights
# only relevant configurations for working with this functionality.
# Map public IP on launch must be enabled for public (Internet accessible) subnets
resource "aws_ec2_subnet" "example" {
map_public_ip_on_launch = true
}
resource "aws_emr_cluster" "example" {
ec2_attributes = {
subnet_id = aws_ec2_subnet.example.id
}
# ... other configuration ...
master_instance_group = {
instance_count = 3
}
# ... other configuration ...
# Master instance count must be set to 3
core_instance_group = {}
release_label = "emr-5.24.1"
termination_protection = true
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.ec2.Subnet;
import com.pulumi.aws.ec2.SubnetArgs;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.aws.emr.inputs.ClusterEc2AttributesArgs;
import com.pulumi.aws.emr.inputs.ClusterMasterInstanceGroupArgs;
import com.pulumi.aws.emr.inputs.ClusterCoreInstanceGroupArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
// This configuration is for illustrative purposes and highlights
// only relevant configurations for working with this functionality.
// Map public IP on launch must be enabled for public (Internet accessible) subnets
var example = new Subnet("example", SubnetArgs.builder()
.mapPublicIpOnLaunch(true)
.build());
var exampleCluster = new Cluster("exampleCluster", ClusterArgs.builder()
.ec2Attributes(ClusterEc2AttributesArgs.builder()
.subnetId(example.id())
.build())
.masterInstanceGroup(ClusterMasterInstanceGroupArgs.builder()
.instanceCount(3)
.build())
.coreInstanceGroup(ClusterCoreInstanceGroupArgs.builder()
.build())
.releaseLabel("emr-5.24.1")
.terminationProtection(true)
.build());
}
}
resources:
# This configuration is for illustrative purposes and highlights
# only relevant configurations for working with this functionality.
# Map public IP on launch must be enabled for public (Internet accessible) subnets
example:
type: aws:ec2:Subnet
properties:
mapPublicIpOnLaunch: true
exampleCluster:
type: aws:emr:Cluster
name: example
properties:
ec2Attributes:
subnetId: ${example.id}
masterInstanceGroup:
instanceCount: 3
coreInstanceGroup: {}
releaseLabel: emr-5.24.1
terminationProtection: true
Import
Using pulumi import, import EMR clusters using the id. For example:
$ pulumi import aws:emr/cluster:Cluster cluster j-123456ABCDEF
Since the API does not return the actual values for Kerberos configurations, environments with those options set will need to use the lifecycle configuration block ignoreChanges argument available to all Pulumi resources to prevent perpetual differences. For example:
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.emr.Cluster("example", {}, {
ignoreChanges: ["kerberosAttributes"],
});
import pulumi
import pulumi_aws as aws
example = aws.emr.Cluster("example", opts = pulumi.ResourceOptions(ignore_changes=["kerberosAttributes"]))
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Emr.Cluster("example", new()
{
}, new CustomResourceOptions
{
IgnoreChanges =
{
"kerberosAttributes",
},
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/emr"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := emr.NewCluster(ctx, "example", nil, pulumi.IgnoreChanges([]string{
"kerberosAttributes",
}))
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_emr_cluster" "example" {
lifecycle {
ignore_changes = [kerberosAttributes]
}
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.emr.Cluster;
import com.pulumi.aws.emr.ClusterArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Cluster("example", ClusterArgs.Empty, CustomResourceOptions.builder()
.ignoreChanges("kerberosAttributes")
.build());
}
}
resources:
example:
type: aws:emr:Cluster
options:
ignoreChanges:
- kerberosAttributes
Constructors
- Cluster(String name, {ClusterArgs? args, CustomResourceOptions? options})
-
Creates a new Cluster.
nameThe Pulumi resource name.argsArguments used to configure this Cluster. The set of arguments for Cluster.optionsResource options controlling this resource's behavior. - Cluster.reference(String urn)
- Creates a typed reference to an existing Cluster resource.
Properties
-
additionalInfo
↔ Output<
String?> -
JSON string for selecting additional features such as adding proxy information. Note: Currently there is no API to retrieve the value of this argument after EMR cluster creation from provider, therefore the provider cannot detect drift from the actual EMR cluster if its value is changed outside the provider.
latefinal
-
applications
↔ Output<
List< String> ?> -
A case-insensitive list of applications for Amazon EMR to install and configure when launching the cluster. For a list of applications available for each Amazon EMR release version, see the Amazon EMR Release Guide.
latefinal
-
arn
↔ Output<
String> -
ARN of the cluster.
latefinal
-
autoscalingRole
↔ Output<
String?> -
IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group.
latefinal
-
autoTerminationPolicy
↔ Output<
ClusterAutoTerminationPolicy?> -
An auto-termination policy for an Amazon EMR cluster. An auto-termination policy defines the amount of idle time in seconds after which a cluster automatically terminates. See Auto Termination Policy Below.
latefinal
-
bootstrapActions
↔ Output<
List< ClusterBootstrapAction> ?> -
Ordered list of bootstrap actions that will be run before Hadoop is started on the cluster nodes. See below.
latefinal
-
childResources
→ Set<
Resource> -
finalinherited
-
clusterState
↔ Output<
String> -
latefinal
-
completionSources
↔ Map<
String, IOutputCompletionSource> -
latefinalinherited
-
configurations
↔ Output<
String?> -
List of configurations supplied for the EMR cluster you are creating. Supply a configuration object for applications to override their default configuration. See AWS Documentation for more information.
latefinal
-
configurationsJson
↔ Output<
String?> -
JSON string for supplying list of configurations for the EMR cluster.
latefinal
-
coreInstanceFleet
↔ Output<
ClusterCoreInstanceFleet> -
Configuration block to use an Instance Fleet for the core node type. Cannot be specified if any
coreInstanceGroupconfiguration blocks are set. Detailed below.latefinal -
coreInstanceGroup
↔ Output<
ClusterCoreInstanceGroup> -
Configuration block to use an Instance Group for the core node type.
latefinal
-
customAmiId
↔ Output<
String?> -
Custom Amazon Linux AMI for the cluster (instead of an EMR-owned AMI). Available in Amazon EMR version 5.7.0 and later.
latefinal
-
ebsRootVolumeSize
↔ Output<
int?> -
Size in GiB of the EBS root device volume of the Linux AMI that is used for each EC2 instance. Available in Amazon EMR version 4.x and later.
latefinal
-
ec2Attributes
↔ Output<
ClusterEc2Attributes?> -
Attributes for the EC2 instances running the job flow. See below.
latefinal
- hashCode → int
-
The hash code for this object.
no setterinherited
-
id
↔ Output<
String> -
getter/setter pairinherited
- isCustom → bool
-
Returns whether this resource is provider-managed.
no setterinherited
- isProtected → bool
-
Returns whether this resource is protected from deletion.
no setterinherited
- isRemote → bool
-
Whether this resource is registered as remote.
no setterinherited
- isResourceReference → bool
-
Whether this instance represents a resource value returned over RPC.
finalinherited
-
keepJobFlowAliveWhenNoSteps
↔ Output<
bool> -
Switch on/off run cluster with no steps or when all steps are complete (default is on)
latefinal
-
kerberosAttributes
↔ Output<
ClusterKerberosAttributes?> -
Kerberos configuration for the cluster. See below.
latefinal
-
listStepsStates
↔ Output<
List< String> ?> -
List of step states used to filter returned steps
latefinal
-
logEncryptionKmsKeyId
↔ Output<
String?> -
AWS KMS customer master key (CMK) key ID or arn used for encrypting log files. This attribute is only available with EMR version 5.30.0 and later, excluding EMR 6.0.0.
latefinal
-
logUri
↔ Output<
String?> -
S3 bucket to write the log files of the job flow. If a value is not provided, logs are not created.
latefinal
-
masterInstanceFleet
↔ Output<
ClusterMasterInstanceFleet> -
Configuration block to use an Instance Fleet for the master node type. Cannot be specified if any
masterInstanceGroupconfiguration blocks are set. Detailed below.latefinal -
masterInstanceGroup
↔ Output<
ClusterMasterInstanceGroup> -
Configuration block to use an Instance Group for the master node type.
latefinal
-
masterPublicDns
↔ Output<
String> -
The DNS name of the master node. If the cluster is on a private subnet, this is the private DNS name. On a public subnet, this is the public DNS name.
latefinal
-
name
↔ Output<
String> -
Name of the job flow.
latefinal
-
osReleaseLabel
↔ Output<
String?> -
Amazon Linux release for all nodes in a cluster launch RunJobFlow request. If not specified, Amazon EMR uses the latest validated Amazon Linux release for cluster launch.
latefinal
-
placementGroupConfigs
↔ Output<
List< ClusterPlacementGroupConfig> ?> -
The specified placement group configuration for an Amazon EMR cluster.
latefinal
-
region
↔ Output<
String> -
Region where this resource will be managed. Defaults to the Region set in the provider configuration.
latefinal
-
releaseLabel
↔ Output<
String> -
Release label for the Amazon EMR release.
latefinal
-
resourceTransforms
→ List<
ResourceTransform> -
Inherited/explicit async transforms.
no setterinherited
- runtimeType → Type
-
A representation of the runtime type of the object.
no setterinherited
-
scaleDownBehavior
↔ Output<
String> -
Way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an
instance groupis resized.latefinal -
securityConfiguration
↔ Output<
String?> -
Security configuration name to attach to the EMR cluster. Only valid for EMR clusters with
releaseLabel4.8.0 or greater.latefinal -
serviceRole
↔ Output<
String> -
IAM role that will be assumed by the Amazon EMR service to access AWS resources.
latefinal
-
stepConcurrencyLevel
↔ Output<
int?> -
Number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with
releaseLabel5.28.0 or greater (default is 1).latefinal -
steps
↔ Output<
List< ClusterStep> > -
List of steps to run when creating the cluster. See below. It is highly recommended to utilize the lifecycle resource options block with
ignoreChangesif other steps are being managed outside of this provider.latefinal -
list of tags to apply to the EMR Cluster. If configured with a provider
defaultTagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.latefinal -
Map of tags assigned to the resource, including those inherited from the provider
defaultTagsconfiguration block.latefinal -
terminationProtection
↔ Output<
bool> -
Switch on/off termination protection (default is
false, except when using multiple master nodes). Before attempting to destroy the resource when termination protection is enabled, this configuration must be applied with its value set tofalse.latefinal -
transformations
→ List<
ResourceTransformation> -
Inherited/explicit legacy transformations.
no setterinherited
-
unhealthyNodeReplacement
↔ Output<
bool?> -
Whether whether Amazon EMR should gracefully replace core nodes that have degraded within the cluster. Default value is
false.latefinal -
urn
↔ Output<
String> -
latefinalinherited
-
visibleToAllUsers
↔ Output<
bool?> -
Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default value is
true.latefinal
Methods
-
failId(
Object error) → void -
Completes this resource ID with an error when registration fails.
inherited
-
failOutputs(
Object error) → void -
Completes all output properties with
error.inherited -
failUrn(
Object error) → void -
Completes this resource URN with an error when registration fails.
inherited
-
getProvider(
String moduleMember) → ProviderResource? -
Returns provider for
moduleMember's package, if configured.inherited -
getResourceName(
) → String -
Returns this resource's logical name.
inherited
-
getResourceType(
) → String -
Returns this resource's Pulumi type token.
inherited
-
noSuchMethod(
Invocation invocation) → dynamic -
Invoked when a nonexistent method or property is accessed.
inherited
-
registerOutput<
T> (String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output< T> -
Registers a dynamic output property for this resource.
inherited
-
resolveId(
String? value, {required bool isKnown}) → void -
Resolves the provider-assigned ID for this resource.
inherited
-
resolveOutputs(
Struct outputs) → void -
Resolves all output properties from a monitor response payload.
inherited
-
resolveUrn(
String value) → void -
Resolves this resource's URN once assigned by the engine.
inherited
-
serializeProperties(
Map< String, dynamic> properties) → Future<Struct> -
Serializes resource properties for RPC transmission.
inherited
-
toString(
) → String -
A string representation of this object.
inherited
Operators
-
operator ==(
Object other) → bool -
The equality operator.
inherited