Job class

Manages a job resource within a Dataproc cluster within GCE. For more information see the official dataproc documentation.

> Note: This resource does not support 'update' and changing any attributes will cause the resource to be recreated.

Example Usage

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const mycluster = new gcp.dataproc.Cluster("mycluster", {
    name: "dproc-cluster-unique-name",
    region: "us-central1",
});
// Submit an example spark job to a dataproc cluster
const spark = new gcp.dataproc.Job("spark", {
    region: mycluster.region,
    forceDelete: true,
    placement: {
        clusterName: mycluster.name,
    },
    sparkConfig: {
        mainClass: "org.apache.spark.examples.SparkPi",
        jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
        args: ["1000"],
        properties: {
            "spark.logConf": "true",
        },
        loggingConfig: {
            driverLogLevels: {
                root: "INFO",
            },
        },
    },
});
// Submit an example pyspark job to a dataproc cluster
const pyspark = new gcp.dataproc.Job("pyspark", {
    region: mycluster.region,
    forceDelete: true,
    placement: {
        clusterName: mycluster.name,
    },
    pysparkConfig: {
        mainPythonFileUri: "gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py",
        properties: {
            "spark.logConf": "true",
        },
    },
});
export const sparkStatus = spark.statuses[0].state;
export const pysparkStatus = pyspark.statuses[0].state;

import pulumi
import pulumi_gcp as gcp

mycluster = gcp.dataproc.Cluster("mycluster",
    name="dproc-cluster-unique-name",
    region="us-central1")
# Submit an example spark job to a dataproc cluster
spark = gcp.dataproc.Job("spark",
    region=mycluster.region,
    force_delete=True,
    placement={
        "cluster_name": mycluster.name,
    },
    spark_config={
        "main_class": "org.apache.spark.examples.SparkPi",
        "jar_file_uris": ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
        "args": ["1000"],
        "properties": {
            "spark.logConf": "true",
        },
        "logging_config": {
            "driver_log_levels": {
                "root": "INFO",
            },
        },
    })
# Submit an example pyspark job to a dataproc cluster
pyspark = gcp.dataproc.Job("pyspark",
    region=mycluster.region,
    force_delete=True,
    placement={
        "cluster_name": mycluster.name,
    },
    pyspark_config={
        "main_python_file_uri": "gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py",
        "properties": {
            "spark.logConf": "true",
        },
    })
pulumi.export("sparkStatus", spark.statuses[0].state)
pulumi.export("pysparkStatus", pyspark.statuses[0].state)

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() =>
{
    var mycluster = new Gcp.Dataproc.Cluster("mycluster", new()
    {
        Name = "dproc-cluster-unique-name",
        Region = "us-central1",
    });

    // Submit an example spark job to a dataproc cluster
    var spark = new Gcp.Dataproc.Job("spark", new()
    {
        Region = mycluster.Region,
        ForceDelete = true,
        Placement = new Gcp.Dataproc.Inputs.JobPlacementArgs
        {
            ClusterName = mycluster.Name,
        },
        SparkConfig = new Gcp.Dataproc.Inputs.JobSparkConfigArgs
        {
            MainClass = "org.apache.spark.examples.SparkPi",
            JarFileUris = new[]
            {
                "file:///usr/lib/spark/examples/jars/spark-examples.jar",
            },
            Args = new[]
            {
                "1000",
            },
            Properties =
            {
                { "spark.logConf", "true" },
            },
            LoggingConfig = new Gcp.Dataproc.Inputs.JobSparkConfigLoggingConfigArgs
            {
                DriverLogLevels =
                {
                    { "root", "INFO" },
                },
            },
        },
    });

    // Submit an example pyspark job to a dataproc cluster
    var pyspark = new Gcp.Dataproc.Job("pyspark", new()
    {
        Region = mycluster.Region,
        ForceDelete = true,
        Placement = new Gcp.Dataproc.Inputs.JobPlacementArgs
        {
            ClusterName = mycluster.Name,
        },
        PysparkConfig = new Gcp.Dataproc.Inputs.JobPysparkConfigArgs
        {
            MainPythonFileUri = "gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py",
            Properties =
            {
                { "spark.logConf", "true" },
            },
        },
    });

    return new Dictionary<string, object?>
    {
        ["sparkStatus"] = spark.Statuses.Apply(statuses => statuses[0].State),
        ["pysparkStatus"] = pyspark.Statuses.Apply(statuses => statuses[0].State),
    };
});

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		mycluster, err := dataproc.NewCluster(ctx, "mycluster", &dataproc.ClusterArgs{
			Name:   pulumi.String("dproc-cluster-unique-name"),
			Region: pulumi.String("us-central1"),
		})
		if err != nil {
			return err
		}
		// Submit an example spark job to a dataproc cluster
		spark, err := dataproc.NewJob(ctx, "spark", &dataproc.JobArgs{
			Region:      mycluster.Region,
			ForceDelete: pulumi.Bool(true),
			Placement: &dataproc.JobPlacementArgs{
				ClusterName: mycluster.Name,
			},
			SparkConfig: &dataproc.JobSparkConfigArgs{
				MainClass: pulumi.String("org.apache.spark.examples.SparkPi"),
				JarFileUris: pulumi.StringArray{
					pulumi.String("file:///usr/lib/spark/examples/jars/spark-examples.jar"),
				},
				Args: pulumi.StringArray{
					pulumi.String("1000"),
				},
				Properties: pulumi.StringMap{
					"spark.logConf": pulumi.String("true"),
				},
				LoggingConfig: &dataproc.JobSparkConfigLoggingConfigArgs{
					DriverLogLevels: pulumi.StringMap{
						"root": pulumi.String("INFO"),
					},
				},
			},
		})
		if err != nil {
			return err
		}
		// Submit an example pyspark job to a dataproc cluster
		pyspark, err := dataproc.NewJob(ctx, "pyspark", &dataproc.JobArgs{
			Region:      mycluster.Region,
			ForceDelete: pulumi.Bool(true),
			Placement: &dataproc.JobPlacementArgs{
				ClusterName: mycluster.Name,
			},
			PysparkConfig: &dataproc.JobPysparkConfigArgs{
				MainPythonFileUri: pulumi.String("gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py"),
				Properties: pulumi.StringMap{
					"spark.logConf": pulumi.String("true"),
				},
			},
		})
		if err != nil {
			return err
		}
		ctx.Export("sparkStatus", spark.Statuses.ApplyT(func(statuses []dataproc.JobStatus) (*string, error) {
			return statuses[0].State, nil
		}).(pulumi.StringPtrOutput))
		ctx.Export("pysparkStatus", pyspark.Statuses.ApplyT(func(statuses []dataproc.JobStatus) (*string, error) {
			return statuses[0].State, nil
		}).(pulumi.StringPtrOutput))
		return nil
	})
}

pulumi {
  required_providers {
    gcp = {
      source = "pulumi/gcp"
    }
  }
}

resource "gcp_dataproc_cluster" "mycluster" {
  name   = "dproc-cluster-unique-name"
  region = "us-central1"
}
# Submit an example spark job to a dataproc cluster
resource "gcp_dataproc_job" "spark" {
  region       = gcp_dataproc_cluster.mycluster.region
  force_delete = true
  placement = {
    cluster_name = gcp_dataproc_cluster.mycluster.name
  }
  spark_config = {
    main_class    = "org.apache.spark.examples.SparkPi"
    jar_file_uris = ["file:///usr/lib/spark/examples/jars/spark-examples.jar"]
    args          = ["1000"]
    properties = {
      "spark.logConf" = "true"
    }
    logging_config = {
      driver_log_levels = {
        "root" = "INFO"
      }
    }
  }
}
# Submit an example pyspark job to a dataproc cluster
resource "gcp_dataproc_job" "pyspark" {
  region       = gcp_dataproc_cluster.mycluster.region
  force_delete = true
  placement = {
    cluster_name = gcp_dataproc_cluster.mycluster.name
  }
  pyspark_config = {
    main_python_file_uri = "gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py"
    properties = {
      "spark.logConf" = "true"
    }
  }
}
# Check out current state of the jobs
output "sparkStatus" {
  value = gcp_dataproc_job.spark.statuses[0].state
}
output "pysparkStatus" {
  value = gcp_dataproc_job.pyspark.statuses[0].state
}

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataproc.Cluster;
import com.pulumi.gcp.dataproc.ClusterArgs;
import com.pulumi.gcp.dataproc.Job;
import com.pulumi.gcp.dataproc.JobArgs;
import com.pulumi.gcp.dataproc.inputs.JobPlacementArgs;
import com.pulumi.gcp.dataproc.inputs.JobSparkConfigArgs;
import com.pulumi.gcp.dataproc.inputs.JobSparkConfigLoggingConfigArgs;
import com.pulumi.gcp.dataproc.inputs.JobPysparkConfigArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var mycluster = new Cluster("mycluster", ClusterArgs.builder()
            .name("dproc-cluster-unique-name")
            .region("us-central1")
            .build());

        // Submit an example spark job to a dataproc cluster
        var spark = new Job("spark", JobArgs.builder()
            .region(mycluster.region())
            .forceDelete(true)
            .placement(JobPlacementArgs.builder()
                .clusterName(mycluster.name())
                .build())
            .sparkConfig(JobSparkConfigArgs.builder()
                .mainClass("org.apache.spark.examples.SparkPi")
                .jarFileUris("file:///usr/lib/spark/examples/jars/spark-examples.jar")
                .args("1000")
                .properties(Map.of("spark.logConf", "true"))
                .loggingConfig(JobSparkConfigLoggingConfigArgs.builder()
                    .driverLogLevels(Map.of("root", "INFO"))
                    .build())
                .build())
            .build());

        // Submit an example pyspark job to a dataproc cluster
        var pyspark = new Job("pyspark", JobArgs.builder()
            .region(mycluster.region())
            .forceDelete(true)
            .placement(JobPlacementArgs.builder()
                .clusterName(mycluster.name())
                .build())
            .pysparkConfig(JobPysparkConfigArgs.builder()
                .mainPythonFileUri("gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py")
                .properties(Map.of("spark.logConf", "true"))
                .build())
            .build());

        ctx.export("sparkStatus", spark.statuses().applyValue(_statuses -> _statuses.get(0).state()));
        ctx.export("pysparkStatus", pyspark.statuses().applyValue(_statuses -> _statuses.get(0).state()));
    }
}

resources:
  mycluster:
    type: gcp:dataproc:Cluster
    properties:
      name: dproc-cluster-unique-name
      region: us-central1
  # Submit an example spark job to a dataproc cluster
  spark:
    type: gcp:dataproc:Job
    properties:
      region: ${mycluster.region}
      forceDelete: true
      placement:
        clusterName: ${mycluster.name}
      sparkConfig:
        mainClass: org.apache.spark.examples.SparkPi
        jarFileUris:
          - file:///usr/lib/spark/examples/jars/spark-examples.jar
        args:
          - '1000'
        properties:
          spark.logConf: 'true'
        loggingConfig:
          driverLogLevels:
            root: INFO
  # Submit an example pyspark job to a dataproc cluster
  pyspark:
    type: gcp:dataproc:Job
    properties:
      region: ${mycluster.region}
      forceDelete: true
      placement:
        clusterName: ${mycluster.name}
      pysparkConfig:
        mainPythonFileUri: gs://dataproc-examples-2f10d78d114f6aaec76462e3c310f31f/src/pyspark/hello-world/hello-world.py
        properties:
          spark.logConf: 'true'
outputs:
  # Check out current state of the jobs
  sparkStatus: ${spark.statuses[0].state}
  pysparkStatus: ${pyspark.statuses[0].state}

Import

This resource does not support import.

Constructors

Job(String name, {JobArgs? args, CustomResourceOptions? options})

Creates a new Job. name The Pulumi resource name. args Arguments used to configure this Job. The set of arguments for Job. options Resource options controlling this resource's behavior.

Job.reference(String urn)

Creates a typed reference to an existing Job resource.

Properties

childResources → Set<Resource>

final, inherited

completionSources ↔ Map<String, IOutputCompletionSource>

late, final, inherited

deletionPolicy ↔ Output<String>

Whether Terraform will be prevented from destroying the resource. Defaults to "DELETE". When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.

late, final

driverControlsFilesUri ↔ Output<String>

If present, the location of miscellaneous control files which may be used as part of job setup and handling. If not present, control files may be placed in the same location as driver_output_uri.

late, final

driverOutputResourceUri ↔ Output<String>

A URI pointing to the location of the stdout of the job's driver program.

late, final

effectiveLabels ↔ Output<Map<String, String>>

All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.

late, final

forceDelete ↔ Output<bool?>

By default, you can only delete inactive jobs within Dataproc. Setting this to true, and calling destroy, will ensure that the job is first cancelled before issuing the delete.

late, final

hadoopConfig ↔ Output<JobHadoopConfig?>

The config of Hadoop job

late, final

hashCode → int

The hash code for this object.

no setter, inherited

hiveConfig ↔ Output<JobHiveConfig?>

The config of hive job

late, final

id ↔ Output<String>

getter/setter pair, inherited

isCustom → bool

Returns whether this resource is provider-managed.

no setter, inherited

isProtected → bool

Returns whether this resource is protected from deletion.

no setter, inherited

isRemote → bool

Whether this resource is registered as remote.

no setter, inherited

isResourceReference → bool

Whether this instance represents a resource value returned over RPC.

final, inherited

labels ↔ Output<Map<String, String>?>

The list of labels (key/value pairs) to add to the job. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource.

late, final

pigConfig ↔ Output<JobPigConfig?>

The config of pag job.

late, final

placement ↔ Output<JobPlacement>

The config of job placement.

late, final

prestoConfig ↔ Output<JobPrestoConfig?>

The config of presto job

late, final

project ↔ Output<String>

The project in which the cluster can be found and jobs subsequently run against. If it is not provided, the provider project is used.

late, final

pulumiLabels ↔ Output<Map<String, String>>

The combination of labels configured directly on the resource and default labels configured on the provider.

late, final

pysparkConfig ↔ Output<JobPysparkConfig?>

The config of pySpark job.

late, final

reference ↔ Output<JobReference>

The reference of the job

late, final

region ↔ Output<String?>

The Cloud Dataproc region. This essentially determines which clusters are available for this job to be submitted to. If not specified, defaults to global.

late, final

resourceTransforms → List<ResourceTransform>

Inherited/explicit async transforms.

no setter, inherited

runtimeType → Type

A representation of the runtime type of the object.

no setter, inherited

scheduling ↔ Output<JobScheduling?>

Optional. Job scheduling configuration.

late, final

sparkConfig ↔ Output<JobSparkConfig?>

The config of the Spark job.

late, final

sparksqlConfig ↔ Output<JobSparksqlConfig?>

The config of SparkSql job

late, final

statuses ↔ Output<List<JobStatus>>

The status of the job.

late, final

transformations → List<ResourceTransformation>

Inherited/explicit legacy transformations.

no setter, inherited

urn ↔ Output<String>

late, final, inherited

waitForCompletion ↔ Output<bool?>

If set to true, Terraform will wait for the job to reach a terminal state (DONE, ERROR, CANCELLED, ATTEMPT_FAILURE). Otherwise, Terraform will consider the job 'created' once it is in the RUNNING state.

late, final

Methods

failId(Object error) → void

Completes this resource ID with an error when registration fails.

inherited

failOutputs(Object error) → void

Completes all output properties with error.

inherited

failUrn(Object error) → void

Completes this resource URN with an error when registration fails.

inherited

getProvider(String moduleMember) → ProviderResource?

Returns provider for moduleMember's package, if configured.

inherited

getResourceName() → String

Returns this resource's logical name.

inherited

getResourceType() → String

Returns this resource's Pulumi type token.

inherited

noSuchMethod(Invocation invocation) → dynamic

Invoked when a nonexistent method or property is accessed.

inherited

registerOutput<T>(String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output<T>

Registers a dynamic output property for this resource.

inherited

resolveId(String? value, {required bool isKnown}) → void

Resolves the provider-assigned ID for this resource.

inherited

resolveOutputs(Struct outputs) → void

Resolves all output properties from a monitor response payload.

inherited

resolveUrn(String value) → void

Resolves this resource's URN once assigned by the engine.

inherited

serializeProperties(Map<String, dynamic> properties) → Future<Struct>

Serializes resource properties for RPC transmission.

inherited

toString() → String

A string representation of this object.

inherited

Operators

operator ==(Object other) → bool

The equality operator.

inherited

Static Methods

get(String name, Input<String> id, {JobState? state, CustomResourceOptions? options}) → Job

Gets an existing Job resource's state with the given name and id.