Crawler class
Manages a Glue Crawler. More information can be found in the AWS Glue Developer Guide
Example Usage
DynamoDB Target Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Crawler("example", {
dynamodbTargets: [{
path: "table-name",
}],
databaseName: exampleAwsGlueCatalogDatabase.name,
name: "example",
role: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Crawler("example",
dynamodb_targets=[{
"path": "table-name",
}],
database_name=example_aws_glue_catalog_database["name"],
name="example",
role=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Crawler("example", new()
{
DynamodbTargets = new[]
{
new Aws.Glue.Inputs.CrawlerDynamodbTargetArgs
{
Path = "table-name",
},
},
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Name = "example",
Role = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewCrawler(ctx, "example", &glue.CrawlerArgs{
DynamodbTargets: glue.CrawlerDynamodbTargetArray{
&glue.CrawlerDynamodbTargetArgs{
Path: pulumi.String("table-name"),
},
},
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Name: pulumi.String("example"),
Role: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "example" {
dynamodb_targets {
path = "table-name"
}
database_name = exampleAwsGlueCatalogDatabase.name
name = "example"
role = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerDynamodbTargetArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Crawler("example", CrawlerArgs.builder()
.dynamodbTargets(CrawlerDynamodbTargetArgs.builder()
.path("table-name")
.build())
.databaseName(exampleAwsGlueCatalogDatabase.name())
.name("example")
.role(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:glue:Crawler
properties:
dynamodbTargets:
- path: table-name
databaseName: ${exampleAwsGlueCatalogDatabase.name}
name: example
role: ${exampleAwsIamRole.arn}
JDBC Target Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Crawler("example", {
jdbcTargets: [{
connectionName: exampleAwsGlueConnection.name,
path: "database-name/%",
}],
databaseName: exampleAwsGlueCatalogDatabase.name,
name: "example",
role: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Crawler("example",
jdbc_targets=[{
"connection_name": example_aws_glue_connection["name"],
"path": "database-name/%",
}],
database_name=example_aws_glue_catalog_database["name"],
name="example",
role=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Crawler("example", new()
{
JdbcTargets = new[]
{
new Aws.Glue.Inputs.CrawlerJdbcTargetArgs
{
ConnectionName = exampleAwsGlueConnection.Name,
Path = "database-name/%",
},
},
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Name = "example",
Role = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewCrawler(ctx, "example", &glue.CrawlerArgs{
JdbcTargets: glue.CrawlerJdbcTargetArray{
&glue.CrawlerJdbcTargetArgs{
ConnectionName: pulumi.Any(exampleAwsGlueConnection.Name),
Path: pulumi.String("database-name/%"),
},
},
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Name: pulumi.String("example"),
Role: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "example" {
jdbc_targets {
connection_name = exampleAwsGlueConnection.name
path = "database-name/%"
}
database_name = exampleAwsGlueCatalogDatabase.name
name = "example"
role = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerJdbcTargetArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Crawler("example", CrawlerArgs.builder()
.jdbcTargets(CrawlerJdbcTargetArgs.builder()
.connectionName(exampleAwsGlueConnection.name())
.path("database-name/%")
.build())
.databaseName(exampleAwsGlueCatalogDatabase.name())
.name("example")
.role(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:glue:Crawler
properties:
jdbcTargets:
- connectionName: ${exampleAwsGlueConnection.name}
path: database-name/%
databaseName: ${exampleAwsGlueCatalogDatabase.name}
name: example
role: ${exampleAwsIamRole.arn}
S3 Target Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Crawler("example", {
s3Targets: [{
path: `s3://${exampleAwsS3Bucket.bucket}`,
}],
databaseName: exampleAwsGlueCatalogDatabase.name,
name: "example",
role: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Crawler("example",
s3_targets=[{
"path": f"s3://{example_aws_s3_bucket['bucket']}",
}],
database_name=example_aws_glue_catalog_database["name"],
name="example",
role=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Crawler("example", new()
{
S3Targets = new[]
{
new Aws.Glue.Inputs.CrawlerS3TargetArgs
{
Path = $"s3://{exampleAwsS3Bucket.Bucket}",
},
},
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Name = "example",
Role = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewCrawler(ctx, "example", &glue.CrawlerArgs{
S3Targets: glue.CrawlerS3TargetArray{
&glue.CrawlerS3TargetArgs{
Path: pulumi.Sprintf("s3://%v", exampleAwsS3Bucket.Bucket),
},
},
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Name: pulumi.String("example"),
Role: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "example" {
s3_targets {
path ="s3://${exampleAwsS3Bucket.bucket}"
}
database_name = exampleAwsGlueCatalogDatabase.name
name = "example"
role = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Crawler("example", CrawlerArgs.builder()
.s3Targets(CrawlerS3TargetArgs.builder()
.path(String.format("s3://%s", exampleAwsS3Bucket.bucket()))
.build())
.databaseName(exampleAwsGlueCatalogDatabase.name())
.name("example")
.role(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:glue:Crawler
properties:
s3Targets:
- path: s3://${exampleAwsS3Bucket.bucket}
databaseName: ${exampleAwsGlueCatalogDatabase.name}
name: example
role: ${exampleAwsIamRole.arn}
Catalog Target Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Crawler("example", {
schemaChangePolicy: {
deleteBehavior: "LOG",
},
catalogTargets: [{
databaseName: exampleAwsGlueCatalogDatabase.name,
tables: [exampleAwsGlueCatalogTable.name],
}],
databaseName: exampleAwsGlueCatalogDatabase.name,
name: "example",
role: exampleAwsIamRole.arn,
configuration: `{
\\"Version\\":1.0,
\\"Grouping\\": {
\\"TableGroupingPolicy\\": \\"CombineCompatibleSchemas\\"
}
}
`,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Crawler("example",
schema_change_policy={
"delete_behavior": "LOG",
},
catalog_targets=[{
"database_name": example_aws_glue_catalog_database["name"],
"tables": [example_aws_glue_catalog_table["name"]],
}],
database_name=example_aws_glue_catalog_database["name"],
name="example",
role=example_aws_iam_role["arn"],
configuration="""{
\"Version\":1.0,
\"Grouping\": {
\"TableGroupingPolicy\": \"CombineCompatibleSchemas\"
}
}
""")
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Crawler("example", new()
{
SchemaChangePolicy = new Aws.Glue.Inputs.CrawlerSchemaChangePolicyArgs
{
DeleteBehavior = "LOG",
},
CatalogTargets = new[]
{
new Aws.Glue.Inputs.CrawlerCatalogTargetArgs
{
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Tables = new[]
{
exampleAwsGlueCatalogTable.Name,
},
},
},
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Name = "example",
Role = exampleAwsIamRole.Arn,
Configuration = @"{
\""Version\"":1.0,
\""Grouping\"": {
\""TableGroupingPolicy\"": \""CombineCompatibleSchemas\""
}
}
",
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewCrawler(ctx, "example", &glue.CrawlerArgs{
SchemaChangePolicy: &glue.CrawlerSchemaChangePolicyArgs{
DeleteBehavior: pulumi.String("LOG"),
},
CatalogTargets: glue.CrawlerCatalogTargetArray{
&glue.CrawlerCatalogTargetArgs{
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Tables: pulumi.StringArray{
exampleAwsGlueCatalogTable.Name,
},
},
},
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Name: pulumi.String("example"),
Role: pulumi.Any(exampleAwsIamRole.Arn),
Configuration: pulumi.String(`{
\"Version\":1.0,
\"Grouping\": {
\"TableGroupingPolicy\": \"CombineCompatibleSchemas\"
}
}
`),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "example" {
schema_change_policy = {
delete_behavior = "LOG"
}
catalog_targets {
database_name = exampleAwsGlueCatalogDatabase.name
tables = [exampleAwsGlueCatalogTable.name]
}
database_name = exampleAwsGlueCatalogDatabase.name
name = "example"
role = exampleAwsIamRole.arn
configuration = "{\n \\\"Version\\\":1.0,\n \\\"Grouping\\\": {\n \\\"TableGroupingPolicy\\\": \\\"CombineCompatibleSchemas\\\"\n }\n}\n"
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerSchemaChangePolicyArgs;
import com.pulumi.aws.glue.inputs.CrawlerCatalogTargetArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Crawler("example", CrawlerArgs.builder()
.schemaChangePolicy(CrawlerSchemaChangePolicyArgs.builder()
.deleteBehavior("LOG")
.build())
.catalogTargets(CrawlerCatalogTargetArgs.builder()
.databaseName(exampleAwsGlueCatalogDatabase.name())
.tables(exampleAwsGlueCatalogTable.name())
.build())
.databaseName(exampleAwsGlueCatalogDatabase.name())
.name("example")
.role(exampleAwsIamRole.arn())
.configuration("""
{
\"Version\":1.0,
\"Grouping\": {
\"TableGroupingPolicy\": \"CombineCompatibleSchemas\"
}
}
""")
.build());
}
}
resources:
example:
type: aws:glue:Crawler
properties:
schemaChangePolicy:
deleteBehavior: LOG
catalogTargets:
- databaseName: ${exampleAwsGlueCatalogDatabase.name}
tables:
- ${exampleAwsGlueCatalogTable.name}
databaseName: ${exampleAwsGlueCatalogDatabase.name}
name: example
role: ${exampleAwsIamRole.arn}
configuration: |
{
\"Version\":1.0,
\"Grouping\": {
\"TableGroupingPolicy\": \"CombineCompatibleSchemas\"
}
}
MongoDB Target Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.glue.Crawler("example", {
mongodbTargets: [{
connectionName: exampleAwsGlueConnection.name,
path: "database-name/%",
}],
databaseName: exampleAwsGlueCatalogDatabase.name,
name: "example",
role: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.glue.Crawler("example",
mongodb_targets=[{
"connection_name": example_aws_glue_connection["name"],
"path": "database-name/%",
}],
database_name=example_aws_glue_catalog_database["name"],
name="example",
role=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Glue.Crawler("example", new()
{
MongodbTargets = new[]
{
new Aws.Glue.Inputs.CrawlerMongodbTargetArgs
{
ConnectionName = exampleAwsGlueConnection.Name,
Path = "database-name/%",
},
},
DatabaseName = exampleAwsGlueCatalogDatabase.Name,
Name = "example",
Role = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := glue.NewCrawler(ctx, "example", &glue.CrawlerArgs{
MongodbTargets: glue.CrawlerMongodbTargetArray{
&glue.CrawlerMongodbTargetArgs{
ConnectionName: pulumi.Any(exampleAwsGlueConnection.Name),
Path: pulumi.String("database-name/%"),
},
},
DatabaseName: pulumi.Any(exampleAwsGlueCatalogDatabase.Name),
Name: pulumi.String("example"),
Role: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "example" {
mongodb_targets {
connection_name = exampleAwsGlueConnection.name
path = "database-name/%"
}
database_name = exampleAwsGlueCatalogDatabase.name
name = "example"
role = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerMongodbTargetArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new Crawler("example", CrawlerArgs.builder()
.mongodbTargets(CrawlerMongodbTargetArgs.builder()
.connectionName(exampleAwsGlueConnection.name())
.path("database-name/%")
.build())
.databaseName(exampleAwsGlueCatalogDatabase.name())
.name("example")
.role(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:glue:Crawler
properties:
mongodbTargets:
- connectionName: ${exampleAwsGlueConnection.name}
path: database-name/%
databaseName: ${exampleAwsGlueCatalogDatabase.name}
name: example
role: ${exampleAwsIamRole.arn}
Configuration Settings Example
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const eventsCrawler = new aws.glue.Crawler("events_crawler", {
s3Targets: [{
path: `s3://${dataLakeBucket.bucket}`,
}],
databaseName: glueDatabase.name,
schedule: "cron(0 1 * * ? *)",
name: `events_crawler_${environmentName}`,
role: glueRole.arn,
tags: tags,
configuration: JSON.stringify({
Grouping: {
TableGroupingPolicy: "CombineCompatibleSchemas",
},
CrawlerOutput: {
Partitions: {
AddOrUpdateBehavior: "InheritFromTable",
},
},
Version: 1,
}),
});
import pulumi
import json
import pulumi_aws as aws
events_crawler = aws.glue.Crawler("events_crawler",
s3_targets=[{
"path": f"s3://{data_lake_bucket['bucket']}",
}],
database_name=glue_database["name"],
schedule="cron(0 1 * * ? *)",
name=f"events_crawler_{environment_name}",
role=glue_role["arn"],
tags=tags,
configuration=json.dumps({
"Grouping": {
"TableGroupingPolicy": "CombineCompatibleSchemas",
},
"CrawlerOutput": {
"Partitions": {
"AddOrUpdateBehavior": "InheritFromTable",
},
},
"Version": 1,
}))
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var eventsCrawler = new Aws.Glue.Crawler("events_crawler", new()
{
S3Targets = new[]
{
new Aws.Glue.Inputs.CrawlerS3TargetArgs
{
Path = $"s3://{dataLakeBucket.Bucket}",
},
},
DatabaseName = glueDatabase.Name,
Schedule = "cron(0 1 * * ? *)",
Name = $"events_crawler_{environmentName}",
Role = glueRole.Arn,
Tags = tags,
Configuration = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["Grouping"] = new Dictionary<string, object?>
{
["TableGroupingPolicy"] = "CombineCompatibleSchemas",
},
["CrawlerOutput"] = new Dictionary<string, object?>
{
["Partitions"] = new Dictionary<string, object?>
{
["AddOrUpdateBehavior"] = "InheritFromTable",
},
},
["Version"] = 1,
}),
});
});
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/glue"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
tmpJSON0, err := json.Marshal(map[string]interface{}{
"Grouping": map[string]string{
"TableGroupingPolicy": "CombineCompatibleSchemas",
},
"CrawlerOutput": map[string]map[string]string{
"Partitions": map[string]string{
"AddOrUpdateBehavior": "InheritFromTable",
},
},
"Version": 1,
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
_, err = glue.NewCrawler(ctx, "events_crawler", &glue.CrawlerArgs{
S3Targets: glue.CrawlerS3TargetArray{
&glue.CrawlerS3TargetArgs{
Path: pulumi.Sprintf("s3://%v", dataLakeBucket.Bucket),
},
},
DatabaseName: pulumi.Any(glueDatabase.Name),
Schedule: pulumi.String("cron(0 1 * * ? *)"),
Name: pulumi.Sprintf("events_crawler_%v", environmentName),
Role: pulumi.Any(glueRole.Arn),
Tags: pulumi.Any(tags),
Configuration: pulumi.String(json0),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_glue_crawler" "events_crawler" {
s3_targets {
path ="s3://${dataLakeBucket.bucket}"
}
database_name = glueDatabase.name
schedule = "cron(0 1 * * ? *)"
name ="events_crawler_${environmentName}"
role = glueRole.arn
tags = tags
configuration = jsonencode({
"Grouping" = {
"TableGroupingPolicy" = "CombineCompatibleSchemas"
}
"CrawlerOutput" = {
"Partitions" = {
"AddOrUpdateBehavior" = "InheritFromTable"
}
}
"Version" = 1
})
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.glue.Crawler;
import com.pulumi.aws.glue.CrawlerArgs;
import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var eventsCrawler = new Crawler("eventsCrawler", CrawlerArgs.builder()
.s3Targets(CrawlerS3TargetArgs.builder()
.path(String.format("s3://%s", dataLakeBucket.bucket()))
.build())
.databaseName(glueDatabase.name())
.schedule("cron(0 1 * * ? *)")
.name(String.format("events_crawler_%s", environmentName))
.role(glueRole.arn())
.tags(tags)
.configuration(serializeJson(
jsonObject(
jsonProperty("Grouping", jsonObject(
jsonProperty("TableGroupingPolicy", "CombineCompatibleSchemas")
)),
jsonProperty("CrawlerOutput", jsonObject(
jsonProperty("Partitions", jsonObject(
jsonProperty("AddOrUpdateBehavior", "InheritFromTable")
))
)),
jsonProperty("Version", 1)
)))
.build());
}
}
resources:
eventsCrawler:
type: aws:glue:Crawler
name: events_crawler
properties:
s3Targets:
- path: s3://${dataLakeBucket.bucket}
databaseName: ${glueDatabase.name}
schedule: cron(0 1 * * ? *)
name: events_crawler_${environmentName}
role: ${glueRole.arn}
tags: ${tags}
configuration:
fn::toJSON:
Grouping:
TableGroupingPolicy: CombineCompatibleSchemas
CrawlerOutput:
Partitions:
AddOrUpdateBehavior: InheritFromTable
Version: 1
Import
Using pulumi import, import Glue Crawlers using name. For example:
$ pulumi import aws:glue/crawler:Crawler MyJob MyJob
Constructors
- Crawler(String name, {CrawlerArgs? args, CustomResourceOptions? options})
-
Creates a new Crawler.
nameThe Pulumi resource name.argsArguments used to configure this Crawler. The set of arguments for Crawler.optionsResource options controlling this resource's behavior. - Crawler.reference(String urn)
- Creates a typed reference to an existing Crawler resource.
Properties
-
arn
↔ Output<
String> -
The ARN of the crawler
latefinal
-
catalogTargets
↔ Output<
List< CrawlerCatalogTarget> ?> -
List of nested AWS Glue Data Catalog target arguments. See Catalog Target below.
latefinal
-
childResources
→ Set<
Resource> -
finalinherited
-
classifiers
↔ Output<
List< String> ?> -
List of custom classifiers. By default, all AWS classifiers are included in a crawl, but these custom classifiers always override the default classifiers for a given classification.
latefinal
-
completionSources
↔ Map<
String, IOutputCompletionSource> -
latefinalinherited
-
configuration
↔ Output<
String?> -
JSON string of configuration information. For more details see Setting Crawler Configuration Options.
latefinal
-
databaseName
↔ Output<
String> -
Glue database where results are written.
latefinal
-
deltaTargets
↔ Output<
List< CrawlerDeltaTarget> ?> -
List of nested Delta Lake target arguments. See Delta Target below.
latefinal
-
description
↔ Output<
String?> -
Description of the crawler.
latefinal
-
dynamodbTargets
↔ Output<
List< CrawlerDynamodbTarget> ?> -
List of nested DynamoDB target arguments. See Dynamodb Target below.
latefinal
- hashCode → int
-
The hash code for this object.
no setterinherited
-
hudiTargets
↔ Output<
List< CrawlerHudiTarget> ?> -
List of nested Hudi target arguments. See Iceberg Target below.
latefinal
-
icebergTargets
↔ Output<
List< CrawlerIcebergTarget> ?> -
List of nested Iceberg target arguments. See Iceberg Target below.
latefinal
-
id
↔ Output<
String> -
getter/setter pairinherited
- isCustom → bool
-
Returns whether this resource is provider-managed.
no setterinherited
- isProtected → bool
-
Returns whether this resource is protected from deletion.
no setterinherited
- isRemote → bool
-
Whether this resource is registered as remote.
no setterinherited
- isResourceReference → bool
-
Whether this instance represents a resource value returned over RPC.
finalinherited
-
jdbcTargets
↔ Output<
List< CrawlerJdbcTarget> ?> -
List of nested JDBC target arguments. See JDBC Target below.
latefinal
-
lakeFormationConfiguration
↔ Output<
CrawlerLakeFormationConfiguration?> -
Specifies Lake Formation configuration settings for the crawler. See Lake Formation Configuration below.
latefinal
-
lineageConfiguration
↔ Output<
CrawlerLineageConfiguration?> -
Specifies data lineage configuration settings for the crawler. See Lineage Configuration below.
latefinal
-
mongodbTargets
↔ Output<
List< CrawlerMongodbTarget> ?> -
List of nested MongoDB target arguments. See MongoDB Target below.
latefinal
-
name
↔ Output<
String> -
Name of the crawler.
latefinal
-
recrawlPolicy
↔ Output<
CrawlerRecrawlPolicy?> -
A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.. See Recrawl Policy below.
latefinal
-
region
↔ Output<
String> -
Region where this resource will be managed. Defaults to the Region set in the provider configuration.
latefinal
-
resourceTransforms
→ List<
ResourceTransform> -
Inherited/explicit async transforms.
no setterinherited
-
role
↔ Output<
String> -
The IAM role friendly name (including path without leading slash), or ARN of an IAM role, used by the crawler to access other resources.
latefinal
- runtimeType → Type
-
A representation of the runtime type of the object.
no setterinherited
-
s3Targets
↔ Output<
List< CrawlerS3Target> ?> -
List of nested Amazon S3 target arguments. See S3 Target below.
latefinal
-
schedule
↔ Output<
String?> -
A cron expression used to specify the schedule. For more information, see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify:
cron(15 12 * * ? *).latefinal -
schemaChangePolicy
↔ Output<
CrawlerSchemaChangePolicy?> -
Policy for the crawler's update and deletion behavior. See Schema Change Policy below.
latefinal
-
securityConfiguration
↔ Output<
String?> -
The name of Security Configuration to be used by the crawler
latefinal
-
tablePrefix
↔ Output<
String?> -
The table prefix used for catalog tables that are created.
latefinal
-
Key-value map of resource tags. .If configured with a provider
defaultTagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.latefinal -
A map of tags assigned to the resource, including those inherited from the provider
defaultTagsconfiguration block.latefinal -
transformations
→ List<
ResourceTransformation> -
Inherited/explicit legacy transformations.
no setterinherited
-
urn
↔ Output<
String> -
latefinalinherited
Methods
-
failId(
Object error) → void -
Completes this resource ID with an error when registration fails.
inherited
-
failOutputs(
Object error) → void -
Completes all output properties with
error.inherited -
failUrn(
Object error) → void -
Completes this resource URN with an error when registration fails.
inherited
-
getProvider(
String moduleMember) → ProviderResource? -
Returns provider for
moduleMember's package, if configured.inherited -
getResourceName(
) → String -
Returns this resource's logical name.
inherited
-
getResourceType(
) → String -
Returns this resource's Pulumi type token.
inherited
-
noSuchMethod(
Invocation invocation) → dynamic -
Invoked when a nonexistent method or property is accessed.
inherited
-
registerOutput<
T> (String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output< T> -
Registers a dynamic output property for this resource.
inherited
-
resolveId(
String? value, {required bool isKnown}) → void -
Resolves the provider-assigned ID for this resource.
inherited
-
resolveOutputs(
Struct outputs) → void -
Resolves all output properties from a monitor response payload.
inherited
-
resolveUrn(
String value) → void -
Resolves this resource's URN once assigned by the engine.
inherited
-
serializeProperties(
Map< String, dynamic> properties) → Future<Struct> -
Serializes resource properties for RPC transmission.
inherited
-
toString(
) → String -
A string representation of this object.
inherited
Operators
-
operator ==(
Object other) → bool -
The equality operator.
inherited