EvaluationJob class

Manages an Amazon Bedrock evaluation job. An evaluation job assesses model or knowledge base performance using either automated metrics or human workers.

> Amazon Bedrock does not support permanently deleting an evaluation job. Destroying this resource stops the job (if it is still running) using the StopEvaluationJob API, then removes it from Terraform state. Set skipDestroy to leave the job in its current state instead.

Example Usage

Automated Model Evaluation

import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const example = new aws.bedrock.EvaluationJob("example", {
    evaluationConfig: {
        automated: {
            datasetMetricConfigs: [{
                dataset: {
                    name: "Builtin.Bold",
                },
                taskType: "Generation",
                metricNames: ["Builtin.Robustness"],
            }],
        },
    },
    inferenceConfig: {
        models: [{
            bedrockModel: {
                modelIdentifier: "amazon.nova-micro-v1:0",
            },
        }],
    },
    outputDataConfig: {
        s3Uri: `s3://${exampleAwsS3Bucket.id}/output/`,
    },
    jobName: "example-job",
    roleArn: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws

example = aws.bedrock.EvaluationJob("example",
    evaluation_config={
        "automated": {
            "dataset_metric_configs": [{
                "dataset": {
                    "name": "Builtin.Bold",
                },
                "task_type": "Generation",
                "metric_names": ["Builtin.Robustness"],
            }],
        },
    },
    inference_config={
        "models": [{
            "bedrock_model": {
                "model_identifier": "amazon.nova-micro-v1:0",
            },
        }],
    },
    output_data_config={
        "s3_uri": f"s3://{example_aws_s3_bucket['id']}/output/",
    },
    job_name="example-job",
    role_arn=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() =>
{
    var example = new Aws.Bedrock.EvaluationJob("example", new()
    {
        EvaluationConfig = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigArgs
        {
            Automated = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedArgs
            {
                DatasetMetricConfigs = new[]
                {
                    new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs
                    {
                        Dataset = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs
                        {
                            Name = "Builtin.Bold",
                        },
                        TaskType = "Generation",
                        MetricNames = new[]
                        {
                            "Builtin.Robustness",
                        },
                    },
                },
            },
        },
        InferenceConfig = new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigArgs
        {
            Models = new[]
            {
                new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigModelArgs
                {
                    BedrockModel = new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigModelBedrockModelArgs
                    {
                        ModelIdentifier = "amazon.nova-micro-v1:0",
                    },
                },
            },
        },
        OutputDataConfig = new Aws.Bedrock.Inputs.EvaluationJobOutputDataConfigArgs
        {
            S3Uri = $"s3://{exampleAwsS3Bucket.Id}/output/",
        },
        JobName = "example-job",
        RoleArn = exampleAwsIamRole.Arn,
    });

});
package main

import (
	"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/bedrock"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := bedrock.NewEvaluationJob(ctx, "example", &bedrock.EvaluationJobArgs{
			EvaluationConfig: &bedrock.EvaluationJobEvaluationConfigArgs{
				Automated: &bedrock.EvaluationJobEvaluationConfigAutomatedArgs{
					DatasetMetricConfigs: bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArray{
						&bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs{
							Dataset: &bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs{
								Name: pulumi.String("Builtin.Bold"),
							},
							TaskType: pulumi.String("Generation"),
							MetricNames: pulumi.StringArray{
								pulumi.String("Builtin.Robustness"),
							},
						},
					},
				},
			},
			InferenceConfig: &bedrock.EvaluationJobInferenceConfigArgs{
				Models: bedrock.EvaluationJobInferenceConfigModelArray{
					&bedrock.EvaluationJobInferenceConfigModelArgs{
						BedrockModel: &bedrock.EvaluationJobInferenceConfigModelBedrockModelArgs{
							ModelIdentifier: pulumi.String("amazon.nova-micro-v1:0"),
						},
					},
				},
			},
			OutputDataConfig: &bedrock.EvaluationJobOutputDataConfigArgs{
				S3Uri: pulumi.Sprintf("s3://%v/output/", exampleAwsS3Bucket.Id),
			},
			JobName: pulumi.String("example-job"),
			RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
		})
		if err != nil {
			return err
		}
		return nil
	})
}
pulumi {
  required_providers {
    aws = {
      source = "pulumi/aws"
    }
  }
}

resource "aws_bedrock_evaluationjob" "example" {
  evaluation_config = {
    automated = {
      dataset_metric_configs = [{
        "dataset" = {
          "name" = "Builtin.Bold"
        }
        "taskType"    = "Generation"
        "metricNames" = ["Builtin.Robustness"]
      }]
    }
  }
  inference_config = {
    models = [{
      "bedrockModel" = {
        "modelIdentifier" = "amazon.nova-micro-v1:0"
      }
    }]
  }
  output_data_config = {
    s3_uri ="s3://${exampleAwsS3Bucket.id}/output/"
  }
  job_name = "example-job"
  role_arn = exampleAwsIamRole.arn
}
package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.bedrock.EvaluationJob;
import com.pulumi.aws.bedrock.EvaluationJobArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigModelArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigModelBedrockModelArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobOutputDataConfigArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var example = new EvaluationJob("example", EvaluationJobArgs.builder()
            .evaluationConfig(EvaluationJobEvaluationConfigArgs.builder()
                .automated(EvaluationJobEvaluationConfigAutomatedArgs.builder()
                    .datasetMetricConfigs(EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs.builder()
                        .dataset(EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs.builder()
                            .name("Builtin.Bold")
                            .build())
                        .taskType("Generation")
                        .metricNames("Builtin.Robustness")
                        .build())
                    .build())
                .build())
            .inferenceConfig(EvaluationJobInferenceConfigArgs.builder()
                .models(EvaluationJobInferenceConfigModelArgs.builder()
                    .bedrockModel(EvaluationJobInferenceConfigModelBedrockModelArgs.builder()
                        .modelIdentifier("amazon.nova-micro-v1:0")
                        .build())
                    .build())
                .build())
            .outputDataConfig(EvaluationJobOutputDataConfigArgs.builder()
                .s3Uri(String.format("s3://%s/output/", exampleAwsS3Bucket.id()))
                .build())
            .jobName("example-job")
            .roleArn(exampleAwsIamRole.arn())
            .build());

    }
}
resources:
  example:
    type: aws:bedrock:EvaluationJob
    properties:
      evaluationConfig:
        automated:
          datasetMetricConfigs:
            - dataset:
                name: Builtin.Bold
              taskType: Generation
              metricNames:
                - Builtin.Robustness
      inferenceConfig:
        models:
          - bedrockModel:
              modelIdentifier: amazon.nova-micro-v1:0
      outputDataConfig:
        s3Uri: s3://${exampleAwsS3Bucket.id}/output/
      jobName: example-job
      roleArn: ${exampleAwsIamRole.arn}

Import

Identity Schema

Required

  • jobArn (String) ARN of the evaluation job.

Using pulumi import, import Bedrock Evaluation Job using the jobArn. For example:

$ pulumi import aws:bedrock/evaluationJob:EvaluationJob example arn:aws:bedrock:us-west-2:123456789012:evaluation-job/abcdefgh1234

Constructors

EvaluationJob(String name, {EvaluationJobArgs? args, CustomResourceOptions? options})
Creates a new EvaluationJob. name The Pulumi resource name. args Arguments used to configure this EvaluationJob. The set of arguments for EvaluationJob. options Resource options controlling this resource's behavior.
EvaluationJob.reference(String urn)
Creates a typed reference to an existing EvaluationJob resource.

Properties

applicationType ↔ Output<String>
Whether the evaluation job evaluates a model or a knowledge base. Valid values: ModelEvaluation, RagEvaluation.
latefinal
childResources → Set<Resource>
finalinherited
completionSources ↔ Map<String, IOutputCompletionSource>
latefinalinherited
createdAt ↔ Output<String>
Date and time the evaluation job was created.
latefinal
customerEncryptionKeyId ↔ Output<String?>
ARN of the customer managed KMS key to use to encrypt the evaluation job.
latefinal
evaluationConfig ↔ Output<EvaluationJobEvaluationConfig>
Configuration for either an automated or human-based evaluation job. See evaluationConfig Block below.
latefinal
failureMessages ↔ Output<List<String>>
List of reasons the evaluation job failed to create, if applicable.
latefinal
hashCode → int
The hash code for this object.
no setterinherited
id ↔ Output<String>
getter/setter pairinherited
inferenceConfig ↔ Output<EvaluationJobInferenceConfig>
Configuration for the inference model, or models, used for the evaluation job. See inferenceConfig Block below.
latefinal
isCustom → bool
Returns whether this resource is provider-managed.
no setterinherited
isProtected → bool
Returns whether this resource is protected from deletion.
no setterinherited
isRemote → bool
Whether this resource is registered as remote.
no setterinherited
isResourceReference → bool
Whether this instance represents a resource value returned over RPC.
finalinherited
jobArn ↔ Output<String>
ARN of the evaluation job.
latefinal
jobDescription ↔ Output<String?>
Description of the evaluation job.
latefinal
jobName ↔ Output<String>
Name for the evaluation job. Must be unique within your AWS account and Region, and consist of lowercase letters, numbers, and hyphens.
latefinal
jobType ↔ Output<String>
Whether the evaluation job is automated or human-based.
latefinal
lastModifiedTime ↔ Output<String>
Date and time the evaluation job was last modified.
latefinal
outputDataConfig ↔ Output<EvaluationJobOutputDataConfig>
Configuration for the Amazon S3 location where the results of the evaluation job are stored. See outputDataConfig Block below.
latefinal
region ↔ Output<String>
Region where this resource will be managed. Defaults to the Region set in the provider configuration.
latefinal
resourceTransforms → List<ResourceTransform>
Inherited/explicit async transforms.
no setterinherited
roleArn ↔ Output<String>
ARN of an IAM service role that Amazon Bedrock can assume to perform tasks on your behalf. See Required permissions for model evaluations.
latefinal
runtimeType → Type
A representation of the runtime type of the object.
no setterinherited
skipDestroy ↔ Output<bool?>
Whether to leave the evaluation job in its current state when destroying the resource, instead of stopping it.
latefinal
status ↔ Output<String>
Current status of the evaluation job.
latefinal
tags ↔ Output<Map<String, String>?>
Map of tags to assign to the evaluation job. If configured with a provider defaultTags configuration block present, tags with matching keys will overwrite those defined at the provider-level.
latefinal
tagsAll ↔ Output<Map<String, String>>
latefinal
timeouts ↔ Output<EvaluationJobTimeouts?>
latefinal
transformations → List<ResourceTransformation>
Inherited/explicit legacy transformations.
no setterinherited
urn ↔ Output<String>
latefinalinherited

Methods

failId(Object error) → void
Completes this resource ID with an error when registration fails.
inherited
failOutputs(Object error) → void
Completes all output properties with error.
inherited
failUrn(Object error) → void
Completes this resource URN with an error when registration fails.
inherited
getProvider(String moduleMember) → ProviderResource?
Returns provider for moduleMember's package, if configured.
inherited
getResourceName() → String
Returns this resource's logical name.
inherited
getResourceType() → String
Returns this resource's Pulumi type token.
inherited
noSuchMethod(Invocation invocation) → dynamic
Invoked when a nonexistent method or property is accessed.
inherited
registerOutput<T>(String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output<T>
Registers a dynamic output property for this resource.
inherited
resolveId(String? value, {required bool isKnown}) → void
Resolves the provider-assigned ID for this resource.
inherited
resolveOutputs(Struct outputs) → void
Resolves all output properties from a monitor response payload.
inherited
resolveUrn(String value) → void
Resolves this resource's URN once assigned by the engine.
inherited
serializeProperties(Map<String, dynamic> properties) → Future<Struct>
Serializes resource properties for RPC transmission.
inherited
toString() → String
A string representation of this object.
inherited

Operators

operator ==(Object other) → bool
The equality operator.
inherited

Static Methods

get(String name, Input<String> id, {EvaluationJobState? state, CustomResourceOptions? options}) → EvaluationJob
Gets an existing EvaluationJob resource's state with the given name and id.