EvaluationJob class
Manages an Amazon Bedrock evaluation job. An evaluation job assesses model or knowledge base performance using either automated metrics or human workers.
> Amazon Bedrock does not support permanently deleting an evaluation job. Destroying this resource stops the job (if it is still running) using the StopEvaluationJob API, then removes it from Terraform state. Set skipDestroy to leave the job in its current state instead.
Example Usage
Automated Model Evaluation
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";
const example = new aws.bedrock.EvaluationJob("example", {
evaluationConfig: {
automated: {
datasetMetricConfigs: [{
dataset: {
name: "Builtin.Bold",
},
taskType: "Generation",
metricNames: ["Builtin.Robustness"],
}],
},
},
inferenceConfig: {
models: [{
bedrockModel: {
modelIdentifier: "amazon.nova-micro-v1:0",
},
}],
},
outputDataConfig: {
s3Uri: `s3://${exampleAwsS3Bucket.id}/output/`,
},
jobName: "example-job",
roleArn: exampleAwsIamRole.arn,
});
import pulumi
import pulumi_aws as aws
example = aws.bedrock.EvaluationJob("example",
evaluation_config={
"automated": {
"dataset_metric_configs": [{
"dataset": {
"name": "Builtin.Bold",
},
"task_type": "Generation",
"metric_names": ["Builtin.Robustness"],
}],
},
},
inference_config={
"models": [{
"bedrock_model": {
"model_identifier": "amazon.nova-micro-v1:0",
},
}],
},
output_data_config={
"s3_uri": f"s3://{example_aws_s3_bucket['id']}/output/",
},
job_name="example-job",
role_arn=example_aws_iam_role["arn"])
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Aws = Pulumi.Aws;
return await Deployment.RunAsync(() =>
{
var example = new Aws.Bedrock.EvaluationJob("example", new()
{
EvaluationConfig = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigArgs
{
Automated = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedArgs
{
DatasetMetricConfigs = new[]
{
new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs
{
Dataset = new Aws.Bedrock.Inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs
{
Name = "Builtin.Bold",
},
TaskType = "Generation",
MetricNames = new[]
{
"Builtin.Robustness",
},
},
},
},
},
InferenceConfig = new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigArgs
{
Models = new[]
{
new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigModelArgs
{
BedrockModel = new Aws.Bedrock.Inputs.EvaluationJobInferenceConfigModelBedrockModelArgs
{
ModelIdentifier = "amazon.nova-micro-v1:0",
},
},
},
},
OutputDataConfig = new Aws.Bedrock.Inputs.EvaluationJobOutputDataConfigArgs
{
S3Uri = $"s3://{exampleAwsS3Bucket.Id}/output/",
},
JobName = "example-job",
RoleArn = exampleAwsIamRole.Arn,
});
});
package main
import (
"github.com/pulumi/pulumi-aws/sdk/v7/go/aws/bedrock"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
_, err := bedrock.NewEvaluationJob(ctx, "example", &bedrock.EvaluationJobArgs{
EvaluationConfig: &bedrock.EvaluationJobEvaluationConfigArgs{
Automated: &bedrock.EvaluationJobEvaluationConfigAutomatedArgs{
DatasetMetricConfigs: bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArray{
&bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs{
Dataset: &bedrock.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs{
Name: pulumi.String("Builtin.Bold"),
},
TaskType: pulumi.String("Generation"),
MetricNames: pulumi.StringArray{
pulumi.String("Builtin.Robustness"),
},
},
},
},
},
InferenceConfig: &bedrock.EvaluationJobInferenceConfigArgs{
Models: bedrock.EvaluationJobInferenceConfigModelArray{
&bedrock.EvaluationJobInferenceConfigModelArgs{
BedrockModel: &bedrock.EvaluationJobInferenceConfigModelBedrockModelArgs{
ModelIdentifier: pulumi.String("amazon.nova-micro-v1:0"),
},
},
},
},
OutputDataConfig: &bedrock.EvaluationJobOutputDataConfigArgs{
S3Uri: pulumi.Sprintf("s3://%v/output/", exampleAwsS3Bucket.Id),
},
JobName: pulumi.String("example-job"),
RoleArn: pulumi.Any(exampleAwsIamRole.Arn),
})
if err != nil {
return err
}
return nil
})
}
pulumi {
required_providers {
aws = {
source = "pulumi/aws"
}
}
}
resource "aws_bedrock_evaluationjob" "example" {
evaluation_config = {
automated = {
dataset_metric_configs = [{
"dataset" = {
"name" = "Builtin.Bold"
}
"taskType" = "Generation"
"metricNames" = ["Builtin.Robustness"]
}]
}
}
inference_config = {
models = [{
"bedrockModel" = {
"modelIdentifier" = "amazon.nova-micro-v1:0"
}
}]
}
output_data_config = {
s3_uri ="s3://${exampleAwsS3Bucket.id}/output/"
}
job_name = "example-job"
role_arn = exampleAwsIamRole.arn
}
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.aws.bedrock.EvaluationJob;
import com.pulumi.aws.bedrock.EvaluationJobArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigModelArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobInferenceConfigModelBedrockModelArgs;
import com.pulumi.aws.bedrock.inputs.EvaluationJobOutputDataConfigArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var example = new EvaluationJob("example", EvaluationJobArgs.builder()
.evaluationConfig(EvaluationJobEvaluationConfigArgs.builder()
.automated(EvaluationJobEvaluationConfigAutomatedArgs.builder()
.datasetMetricConfigs(EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigArgs.builder()
.dataset(EvaluationJobEvaluationConfigAutomatedDatasetMetricConfigDatasetArgs.builder()
.name("Builtin.Bold")
.build())
.taskType("Generation")
.metricNames("Builtin.Robustness")
.build())
.build())
.build())
.inferenceConfig(EvaluationJobInferenceConfigArgs.builder()
.models(EvaluationJobInferenceConfigModelArgs.builder()
.bedrockModel(EvaluationJobInferenceConfigModelBedrockModelArgs.builder()
.modelIdentifier("amazon.nova-micro-v1:0")
.build())
.build())
.build())
.outputDataConfig(EvaluationJobOutputDataConfigArgs.builder()
.s3Uri(String.format("s3://%s/output/", exampleAwsS3Bucket.id()))
.build())
.jobName("example-job")
.roleArn(exampleAwsIamRole.arn())
.build());
}
}
resources:
example:
type: aws:bedrock:EvaluationJob
properties:
evaluationConfig:
automated:
datasetMetricConfigs:
- dataset:
name: Builtin.Bold
taskType: Generation
metricNames:
- Builtin.Robustness
inferenceConfig:
models:
- bedrockModel:
modelIdentifier: amazon.nova-micro-v1:0
outputDataConfig:
s3Uri: s3://${exampleAwsS3Bucket.id}/output/
jobName: example-job
roleArn: ${exampleAwsIamRole.arn}
Import
Identity Schema
Required
jobArn(String) ARN of the evaluation job.
Using pulumi import, import Bedrock Evaluation Job using the jobArn. For example:
$ pulumi import aws:bedrock/evaluationJob:EvaluationJob example arn:aws:bedrock:us-west-2:123456789012:evaluation-job/abcdefgh1234
Constructors
- EvaluationJob(String name, {EvaluationJobArgs? args, CustomResourceOptions? options})
-
Creates a new EvaluationJob.
nameThe Pulumi resource name.argsArguments used to configure this EvaluationJob. The set of arguments for EvaluationJob.optionsResource options controlling this resource's behavior. - EvaluationJob.reference(String urn)
- Creates a typed reference to an existing EvaluationJob resource.
Properties
-
applicationType
↔ Output<
String> -
Whether the evaluation job evaluates a model or a knowledge base. Valid values:
ModelEvaluation,RagEvaluation.latefinal -
childResources
→ Set<
Resource> -
finalinherited
-
completionSources
↔ Map<
String, IOutputCompletionSource> -
latefinalinherited
-
createdAt
↔ Output<
String> -
Date and time the evaluation job was created.
latefinal
-
customerEncryptionKeyId
↔ Output<
String?> -
ARN of the customer managed KMS key to use to encrypt the evaluation job.
latefinal
-
evaluationConfig
↔ Output<
EvaluationJobEvaluationConfig> -
Configuration for either an automated or human-based evaluation job. See
evaluationConfigBlock below.latefinal -
failureMessages
↔ Output<
List< String> > -
List of reasons the evaluation job failed to create, if applicable.
latefinal
- hashCode → int
-
The hash code for this object.
no setterinherited
-
id
↔ Output<
String> -
getter/setter pairinherited
-
inferenceConfig
↔ Output<
EvaluationJobInferenceConfig> -
Configuration for the inference model, or models, used for the evaluation job. See
inferenceConfigBlock below.latefinal - isCustom → bool
-
Returns whether this resource is provider-managed.
no setterinherited
- isProtected → bool
-
Returns whether this resource is protected from deletion.
no setterinherited
- isRemote → bool
-
Whether this resource is registered as remote.
no setterinherited
- isResourceReference → bool
-
Whether this instance represents a resource value returned over RPC.
finalinherited
-
jobArn
↔ Output<
String> -
ARN of the evaluation job.
latefinal
-
jobDescription
↔ Output<
String?> -
Description of the evaluation job.
latefinal
-
jobName
↔ Output<
String> -
Name for the evaluation job. Must be unique within your AWS account and Region, and consist of lowercase letters, numbers, and hyphens.
latefinal
-
jobType
↔ Output<
String> -
Whether the evaluation job is automated or human-based.
latefinal
-
lastModifiedTime
↔ Output<
String> -
Date and time the evaluation job was last modified.
latefinal
-
outputDataConfig
↔ Output<
EvaluationJobOutputDataConfig> -
Configuration for the Amazon S3 location where the results of the evaluation job are stored. See
outputDataConfigBlock below.latefinal -
region
↔ Output<
String> -
Region where this resource will be managed. Defaults to the Region set in the provider configuration.
latefinal
-
resourceTransforms
→ List<
ResourceTransform> -
Inherited/explicit async transforms.
no setterinherited
-
roleArn
↔ Output<
String> -
ARN of an IAM service role that Amazon Bedrock can assume to perform tasks on your behalf. See Required permissions for model evaluations.
latefinal
- runtimeType → Type
-
A representation of the runtime type of the object.
no setterinherited
-
skipDestroy
↔ Output<
bool?> -
Whether to leave the evaluation job in its current state when destroying the resource, instead of stopping it.
latefinal
-
status
↔ Output<
String> -
Current status of the evaluation job.
latefinal
-
Map of tags to assign to the evaluation job. If configured with a provider
defaultTagsconfiguration block present, tags with matching keys will overwrite those defined at the provider-level.latefinal -
latefinal
-
timeouts
↔ Output<
EvaluationJobTimeouts?> -
latefinal
-
transformations
→ List<
ResourceTransformation> -
Inherited/explicit legacy transformations.
no setterinherited
-
urn
↔ Output<
String> -
latefinalinherited
Methods
-
failId(
Object error) → void -
Completes this resource ID with an error when registration fails.
inherited
-
failOutputs(
Object error) → void -
Completes all output properties with
error.inherited -
failUrn(
Object error) → void -
Completes this resource URN with an error when registration fails.
inherited
-
getProvider(
String moduleMember) → ProviderResource? -
Returns provider for
moduleMember's package, if configured.inherited -
getResourceName(
) → String -
Returns this resource's logical name.
inherited
-
getResourceType(
) → String -
Returns this resource's Pulumi type token.
inherited
-
noSuchMethod(
Invocation invocation) → dynamic -
Invoked when a nonexistent method or property is accessed.
inherited
-
registerOutput<
T> (String propertyName, {Object? decoder(Object?)?, bool isSecret = false}) → Output< T> -
Registers a dynamic output property for this resource.
inherited
-
resolveId(
String? value, {required bool isKnown}) → void -
Resolves the provider-assigned ID for this resource.
inherited
-
resolveOutputs(
Struct outputs) → void -
Resolves all output properties from a monitor response payload.
inherited
-
resolveUrn(
String value) → void -
Resolves this resource's URN once assigned by the engine.
inherited
-
serializeProperties(
Map< String, dynamic> properties) → Future<Struct> -
Serializes resource properties for RPC transmission.
inherited
-
toString(
) → String -
A string representation of this object.
inherited
Operators
-
operator ==(
Object other) → bool -
The equality operator.
inherited
Static Methods
-
get(
String name, Input< String> id, {EvaluationJobState? state, CustomResourceOptions? options}) → EvaluationJob -
Gets an existing EvaluationJob resource's state with the given
nameandid.