create_evaluator ¶

def create_evaluator(
    self,
    evaluator_name: "capo_bedrock_agentcore_control.types.custom_evaluator_name.CustomEvaluatorName",
    evaluator_config: "capo_bedrock_agentcore_control.types.evaluator_config.EvaluatorConfig",
    level: "capo_bedrock_agentcore_control.types.evaluator_level.EvaluatorLevel",
    *,
    config_overrides: Optional[BedrockAgentCoreControlClientConfig] = None,
    client_token: Optional[
        "capo_bedrock_agentcore_control.types.client_token.ClientToken"
    ] = None,
    description: Optional[
        "capo_bedrock_agentcore_control.types.evaluator_description.EvaluatorDescription"
    ] = None,
    kms_key_arn: Optional[
        "capo_bedrock_agentcore_control.types.kms_key_arn.KmsKeyArn"
    ] = None,
    tags: Optional["capo_bedrock_agentcore_control.types.tags_map.TagsMap"] = None,
) -> "capo_bedrock_agentcore_control.types.create_evaluator_response.CreateEvaluatorResponse":
    """<p> Creates a custom evaluator for agent quality assessment. Custom evaluators can use either LLM-as-a-Judge configurations with user-defined prompts, rating scales, and model settings, or code-based configurations with customer-managed Lambda functions to evaluate agent performance at tool call, trace, or session levels. </p>

    Args:
        client_token: <p>A unique, case-sensitive identifier to ensure that the API request completes no more than one time. If you don't specify this field, a value is randomly generated for you. If this token matches a previous request, the service ignores the request, but doesn't return an error. For more information, see <a href="https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html">Ensuring idempotency</a>.</p>
        evaluator_name: <p> The name of the evaluator. Must be unique within your account. </p>
        description: <p> The description of the evaluator that explains its purpose and evaluation criteria. </p>
        evaluator_config: <p> The configuration for the evaluator. Specify either LLM-as-a-Judge settings with instructions, rating scale, and model configuration, or code-based settings with a customer-managed Lambda function. </p>
        level: <p> The evaluation level that determines the scope of evaluation. Valid values are <code>TOOL_CALL</code> for individual tool invocations, <code>TRACE</code> for single request-response interactions, or <code>SESSION</code> for entire conversation sessions. </p>
        kms_key_arn: <p> The Amazon Resource Name (ARN) of a customer managed KMS key to use for encrypting sensitive evaluator data, including instructions and rating scale. If you don't specify a KMS key, the evaluator data is encrypted with an Amazon Web Services owned key. Only symmetric encryption KMS keys are supported. For more information, see <a href="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/evaluations-encryption.html">Encryption at rest for AgentCore Evaluations</a>. </p>
        tags: <p>A map of tag keys and values to assign to an AgentCore Evaluator. Tags enable you to categorize your resources in different ways, for example, by purpose, owner, or environment.</p>

    Raises:
        capo_bedrock_agentcore_control.errors.access_denied_exception.AccessDeniedException: <p>This exception is thrown when a request is denied per access permissions</p>
        capo_bedrock_agentcore_control.errors.conflict_exception.ConflictException: <p>This exception is thrown when there is a conflict performing an operation</p>
        capo_bedrock_agentcore_control.errors.internal_server_exception.InternalServerException: <p>This exception is thrown if there was an unexpected error during processing of request</p>
        capo_bedrock_agentcore_control.errors.service_quota_exceeded_exception.ServiceQuotaExceededException: <p>This exception is thrown when a request is made beyond the service quota</p>
        capo_bedrock_agentcore_control.errors.throttling_exception.ThrottlingException: <p>This exception is thrown when the number of requests exceeds the limit</p>
        capo_bedrock_agentcore_control.errors.validation_exception.ValidationException: <p>The input fails to satisfy the constraints specified by the service.</p>
        capo_bedrock_agentcore_control.errors.UnknownServiceError: The service returned an error code this client does not model.
    """

    def _handler(
        req: "OperationRequest[capo_bedrock_agentcore_control.types.create_evaluator_request.CreateEvaluatorRequest]",
    ) -> OperationResponse[
        "capo_bedrock_agentcore_control.types.create_evaluator_response.CreateEvaluatorResponse"
    ]:
        import capo_bedrock_agentcore_control._operations.amazon_bedrock_agent_core_control.create_evaluator

        output, http_response = (
            capo_bedrock_agentcore_control._operations.amazon_bedrock_agent_core_control.create_evaluator.create_evaluator(
                req.options, req.input
            )
        )
        return OperationResponse(output=output, response=http_response)

    def _redact_input(
        input_: "capo_bedrock_agentcore_control.types.create_evaluator_request.CreateEvaluatorRequest",
    ) -> dict[str, Any]:
        import capo_bedrock_agentcore_control.types.create_evaluator_request

        return capo_bedrock_agentcore_control.types.create_evaluator_request.redact_sensitive(
            input_
        )

    interceptors_, options_ = self.operation_options(
        config_overrides,
        OperationLogging("CreateEvaluator", redact_input=_redact_input),
    )
    input_: capo_bedrock_agentcore_control.types.create_evaluator_request.CreateEvaluatorRequest = {
        "evaluator_name": evaluator_name,
        "evaluator_config": evaluator_config,
        "level": level,
    }
    if client_token is None:
        client_token = str(uuid.uuid4())
    input_["client_token"] = client_token
    if description is not None:
        input_["description"] = description
    if kms_key_arn is not None:
        input_["kms_key_arn"] = kms_key_arn
    if tags is not None:
        input_["tags"] = tags

    response = execute_pipeline(
        OperationRequest(input=input_, options=options_),
        handler=_handler,
        interceptors=list(interceptors_),
    )
    response.response.close()
    return response.output