evaluate async ¶

async def evaluate(
    self,
    evaluator_id: "capo_bedrock_agentcore.types.evaluator_id.EvaluatorId",
    evaluation_input: "capo_bedrock_agentcore.types.evaluation_input.EvaluationInput",
    *,
    config_overrides: Optional[AsyncBedrockAgentCoreClientConfig] = None,
    evaluation_target: Optional[
        "capo_bedrock_agentcore.types.evaluation_target.EvaluationTarget"
    ] = None,
    evaluation_reference_inputs: Optional[
        "capo_bedrock_agentcore.types.evaluation_reference_inputs.EvaluationReferenceInputs"
    ] = None,
) -> "capo_bedrock_agentcore.types.evaluate_response.EvaluateResponse":
    """<p> Performs on-demand evaluation of agent traces using a specified evaluator. This synchronous API accepts traces in OpenTelemetry format and returns immediate scoring results with detailed explanations.</p>

    Args:
        evaluator_id: <p> The unique identifier of the evaluator to use for scoring. Can be a built-in evaluator (e.g., <code>Builtin.Helpfulness</code>, <code>Builtin.Correctness</code>) or a custom evaluator Id created through the control plane API. </p>
        evaluation_input: <p> The input data containing agent session spans to be evaluated. Includes a list of spans in OpenTelemetry format from supported frameworks like Strands (AgentCore Runtime) or LangGraph with OpenInference instrumentation. </p>
        evaluation_target: <p> The specific trace or span IDs to evaluate within the provided input. Allows targeting evaluation at different levels: individual tool calls, single request-response interactions (traces), or entire conversation sessions. </p>
        evaluation_reference_inputs: <p> Ground truth data to compare against agent responses during evaluation. Allows to provide expected responses, assertions, and expected tool trajectories at different evaluation levels. Session-level reference inputs apply to the entire conversation, while trace-level reference inputs target specific request-response interactions identified by trace ID. </p>

    Raises:
        capo_bedrock_agentcore.errors.access_denied_exception.AccessDeniedException: <p>The exception that occurs when you do not have sufficient permissions to perform an action. Verify that your IAM policy includes the necessary permissions for the operation you are trying to perform.</p>
        capo_bedrock_agentcore.errors.conflict_exception.ConflictException: <p>The exception that occurs when the request conflicts with the current state of the resource. This can happen when trying to modify a resource that is currently being modified by another request, or when trying to create a resource that already exists.</p>
        capo_bedrock_agentcore.errors.duplicate_id_exception.DuplicateIdException: <p> An exception thrown when attempting to create a resource with an identifier that already exists.</p>
        capo_bedrock_agentcore.errors.internal_server_exception.InternalServerException: <p>The exception that occurs when the service encounters an unexpected internal error. This is a temporary condition that will resolve itself with retries. We recommend implementing exponential backoff retry logic in your application.</p>
        capo_bedrock_agentcore.errors.resource_not_found_exception.ResourceNotFoundException: <p>The exception that occurs when the specified resource does not exist. This can happen when using an invalid identifier or when trying to access a resource that has been deleted.</p>
        capo_bedrock_agentcore.errors.service_quota_exceeded_exception.ServiceQuotaExceededException: <p>The exception that occurs when the request would cause a service quota to be exceeded. Review your service quotas and either reduce your request rate or request a quota increase.</p>
        capo_bedrock_agentcore.errors.throttling_exception.ThrottlingException: <p>The exception that occurs when the request was denied due to request throttling. This happens when you exceed the allowed request rate for an operation. Reduce the frequency of requests or implement exponential backoff retry logic in your application.</p>
        capo_bedrock_agentcore.errors.unauthorized_exception.UnauthorizedException: <p>This exception is thrown when the JWT bearer token is invalid or not found for OAuth bearer token based access</p>
        capo_bedrock_agentcore.errors.validation_exception.ValidationException: <p>The exception that occurs when the input fails to satisfy the constraints specified by the service. Check the error message for details about which input parameter is invalid and correct your request.</p>
        capo_bedrock_agentcore.errors.UnknownServiceError: The service returned an error code this client does not model.
    """

    async def _handler(
        req: "AsyncOperationRequest[capo_bedrock_agentcore.types.evaluate_request.EvaluateRequest]",
    ) -> AsyncOperationResponse[
        "capo_bedrock_agentcore.types.evaluate_response.EvaluateResponse"
    ]:
        import capo_bedrock_agentcore._operations.amazon_bedrock_agent_core.evaluate

        (
            output,
            http_response,
        ) = await capo_bedrock_agentcore._operations.amazon_bedrock_agent_core.evaluate.async_evaluate(
            req.options, req.input
        )
        return AsyncOperationResponse(output=output, response=http_response)

    def _redact_input(
        input_: "capo_bedrock_agentcore.types.evaluate_request.EvaluateRequest",
    ) -> dict[str, Any]:
        import capo_bedrock_agentcore.types.evaluate_request

        return capo_bedrock_agentcore.types.evaluate_request.redact_sensitive(
            input_
        )

    def _redact_output(
        output: "capo_bedrock_agentcore.types.evaluate_response.EvaluateResponse",
    ) -> dict[str, Any]:
        import capo_bedrock_agentcore.types.evaluate_response

        return capo_bedrock_agentcore.types.evaluate_response.redact_sensitive(
            output
        )

    interceptors_, options_ = self.operation_options(
        config_overrides,
        OperationLogging(
            "Evaluate", redact_input=_redact_input, redact_output=_redact_output
        ),
    )
    input_: capo_bedrock_agentcore.types.evaluate_request.EvaluateRequest = {
        "evaluator_id": evaluator_id,
        "evaluation_input": evaluation_input,
    }
    if evaluation_target is not None:
        input_["evaluation_target"] = evaluation_target
    if evaluation_reference_inputs is not None:
        input_["evaluation_reference_inputs"] = evaluation_reference_inputs

    response = await aexecute_pipeline(
        AsyncOperationRequest(input=input_, options=options_),
        handler=_handler,
        interceptors=list(interceptors_),
    )
    await response.response.aclose()
    return response.output