invoke_model_with_response_stream async ¶

@asynccontextmanager
async def invoke_model_with_response_stream(
    self,
    model_id: "capo_bedrock_runtime.types.invoke_model_identifier.InvokeModelIdentifier",
    *,
    config_overrides: Optional[AsyncBedrockRuntimeClientConfig] = None,
    body: Optional["capo_bedrock_runtime.types.body.Body"] = None,
    content_type: Optional["capo_bedrock_runtime.types.mime_type.MimeType"] = None,
    accept: Optional["capo_bedrock_runtime.types.mime_type.MimeType"] = None,
    trace: Optional["capo_bedrock_runtime.types.trace.Trace"] = None,
    guardrail_identifier: Optional[
        "capo_bedrock_runtime.types.guardrail_identifier.GuardrailIdentifier"
    ] = None,
    guardrail_version: Optional[
        "capo_bedrock_runtime.types.guardrail_version.GuardrailVersion"
    ] = None,
    performance_config_latency: Optional[
        "capo_bedrock_runtime.types.performance_config_latency.PerformanceConfigLatency"
    ] = None,
    service_tier: Optional[
        "capo_bedrock_runtime.types.service_tier_type.ServiceTierType"
    ] = None,
    request_metadata: Optional[
        "capo_bedrock_runtime.types.request_metadata_json.RequestMetadataJson"
    ] = None,
) -> "AsyncGenerator[capo_bedrock_runtime.types.invoke_model_with_response_stream_response.InvokeModelWithResponseStreamResponse]":
    """<p>Invoke the specified Amazon Bedrock model to run inference using the prompt and inference parameters provided in the request body. The response is returned in a stream.</p> <p>To see if a model supports streaming, call <a href="https://docs.aws.amazon.com/bedrock/latest/APIReference/API_GetFoundationModel.html">GetFoundationModel</a> and check the <code>responseStreamingSupported</code> field in the response.</p> <note> <p>The CLI doesn't support streaming operations in Amazon Bedrock, including <code>InvokeModelWithResponseStream</code>.</p> </note> <p>For example code, see <i>Invoke model with streaming code example</i> in the <i>Amazon Bedrock User Guide</i>. </p> <p>This operation requires permissions to perform the <code>bedrock:InvokeModelWithResponseStream</code> action. </p> <important> <p>To deny all inference access to resources that you specify in the modelId field, you need to deny access to the <code>bedrock:InvokeModel</code> and <code>bedrock:InvokeModelWithResponseStream</code> actions. Doing this also denies access to the resource through the Converse API actions (<a href="https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html">Converse</a> and <a href="https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html">ConverseStream</a>). For more information see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html#security_iam_id-based-policy-examples-deny-inference">Deny access for inference on specific models</a>. </p> </important> <p>For troubleshooting some of the common errors you might encounter when using the <code>InvokeModelWithResponseStream</code> API, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html">Troubleshooting Amazon Bedrock API Error Codes</a> in the Amazon Bedrock User Guide</p>

    Args:
        body: <p>The prompt and inference parameters in the format specified in the <code>contentType</code> in the header. You must provide the body in JSON format. To see the format and content of the request and response bodies for different models, refer to <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html">Inference parameters</a>. For more information, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/api-methods-run.html">Run inference</a> in the Bedrock User Guide.</p>
        content_type: <p>The MIME type of the input data in the request. You must specify <code>application/json</code>.</p>
        accept: <p>The desired MIME type of the inference body in the response. The default value is <code>application/json</code>.</p>
        model_id: <p>The unique identifier of the model to invoke to run inference.</p> <p>The <code>modelId</code> to provide depends on the type of model or throughput that you use:</p> <ul> <li> <p>If you use a base model, specify the model ID or its ARN. For a list of model IDs for base models, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html#model-ids-arns">Amazon Bedrock base model IDs (on-demand throughput)</a> in the Amazon Bedrock User Guide.</p> </li> <li> <p>If you use an inference profile, specify the inference profile ID or its ARN. For a list of inference profile IDs, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/cross-region-inference-support.html">Supported Regions and models for cross-region inference</a> in the Amazon Bedrock User Guide.</p> </li> <li> <p>If you use a provisioned model, specify the ARN of the Provisioned Throughput. For more information, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/prov-thru-use.html">Run inference using a Provisioned Throughput</a> in the Amazon Bedrock User Guide.</p> </li> <li> <p>If you use a custom model, specify the ARN of the custom model deployment (for on-demand inference) or the ARN of your provisioned model (for Provisioned Throughput). For more information, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-use.html">Use a custom model in Amazon Bedrock</a> in the Amazon Bedrock User Guide.</p> </li> <li> <p>If you use an <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html">imported model</a>, specify the ARN of the imported model. You can get the model ARN from a successful call to <a href="https://docs.aws.amazon.com/bedrock/latest/APIReference/API_CreateModelImportJob.html">CreateModelImportJob</a> or from the Imported models page in the Amazon Bedrock console.</p> </li> </ul>
        trace: <p>Specifies whether to enable or disable the Bedrock trace. If enabled, you can see the full Bedrock trace.</p>
        guardrail_identifier: <p>The unique identifier of the guardrail that you want to use. If you don't provide a value, no guardrail is applied to the invocation.</p> <p>An error is thrown in the following situations.</p> <ul> <li> <p>You don't provide a guardrail identifier but you specify the <code>amazon-bedrock-guardrailConfig</code> field in the request body.</p> </li> <li> <p>You enable the guardrail but the <code>contentType</code> isn't <code>application/json</code>.</p> </li> <li> <p>You provide a guardrail identifier, but <code>guardrailVersion</code> isn't specified.</p> </li> </ul>
        guardrail_version: <p>The version number for the guardrail. The value can also be <code>DRAFT</code>.</p>
        performance_config_latency: <p>Model performance settings for the request.</p>
        service_tier: <p>Specifies the processing tier type used for serving the request.</p>
        request_metadata: <p>Key-value pairs that you can use to filter invocation logs.</p>

    Raises:
        capo_bedrock_runtime.errors.access_denied_exception.AccessDeniedException: <p>The request is denied because you do not have sufficient permissions to perform the requested action. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-access-denied">AccessDeniedException</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.internal_server_exception.InternalServerException: <p>An internal server error occurred. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-internal-failure">InternalFailure</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.model_error_exception.ModelErrorException: <p>The request failed due to an error while processing the model.</p>
        capo_bedrock_runtime.errors.model_not_ready_exception.ModelNotReadyException: <p>The model specified in the request is not ready to serve inference requests. The AWS SDK will automatically retry the operation up to 5 times. For information about configuring automatic retries, see <a href="https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html">Retry behavior</a> in the <i>AWS SDKs and Tools</i> reference guide.</p>
        capo_bedrock_runtime.errors.model_stream_error_exception.ModelStreamErrorException: <p>An error occurred while streaming the response. Retry your request.</p>
        capo_bedrock_runtime.errors.model_timeout_exception.ModelTimeoutException: <p>The request took too long to process. Processing time exceeded the model timeout length.</p>
        capo_bedrock_runtime.errors.resource_not_found_exception.ResourceNotFoundException: <p>The specified resource ARN was not found. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-resource-not-found">ResourceNotFound</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.service_quota_exceeded_exception.ServiceQuotaExceededException: <p>Your request exceeds the service quota for your account. You can view your quotas at <a href="https://docs.aws.amazon.com/servicequotas/latest/userguide/gs-request-quota.html">Viewing service quotas</a>. You can resubmit your request later.</p>
        capo_bedrock_runtime.errors.service_unavailable_exception.ServiceUnavailableException: <p>The service isn't currently available. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-service-unavailable">ServiceUnavailable</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.throttling_exception.ThrottlingException: <p>Your request was denied due to exceeding the account quotas for <i>Amazon Bedrock</i>. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-throttling-exception">ThrottlingException</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.validation_exception.ValidationException: <p>The input fails to satisfy the constraints specified by <i>Amazon Bedrock</i>. For troubleshooting this error, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html#ts-validation-error">ValidationError</a> in the Amazon Bedrock User Guide</p>
        capo_bedrock_runtime.errors.UnknownServiceError: The service returned an error code this client does not model.
    """

    async def _handler(
        req: "AsyncOperationRequest[capo_bedrock_runtime.types.invoke_model_with_response_stream_request.InvokeModelWithResponseStreamRequest]",
    ) -> AsyncOperationResponse[
        "capo_bedrock_runtime.types.invoke_model_with_response_stream_response.InvokeModelWithResponseStreamResponse"
    ]:
        import capo_bedrock_runtime._operations.amazon_bedrock_frontend_service.invoke_model_with_response_stream

        (
            output,
            http_response,
        ) = await capo_bedrock_runtime._operations.amazon_bedrock_frontend_service.invoke_model_with_response_stream.async_invoke_model_with_response_stream(
            req.options, req.input
        )
        return AsyncOperationResponse(output=output, response=http_response)

    def _redact_input(
        input_: "capo_bedrock_runtime.types.invoke_model_with_response_stream_request.InvokeModelWithResponseStreamRequest",
    ) -> dict[str, Any]:
        import capo_bedrock_runtime.types.invoke_model_with_response_stream_request

        return capo_bedrock_runtime.types.invoke_model_with_response_stream_request.redact_sensitive(
            input_
        )

    def _redact_output(
        output: "capo_bedrock_runtime.types.invoke_model_with_response_stream_response.InvokeModelWithResponseStreamResponse",
    ) -> dict[str, Any]:
        import capo_bedrock_runtime.types.invoke_model_with_response_stream_response

        return capo_bedrock_runtime.types.invoke_model_with_response_stream_response.redact_sensitive(
            output
        )

    interceptors_, options_ = self.operation_options(
        config_overrides,
        OperationLogging(
            "InvokeModelWithResponseStream",
            redact_input=_redact_input,
            redact_output=_redact_output,
        ),
    )
    input_: capo_bedrock_runtime.types.invoke_model_with_response_stream_request.InvokeModelWithResponseStreamRequest = {
        "model_id": model_id
    }
    if body is not None:
        input_["body"] = body
    if content_type is not None:
        input_["content_type"] = content_type
    if accept is not None:
        input_["accept"] = accept
    if trace is not None:
        input_["trace"] = trace
    if guardrail_identifier is not None:
        input_["guardrail_identifier"] = guardrail_identifier
    if guardrail_version is not None:
        input_["guardrail_version"] = guardrail_version
    if performance_config_latency is not None:
        input_["performance_config_latency"] = performance_config_latency
    if service_tier is not None:
        input_["service_tier"] = service_tier
    if request_metadata is not None:
        input_["request_metadata"] = request_metadata

    response = await aexecute_pipeline(
        AsyncOperationRequest(input=input_, options=options_),
        handler=_handler,
        interceptors=list(interceptors_),
    )
    try:
        yield response.output
    finally:
        await response.response.aclose()