AWS Secret Manager throws HttpErrorResponseException after many requests

Question

I'm kind lost for a moment I have the following code on .NET 6, and everything looks right, and the application even work well, but after like 100 connection on the API, I always getting the error as follow:

Amazon.SecretsManager.AmazonSecretsManagerException: The service returned an error. See inner exception for details. ---> Amazon.Runtime.Internal.HttpErrorResponseException: Exception of type 'Amazon.Runtime.Internal.HttpErrorResponseException' was thrown. at Amazon.Runtime.HttpWebRequestMessage.GetResponseAsync(CancellationToken cancellationToken) at Amazon.Runtime.Internal.HttpHandler1.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.Unmarshaller.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.ErrorHandler.InvokeAsync[T](IExecutionContext executionContext) --- End of inner exception stack trace --- at Amazon.Runtime.Internal.HttpErrorResponseExceptionHandler.HandleExceptionStream(IRequestContext requestContext, IWebResponseData httpErrorResponse, HttpErrorResponseException exception, Stream responseStream) at Amazon.Runtime.Internal.HttpErrorResponseExceptionHandler.HandleExceptionAsync(IExecutionContext executionContext, HttpErrorResponseException exception) at Amazon.Runtime.Internal.ExceptionHandler1.HandleAsync(IExecutionContext executionContext, Exception exception) at Amazon.Runtime.Internal.ErrorHandler.ProcessExceptionAsync(IExecutionContext executionContext, Exception exception) at Amazon.Runtime.Internal.ErrorHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.EndpointDiscoveryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.EndpointDiscoveryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CredentialsRetriever.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.RetryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.RetryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.ErrorCallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.MetricsHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.SecretsManager.Extensions.Caching.SecretCacheItem.ExecuteRefreshAsync() at Amazon.SecretsManager.Extensions.Caching.SecretCacheObject1.RefreshAsync() at Amazon.SecretsManager.Extensions.Caching.SecretCacheObject1.GetSecretValue() at Amazon.SecretsManager.Extensions.Caching.SecretsManagerCache.GetSecretString(String secretId)

The C# code that I have is:

Amazon Module:

public static class AmazonModule
{
    public static void AmazonServices(this IServiceCollection services, IConfiguration configuration)
    {
        
        services.AddSingleton<IAmazonSecretsManager>(config => new AmazonSecretsManagerClient(RegionEndpoint.GetBySystemName(_section[Constants.AWS_SECRETMANAGER_REGION])));
    
        var assemblyToScan = Assembly.GetAssembly(typeof(SecretsManagerService));

        services.RegisterAssemblyPublicNonGenericClasses(assemblyToScan)
            .Where(t => t.Name.EndsWith("Service"))
            .AsPublicImplementedInterfaces(ServiceLifetime.Scoped);
      
        services.AddCognitoIdentity();
    }
}

Database Module:

public static class DatabaseModule
{
    public static void RegisterDatabaseModule(this IServiceCollection services, IConfiguration configuration)
    {
        services.AddDbContext<BoletoPixDataContext>((ctx, options) =>
        {
            string connection = "";
            using (IServiceScope scope = ctx.GetRequiredService<IServiceScopeFactory>().CreateScope())
            {
                var secretsManagerService = ctx.GetService<ISecretsManagerService>();
                connection = secretsManagerService.GetSecret<ConnectionString>(configuration[Constants.AWS_CONFIG_SECRETMANAGER_SECTION + ":" + Constants.AWS_SECRETMANAGER_DATABASE]);
            }

            options.UseSqlServer(connection);


            var auditNet = Convert.ToBoolean(configuration.GetSection("AuditNetEntity").Value);
            if (auditNet)
            {
                Configuration
                    .Setup()
                    .ForAnyContext(config => config
                        .IncludeEntityObjects()
                        .AuditEventType("{database}_{context}"))
                    .UseOptOut();
                options.AddInterceptors(new AuditSaveChangesInterceptor());
            }
        });
    }
}

SecretsManagerService:

public class SecretsManagerService : ISecretsManagerService
{
    //Ommited for brevity
    public SecretsManagerService(IConfiguration configuration, Func<object, LoggerConfig> loggerConfig, IAmazonSecretsManager amazonSecretsManager)
    {
        _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
        var section = _configuration.GetSection(Constants.AWS_CONFIG_SECRETMANAGER_SECTION);
        _loggerConfig = loggerConfig(this) ?? throw new ArgumentNullException(nameof(loggerConfig));
        _client = amazonSecretsManager ?? throw new ArgumentNullException(nameof(amazonSecretsManager));
        cache = new SecretsManagerCache(_client);
    }
    
    public string GetSecret<T>(string secretName) where T : IConfigs
    {
        var cachedString = GetCachedSecret(secretName);
        if (string.IsNullOrEmpty(cachedString))
        {
            var request = new GetSecretValueRequest()
            {
                SecretId = secretName,
                VersionStage = "AWSCURRENT"
            };

            try
            {
                LoggerSingleton.Info(_loggerConfig, "Local Request");
                var response = _client.GetSecretValueAsync(request).Result;
                
                var connection = JsonConvert.DeserializeObject<T>(response.SecretString);
                return connection?.GetString() ?? "";
            }
            catch (Exception ex)
            {
                LoggerSingleton.Info(_loggerConfig, "Error on AWS SECRET " + ex.Message);
                throw;
            }
        }
        else
        {
            var connection = JsonConvert.DeserializeObject<ConnectionString>(cachedString);
            return connection?.GetString() ?? "";
        }


    }
    
    public string GetCachedSecret(string secretName)
    {
        if (cache != null)
        {
            var mySecret = cache.GetSecretString(secretName).Result;
            return mySecret;
        }
        return "";
    }
}

All the class are well registered on the container the Program.cs is something like:

builder.Services.AmazonServices(configuration);
builder.Services.RegisterDatabaseModule(configuration);

Anyone could point, where I'm doing wrong?

Did you manage to solve it? If so, please share the solution — Alex Sherzhukov, Jun 23 '22 at 11:57
I am having the same problem as you have found a solution? thanks — NHARI Med, Feb 21 '23 at 08:26

AWS Secret Manager throws HttpErrorResponseException after many requests

0 Answers0