Мы пытаемся защитить наш кластер ElastiCache для трафика Spring Boot c, включив шифрование SSL. Приложение Spring Boot успешно запускается, но при попытке регистрации в Eureka и Spring Boot Admin RedisHealthIndicator завершается неудачно с исключением:
2020-02-28 12:31:32.742 WARN [app,,,] 8894 --- [nio-8081-exec-1] o.s.b.a.redis.RedisHealthIndicator : Redis health check failed
org.springframework.data.redis.RedisConnectionFailureException: Unable to connect to Redis; nested exception is io.lettuce.core.RedisException: Cannot retrieve initial cluster partitions from initial URIs [RedisURI [host='my-cluster-url.amazonaws.com', port=6379]]
at org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory$SharedConnection.getNativeConnection(LettuceConnectionFactory.java:1111)
at org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory$SharedConnection.getConnection(LettuceConnectionFactory.java:1084)
at org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory.getClusterConnection(LettuceConnectionFactory.java:363)
at org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory.getConnection(LettuceConnectionFactory.java:333)
at org.springframework.data.redis.core.RedisConnectionUtils.doGetConnection(RedisConnectionUtils.java:132)
at org.springframework.data.redis.core.RedisConnectionUtils.getConnection(RedisConnectionUtils.java:95)
at org.springframework.data.redis.core.RedisConnectionUtils.getConnection(RedisConnectionUtils.java:82)
at org.springframework.boot.actuate.redis.RedisHealthIndicator.doHealthCheck(RedisHealthIndicator.java:56)
at org.springframework.boot.actuate.health.AbstractHealthIndicator.health(AbstractHealthIndicator.java:84)
at org.springframework.boot.actuate.health.CompositeHealthIndicator.health(CompositeHealthIndicator.java:98)
at org.springframework.boot.actuate.health.CompositeHealthIndicator.health(CompositeHealthIndicator.java:98)
at org.springframework.boot.actuate.health.HealthEndpoint.health(HealthEndpoint.java:50)
at org.springframework.boot.actuate.health.HealthEndpointWebExtension.health(HealthEndpointWebExtension.java:54)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:282)
at org.springframework.boot.actuate.endpoint.invoke.reflect.ReflectiveOperationInvoker.invoke(ReflectiveOperationInvoker.java:76)
at org.springframework.boot.actuate.endpoint.annotation.AbstractDiscoveredOperation.invoke(AbstractDiscoveredOperation.java:61)
at org.springframework.boot.actuate.endpoint.web.servlet.AbstractWebMvcEndpointHandlerMapping$ServletWebOperationAdapter.handle(AbstractWebMvcEndpointHandlerMapping.java:294)
at org.springframework.boot.actuate.endpoint.web.servlet.AbstractWebMvcEndpointHandlerMapping$OperationHandler.handle(AbstractWebMvcEndpointHandlerMapping.java:355)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:189)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:102)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:892)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:797)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.boot.actuate.autoconfigure.web.servlet.CompositeHandlerAdapter.handle(CompositeHandlerAdapter.java:58)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1038)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:942)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1005)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:897)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:645)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:882)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:750)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:209)
at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:178)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:200)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:678)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:834)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1415)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
Caused by: io.lettuce.core.RedisException: Cannot retrieve initial cluster partitions from initial URIs [RedisURI [host='my-cluster-urlamazonaws.com', port=6379]]
at io.lettuce.core.cluster.RedisClusterClient.loadPartitions(RedisClusterClient.java:865)
at io.lettuce.core.cluster.RedisClusterClient.initializePartitions(RedisClusterClient.java:819)
at io.lettuce.core.cluster.RedisClusterClient.connect(RedisClusterClient.java:345)
at org.springframework.data.redis.connection.lettuce.ClusterConnectionProvider.getConnection(ClusterConnectionProvider.java:85)
at org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory$SharedConnection.getNativeConnection(LettuceConnectionFactory.java:1104)
... 62 common frames omitted
Caused by: io.lettuce.core.RedisCommandInterruptedException: Command interrupted
at io.lettuce.core.cluster.topology.ClusterTopologyRefresh.loadViews(ClusterTopologyRefresh.java:109)
at io.lettuce.core.cluster.RedisClusterClient.doLoadPartitions(RedisClusterClient.java:871)
at io.lettuce.core.cluster.RedisClusterClient.loadPartitions(RedisClusterClient.java:844)
... 66 common frames omitted
Caused by: java.lang.InterruptedException: null
at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:347)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
at io.lettuce.core.cluster.topology.RefreshFutures.awaitAll(RefreshFutures.java:51)
at io.lettuce.core.cluster.topology.AsyncConnections.get(AsyncConnections.java:80)
at io.lettuce.core.cluster.topology.ClusterTopologyRefresh.loadViews(ClusterTopologyRefresh.java:73)
... 68 common frames omitted
Настройка:
- Spring Boot версии 2.1.4. RELEASE
- ElastiCache - Redis 5.0.3
Рабочий процесс:
- Запускается приложение и подключается к кластеру ElastiCache
2020-02-28 12:29:31.786 DEBUG [app,,,] 8894 --- [nfoReplicator-0] io.lettuce.core.RedisClient : Connecting to Redis at my-cluster-url.amazonaws.com:6379
2020-02-28 12:29:32.633 DEBUG [app,,,] 8894 --- [ioEventLoop-8-1] io.lettuce.core.RedisClient : Connecting to Redis at my-cluster-url.amazonaws.com:6379: Success
- Через некоторое время регистрируется предупреждение 10-20 с
Redis health check failed и приложение помечается как мертвое в Eureka, даже если оно все еще пытается повторно подключиться к ElastiCache и может видеть кластер.
2020-02-28 12:31:48.159 DEBUG [app,,,] 30409 --- [nio-8081-exec-1] o.s.d.redis.core.RedisConnectionUtils : Opening RedisConnection
2020-02-28 12:31:48.162 DEBUG [app,,,] 30409 --- [nio-8081-exec-1] i.l.c.c.PooledClusterConnectionProvider : getConnection(WRITE, my-cluster-url.amazonaws.com, 6379)
2020-02-28 12:31:48.165 DEBUG [app,,,] 30409 --- [ioEventLoop-9-1] i.l.core.protocol.RedisStateMachine : Decoded LatencyMeteredCommand [type=CLUSTER, output=StatusOutput [output=cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:4
cluster_size:2
cluster_current_epoch:3
cluster_my_epoch:3
cluster_stats_messages_ping_sent:2468227
cluster_stats_messages_pong_sent:2527673
cluster_stats_messages_meet_sent:3
cluster_stats_messages_sent:4995903
cluster_stats_messages_ping_received:2527667
cluster_stats_messages_pong_received:2468229
cluster_stats_messages_meet_received:3
cluster_stats_messages_fail_received:1
cluster_stats_messages_received:4995900
, error='null'], commandType=io.lettuce.core.protocol.AsyncCommand], empty stack: true
Конфигурация:
LettuceClientConfiguration lettuceClientConfiguration = LettuceClientConfiguration.builder().useSsl().disablePeerVerification().build();
return new LettuceConnectionFactory(redisClusterConfiguration, lettuceClientConfiguration);
Пробовал установить .startTls() - не помогло. Пытался установить .pingBeforeActivateConnection(true) - не помогло.
Я думаю, проблема внутри spring-boot-actuator сама по себе, поэтому одним из возможных решений может быть попытка переопределить RedisHealthIndicator .