У меня есть два узла, настроенных в режиме кластеризации домена, в настоящее время я обновляюсь с 4.3.0, который работает нормально, до 4.4.0, после обновления node1 keycloak-master запускается без проблем, но когда я пытаюсь запустить узел 2 - keycloak-не работает, с исключениями, связанными с StateTransfer бесконечных кэшей.Эта проблема не возникала в нашей среде разработки, но возникла в рабочей среде, где мы выполняем тесты входа в систему
Feb 6 08:48:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[0m#033[0m08:48:49,355 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 56) WFLYCLINF0002: Started users cache from keycloak container#033[0m
Feb 6 08:48:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[0m#033[0m08:48:49,355 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 55) WFLYCLINF0002: Started keys cache from keycloak container#033[0m
Feb 6 08:48:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[0m#033[0m08:48:49,355 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 51) WFLYCLINF0002: Started authorization cache from keycloak container#033[0m
Feb 6 08:48:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[0m#033[0m08:48:49,364 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 50) WFLYCLINF0002: Started realms cache from keycloak container#033[0m
Feb 6 08:51:40 node2.example.org sshd[6479]: Connection closed by 100.127.173.128 port 45404 [preauth]
Feb 6 08:51:40 node2.example.org rsyslogd: omkafka: action will suspended due to kafka error -195: Local: Broker transport failure [v8.35.0 try http://www.rsyslog.com/e/2422 ]
Feb 6 08:51:40 node2.example.org rsyslogd: action 'action 10' suspended (module 'omkafka'), retry 0. There should be messages before this one giving the reason for suspension. [v8.35.0 try http://www.rsyslog.com/e/2007 ]
Feb 6 08:51:40 node2.example.org rsyslogd: action 'action 10' resumed (module 'omkafka') [v8.35.0 try http://www.rsyslog.com/e/2359 ]
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[0m#033[31m08:52:49,435 ERROR [org.jboss.msc.service.fail] (ServerService Thread Pool -- 57) MSC000001: Failed to start service org.wildfly.clustering.infinispan.cache.keycloak.work: org.jboss.msc.service.StartException in service org.wildfly.clustering.infinispan.cache.keycloak.work: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete() throws java.lang.Exception on object of type StateTransferManagerImpl#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.service.FunctionalValueService.start(FunctionalValueService.java:70)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.service.AsynchronousServiceBuilder.lambda$start$0(AsynchronousServiceBuilder.java:99)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1349)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at java.lang.Thread.run(Thread.java:748)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.threads.JBossThread.run(JBossThread.java:485)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31mCaused by: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete() throws java.lang.Exception on object of type StateTransferManagerImpl#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:83)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.security.Security.doPrivileged(Security.java:76)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:229)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1020)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:644)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:589)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:475)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:461)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:447)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:86)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.infinispan.spi.service.CacheBuilder.get(CacheBuilder.java:71)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.infinispan.spi.service.CacheBuilder.get(CacheBuilder.java:54)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.service.SuppliedValueService.lambda$new$0(SuppliedValueService.java:42)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.wildfly.clustering.service.FunctionalValueService.start(FunctionalValueService.java:68)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011... 7 more#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31mCaused by: org.infinispan.commons.CacheException: Initial state transfer timed out for cache work on node2.example.org:server-node2#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:233)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at java.lang.reflect.Method.invoke(Method.java:498)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#011... 31 more#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#033[0m
Feb 6 08:52:49 node2.example.org domain.sh[6059]: [Server:server-node2] #033[31m#033[0m#033[31m08:52:49,451 ERROR [org.jboss.msc.service.fail] (ServerService Thread Pool -- 62) MSC000001: Failed to start service org.wildfly.clustering.infinispan.cache.ejb.client-mappings: org.jboss.msc.service.StartException in service org.wildfly.clustering.infinispan.cache.ejb.client-mappings: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete() throws java.lang.Exception on object of type StateTransferManagerImpl#033[0m
это журнал с узла 1:
Feb 7 09:19:33 node1.example.org domain.sh[13122]: [Server:server-node1] #033[0m#033[0m09:19:33,644 INFO [org.infinispan.CLUSTER] (transport-thread--p15-t8) [Context=offlineSessions][Scope=master:server-
idp-green01]ISPN100003: Node master:server-node1 finished rebalance phase with topology id 2#033[0m
Feb 7 09:19:33 node1.example.org domain.sh[13122]: [Server:server-node1] #033[0m#033[0m09:19:33,644 INFO [org.infinispan.CLUSTER] (transport-thread--p15-t23) [Context=loginFailures][Scope=master:server-i
dp-green01]ISPN100003: Node master:server-node1 finished rebalance phase with topology id 2#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[0m#033[31m09:20:50,407 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p9-t1) ISPN00013
6: Error executing command RemoveCommand, writing keys [869d1fde-2aad-4bca-8db7-64e05098d083]: org.infinispan.util.concurrent.TimeoutException: ISPN000476: Timed out waiting for responses for request 2 from node2.stage.bdc1.hu.sec.
in.pan-net.eu:server-node2#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.impl.MultiTargetRequest.onTimeout(MultiTargetRequest.java:167)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:87)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:22)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.FutureTask.run(FutureTask.java:266)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180
)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)#033[0
m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.lang.Thread.run(Thread.java:748)#033[0m
Feb 7 09:20:50 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#033[0m#033[31m09:21:00,445 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p9-t1) ISPN000136: Error executing command PutKeyValueCommand, writing keys [869d1fde-2aad-4bca-8db7-64e05098d083]: org.infinispan.util.concurrent.TimeoutException: ISPN000476: Timed out waiting for responses for request 3 from node2.example.org:server-node2#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.impl.MultiTargetRequest.onTimeout(MultiTargetRequest.java:167)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:87)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:22)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.FutureTask.run(FutureTask.java:266)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#011at java.lang.Thread.run(Thread.java:748)#033[0m
Feb 7 09:21:00 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#033[0m
Feb 7 09:21:10 node1.example.org domain.sh[13122]: [Server:server-node1] #033[31m#033[0m#033[31m09:21:10,454 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p9-t1) ISPN000136: Error executing command PutKeyValueCommand, writing keys [0352cd4f-9979-49ca-baa7-4f504bd7156c]: org.infinispan.util.concurrent.TimeoutException: ISPN000476: Timed out waiting for responses for request 4 from node2.example.org:server-node2#033[0m