Исключение тайм-аута репликации часто возникает в реплицированном кэше infinispan, добавленном в кластер wildfly 10 (2 узла)

Мы настраиваем кластерную среду, используя wildfly 10 из 2 узлов. Мы добавили новый контейнер кеша и реплицированный кеш с именем userSessionData для обмена данными между двумя серверами. Используется Standalone-full-ha.xml. В нашей среде разработки все работает нормально.

Но в клиентской среде pre-prod довольно часто мы получаем исключение тайм-аута репликации. В нашей среде разработки мы используем настройку удаленного тайм-аута по умолчанию, то есть 17500 мс, что, как мне кажется, соответствует таймауту репликации. В клиентской среде pre-prod мы увеличили его до 60000 мс, т.е. 1 мин, но по-прежнему довольно часто получаем одно и то же исключение. Один из сценариев проблемы с тайм-аутом: - Когда пользователь входит в приложение, мы помещаем данные пользовательского сеанса в кеш. Во время записи в кэш возникает исключение.

Я приложил конфигурацию infinispan, коды Java, используемые при записи в кеш, и исключение. Тоже наклеил ниже.

Ниже приведена конфигурация Infinispan для добавленного кеша.

<cache-container name="userSessionData_cache" default-cache="default" jndi-name="infinispan/userSessionData_cache">
                <transport lock-timeout="60000"/>
                <replicated-cache name="userSessionData" jndi-name="infinispan/userSessionData_cache/userSessionData" mode="SYNC" remote-timeout="60000">
                    <transaction mode="BATCH"/>
                </replicated-cache>
</cache-container>

Ниже приведен код класса Java, который будет вызывать другой класс Java, используемый для кеширования:

public class UserSessionTracker {

    private UserSessionTracker() {
    }
    private static UserSessionTracker instance = new UserSessionTracker();

    public static void putUserSession(String userName, String sessionId) {
        synchronized (instance) {
                UserSessionDataCacheManager.getInstance().putUserSession(userName, sessionId);
        }
    }
 }

Ниже приведен java-класс, используемый для кеширования (используется шаблон Singleton для класса):

@ManagedBean(name = "userSessionDataCacheManager", eager = true)
@ApplicationScoped
public class UserSessionDataCacheManager {

@Resource(name = "java:jboss/infinispan/userSessionData_cache/userSessionData")
    private Cache<String, java.util.Hashtable<String, Object>> entityCache = null;

    private static UserSessionDataCacheManager configInstance = null;

    public UserSessionDataCacheManager() {
        configInstance = this;
    }

    @PostConstruct
    public void start() {
System.out.println("Cache name: " + entityCache.getName());
    }

    public static UserSessionDataCacheManager getInstance() {
        if (configInstance == null) {
            createInstance();
        }
        return configInstance;
    }

    private static void createInstance() {
        configInstance = new UserSessionDataCacheManager();
    }

    public void putUserSession(String userName, String sessionId) {
System.out.println("inside  putUserSession");
    java.util.Hashtable<String, Object> sessionData = new java.util.Hashtable<String, Object>();
        sessionData.put("Session", sessionId);
        entityCache.put(userName, sessionData);

    }
}

Ниже приведен журнал исключений.

2017-07-27 11:53:54,539 INFO  [stdout] (default task-15) inside putUserSession

2017-07-27 11:54:54,571 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (default task-15) ISPN000136: Error executing command PrepareCommand, writing keys [F2959267]: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)

2017-07-27 11:54:54,571 ERROR [org.infinispan.transaction.impl.TransactionCoordinator] (default task-15) ISPN000097: Error while processing a prepare in a single-phase transaction: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)

2017-07-27 11:55:54,587 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (default task-15) ISPN000136: Error executing command RollbackCommand, writing keys [F2959267]: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)

2017-07-27 11:55:54,587 WARN  [org.infinispan.transaction.impl.TransactionCoordinator] (default task-15) ISPN000141: Could not rollback prepared 1PC transaction. This transaction will be rolled back by the recovery process, if enabled. Transaction: LocalXaTransaction{xid=DummyXid{id=77}} LocalTransaction{remoteLockedNodes=null, isMarkedForRollback=false, lockedKeys=[], backupKeyLocks=[], topologyId=3, stateTransferFlag=null} org.infinispan.transaction.xa.LocalXaTransaction@8b: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)

2017-07-27 11:55:54,587 WARN  [org.infinispan.transaction.tm.DummyTransaction] (default task-15) ISPN000112: exception while committing: javax.transaction.xa.XAException
    at org.infinispan.transaction.impl.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:207)
    at org.infinispan.transaction.impl.TransactionCoordinator.commit(TransactionCoordinator.java:159)
    at org.infinispan.transaction.xa.TransactionXaAdapter.commit(TransactionXaAdapter.java:114)
    at org.infinispan.transaction.tm.DummyTransaction.finishResource(DummyTransaction.java:401)
    at org.infinispan.transaction.tm.DummyTransaction.commitResources(DummyTransaction.java:448)
    at org.infinispan.transaction.tm.DummyTransaction.runCommit(DummyTransaction.java:321)
    at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:108)
    at org.wildfly.clustering.ee.infinispan.InfinispanBatch.close(InfinispanBatch.java:71)
    at org.wildfly.clustering.web.undertow.session.DistributableSession.requestDone(DistributableSession.java:81)
    at io.undertow.servlet.spec.ServletContextImpl.updateSessionAccessTime(ServletContextImpl.java:814)
    at io.undertow.servlet.spec.HttpServletResponseImpl.responseDone(HttpServletResponseImpl.java:565)
    at io.undertow.servlet.spec.HttpServletResponseImpl.sendRedirect(HttpServletResponseImpl.java:202)
    at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61)
    at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
    at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
    at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
    at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
    at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131)
    at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
    at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
    at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
    at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
    at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50)
    at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:292)
    at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:81)
    at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:138)
    at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:135)
    at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48)
    at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:272)
    at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81)
    at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:104)
    at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202)
    at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:805)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)
Caused by: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    ... 3 more

2017-07-27 11:55:54,712 WARN  [org.wildfly.clustering.web.undertow] (default task-15) javax.transaction.HeuristicMixedException: org.infinispan.commons.CacheException: javax.transaction.HeuristicMixedException
    at org.wildfly.clustering.ee.infinispan.InfinispanBatch.close(InfinispanBatch.java:74)
    at org.wildfly.clustering.web.undertow.session.DistributableSession.requestDone(DistributableSession.java:81)
    at io.undertow.servlet.spec.ServletContextImpl.updateSessionAccessTime(ServletContextImpl.java:814)
    at io.undertow.servlet.spec.HttpServletResponseImpl.responseDone(HttpServletResponseImpl.java:565)
    at io.undertow.servlet.spec.HttpServletResponseImpl.sendRedirect(HttpServletResponseImpl.java:202)
    at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:61)
    at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
    at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
    at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
    at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
    at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131)
    at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
    at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
    at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
    at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
    at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50)
    at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
    at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:292)
    at io.undertow.servlet.handlers.ServletInitialHandler.access$100(ServletInitialHandler.java:81)
    at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:138)
    at io.undertow.servlet.handlers.ServletInitialHandler$2.call(ServletInitialHandler.java:135)
    at io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:48)
    at io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.api.LegacyThreadSetupActionWrapper$1.call(LegacyThreadSetupActionWrapper.java:44)
    at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:272)
    at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81)
    at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:104)
    at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202)
    at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:805)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)
Caused by: javax.transaction.HeuristicMixedException
    at org.infinispan.transaction.tm.DummyTransaction.finishResource(DummyTransaction.java:439)
    at org.infinispan.transaction.tm.DummyTransaction.commitResources(DummyTransaction.java:448)
    at org.infinispan.transaction.tm.DummyTransaction.runCommit(DummyTransaction.java:321)
    at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:108)
    at org.wildfly.clustering.ee.infinispan.InfinispanBatch.close(InfinispanBatch.java:71)
    ... 161 more
Caused by: javax.transaction.xa.XAException
    at org.infinispan.transaction.impl.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:207)
    at org.infinispan.transaction.impl.TransactionCoordinator.commit(TransactionCoordinator.java:159)
    at org.infinispan.transaction.xa.TransactionXaAdapter.commit(TransactionXaAdapter.java:114)
    at org.infinispan.transaction.tm.DummyTransaction.finishResource(DummyTransaction.java:401)
    ... 165 more
Caused by: org.infinispan.util.concurrent.TimeoutException: Replication timeout for fit-r3stems01
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:801)
    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
    at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
    at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
    at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
    at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:47)
    at org.infinispan.remoting.transport.jgroups.RspListFuture.call(RspListFuture.java:16)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
    ... 3 more