Jedis lib не освобождает соединения и не блокирует приложение - PullRequest
Новая
       39

Jedis lib не освобождает соединения и не блокирует приложение

0 голосов
/ 27 декабря 2018

Я использую сервер Redis для распределенной координации и хранения некоторых данных кэша.redis_version: 4.0.2

Я использую jedis lib 

compile group: 'redis.clients', name: 'jedis', version: '2.8.1'

Проблема, с которой я столкнулся: мой пул со временем истощается, и из-за этого мой конвейер обработки блокируется.

Вот поток приложения:

Я читаю данные из очереди Oracle в непрерывном цикле выполнения, выполняю операции с базами данных над этими данными, затем отслеживаю состояние единого числа (которое я получаю из Oracleочереди), а затем отследите его при его перемещении из БД в Kinesis в сегменты S3.

Приведенный выше контекст был дан для приблизительных затрат на конвейерное приложение.

Когда я начал масштабировать приложение, янажмите на проблему с ресурсом redis, так как после обработки, скажем, 20 элементов мой поток просто будет бесконечно ждать. После выполнения дампа потока я обнаружил, что он ожидает DBCP BlockingQueue, который внутренне используется jedis для управления пулом соединений.

являются фрагментами кода из моего приложения, где я использую приложение. 

package com.here.umob.realtime.sync.impl;

import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.here.umob.realtime.sync.api.ICacheConnectionProvider;
import com.here.umob.realtime.sync.configuration.IConfigurationPropertyInitializer;
import org.apache.log4j.Logger;
import redis.clients.jedis.*;
import redis.clients.jedis.exceptions.JedisConnectionException;

import java.time.Duration;
import java.util.Objects;

/**
 * @author marwaha
 *
 */
@Singleton
public class CacheConnectionProviderImpl implements ICacheConnectionProvider {

    private IConfigurationPropertyInitializer configurationPropertyInitializer;
    private JedisPool jedisPool = null;

    private JedisCluster jedisCluster = null;

    private Logger logger = Logger.getLogger(CacheConnectionProviderImpl.class);

    @Inject
    public CacheConnectionProviderImpl(IConfigurationPropertyInitializer configurationPropertyInitializer) {
        this.configurationPropertyInitializer = configurationPropertyInitializer;
        initializeRedisPoolConnection();
        // initializeRedisClusterInstace();
    }

    private void initializeRedisClusterInstace() {
        jedisCluster = new JedisCluster(
                new HostAndPort(configurationPropertyInitializer.getConfiguration().get().getCacheServiceaddress(),
                        configurationPropertyInitializer.getConfiguration().get().getCacheServicePort()));

    }

    private void initializeRedisPoolConnection() {
        logger.info("Initializing redis connection config");
        JedisPoolConfig poolConfig = new JedisPoolConfig();
        poolConfig.setMaxTotal(128);
        poolConfig.setMaxIdle(128);
        poolConfig.setMinIdle(16);
        poolConfig.setTestOnBorrow(true);
        poolConfig.setTestOnReturn(true);
        poolConfig.setTestWhileIdle(true);
        poolConfig.setMinEvictableIdleTimeMillis(Duration.ofSeconds(60).toMillis());
        poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofSeconds(30).toMillis());
        poolConfig.setNumTestsPerEvictionRun(3);
        poolConfig.setBlockWhenExhausted(true);
        jedisPool = new JedisPool(poolConfig,
                configurationPropertyInitializer.getConfiguration().get().getCacheServiceaddress(),
                configurationPropertyInitializer.getConfiguration().get().getCacheServicePort());

    }

    @Override
    public Jedis getCacheConnection() {
        if (Objects.isNull(jedisPool) || !jedisPool.getResource().isConnected())
            initializeRedisPoolConnection();

        return jedisPool.getResource();
    }

    @Override
    public void resetCacheConnection() {

        if (!jedisPool.getResource().isConnected()) {
            try {
                jedisPool.close();
            } catch (JedisConnectionException connectionException) {
                logger.error("Error closing redis  pool");
            }

        }
        initializeRedisPoolConnection();
    }

Использование вышеупомянутого пула: 

public boolean updateState(String workSpace, String changeId, String state) {
        int retries = 0;
        boolean retry = false;
        do {
            logger.info(String.format("Updating state in redis with key %s, and state %s and workspace %s", changeId, state,workSpace));

            try (Jedis jedis = cacheConnectionProvider.getCacheConnection()) {
                String key = workSpace + "-" + changeId;
                String currentState = jedis.hget(workSpace + region, key);
                logger.info(String.format("Current state of change id key %s is %s and is being updated to %s", key,
                        currentState, state));
                jedis.hset(workSpace + region, key, state);
                if (state.equals(StateConstants.retrievedFromJMS))
                    jedis.rpush(workSpace, changeId);

            } catch (JedisException je) {
                logger.error(
                        "Error In connecting to redis cluster" + " " + je.getMessage() + " retrying" + " " + retry);
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                cacheConnectionProvider.resetCacheConnection();
                retry = true;
            }


        } while (retries++ < MAX_RETRIES && retry);
        return false;

    }

Использование try-with я предполагаю, чторесурс Jedis должен закрываться автоматически, и мне не нужно делать явный вызов.

Также я дал инициализацию пула подключений, параметры, чтобы более широкая аудитория могла взглянуть и увидеть, что не так.

Информация о соединении С redis: 

10.126.102.30:9300> client list
id=185152 addr=10.222.127.180:64932 fd=59 name= age=19 idle=19 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185126 addr=10.197.6.80:33496 fd=18 name= age=66 idle=66 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185104 addr=10.197.6.80:57506 fd=11 name= age=247 idle=247 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185153 addr=10.222.127.180:64934 fd=60 name= age=18 idle=18 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185148 addr=10.222.127.180:64928 fd=55 name= age=22 idle=17 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185131 addr=10.197.6.80:34270 fd=23 name= age=34 idle=34 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185117 addr=10.197.6.80:57998 fd=36 name= age=225 idle=225 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185158 addr=10.222.127.180:64939 fd=65 name= age=11 idle=11 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185154 addr=10.222.127.180:64935 fd=61 name= age=17 idle=17 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185160 addr=10.222.127.180:64943 fd=67 name= age=5 idle=5 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185156 addr=10.222.127.180:64937 fd=63 name= age=13 idle=13 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185125 addr=10.197.6.80:33460 fd=37 name= age=68 idle=68 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185112 addr=10.197.6.80:57758 fd=8 name= age=235 idle=235 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185101 addr=10.197.6.80:57200 fd=10 name= age=257 idle=257 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185109 addr=10.197.6.80:57706 fd=17 name= age=237 idle=237 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185136 addr=10.222.127.180:64918 fd=44 name= age=32 idle=31 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185137 addr=10.222.127.180:64919 fd=45 name= age=32 idle=31 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185138 addr=10.222.127.180:64920 fd=46 name= age=32 idle=31 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185147 addr=10.222.127.180:64927 fd=54 name= age=23 idle=17 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185139 addr=10.222.127.180:64921 fd=47 name= age=32 idle=31 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185128 addr=10.197.6.80:33790 fd=20 name= age=54 idle=54 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185121 addr=10.197.6.80:32962 fd=35 name= age=88 idle=88 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185122 addr=10.197.6.80:33142 fd=14 name= age=77 idle=77 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185150 addr=10.222.127.180:64930 fd=57 name= age=22 idle=18 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185159 addr=10.222.127.180:64942 fd=66 name= age=5 idle=5 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185120 addr=10.197.6.80:58532 fd=29 name= age=201 idle=201 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185124 addr=10.197.6.80:33416 fd=13 name= age=70 idle=70 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185105 addr=10.197.6.80:57534 fd=27 name= age=245 idle=245 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185123 addr=10.197.6.80:33316 fd=12 name= age=74 idle=74 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185162 addr=10.222.125.33:64104 fd=69 name= age=4 idle=0 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=32768 obl=0 oll=0 omem=0 events=r cmd=client
id=185118 addr=10.197.6.80:58102 fd=15 name= age=221 idle=221 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185111 addr=10.197.6.80:57750 fd=31 name= age=235 idle=235 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185106 addr=10.197.6.80:57590 fd=28 name= age=241 idle=241 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185116 addr=10.197.6.80:57962 fd=34 name= age=225 idle=225 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185103 addr=10.197.6.80:57500 fd=19 name= age=247 idle=247 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185110 addr=10.197.6.80:57730 fd=22 name= age=236 idle=236 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185115 addr=10.197.6.80:57860 fd=25 name= age=228 idle=228 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185130 addr=10.197.6.80:34068 fd=21 name= age=42 idle=42 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185108 addr=10.197.6.80:57676 fd=39 name= age=238 idle=238 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185102 addr=10.197.6.80:57488 fd=26 name= age=248 idle=248 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185149 addr=10.222.127.180:64929 fd=56 name= age=22 idle=18 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185151 addr=10.222.127.180:64931 fd=58 name= age=21 idle=21 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185129 addr=10.197.6.80:33940 fd=7 name= age=48 idle=48 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185114 addr=10.197.6.80:57798 fd=32 name= age=233 idle=233 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185113 addr=10.197.6.80:57766 fd=33 name= age=235 idle=235 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185127 addr=10.197.6.80:33626 fd=16 name= age=60 idle=60 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185132 addr=10.222.127.180:64914 fd=40 name= age=32 idle=32 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185155 addr=10.222.127.180:64936 fd=62 name= age=17 idle=17 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185133 addr=10.222.127.180:64915 fd=41 name= age=32 idle=32 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185134 addr=10.222.127.180:64916 fd=42 name= age=32 idle=32 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185135 addr=10.222.127.180:64917 fd=43 name= age=32 idle=32 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185107 addr=10.197.6.80:57624 fd=30 name= age=240 idle=240 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185119 addr=10.197.6.80:58120 fd=9 name= age=220 idle=220 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185161 addr=10.222.127.180:64944 fd=68 name= age=4 idle=4 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185140 addr=10.222.127.180:64922 fd=48 name= age=31 idle=22 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185146 addr=10.222.127.180:64926 fd=53 name= age=23 idle=23 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185141 addr=10.222.127.180:64923 fd=50 name= age=31 idle=22 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185142 addr=10.222.127.180:64924 fd=51 name= age=31 idle=22 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185143 addr=10.222.127.180:64925 fd=52 name= age=31 idle=22 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=ping
id=185144 addr=10.197.6.80:34360 fd=24 name= age=29 idle=29 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185145 addr=10.197.6.80:34528 fd=38 name= age=23 idle=23 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL
id=185157 addr=10.222.127.180:64938 fd=64 name= age=12 idle=12 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=NULL

и Thread

dump: 

"RealTimeSync-1" #24 prio=5 os_prio=31 tid=0x00007fd81cd89800 nid=0x6307 waiting on condition [0x0000700006f9f000]
   java.lang.Thread.State: WAITING (parking)
        at sun.misc.Unsafe.park(Native Method)
        - parking to wait for  <0x00000006c074a828> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at org.apache.commons.pool2.impl.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:583)
        at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:442)
        at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:363)
        at redis.clients.util.Pool.getResource(Pool.java:49)
        at redis.clients.jedis.JedisPool.getResource(JedisPool.java:99)
        at com.umob.realtime.sync.impl.CacheConnectionProviderImpl.getCacheConnection(CacheConnectionProviderImpl.java:69)
        at com.umob.realtime.sync.impl.ItemProcessedStatusCacheImpl.updateLastProcessedSequenceItemJedis(ItemProcessedStatusCacheImpl.java:87)
        at com.umob.realtime.sync.impl.QueueItemProcessorImpl.processQueue(QueueItemProcessorImpl.java:187)
        at com.umob.realtime.sync.impl.QueueItemProcessorImpl.run(QueueItemProcessorImpl.java:229)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
...