Kubernetes / Docker redis отказано в новых подключениях (существующие отлично работают) - PullRequest
0 голосов
/ 19 июня 2020
• 1000 *, из статистики я вижу только 22 подключенных клиента и несколько новых подключений

Я думал, что это может быть net.core.somaxconn или ulimit, но я попытался воспроизвести его локально, но смог подключить до 10000 клиентов без каких-либо проблем, память выглядит нормально, иначе я предполагаю, что это приведет к сбою / OOM процесса, я предполагаю, что высокая загрузка процессора может вызвать тайм-аут, но не отказ в соединении,
какие-либо мысли о том, как его отлаживать?

Redis Pod YAML

apiVersion: v1
kind: Pod
metadata:
  annotations:
    checksum/configmap: 77348d3248abead3b65da5ba143e2e7b579859fd0011eb15218462139dad6e7b
    checksum/health: 33d0c6a43d8f0b99a2157d8c8b15638a8a482c210ec193ca937614f03f1309ed
    checksum/secret: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
  creationTimestamp: "2020-06-19T07:44:49Z"
  generateName: redis-master-
  labels:
    app: redis
    chart: redis-8.0.12
    controller-revision-hash: redis-master-cb87b9644
    release: prod
    role: master
    statefulset.kubernetes.io/pod-name: redis-master-0
  name: redis-master-0
  namespace: default
  ownerReferences:
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: StatefulSet
    name: redis-master
    uid: 3a5e38a1-0fd0-11ea-b359-025000000001
  resourceVersion: "17975259"
  selfLink: /api/v1/namespaces/default/pods/redis-master-0
  uid: c1296561-b200-11ea-bd3d-025000000001
spec:
  containers:
  - command:
    - /bin/bash
    - -c
    - "if [[ -n $REDIS_PASSWORD_FILE ]]; then\n  password_aux=`cat ${REDIS_PASSWORD_FILE}`\n
      \ export REDIS_PASSWORD=$password_aux\nfi\nif [[ ! -f /opt/bitnami/redis/etc/master.conf
      ]];then\n  cp /opt/bitnami/redis/mounted-etc/master.conf /opt/bitnami/redis/etc/master.conf\nfi\nif
      [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then\n  cp /opt/bitnami/redis/mounted-etc/redis.conf
      /opt/bitnami/redis/etc/redis.conf\nfi          \nARGS=(\"--port\" \"${REDIS_PORT}\")\nARGS+=(\"--protected-mode\"
      \"no\")\nARGS+=(\"--include\" \"/opt/bitnami/redis/etc/redis.conf\")\nARGS+=(\"--include\"
      \"/opt/bitnami/redis/etc/master.conf\")\n/run.sh ${ARGS[@]}\n"
    env:
    - name: REDIS_REPLICATION_MODE
      value: master
    - name: ALLOW_EMPTY_PASSWORD
      value: "yes"
    - name: REDIS_PORT
      value: "6379"
    image: docker.io/bitnami/redis:5.0.5-debian-9-r36
    imagePullPolicy: IfNotPresent
    livenessProbe:
      exec:
        command:
        - sh
        - -c
        - /health/ping_local.sh 5
      failureThreshold: 5
      initialDelaySeconds: 5
      periodSeconds: 5
      successThreshold: 1
      timeoutSeconds: 5
    name: redis
    ports:
    - containerPort: 6379
      name: redis
      protocol: TCP
    readinessProbe:
      exec:
        command:
        - sh
        - -c
        - /health/ping_local.sh 5
      failureThreshold: 5
      initialDelaySeconds: 5
      periodSeconds: 5
      successThreshold: 1
      timeoutSeconds: 1
    resources:
      limits:
        memory: 2Gi
      requests:
        memory: 256Mi
    securityContext:
      runAsUser: 1001
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /health
      name: health
    - mountPath: /data
      name: redis-data
    - mountPath: /opt/bitnami/redis/mounted-etc
      name: config
    - mountPath: /opt/bitnami/redis/etc/
      name: redis-tmp-conf
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: default-token-bmppc
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  hostname: redis-master-0
  imagePullSecrets:
  - name: map[name:registrykey]
  nodeName: docker-desktop
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 1001
  serviceAccount: default
  serviceAccountName: default
  subdomain: redis-headless
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - name: redis-data
    persistentVolumeClaim:
      claimName: redis-data-redis-master-0
  - configMap:
      defaultMode: 493
      name: redis-health
    name: health
  - configMap:
      defaultMode: 420
      name: redis
    name: config
  - emptyDir: {}
    name: redis-tmp-conf
  - name: default-token-bmppc
    secret:
      defaultMode: 420
      secretName: default-token-bmppc

Redis Stats (один из существующих потоков выводит это каждые несколько минут)

# Server
redis_version:5.0.5
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:24f9cd9802bc0a06
redis_mode:standalone
os:Linux 4.4.0-1100-aws x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:6.3.0
process_id:1
run_id:2e4fafe374c8f89a28e5efa0d44da647c7ee3f79
tcp_port:6379
uptime_in_seconds:7423470
uptime_in_days:85
hz:10
configured_hz:10
lru_clock:15361330
executable:/opt/bitnami/redis/bin/redis-server
config_file:/opt/bitnami/redis/etc/redis.conf

# Clients
connected_clients:22
client_recent_max_input_buffer:2
client_recent_max_output_buffer:0
blocked_clients:2

# Memory
used_memory:824062816
used_memory_human:785.89M
used_memory_rss:847245312
used_memory_rss_human:808.00M
used_memory_peak:831404736
used_memory_peak_human:792.89M
used_memory_peak_perc:99.12%
used_memory_overhead:89645730
used_memory_startup:791496
used_memory_dataset:734417086
used_memory_dataset_perc:89.21%
allocator_allocated:824194008
allocator_active:824901632
allocator_resident:849993728
total_system_memory:16340549632
total_system_memory_human:15.22G
used_memory_lua:35840
used_memory_lua_human:35.00K
used_memory_scripts:448
used_memory_scripts_human:448B
number_of_cached_scripts:2
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.00
allocator_frag_bytes:707624
allocator_rss_ratio:1.03
allocator_rss_bytes:25092096
rss_overhead_ratio:1.00
rss_overhead_bytes:-2748416
mem_fragmentation_ratio:1.03
mem_fragmentation_bytes:23224264
mem_not_counted_for_evict:1218
mem_replication_backlog:0
mem_clients_slaves:0
mem_clients_normal:536144
mem_aof_buffer:1218
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0

# Persistence
loading:0
rdb_changes_since_last_save:352987925
rdb_bgsave_in_progress:0
rdb_last_save_time:1584996164
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:0
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:86
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:3002368
aof_current_size:643062882
aof_base_size:521502964
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:359

# Stats
total_connections_received:5395254
total_commands_processed:790981924
instantaneous_ops_per_sec:17
total_net_input_bytes:128961701343
total_net_output_bytes:42336097542
instantaneous_input_kbps:1.04
instantaneous_output_kbps:0.18
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:27448
expired_stale_perc:0.00
expired_time_cap_reached_count:0
evicted_keys:0
keyspace_hits:215378512
keyspace_misses:145834885
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:270570
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0

# Replication
role:master
connected_slaves:0
master_replid:bec5aac990fe73bfceb306838d911ba3a4545ec6
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

#
 CPU
used_cpu_sys:26901.988000
used_cpu_user:990782.152000
used_cpu_sys_children:137.836000
used_cpu_user_children:1326.688000

# Commandstats
cmdstat_rpoplpush:calls=24004,usec=916109,usec_per_call=38.16
cmdstat_hincrby:calls=242161098,usec=1621116434,usec_per_call=6.69
cmdstat_info:calls=24743,usec=272596086,usec_per_call=11017.10
cmdstat_sadd:calls=1788202,usec=26694448,usec_per_call=14.93
cmdstat_hget:calls=36271409,usec=527003855,usec_per_call=14.53
cmdstat_rename:calls=8,usec=52,usec_per_call=6.50
cmdstat_sdiff:calls=2,usec=96,usec_per_call=48.00
cmdstat_rpush:calls=2219720,usec=142033543,usec_per_call=63.99
cmdstat_brpoplpush:calls=18425367,usec=523846234,usec_per_call=28.43
cmdstat_exists:calls=13097339,usec=117464452,usec_per_call=8.97
cmdstat_zadd:calls=785,usec=444394,usec_per_call=566.11
cmdstat_lrange:calls=18130,usec=100941,usec_per_call=5.57
cmdstat_hset:calls=6844044,usec=187927053,usec_per_call=27.46
cmdstat_select:calls=2371237,usec=15328732,usec_per_call=6.46
cmdstat_expire:calls=4438329,usec=65649053,usec_per_call=14.79
cmdstat_ping:calls=8157340,usec=20556877,usec_per_call=2.52
cmdstat_watch:calls=20844180,usec=513086491,usec_per_call=24.62
cmdstat_decrby:calls=2201527,usec=15442412,usec_per_call=7.01
cmdstat_lrem:calls=9765096,usec=343373099,usec_per_call=35.16
cmdstat_hkeys:calls=8145,usec=308955,usec_per_call=37.93
cmdstat_evalsha:calls=2451255,usec=824307343,usec_per_call=336.28
cmdstat_zrangebyscore:calls=20844174,usec=816391113,usec_per_call=39.17
cmdstat_scard:calls=16,usec=6488,usec_per_call=405.50
cmdstat_lpush:calls=7132778,usec=133423873,usec_per_call=18.71
cmdstat_exec:calls=2004,usec=422236,usec_per_call=210.70
cmdstat_zrem:calls=785,usec=134329,usec_per_call=171.12
cmdstat_set:calls=61770783,usec=334229160,usec_per_call=5.41
cmdstat_get:calls=282532307,usec=1716913145,usec_per_call=6.08
cmdstat_del:calls=2373249,usec=38018613,usec_per_call=16.02
cmdstat_llen:calls=8441873,usec=25837214,usec_per_call=3.06
cmdstat_blpop:calls=7735374,usec=247063170,usec_per_call=31.94
cmdstat_discard:calls=20842170,usec=138960731,usec_per_call=6.67
cmdstat_multi:calls=2004,usec=1367,usec_per_call=0.68
cmdstat_eval:calls=2,usec=176,usec_per_call=88.00
cmdstat_keys:calls=5990610,usec=1263983416571,usec_per_call=210994.11
cmdstat_incrby:calls=2201528,usec=67602672,usec_per_call=30.71

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=1788471,expires=7,avg_ttl=15045772
db1:keys=1,expires=0,avg_ttl=0

Redis Pod Logs

{"log":"1:M 17 Jun 2020 18:45:17.347 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:17.347417952Z"}
{"log":"1:M 17 Jun 2020 18:45:17.982 - DB 0: 1788449 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:17.982245331Z"}
{"log":"1:M 17 Jun 2020 18:45:17.982 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:17.982273468Z"}
{"log":"1:M 17 Jun 2020 18:45:17.982 - 22 clients connected (0 replicas), 823973496 bytes in use\n","stream":"stdout","time":"2020-06-17T18:45:17.982277588Z"}
{"log":"1:M 17 Jun 2020 18:45:22.013 - Accepted 127.0.0.1:48006\n","stream":"stdout","time":"2020-06-17T18:45:22.013232896Z"}
{"log":"1:M 17 Jun 2020 18:45:22.013 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:22.013693738Z"}
{"log":"1:M 17 Jun 2020 18:45:22.619 - Accepted 127.0.0.1:48010\n","stream":"stdout","time":"2020-06-17T18:45:22.61921419Z"}
{"log":"1:M 17 Jun 2020 18:45:23.212 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:23.212619397Z"}
{"log":"1:M 17 Jun 2020 18:45:27.297 - Accepted 127.0.0.1:48128\n","stream":"stdout","time":"2020-06-17T18:45:27.297986674Z"}
{"log":"1:M 17 Jun 2020 18:45:27.297 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:27.298009831Z"}
{"log":"1:M 17 Jun 2020 18:45:27.866 - Accepted 127.0.0.1:48148\n","stream":"stdout","time":"2020-06-17T18:45:27.867116201Z"}
{"log":"1:M 17 Jun 2020 18:45:27.867 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:27.867594001Z"}
{"log":"1:M 17 Jun 2020 18:45:32.455 - Accepted 127.0.0.1:48266\n","stream":"stdout","time":"2020-06-17T18:45:32.455977779Z"}
{"log":"1:M 17 Jun 2020 18:45:32.455 - Accepted 127.0.0.1:48274\n","stream":"stdout","time":"2020-06-17T18:45:32.456012723Z"}
{"log":"1:M 17 Jun 2020 18:45:32.456 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:32.456912078Z"}
{"log":"1:M 17 Jun 2020 18:45:32.456 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:32.457894603Z"}
{"log":"1:M 17 Jun 2020 18:45:37.443 - Accepted 127.0.0.1:48644\n","stream":"stdout","time":"2020-06-17T18:45:37.443850298Z"}
{"log":"1:M 17 Jun 2020 18:45:37.443 - Accepted 127.0.0.1:48652\n","stream":"stdout","time":"2020-06-17T18:45:37.443885573Z"}
{"log":"1:M 17 Jun 2020 18:45:37.985 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:37.985153612Z"}
{"log":"1:M 17 Jun 2020 18:45:37.985 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:37.985179371Z"}
{"log":"1:M 17 Jun 2020 18:45:38.520 - DB 0: 1788451 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:38.520158133Z"}
{"log":"1:M 17 Jun 2020 18:45:38.520 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:38.520179568Z"}
{"log":"1:M 17 Jun 2020 18:45:38.520 - 22 clients connected (0 replicas), 824138256 bytes in use\n","stream":"stdout","time":"2020-06-17T18:45:38.520182764Z"}
{"log":"1:M 17 Jun 2020 18:45:42.253 - Accepted 127.0.0.1:48714\n","stream":"stdout","time":"2020-06-17T18:45:42.253712398Z"}
{"log":"1:M 17 Jun 2020 18:45:42.254 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:42.254367162Z"}
{"log":"1:M 17 Jun 2020 18:45:42.805 - Accepted 127.0.0.1:48718\n","stream":"stdout","time":"2020-06-17T18:45:42.805481366Z"}
{"log":"1:M 17 Jun 2020 18:45:43.345 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:43.345846992Z"}
{"log":"1:M 17 Jun 2020 18:45:46.925 - Accepted 127.0.0.1:48778\n","stream":"stdout","time":"2020-06-17T18:45:46.925360782Z"}
{"log":"1:M 17 Jun 2020 18:45:46.925 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:46.92560591Z"}
{"log":"1:M 17 Jun 2020 18:45:47.316 - Accepted 127.0.0.1:48802\n","stream":"stdout","time":"2020-06-17T18:45:47.316589733Z"}
{"log":"1:M 17 Jun 2020 18:45:47.316 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:47.31694214Z"}
{"log":"1:M 17 Jun 2020 18:45:48.316 - DB 0: 1788459 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:48.316830035Z"}
{"log":"1:M 17 Jun 2020 18:45:48.316 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:48.316850428Z"}
{"log":"1:M 17 Jun 2020 18:45:48.316 - 22 clients connected (0 replicas), 823976712 bytes in use\n","stream":"stdout","time":"2020-06-17T18:45:48.316853431Z"}
{"log":"1:M 17 Jun 2020 18:45:51.907 - Accepted 127.0.0.1:48874\n","stream":"stdout","time":"2020-06-17T18:45:51.90776359Z"}
{"log":"1:M 17 Jun 2020 18:45:51.907 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:51.908031401Z"}
{"log":"1:M 17 Jun 2020 18:45:52.323 - Accepted 127.0.0.1:48878\n","stream":"stdout","time":"2020-06-17T18:45:52.324017264Z"}
{"log":"1:M 17 Jun 2020 18:45:52.324 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:52.324309956Z"}
{"log":"1:M 17 Jun 2020 18:45:53.331 - DB 0: 1788460 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:53.33130562Z"}
{"log":"1:M 17 Jun 2020 18:45:53.331 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:53.331326037Z"}
{"log":"1:M 17 Jun 2020 18:45:53.331 - 22 clients connected (0 replicas), 823936224 bytes in use\n","stream":"stdout","time":"2020-06-17T18:45:53.331329161Z"}
{"log":"1:M 17 Jun 2020 18:45:56.924 - Accepted 127.0.0.1:49224\n","stream":"stdout","time":"2020-06-17T18:45:56.924651232Z"}
{"log":"1:M 17 Jun 2020 18:45:56.924 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:56.924926435Z"}
{"log":"1:M 17 Jun 2020 18:45:57.336 - Accepted 127.0.0.1:49254\n","stream":"stdout","time":"2020-06-17T18:45:57.336590506Z"}
{"log":"1:M 17 Jun 2020 18:45:57.336 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:45:57.336777539Z"}
{"log":"1:M 17 Jun 2020 18:45:58.356 - DB 0: 1788460 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:58.357003077Z"}
{"log":"1:M 17 Jun 2020 18:45:58.356 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:45:58.357029903Z"}
{"log":"1:M 17 Jun 2020 18:45:58.356 - 22 clients connected (0 replicas), 823895272 bytes in use\n","stream":"stdout","time":"2020-06-17T18:45:58.357034349Z"}
{"log":"1:M 17 Jun 2020 18:46:02.042 - Accepted 127.0.0.1:49594\n","stream":"stdout","time":"2020-06-17T18:46:02.042501452Z"}
{"log":"1:M 17 Jun 2020 18:46:02.042 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:02.042748815Z"}
{"log":"1:M 17 Jun 2020 18:46:02.328 - Accepted 127.0.0.1:49612\n","stream":"stdout","time":"2020-06-17T18:46:02.328374114Z"}
{"log":"1:M 17 Jun 2020 18:46:02.328 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:02.328658084Z"}
{"log":"1:M 17 Jun 2020 18:46:03.375 - DB 0: 1788460 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:03.375225273Z"}
{"log":"1:M 17 Jun 2020 18:46:03.375 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:03.375249792Z"}
{"log":"1:M 17 Jun 2020 18:46:03.375 - 22 clients connected (0 replicas), 823936224 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:03.375254041Z"}
{"log":"1:M 17 Jun 2020 18:46:06.952 - Accepted 127.0.0.1:49686\n","stream":"stdout","time":"2020-06-17T18:46:06.953059506Z"}
{"log":"1:M 17 Jun 2020 18:46:06.953 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:06.953224847Z"}
{"log":"1:M 17 Jun 2020 18:46:07.348 - Accepted 127.0.0.1:49694\n","stream":"stdout","time":"2020-06-17T18:46:07.349016118Z"}
{"log":"1:M 17 Jun 2020 18:46:07.349 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:07.349382274Z"}
{"log":"1:M 17 Jun 2020 18:46:08.391 - DB 0: 1788460 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:08.391698311Z"}
{"log":"1:M 17 Jun 2020 18:46:08.391 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:08.391739397Z"}
{"log":"1:M 17 Jun 2020 18:46:08.391 - 22 clients connected (0 replicas), 823892832 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:08.391745272Z"}
{"log":"1:M 17 Jun 2020 18:46:11.965 - Accepted 127.0.0.1:49780\n","stream":"stdout","time":"2020-06-17T18:46:11.965395852Z"}
{"log":"1:M 17 Jun 2020 18:46:11.965 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:11.965808732Z"}
{"log":"1:M 17 Jun 2020 18:46:12.354 - Accepted 127.0.0.1:49784\n","stream":"stdout","time":"2020-06-17T18:46:12.354725886Z"}
{"log":"1:M 17 Jun 2020 18:46:12.358 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:12.367203745Z"}
{"log":"1:M 17 Jun 2020 18:46:13.560 - DB 0: 1788462 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:13.560915216Z"}
{"log":"1:M 17 Jun 2020 18:46:13.560 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:13.560929844Z"}
{"log":"1:M 17 Jun 2020 18:46:13.560 - 22 clients connected (0 replicas), 823976400 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:13.560933724Z"}
{"log":"1:M 17 Jun 2020 18:46:17.262 - Accepted 127.0.0.1:50058\n","stream":"stdout","time":"2020-06-17T18:46:17.262385128Z"}
{"log":"1:M 17 Jun 2020 18:46:17.263 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:17.264188739Z"}
{"log":"1:M 17 Jun 2020 18:46:18.000 - Accepted 127.0.0.1:50110\n","stream":"stdout","time":"2020-06-17T18:46:18.000348733Z"}
{"log":"1:M 17 Jun 2020 18:46:18.630 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:18.630798563Z"}
{"log":"1:M 17 Jun 2020 18:46:22.135 - Accepted 127.0.0.1:50182\n","stream":"stdout","time":"2020-06-17T18:46:22.135347553Z"}
{"log":"1:M 17 Jun 2020 18:46:22.680 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:22.680803014Z"}
{"log":"1:M 17 Jun 2020 18:46:22.680 - Accepted 127.0.0.1:50186\n","stream":"stdout","time":"2020-06-17T18:46:22.680827274Z"}
{"log":"1:M 17 Jun 2020 18:46:22.680 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:22.680841194Z"}
{"log":"1:M 17 Jun 2020 18:46:27.147 - Accepted 127.0.0.1:50252\n","stream":"stdout","time":"2020-06-17T18:46:27.148067254Z"}
{"log":"1:M 17 Jun 2020 18:46:27.850 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:27.851009785Z"}
{"log":"1:M 17 Jun 2020 18:46:27.850 - Accepted 127.0.0.1:50258\n","stream":"stdout","time":"2020-06-17T18:46:27.85103785Z"}
{"log":"1:M 17 Jun 2020 18:46:27.851 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:27.851519079Z"}
{"log":"1:M 17 Jun 2020 18:46:32.471 - Accepted 127.0.0.1:50322\n","stream":"stdout","time":"2020-06-17T18:46:32.471106242Z"}
{"log":"1:M 17 Jun 2020 18:46:32.471 - Accepted 127.0.0.1:50332\n","stream":"stdout","time":"2020-06-17T18:46:32.471178694Z"}
{"log":"1:M 17 Jun 2020 18:46:32.471 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:32.471945959Z"}
{"log":"1:M 17 Jun 2020 18:46:32.471 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:32.472949161Z"}
{"log":"1:M 17 Jun 2020 18:46:36.981 - Accepted 127.0.0.1:50652\n","stream":"stdout","time":"2020-06-17T18:46:36.981729179Z"}
{"log":"1:M 17 Jun 2020 18:46:36.981 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:36.982135722Z"}
{"log":"1:M 17 Jun 2020 18:46:37.642 - Accepted 127.0.0.1:50716\n","stream":"stdout","time":"2020-06-17T18:46:37.64291409Z"}
{"log":"1:M 17 Jun 2020 18:46:37.643 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:37.643276813Z"}
{"log":"1:M 17 Jun 2020 18:46:38.213 - DB 0: 1788465 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:38.213248724Z"}
{"log":"1:M 17 Jun 2020 18:46:38.213 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:38.213322448Z"}
{"log":"1:M 17 Jun 2020 18:46:38.213 - 22 clients connected (0 replicas), 824141736 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:38.213381915Z"}
{"log":"1:M 17 Jun 2020 18:46:41.924 - Accepted 127.0.0.1:50866\n","stream":"stdout","time":"2020-06-17T18:46:41.924577095Z"}
{"log":"1:M 17 Jun 2020 18:46:41.924 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:41.925003067Z"}
{"log":"1:M 17 Jun 2020 18:46:42.342 - Accepted 127.0.0.1:50878\n","stream":"stdout","time":"2020-06-17T18:46:42.342633941Z"}
{"log":"1:M 17 Jun 2020 18:46:42.342 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:42.343004442Z"}
{"log":"1:M 17 Jun 2020 18:46:46.260 - DB 0: 1788466 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:46.260438061Z"}
{"log":"1:M 17 Jun 2020 18:46:46.260 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:46.26046765Z"}
{"log":"1:M 17 Jun 2020 18:46:46.260 - 22 clients connected (0 replicas), 823978192 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:46.260471755Z"}
{"log":"1:M 17 Jun 2020 18:46:46.955 - Accepted 127.0.0.1:50956\n","stream":"stdout","time":"2020-06-17T18:46:46.955680213Z"}
{"log":"1:M 17 Jun 2020 18:46:46.958 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:46.958752513Z"}
{"log":"1:M 17 Jun 2020 18:46:47.405 - Accepted 127.0.0.1:50968\n","stream":"stdout","time":"2020-06-17T18:46:47.405270915Z"}
{"log":"1:M 17 Jun 2020 18:46:47.405 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:47.405735798Z"}
{"log":"1:M 17 Jun 2020 18:46:51.291 - DB 0: 1788467 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:51.291938983Z"}
{"log":"1:M 17 Jun 2020 18:46:51.291 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:51.291974787Z"}
{"log":"1:M 17 Jun 2020 18:46:51.291 - 22 clients connected (0 replicas), 823937632 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:51.291978181Z"}
{"log":"1:M 17 Jun 2020 18:46:51.964 - Accepted 127.0.0.1:51064\n","stream":"stdout","time":"2020-06-17T18:46:51.964149441Z"}
{"log":"1:M 17 Jun 2020 18:46:51.964 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:51.964480515Z"}
{"log":"1:M 17 Jun 2020 18:46:52.356 - Accepted 127.0.0.1:51070\n","stream":"stdout","time":"2020-06-17T18:46:52.356459307Z"}
{"log":"1:M 17 Jun 2020 18:46:52.356 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:52.356899686Z"}
{"log":"1:M 17 Jun 2020 18:46:56.302 - DB 0: 1788469 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:56.302874653Z"}
{"log":"1:M 17 Jun 2020 18:46:56.302 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:46:56.302894465Z"}
{"log":"1:M 17 Jun 2020 18:46:56.302 - 22 clients connected (0 replicas), 823938432 bytes in use\n","stream":"stdout","time":"2020-06-17T18:46:56.302897751Z"}
{"log":"1:M 17 Jun 2020 18:46:56.920 - Accepted 127.0.0.1:51390\n","stream":"stdout","time":"2020-06-17T18:46:56.920809107Z"}
{"log":"1:M 17 Jun 2020 18:46:56.921 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:56.921146389Z"}
{"log":"1:M 17 Jun 2020 18:46:57.332 - Accepted 127.0.0.1:51414\n","stream":"stdout","time":"2020-06-17T18:46:57.332651562Z"}
{"log":"1:M 17 Jun 2020 18:46:57.332 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:46:57.332974816Z"}
{"log":"1:M 17 Jun 2020 18:47:01.313 - DB 0: 1788469 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:01.314334687Z"}
{"log":"1:M 17 Jun 2020 18:47:01.313 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:01.314356908Z"}
{"log":"1:M 17 Jun 2020 18:47:01.313 - 22 clients connected (0 replicas), 823856528 bytes in use\n","stream":"stdout","time":"2020-06-17T18:47:01.314361827Z"}
{"log":"1:M 17 Jun 2020 18:47:01.919 - Accepted 127.0.0.1:51462\n","stream":"stdout","time":"2020-06-17T18:47:01.919955026Z"}
{"log":"1:M 17 Jun 2020 18:47:01.920 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:01.920170701Z"}
{"log":"1:M 17 Jun 2020 18:47:02.327 - Accepted 127.0.0.1:51472\n","stream":"stdout","time":"2020-06-17T18:47:02.327818835Z"}
{"log":"1:M 17 Jun 2020 18:47:02.328 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:02.328068416Z"}
{"log":"1:M 17 Jun 2020 18:47:06.334 - DB 0: 1788470 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:06.334684602Z"}
{"log":"1:M 17 Jun 2020 18:47:06.334 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:06.33470951Z"}
{"log":"1:M 17 Jun 2020 18:47:06.334 - 22 clients connected (0 replicas), 824060464 bytes in use\n","stream":"stdout","time":"2020-06-17T18:47:06.334712943Z"}
{"log":"1:M 17 Jun 2020 18:47:06.919 - Accepted 127.0.0.1:51556\n","stream":"stdout","time":"2020-06-17T18:47:06.919532484Z"}
{"log":"1:M 17 Jun 2020 18:47:06.919 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:06.919911286Z"}
{"log":"1:M 17 Jun 2020 18:47:07.322 - Accepted 127.0.0.1:51562\n","stream":"stdout","time":"2020-06-17T18:47:07.322282052Z"}
{"log":"1:M 17 Jun 2020 18:47:07.323 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:07.32311733Z"}
{"log":"1:M 17 Jun 2020 18:47:11.358 - DB 0: 1788470 keys (7 volatile) in 2097152 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:11.358698315Z"}
{"log":"1:M 17 Jun 2020 18:47:11.358 - DB 1: 1 keys (0 volatile) in 4 slots HT.\n","stream":"stdout","time":"2020-06-17T18:47:11.358739606Z"}
{"log":"1:M 17 Jun 2020 18:47:11.358 - 22 clients connected (0 replicas), 823937608 bytes in use\n","stream":"stdout","time":"2020-06-17T18:47:11.358744701Z"}
{"log":"1:M 17 Jun 2020 18:47:11.926 - Accepted 127.0.0.1:51638\n","stream":"stdout","time":"2020-06-17T18:47:11.926201331Z"}
{"log":"1:M 17 Jun 2020 18:47:11.926 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:11.926433347Z"}
{"log":"1:M 17 Jun 2020 18:47:12.557 - Accepted 127.0.0.1:51642\n","stream":"stdout","time":"2020-06-17T18:47:12.557644734Z"}
{"log":"1:M 17 Jun 2020 18:47:13.071 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:13.071323035Z"}
{"log":"1:M 17 Jun 2020 18:47:16.980 - Accepted 127.0.0.1:51960\n","stream":"stdout","time":"2020-06-17T18:47:16.980598658Z"}
{"log":"1:M 17 Jun 2020 18:47:17.516 - Client closed connection\n","stream":"stdout","time":"2020-06-17T18:47:17.516754563Z"}
...