Elasticsearch вылетает через некоторое время с сообщением «все осколки не пройдены» - PullRequest
Новая
       15

Elasticsearch вылетает через некоторое время с сообщением «все осколки не пройдены»

0 голосов
/ 30 мая 2019

У меня есть приложение, которое отправляет максимальное количество 600 сообщений журнала каждые 2 минуты в пакетах по 100 (отправляет 100 журналов, 2 секунды пауза ...)

Это приложение отправляет журналы на Logstash -> ElasticSearch -> Kibana.

Через некоторое время (например, 10 минут, иногда даже меньше), ElasticSearch вылетает со следующим сообщением: 

org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed,
    "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:296) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:139) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:259) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.InitialSearchPhase.onShardFailure(InitialSearchPhase.java:105) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.InitialSearchPhase.access$200(InitialSearchPhase.java:50) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.InitialSearchPhase$2.onFailure(InitialSearchPhase.java:273) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.SearchExecutionStatsCollector.onFailure(SearchExecutionStatsCollector.java:73) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:59) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.search.SearchTransportService$ConnectionCountingHandler.handleException(SearchTransportService.java:441) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1236) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1210) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.transport.TaskTransportChannel.sendResponse(TaskTransportChannel.java:60) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.action.support.ChannelActionListener.onFailure(ChannelActionListener.java:56) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.search.SearchService$2.onFailure(SearchService.java:366) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:360) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:354) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1069) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.1.1.jar:7.1.1]",
    "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.1.1.jar:7.1.1]",
    "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
    "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
    "at java.lang.Thread.run(Thread.java:835) [?:?]"

Мой docker-compose.yml файл это: 

services:

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
    container_name: elasticsearch
    ports:
      - "9200:9200"
    volumes:
      - ./elastic.yml:/config/elastic.yml
    environment:
      - node.name=elasticsearch
      - cluster.initial_master_nodes=elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:7.1.1
    container_name: kibana
    ports:
      - "5601:5601"
    volumes:
      - ./kibana.yml:/config/kibana.yml
    links:
      - elasticsearch
    depends_on:
      - elasticsearch


  logstash:
    image: docker.elastic.co/logstash/logstash:7.1.1
    container_name: logstash
    links:
      - elasticsearch
    volumes:
      - ./logstash.conf:/config/logstash.conf
    command: logstash -f /config/logstash.conf
    ports:
      - "9500:9500"
      - "9600:9600/udp"
      - "9700:9700"
    depends_on:
    - elasticsearch

volumes:
  elasticsearch:
    driver: local

У меня есть последние изображения для контейнеров ...

...