Я использовал docker -compose для запуска ELKB. Моя основная задача - запустить контейнерыasticsearch и logsta sh. Контейнер Logsta sh должен быть успешно соединен с эластичным поиском и передавать журналы в эластичный поиск для дальнейшего поиска или обработки.
Но по неосознанной причине контейнер Logsta sh должен часто останавливаться. Мне нужно сохраниться в контейнере logsta sh иasticsearch, но этого не происходит.
Я не знаю, в чем причина периодического закрытия контейнера logsta sh.
Я использую эластичный поиск: 7.6.3 и logsta sh: 7.6.3
Пожалуйста, просмотрите приведенный ниже код и укажите, где я допустил ошибку.
docker -compose.yml
# Docker version 19.03.5
# docker-compose version 1.25.3
version: "3.7"
services:
elasticsearch:
container_name: elasticsearch
build:
context: ./elasticsearch
dockerfile: Dockerfile
ports:
- 9200:9200
- 9300:9300
volumes:
- ./elasticsearch/data:/usr/share/elasticsearch/data:rw
- ./elasticsearch/logs:/usr/share/elasticsearch/logs:rw
restart: always
ulimits:
memlock:
soft: -1
hard: -1
networks:
- elkb
logstash:
container_name: logstash
build:
context: ./logstash
dockerfile: Dockerfile
ports:
- 9600:9600
- 5000:5000/udp
- 5000:5000/tcp
volumes:
- ./logstash/input-logs:/usr/share/logstash/logs
- ./logstash/data:/var/lib/logstash:rw
- ./logstash/logs:/var/logs/logstash:rw
restart: always
ulimits:
memlock:
soft: -1
hard: -1
networks:
- elkb
links:
- elasticsearch
depends_on:
- elasticsearch
networks:
elk:
driver: bridge
volumes:
elasticsearch:
Файл поиска Elasticsearch
FROM docker.elastic.co/elasticsearch/elasticsearch:7.6.2
/usr/share/elasticsearch/config/elasticsearch.yml
RUN mkdir -p /var/log/elasticsearch
RUN chown -R elasticsearch:elasticsearch /var/log/elasticsearch
RUN mkdir -p /var/lib/elasticsearch
RUN chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
EXPOSE 9200
EXPOSE 9300
asticsearch.yml
cluster.name: es_cluster
node.name: es_node_1
bootstrap.memory_lock: true
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["0.0.0.0"]
cluster.initial_master_nodes: ["es_node_1"]
Logsta sh Dockerfile
FROM docker.elastic.co/logstash/logstash:7.6.2
COPY logstash.yml /usr/share/logstash/config/logstash.yml
COPY ./pipeline/logstash.conf /usr/share/logstash/pipeline/logstash.conf
EXPOSE 9600
logsta sh .yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: "http://elasticsearch:9200"
xpack.monitoring.enabled: true
logsta sh .conf
input{
stdin{}
}
output{
elasticsearch {
hosts => ["http://elasticsearch:9200"]
}
}
Контейнерные журналы logsta sh
container_logstash | WARNING: An illegal reflective access operation has occurred
container_logstash | WARNING: Illegal reflective access by com.headius.backport9.modules.Modules (file:/usr/share/logstash/logstash-core/lib/jars/jruby-complete-9.2.9.0.jar) to method sun.nio.ch.NativeThread.signal(long)
container_logstash | WARNING: Please consider reporting this to the maintainers of com.headius.backport9.modules.Modules
container_logstash | WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
container_logstash | WARNING: All illegal access operations will be denied in a future release
container_logstash | Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
container_logstash | [2020-04-25T14:50:33,271][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.6.2"}
container_logstash | [2020-04-25T14:50:34,013][INFO ][logstash.licensechecker.licensereader] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}}
container_logstash | [2020-04-25T14:50:34,127][WARN ][logstash.licensechecker.licensereader] Restored connection to ES instance {:url=>"http://elasticsearch:9200/"}
container_logstash | [2020-04-25T14:50:34,157][INFO ][logstash.licensechecker.licensereader] ES Output version determined {:es_version=>7}
container_logstash | [2020-04-25T14:50:34,160][WARN ][logstash.licensechecker.licensereader] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
container_logstash | [2020-04-25T14:50:34,243][INFO ][logstash.monitoring.internalpipelinesource] Monitoring License OK
container_logstash | [2020-04-25T14:50:34,244][INFO ][logstash.monitoring.internalpipelinesource] Validated license for monitoring. Enabling monitoring pipeline.
container_logstash | [2020-04-25T14:50:34,982][INFO ][org.reflections.Reflections] Reflections took 22 ms to scan 1 urls, producing 20 keys and 40 values
container_logstash | [2020-04-25T14:50:35,126][INFO ][logstash.outputs.elasticsearch][main] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}}
container_logstash | [2020-04-25T14:50:35,134][WARN ][logstash.outputs.elasticsearch][main] Restored connection to ES instance {:url=>"http://elasticsearch:9200/"}
container_logstash | [2020-04-25T14:50:35,138][INFO ][logstash.outputs.elasticsearch][main] ES Output version determined {:es_version=>7}
container_logstash | [2020-04-25T14:50:35,138][WARN ][logstash.outputs.elasticsearch][main] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
container_logstash | [2020-04-25T14:50:35,159][INFO ][logstash.outputs.elasticsearch][main] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["http://elasticsearch:9200"]}
container_logstash | [2020-04-25T14:50:35,182][INFO ][logstash.outputs.elasticsearch][main] Using default mapping template
container_logstash | [2020-04-25T14:50:35,206][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge][main] A gauge metric of an unknown type (org.jruby.specialized.RubyArrayOneObject) has been created for key: cluster_uuids. This may result in invalid serialization. It is recommended to log an issue to the responsible developer/development team.
container_logstash | [2020-04-25T14:50:35,213][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>6, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>750, "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf"], :thread=>"#<Thread:0x27747a5a run>"}
container_logstash | [2020-04-25T14:50:35,213][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1, "index.lifecycle.name"=>"logstash-policy", "index.lifecycle.rollover_alias"=>"logstash"}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
container_logstash | [2020-04-25T14:50:35,711][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
container_logstash | [2020-04-25T14:50:35,738][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
container_logstash | [2020-04-25T14:50:36,233][WARN ][logstash.outputs.elasticsearch] You are using a deprecated config setting "document_type" set in elasticsearch. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature If you have any questions about this, please visit the #logstash channel on freenode irc. {:name=>"document_type", :plugin=><LogStash::Outputs::ElasticSearch bulk_path=>"/_monitoring/bulk?system_id=logstash&system_api_version=7&interval=1s", hosts=>[http://elasticsearch:9200], sniffing=>false, manage_template=>false, id=>"ebdd88635541942b096027ed79be84efc3dd562a5f0e1b78fca83c7b5c9a1a7c", document_type=>"%{[@metadata][document_type]}", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_031a6e38-cafd-42f9-b689-b577ba9acc88", enable_metric=>true, charset=>"UTF-8">, workers=>1, template_name=>"logstash", template_overwrite=>false, doc_as_upsert=>false, script_type=>"inline", script_lang=>"painless", script_var_name=>"event", scripted_upsert=>false, retry_initial_interval=>2, retry_max_interval=>64, retry_on_conflict=>1, ilm_enabled=>"auto", ilm_rollover_alias=>"logstash", ilm_pattern=>"{now/d}-000001", ilm_policy=>"logstash-policy", action=>"index", ssl_certificate_verification=>true, sniffing_delay=>5, timeout=>60, pool_max=>1000, pool_max_per_route=>100, resurrect_delay=>5, validate_after_inactivity=>10000, http_compression=>false>}
container_logstash | [2020-04-25T14:50:36,246][INFO ][logstash.outputs.elasticsearch][.monitoring-logstash] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}}
container_logstash | [2020-04-25T14:50:36,250][WARN ][logstash.outputs.elasticsearch][.monitoring-logstash] Restored connection to ES instance {:url=>"http://elasticsearch:9200/"}
container_logstash | [2020-04-25T14:50:36,253][INFO ][logstash.outputs.elasticsearch][.monitoring-logstash] ES Output version determined {:es_version=>7}
container_logstash | [2020-04-25T14:50:36,253][WARN ][logstash.outputs.elasticsearch][.monitoring-logstash] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
container_logstash | [2020-04-25T14:50:36,268][INFO ][logstash.outputs.elasticsearch][.monitoring-logstash] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["http://elasticsearch:9200"]}
container_logstash | [2020-04-25T14:50:36,271][INFO ][logstash.javapipeline ][.monitoring-logstash] Starting pipeline {:pipeline_id=>".monitoring-logstash", "pipeline.workers"=>1, "pipeline.batch.size"=>2, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>2, "pipeline.sources"=>["monitoring pipeline"], :thread=>"#<Thread:0x6e9553e7 run>"}
container_logstash | [2020-04-25T14:50:36,288][INFO ][logstash.javapipeline ][.monitoring-logstash] Pipeline started {"pipeline.id"=>".monitoring-logstash"}
container_logstash | [2020-04-25T14:50:36,294][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:".monitoring-logstash"], :non_running_pipelines=>[:main]}
container_logstash | [2020-04-25T14:50:36,398][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
container_logstash | [2020-04-25T14:50:37,402][INFO ][logstash.javapipeline ] Pipeline terminated {"pipeline.id"=>".monitoring-logstash"}
container_logstash | [2020-04-25T14:50:38,337][INFO ][logstash.runner ] Logstash shut down.
Пожалуйста, дайте мне знать, если вам нужно больше разъяснений или нужна дополнительная информация.
Спасибо за разрешение.