Вчера, когда я просыпаюсь, я получаю слишком много писем от клиентов, что они не могут видеть никаких данных. Позже я обнаружил, что все данные поиска elasti c исчезли. Затем я должен переиндексировать все данные, и это занимает слишком много времени. Я не могу понять, почему это произошло. Вот записи в журнале:
Я использую эти версии в своем приложении rails:
asticsearch (1.0.2)
[2020-03-24 12:37:28,754][WARN ][transport.netty ] [ubuntu.members.linode.com] exception caught on transport layer [[id: 0x76418f78, /124.60.24.819:64731 :> /171.101.201.119:9300]], closing connection
java.io.StreamCorruptedException: invalid internal transport message format
at org.elasticsearch.transport.netty.SizeHeaderFrameDecoder.decode(SizeHeaderFrameDecoder.java:46)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:425)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:482)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.elasticsearch.common.netty.channel.Channels.fireChannelDisconnected(Channels.java:396)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:81)
at org.elasticsearch.common.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:574)
at org.elasticsearch.common.netty.channel.Channels.close(Channels.java:812)
at org.elasticsearch.common.netty.channel.AbstractChannel.close(AbstractChannel.java:197)
at org.elasticsearch.transport.netty.NettyTransport.exceptionCaught(NettyTransport.java:532)
at org.elasticsearch.transport.netty.MessageChannelHandler.exceptionCaught(MessageChannelHandler.java:229)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.exceptionCaught(FrameDecoder.java:377)
at org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.elasticsearch.common.netty.channel.Channels.fireExceptionCaught(Channels.java:525)
at org.elasticsearch.common.netty.channel.AbstractChannelSink.exceptionCaught(AbstractChannelSink.java:48)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.notifyHandlerException(DefaultChannelPipeline.java:658)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:566)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at org.elasticsearch.common.netty.OpenChannelsHandler.handleUpstream(OpenChannelsHandler.java:74)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at org.elasticsearch.common.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.elasticsearch.common.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1152)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:622)
at java.lang.Thread.run(Thread.java:748)
[2020-03-24 23:11:35,216][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [samples] deleting index
[2020-03-24 23:11:35,244][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [cgi] deleting index
[2020-03-24 23:11:35,263][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [messagebroker] deleting index
[2020-03-24 23:11:35,278][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [acts_as_taggable_on_tags] deleting index
[2020-03-24 23:11:35,307][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [flex2gateway] deleting index
[2020-03-24 23:11:35,325][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [tasks] deleting index
[2020-03-24 23:11:35,389][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [website] deleting index
[2020-03-24 23:11:35,406][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [contact_associations] deleting index
[2020-03-24 23:11:35,540][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [decisions] deleting index
[2020-03-24 23:11:35,573][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [lcds] deleting index
[2020-03-24 23:11:35,589][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [lcds-samples] deleting index
[2020-03-24 23:11:35,603][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [blazeds] deleting index
[2020-03-24 23:20:48,473][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [tasks] creating index, cause [auto(index api)], shards [5]/[1], mappings []
[2020-03-24 23:20:48,557][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [tasks] update_mapping [task] (dynamic)
[2020-03-24 23:20:48,829][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [meetings] update_mapping [meeting] (dynamic)
[2020-03-24 23:23:16,449][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [tasks] update_mapping [task] (dynamic)
[2020-03-24 23:23:16,470][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [tasks] update_mapping [task] (dynamic)
[2020-03-24 23:24:10,820][INFO ][cluster.metadata ] [ubuntu.members.linode.com] [acts_as_taggable_on_tags] creating index, cause [auto(index api)], shards [5]/[1], mappings []
Таинственным образом это произошло как на постановочные и производственные серверы. Они размещены на разных серверах