Сбой записи в HBase с RetriesExhaustedWithDetailsException - PullRequest
Новая
       9

Сбой записи в HBase с RetriesExhaustedWithDetailsException

0 голосов
/ 23 января 2020

У нас есть два кластера HBase для управления операциями в случае бедствий. Наш клиентский код HBase успешно записывает данные в один кластер, но при записи данных в резервный кластер он выдает ниже исключения после успешной записи нескольких записей. 

2020-01-21 10:57:16,998 ERROR [org.springframework.kafka.KafkaListenerEndpointContainer#0-25-C-1] c.u.n.h.h.s.MutatorTransactionStepMessageHandler [MutatorTransactionStepMessageHandler.java:84] Error submitting step e1ce655d-26d3-4b23-b940-a88b9e2c6376, retrying.
org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 1 action: IOException: 1 time, servers with issues: null, 
    at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.makeException(AsyncProcess.java:260) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.AsyncProcess$BatchErrors.access$2400(AsyncProcess.java:240) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1889) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:247) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:153) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:120) ~[hbase-client-1.3.1.jar:1.3.1]
    at com.uprr.netcontrol.hds.hbase.step.MutatorTransactionStepMessageHandler.process(MutatorTransactionStepMessageHandler.java:79) ~[hbase-btt-service-index-step-1_0-1.1.1.jar:na]
    at sun.reflect.GeneratedMethodAccessor49.invoke(Unknown Source) ~[na:na]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_121]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_121]
    at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:171) [spring-messaging-5.1.7.RELEASE.jar:5.1.7.RELEASE]
    at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:120) [spring-messaging-5.1.7.RELEASE.jar:5.1.7.RELEASE]
    at org.springframework.kafka.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:48) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:283) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:79) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:50) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeOnMessage(KafkaMessageListenerContainer.java:1263) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeOnMessage(KafkaMessageListenerContainer.java:1256) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeRecordListener(KafkaMessageListenerContainer.java:1217) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeWithRecords(KafkaMessageListenerContainer.java:1198) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeRecordListener(KafkaMessageListenerContainer.java:1118) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeListener(KafkaMessageListenerContainer.java:933) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:749) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:698) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_121]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_121]
    at java.lang.Thread.run(Thread.java:745) [na:1.8.0_121]
2020-01-21 10:57:17,506 ERROR [org.springframework.kafka.KafkaListenerEndpointContainer#0-8-C-1] o.a.h.h.c.AsyncProcess [AsyncProcess.java:432] Failed to get region location 
java.io.IOException: Failed to get result within timeout, timeout=60000ms
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:206) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:212) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:228) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:202) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1277) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1183) ~[hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:422) [hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:371) [hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:237) [hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:153) [hbase-client-1.3.1.jar:1.3.1]
    at org.apache.hadoop.hbase.client.BufferedMutatorImpl.mutate(BufferedMutatorImpl.java:120) [hbase-client-1.3.1.jar:1.3.1]
    at com.uprr.netcontrol.hds.hbase.step.MutatorTransactionStepMessageHandler.process(MutatorTransactionStepMessageHandler.java:79) [hbase-btt-service-index-step-1_0-1.1.1.jar:na]
    at sun.reflect.GeneratedMethodAccessor49.invoke(Unknown Source) ~[na:na]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_121]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_121]
    at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:171) [spring-messaging-5.1.7.RELEASE.jar:5.1.7.RELEASE]
    at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:120) [spring-messaging-5.1.7.RELEASE.jar:5.1.7.RELEASE]
    at org.springframework.kafka.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:48) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:283) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:79) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:50) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeOnMessage(KafkaMessageListenerContainer.java:1263) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeOnMessage(KafkaMessageListenerContainer.java:1256) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeRecordListener(KafkaMessageListenerContainer.java:1217) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeWithRecords(KafkaMessageListenerContainer.java:1198) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeRecordListener(KafkaMessageListenerContainer.java:1118) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeListener(KafkaMessageListenerContainer.java:933) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.pollAndInvoke(KafkaMessageListenerContainer.java:749) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.run(KafkaMessageListenerContainer.java:698) [spring-kafka-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_121]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_121]
    at java.lang.Thread.run(Thread.java:745) [na:1.8.0_121]

Мы попытались увеличить интервал ожидания RP C и также попытался настроить нижеприведенные реквизиты. 

<property>
<name>hbase.thrift.connection.max-idletime</name>
<value>1800000</value>
</property>

hbase.regionserver.lease.period
hbase.rpc.timeout
hbase.client.scanner.timeout.period

hbase hbck -details 'ns:table'

Ни один из них не назвал причину root, стоящую за этой проблемой. Любая помощь приветствуется.

Добро пожаловать на сайт PullRequest, где вы можете задавать вопросы и получать ответы от других членов сообщества.

Похожие темы

...