У меня есть конвейер в NIFI , и все работает нормально, за исключением иногда (очень редко) некоторой записи go в очередь повторов в PUTHIVESTREAMING процессоре при попытке записи в HIVE . Журналы nifi показывают исключение
ERROR [put-hive-streaming-0] o.a.h.h.streaming.AbstractRecordWriter Unable to close org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater[hdfs://hdfscluster/user/hive/metastore/bi_sureyield_db.db/bi_sureyield_events/year=2020/month=1/day=10/delta_127318799_127318808/bucket_00000] due to: Column has wrong number of index entries found: 0 expected: 1
java.lang.IllegalArgumentException: Column has wrong number of index entries found: 0 expected: 1
at org.apache.orc.impl.WriterImpl$TreeWriter.writeStripe(WriterImpl.java:695)
at org.apache.orc.impl.WriterImpl$StructTreeWriter.writeStripe(WriterImpl.java:2147)
at org.apache.orc.impl.WriterImpl.flushStripe(WriterImpl.java:2661)
at org.apache.orc.impl.WriterImpl.close(WriterImpl.java:2834)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.close(WriterImpl.java:321)
at org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater.close(OrcRecordUpdater.java:502)
at org.apache.hive.hcatalog.streaming.AbstractRecordWriter.closeBatch(AbstractRecordWriter.java:218)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl$6.run(HiveEndPoint.java:998)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl$6.run(HiveEndPoint.java:995)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.closeImpl(HiveEndPoint.java:994)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.markDead(HiveEndPoint.java:760)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.commit(HiveEndPoint.java:854)
at org.apache.nifi.util.hive.HiveWriter$4.call(HiveWriter.java:237)
at org.apache.nifi.util.hive.HiveWriter$4.call(HiveWriter.java:234)
at org.apache.nifi.util.hive.HiveWriter.lambda$null$3(HiveWriter.java:373)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.nifi.util.hive.HiveWriter.lambda$callWithTimeout$4(HiveWriter.java:373)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-04-29 14:55:39,542 ERROR [put-hive-streaming-0] o.a.h.h.streaming.AbstractRecordWriter Unable to close org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater[hdfs://hdfscluster/user/hive/metastore/bi_sureyield_db.db/bi_sureyield_events/year=2020/month=1/day=10/delta_127318799_127318808/bucket_00001] due to: null
java.nio.channels.ClosedChannelException: null
at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1521)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:104)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at org.apache.orc.impl.PhysicalFsWriter$BufferedStream.spillToDiskAndClear(PhysicalFsWriter.java:286)
at org.apache.orc.impl.PhysicalFsWriter.finalizeStripe(PhysicalFsWriter.java:337)
at org.apache.orc.impl.WriterImpl.flushStripe(WriterImpl.java:2665)
at org.apache.orc.impl.WriterImpl.close(WriterImpl.java:2834)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.close(WriterImpl.java:321)
at org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater.close(OrcRecordUpdater.java:502)
at org.apache.hive.hcatalog.streaming.AbstractRecordWriter.closeBatch(AbstractRecordWriter.java:218)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl$6.run(HiveEndPoint.java:998)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl$6.run(HiveEndPoint.java:995)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.closeImpl(HiveEndPoint.java:994)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.markDead(HiveEndPoint.java:760)
at org.apache.hive.hcatalog.streaming.HiveEndPoint$TransactionBatchImpl.commit(HiveEndPoint.java:854)
at org.apache.nifi.util.hive.HiveWriter$4.call(HiveWriter.java:237)
at org.apache.nifi.util.hive.HiveWriter$4.call(HiveWriter.java:234)
at org.apache.nifi.util.hive.HiveWriter.lambda$null$3(HiveWriter.java:373)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.nifi.util.hive.HiveWriter.lambda$callWithTimeout$4(HiveWriter.java:373)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Я не могу понять, где проблема и как ее исправить.