Исключение при сохранении фрейма данных в HDFS - org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException - PullRequest
Новая
       7

Исключение при сохранении фрейма данных в HDFS - org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException

0 голосов
/ 19 сентября 2019

При использовании spark я записываю таблицу данных в таблицу HDFS.Я использую оператор вставки в спарк SQL для записи данных в таблицу.Но получается ошибка ниже: 

User class threw exception: org.apache.spark.SparkException: Job aborted due to stage failure: Task 144 in stage 27.1 failed 4 times, most recent failure: Lost task 144.3 in stage 27.1 (TID 181665, cdts04hdfc06p.rxcorp.com, executor 506): org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException): Failed to create file [/user/hive/warehouse/prod_df2_lad_batch.db/ss_lad_pat_seq/.hive-staging_hive_2019-09-16_22-27-56_435_7609238235778059432-1/-ext-10000/_temporary/0/_temporary/attempt_201909162227_0027_m_000344_3/week_id=20190830/part-00344] for [DFSClient_attempt_201909162227_0027_m_001541_1_-1711340928_161] for client [10.75.128.110], because this file is already being created by [DFSClient_NONMAPREDUCE_-2112517704_159] on [10.75.128.141] 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.recoverLeaseInternal(FSNamesystem.java:3344) 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:3002) 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2890) 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2774) 
    at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:610) 
    at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.create(AuthorizationProviderProxyClientProtocol.java:117) 
    at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:413) 
    at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) 
    at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617) 
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2281) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2277) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:422) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924) 
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2275) 
    at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:248) 
    at org.apache.spark.sql.hive.SparkHiveDynamicPartitionWriterContainer.org$apache$spark$sql$hive$SparkHiveDynamicPartitionWriterContainer$$newWriter$1(hiveWriterContainers.scala:240) 
    at org.apache.spark.sql.hive.SparkHiveDynamicPartitionWriterContainer$$anonfun$getLocalFileWriter$1.apply(hiveWriterContainers.scala:249) 
    at org.apache.spark.sql.hive.SparkHiveDynamicPartitionWriterContainer$$anonfun$getLocalFileWriter$1.apply(hiveWriterContainers.scala:249) 
    at scala.collection.mutable.MapLike$class.getOrElseUpdate(MapLike.scala:189) 
    at scala.collection.mutable.AbstractMap.getOrElseUpdate(Map.scala:91) 
    at org.apache.spark.sql.hive.SparkHiveDynamicPartitionWriterContainer.getLocalFileWriter(hiveWriterContainers.scala:249) 
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1$1.apply(InsertIntoHiveTable.scala:178) 
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1$1.apply(InsertIntoHiveTable.scala:170) 
    at scala.collection.Iterator$class.foreach(Iterator.scala:727) 
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) 
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1(InsertIntoHiveTable.scala:170) 
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:150) 
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:150) 
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) 
    at org.apache.spark.scheduler.Task.run(Task.scala:89) 
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242) 
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 
    at java.lang.Thread.run(Thread.java:745)
...