Только что обновил с версии 2.2.1 до 2.4.4 и получаю следующую ошибку - он продолжает повторять эту же ошибку. Другие темы предлагали проблемы с памятью, но я работаю с очень маленькими файлами (4 kb как csv). Чего мне не хватает?
20/01/08 15:57:52 WARN FileOutputCommitter: Could not delete file:<file_name>/_temporary/0/_temporary/attempt_20200108155748_0032_m_000000_129
20/01/08 15:57:52 ERROR FileFormatWriter: Job job_20200108155748_0032 aborted.
20/01/08 15:57:52 WARN TaskSetManager: Lost task 0.97 in stage 32.0 (TID 129, localhost, executor driver): TaskCommitDenied (Driver denied task commit) for job: 32, partition: 0, attemptNumber: 97
20/01/08 15:57:52 ERROR Utils: Aborting task
org.apache.spark.executor.CommitDeniedException: attempt_20200108155748_0032_m_000000_130: Not committed because the driver did not authorize commit
at org.apache.spark.mapred.SparkHadoopMapRedUtil$.commitTask(SparkHadoopMapRedUtil.scala:84)
at org.apache.spark.internal.io.HadoopMapReduceCommitProtocol.commitTask(HadoopMapReduceCommitProtocol.scala:225)
at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.commit(FileFormatDataWriter.scala:78)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask$3.apply(FileFormatWriter.scala:247)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask$3.apply(FileFormatWriter.scala:242)
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1394)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:248)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:170)
at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:169)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:123)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)