После создания спарк-кластера в Azure HDInsight выполните следующие действия: https://docs.microsoft.com/en-us/azure/hdinsight/spark/apache-spark-jupyter-spark-sql
Я установил Azure Toolkit для IntelliJ, следуя этому руководству: https://docs.microsoft.com/en-us/azure/hdinsight/spark/apache-spark-intellij-tool-plugin
Когда я отправляю свою искровую работу из Intellij, я получаю следующее исключение:
INFO: Create Spark helper interactive session...
ERROR: com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "progress" (class com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement), not marked as ignorable (4 known properties: "state", "id", "code", "output"])
at [Source: {"id":0,"code":"import java.io._\nimport java.util.Base64\n\nval jarOutput = \"/SparkSubmission/2018/07/03/e2e54d54-e1ec-40b4-957b-04291ed43a26/SparkCluster.jar\"\nval fs = org.apache.hadoop.fs.FileSystem.get(sc.hadoopConfiguration)\nval jarFileOutput = fs.create(new org.apache.hadoop.fs.Path(jarOutput), true)\nval out = new DataOutputStream(new BufferedOutputStream(jarFileOutput))\n\ndef writePage(encodedBase64: String) = {\n val pageBytes = Base64.getDecoder.decode(encodedBase64)\n\n out.write(pageBytes, 0, pageBytes.size)\n}","state":"waiting","output":null,"progress":0.0}; line: 1, column: 588] (through reference chain: com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement["progress"])
stack trace: java.lang.RuntimeException: com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "progress" (class com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement), not marked as ignorable (4 known properties: "state", "id", "code", "output"])
at [Source: {"id":0,"code":"import java.io._\nimport java.util.Base64\n\nval jarOutput = \"/SparkSubmission/2018/07/03/e2e54d54-e1ec-40b4-957b-04291ed43a26/SparkCluster.jar\"\nval fs = org.apache.hadoop.fs.FileSystem.get(sc.hadoopConfiguration)\nval jarFileOutput = fs.create(new org.apache.hadoop.fs.Path(jarOutput), true)\nval out = new DataOutputStream(new BufferedOutputStream(jarFileOutput))\n\ndef writePage(encodedBase64: String) = {\n val pageBytes = Base64.getDecoder.decode(encodedBase64)\n\n out.write(pageBytes, 0, pageBytes.size)\n}","state":"waiting","output":null,"progress":0.0}; line: 1, column: 588] (through reference chain: com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement["progress"])
at rx.exceptions.Exceptions.propagate(Exceptions.java:58)
at com.microsoft.azure.hdinsight.sdk.common.HttpObservable.convertJsonResponseToObject(HttpObservable.java:242)
at com.microsoft.azure.hdinsight.sdk.common.HttpObservable.lambda$post$9(HttpObservable.java:316)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:69)
at rx.internal.operators.OperatorMerge$MergeSubscriber.emitScalar(OperatorMerge.java:395)
at rx.internal.operators.OperatorMerge$MergeSubscriber.tryEmit(OperatorMerge.java:355)
at rx.internal.operators.OperatorMerge$InnerSubscriber.onNext(OperatorMerge.java:846)
at rx.observers.Subscribers$5.onNext(Subscribers.java:235)
at rx.internal.operators.OperatorDoAfterTerminate$1.onNext(OperatorDoAfterTerminate.java:50)
at rx.internal.util.ScalarSynchronousObservable$WeakSingleProducer.request(ScalarSynchronousObservable.java:276)
at rx.Subscriber.setProducer(Subscriber.java:211)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:138)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:129)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:94)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:32)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:248)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:148)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.producers.SingleDelayedProducer.emit(SingleDelayedProducer.java:102)
at rx.internal.producers.SingleDelayedProducer.setValue(SingleDelayedProducer.java:85)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:48)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:248)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:148)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OnSubscribeFilter$FilterSubscriber.onNext(OnSubscribeFilter.java:76)
at rx.internal.operators.OperatorTakeUntilPredicate$ParentSubscriber.onNext(OperatorTakeUntilPredicate.java:61)
at rx.internal.operators.OnSubscribeRedo$2$1.onNext(OnSubscribeRedo.java:244)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OperatorSwitchIfEmpty$ParentSubscriber.onNext(OperatorSwitchIfEmpty.java:90)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OperatorMerge$MergeSubscriber.emitScalar(OperatorMerge.java:395)
at rx.internal.operators.OperatorMerge$MergeSubscriber.tryEmit(OperatorMerge.java:355)
at rx.internal.operators.OperatorMerge$InnerSubscriber.onNext(OperatorMerge.java:846)
at rx.observers.Subscribers$5.onNext(Subscribers.java:235)
at rx.internal.operators.OperatorDoAfterTerminate$1.onNext(OperatorDoAfterTerminate.java:50)
at rx.internal.util.ScalarSynchronousObservable$WeakSingleProducer.request(ScalarSynchronousObservable.java:276)
at rx.Subscriber.setProducer(Subscriber.java:211)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:138)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:129)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:94)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:32)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:248)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:148)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.producers.SingleDelayedProducer.emit(SingleDelayedProducer.java:102)
at rx.internal.producers.SingleDelayedProducer.setValue(SingleDelayedProducer.java:85)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:48)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeRedo$2.call(OnSubscribeRedo.java:273)
at rx.internal.schedulers.TrampolineScheduler$InnerCurrentThreadScheduler.enqueue(TrampolineScheduler.java:73)
at rx.internal.schedulers.TrampolineScheduler$InnerCurrentThreadScheduler.schedule(TrampolineScheduler.java:52)
at rx.internal.operators.OnSubscribeRedo$4$1.onNext(OnSubscribeRedo.java:336)
at rx.internal.operators.OperatorDelay$1$3.call(OperatorDelay.java:87)
at rx.internal.schedulers.EventLoopsScheduler$EventLoopWorker$2.call(EventLoopsScheduler.java:189)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:55)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "progress" (class com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement), not marked as ignorable (4 known properties: "state", "id", "code", "output"])
at [Source: {"id":0,"code":"import java.io._\nimport java.util.Base64\n\nval jarOutput = \"/SparkSubmission/2018/07/03/e2e54d54-e1ec-40b4-957b-04291ed43a26/SparkCluster.jar\"\nval fs = org.apache.hadoop.fs.FileSystem.get(sc.hadoopConfiguration)\nval jarFileOutput = fs.create(new org.apache.hadoop.fs.Path(jarOutput), true)\nval out = new DataOutputStream(new BufferedOutputStream(jarFileOutput))\n\ndef writePage(encodedBase64: String) = {\n val pageBytes = Base64.getDecoder.decode(encodedBase64)\n\n out.write(pageBytes, 0, pageBytes.size)\n}","state":"waiting","output":null,"progress":0.0}; line: 1, column: 588] (through reference chain: com.microsoft.azure.hdinsight.sdk.rest.livy.interactive.Statement["progress"])
at com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62)
at com.fasterxml.jackson.databind.DeserializationContext.handleUnknownProperty(DeserializationContext.java:834)
at com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1093)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1478)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1456)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:282)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:140)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3798)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2842)
at com.microsoft.azure.hdinsight.sdk.rest.ObjectConvertUtils.convertJsonToObject(ObjectConvertUtils.java:46)
at com.microsoft.azure.hdinsight.sdk.common.HttpObservable.convertJsonResponseToObject(HttpObservable.java:238)
... 114 more
Caused by: rx.exceptions.OnErrorThrowable$OnNextValue: OnError while emitting onNext value: com.microsoft.azure.hdinsight.sdk.common.HttpResponse.class
at rx.exceptions.OnErrorThrowable.addValueAsLastCause(OnErrorThrowable.java:118)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:73)
... 112 more
Caused by: rx.exceptions.OnErrorThrowable$OnNextValue: OnError while emitting onNext value: com.microsoft.azure.hdinsight.sdk.common.livy.interactive.SparkSession.class
at rx.exceptions.OnErrorThrowable.addValueAsLastCause(OnErrorThrowable.java:118)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:73)
at rx.observers.Subscribers$5.onNext(Subscribers.java:235)
at rx.internal.operators.OperatorDoAfterTerminate$1.onNext(OperatorDoAfterTerminate.java:50)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.operators.OperatorMerge$MergeSubscriber.emitScalar(OperatorMerge.java:395)
at rx.internal.operators.OperatorMerge$MergeSubscriber.tryEmit(OperatorMerge.java:355)
at rx.internal.operators.OperatorMerge$InnerSubscriber.onNext(OperatorMerge.java:846)
at rx.observers.Subscribers$5.onNext(Subscribers.java:235)
at rx.internal.operators.OperatorDoAfterTerminate$1.onNext(OperatorDoAfterTerminate.java:50)
at rx.internal.util.ScalarSynchronousObservable$WeakSingleProducer.request(ScalarSynchronousObservable.java:276)
at rx.Subscriber.setProducer(Subscriber.java:211)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.Subscriber.setProducer(Subscriber.java:205)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:138)
at rx.internal.util.ScalarSynchronousObservable$JustOnSubscribe.call(ScalarSynchronousObservable.java:129)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:94)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:32)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:248)
at rx.internal.operators.OperatorMerge$MergeSubscriber.onNext(OperatorMerge.java:148)
at rx.internal.operators.OnSubscribeMap$MapSubscriber.onNext(OnSubscribeMap.java:77)
at rx.internal.producers.SingleDelayedProducer.emit(SingleDelayedProducer.java:102)
at rx.internal.producers.SingleDelayedProducer.setValue(SingleDelayedProducer.java:85)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:48)
at rx.internal.operators.OnSubscribeFromCallable.call(OnSubscribeFromCallable.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:94)
at rx.internal.operators.OnSubscribeUsing.call(OnSubscribeUsing.java:32)
at rx.Observable.unsafeSubscribe(Observable.java:10142)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48)
at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48)
at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30)
at rx.Observable.subscribe(Observable.java:10238)
at rx.Observable.subscribe(Observable.java:10205)
at rx.observables.BlockingObservable.blockForSingle(BlockingObservable.java:444)
at rx.observables.BlockingObservable.single(BlockingObservable.java:341)
at com.microsoft.azure.hdinsight.spark.jobs.JobUtils.uploadFileToHDFS(JobUtils.java:565)
at com.microsoft.azure.hdinsight.spark.jobs.JobUtils.uploadFileToCluster(JobUtils.java:574)
at com.microsoft.azure.hdinsight.spark.jobs.JobUtils.lambda$deployArtifact$10(JobUtils.java:616)
at rx.Single.subscribe(Single.java:1876)
at rx.Single$13$1.call(Single.java:1948)
at rx.internal.schedulers.ScheduledAction.run(ScheduledAction.java:55)
at rx.internal.schedulers.ExecutorScheduler$ExecutorSchedulerWorker.run(ExecutorScheduler.java:107)
at com.microsoft.intellij.rxjava.IdeaSchedulers$1.run(IdeaSchedulers.java:51)
at com.intellij.openapi.progress.impl.CoreProgressManager$TaskRunnable.run(CoreProgressManager.java:718)
at com.intellij.openapi.progress.impl.CoreProgressManager.lambda$runProcess$1(CoreProgressManager.java:170)
at com.intellij.openapi.progress.impl.CoreProgressManager.registerIndicatorAndRun(CoreProgressManager.java:548)
at com.intellij.openapi.progress.impl.CoreProgressManager.executeProcessUnderProgress(CoreProgressManager.java:493)
at com.intellij.openapi.progress.impl.ProgressManagerImpl.executeProcessUnderProgress(ProgressManagerImpl.java:94)
at com.intellij.openapi.progress.impl.CoreProgressManager.runProcess(CoreProgressManager.java:157)
at com.intellij.openapi.progress.impl.ProgressManagerImpl$2.run(ProgressManagerImpl.java:165)
at com.intellij.openapi.application.impl.ApplicationImpl$1.run(ApplicationImpl.java:330)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
... 3 more
Есть идеи, как мне решить эту проблему?