data.registerTempTable('tweets')
sqlContext.registerJavaFunction("clean","a.b.c.Cleaner")
sqlContext.sql("SELECT clean(tweet) from tweets").show()
Приведенный выше код успешно работает на jupyter, но не на zeppelin.Я загрузил jar "abcCleaner" с помощью z.load (..), таблица твитов также успешно регистрируется, она видна в контексте sql.
Вот трассировка стека из zeppelin:
Py4JJavaError: Произошла ошибка при вызове o1693.showString.: org.apache.spark.SparkException: задание прервано из-за сбоя этапа: задание 0 на этапе 27.0 не выполнено 4 раза, последний сбой: потерянное задание 0.3 на этапе 27.0 (TID 306, ip-172-31-14-210.eu-central-1.compute.internal, исполнитель 10): java.lang.ClassCastException: невозможно назначить экземпляр scala.collection.immutable.List $ SerializationProxy для поля org.apache.spark.rdd.RDD.org $ apache $ spark $rdd $ RDD $$ dependencies_ типа scala.collection.Seq в экземпляре org.apache.spark.rdd.MapPartitionsRDD в java.io.ObjectStreamClass $ FieldReflector.setObjFieldValues (ObjectStreamClass.java:2287) в java.ream.ObBjectOject.(ObjectStreamClass.java:1417) в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2292) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210) в java.io.ObjectInputStream.jj2068) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572) в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2286) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068) в java.io.ObjectInputStream.jj) в java.io.ObjectInputStream.readObject (ObjectInputStream.java:430) в scala.collection.immutable.List $ SerializationProxy.readObject (List.scala: 490) в sun.reflect.GeneratedMethodAccessor2.invoke (неизвестный источник) на солнце.refle.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) в java.lang.reflect.Method.invoke (Method.java:498) в java.io.ObjectStreamClass.invokeReadObject (ObjectStreamStjectOject.java) java..readSerialData (ObjectInputStream.java:2177) при java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572) в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.Java: 2286) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572) в java.io.ream.ObjectInput.java: 2286) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068) в java.io.ObjectInputStream.readObject0.jj: ObjectInputв java.io.ObjectInputStream.readObject (ObjectInputStream.java:430) в org.apache.spark.serializer.JavaDeserializationStream.readObject (JavaSerializer.scala: 75) в org.apache.spark.serializer.JavaSerializerInstance.alaseizersc (Java): 114) в org.apache.spark.scheduler.ResultTask.runTask (ResultTask.scala: 83) в org.apache.spark.scheduler.Task.run (Task.scala: 121) в org.apache.spark.executor.Исполнитель $ TaskRunner $$ anonfun $ 10.apply (Executor.scala: 402) по адресу org.apache.spark.util.Utils $ .tryWithSafeFinсоюзник (Utils.scala: 1360) в org.apache.spark.executor.Executor $ TaskRunner.run (Executor.scala: 408) в java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1149) в java.util.concurrent.ThreadPoolExecutor $ Worker.run (ThreadPoolExecutor.java:624) в java.lang.Thread.run (Thread.java:748) Отслеживание стека драйверов:в org.apache.spark.scheduler.DAGScheduler.org $ apache $ spark $ планировщик $ DAGScheduler $$ failJobAndIndependentStages (DAGScheduler.scala: 2039) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ abortSage $.scala: 2027) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ abortStage $ 1.apply (DAGScheduler.scala: 2026) в scala.collection.mutable.ResizableArray $ class.foreach (ResizableArray.scala: 59) вscala.collection.mutable.ArrayBuffer.foreach (ArrayBuffer.scala: 48) в org.apache.spark.scheduler.DAGScheduler.abortStage (DAGScheduler.scala: 2026) в org.apache.spark.scheduler.DAGScheduler $$TileSkF$ 1.apply (DAGScheduler.scala: 966) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ handleTaskSetFailed $ 1.apply (DAGScheduler.scala: 966) в scala.Option.foreach (Option.scala: 257) в org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed (DAGScheduler.scala: 966) в org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive (DAGScheduler.scala: 2260) в орг.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive (DAGScheduler.scala: 2209) в org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive (DAGScheduler.scala: 2198) в org.apache.venton $ $.выполнить (EventLoop.scala: 49) в org.apache.spark.scheduler.DAGScheduler.runJob (DAGScheduler.scala: 777) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 2061) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 2082) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 2101) в org.apache.spark.sql.execution.SparkPlan.executeTake (SparkPlan.scala 36)org.apache.spark.sql.execution.CollectLimitExec.executeCollect (limit.scala: 38) в org.apache.spark.sql.Dataset.org $ apache $ spark $ sql $ Набор данных $$ collectFromPlan (Dataset.scala: 3384)в org.apache.spark.sql.Dataset $$ anonfun $ head $ 1.apply (Dataset.scala: 2545) в org.apache.spark.sql.Dataset $$ anonfun $ head $ 1.apply (Dataset.scala: 2545)в org.apache.spark.sql.Dataset $$ anonfun $ 53.apply (Dataset.scala: 3365) в org.apache.spark.sql.execution.SQLExecution $$ anonfun $ withNewExecutionId $ 1.apply (SQLExecution.scala: 78) в org.apache.spark.sql.execution.SQLExecution $ .withSQLConfPropagated (SQLExecuala.scala: SQLExecution.scala: SQLExecution.sc).spark.sql.execution.SQLExecution $ .withNewExecutionId (SQLExecution.scala: 73) в org.apache.spark.sql.Dataset.withAction (Dataset.scala: 3364) в org.apache.spark.sql.Dataset.head (Dataset.scala: 2545) в org.apache.spark.sql.Dataset.take (Dataset.scala: 2759) в org.apache.spark.sql.Dataset.getRows (Dataset.scala: 255) в org.apache.spark.sql.Dataset.showString (Dataset.(Method.java:498) в py4j.reflection.MethodInvoker.invoke (MethodInvoker.java:244) в py4j.reflection.ReflectionEngine.invoke (ReflectionEngine.java:357) в py4j.Gateway.invoke (Gateway.java:282)на py4j.commands.AbstractCommand.invokeМетод (AbstractCommand.java:132) в py4j.commands.CallCommand.execute (CallCommand.java:79) в py4j.GatewayConnection.run (GatewayConnection.java:238) в java.lang.Thread.run (Thread.java:748)) Причина: java.lang.ClassCastException: невозможно назначить экземпляр scala.collection.immutable.List $ SerializationProxy для поля org.apache.spark.rdd.RDD.org $ apache $ spark $ rdd $ RDD $$ dependencies_ типа scala.collection.Seq в экземпляре org.apache.spark.rdd.MapPartitionsRDD в java.io.ObjectStreamClass $ FieldReflector.setObjFieldValues (ObjectStreamClass.java:2287) в java.io.ObjectStreamClass.setObjFream14lass.set.bjFieldVlass.alues (java.io.ObjectStreamClass.java:2287).io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2292)в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210)
в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068)
в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572)
в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2286)
в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210)
в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068)
в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572)
в java.io.ObjectInputStream.readObject (ObjectInputStream.java:430)
в scala.collection.immutable.List $ SerializationProxy.readObject (List.scala: 490)
at sun.reflect.GeneratedMethodAccessor2.invoke (неизвестный источник)
at sun.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43)
в java.lang.reflect.Method.invoke (Method.java:498)
в java.io.ObjectStreamClass.invokeReadObject (ObjectStreamClass.java:1170)
в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2177)
в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068)
в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572)
в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2286)
в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210)
в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068)
в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572)
в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2286)
в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2210)
в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2068)
в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1572)
в java.io.ObjectInputStream.readObject (ObjectInputStream.java:430)
в org.apache.spark.serializer.JavaDeserializationStream.readObject (JavaSerializer.scala: 75)
в org.apache.spark.serializer.JavaSerializerInstance.deserialize (JavaSerializer.scala: 114)
в org.apache.spark.scheduler.ResultTask.runTask (ResultTask.scala: 83)
в org.apache.spark.scheduler.Task.run (Task.scala: 121)
в org.apache.spark.executor.Executor $ TaskRunner $$ anonfun $ 10.apply (Executor.scala: 402)
в org.apache.spark.util.Utils $ .tryWithSafeFinally (Utils.scala: 1360)
в org.apache.spark.executor.Executor $ TaskRunner.run (Executor.scala: 408)
в java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1149)
в java.util.concurrent.ThreadPoolExecutor $ Worker.run (ThreadPoolExecutor.java:624)
... еще 1
()
Ps: одинаковая искра (2.4.0) установлена на обоих