Попытка чтения / записи из / в красное смещение (данные в s3).Но получая странную ошибку при доступе к фрейму данных.Я мог видеть, как создается фрейм данных, и он может получить доступ к данным, поскольку он выводит имена столбцов таблицы
scala> :require /home/hadoop/spark-redshift_2.10-2.0.1.jar
Added '/home/hadoop/spark-redshift_2.10-2.0.1.jar' to classpath.
scala> :require /home/hadoop/RedshiftJDBC41-1.2.12.1017.jar
Added '/home/hadoop/RedshiftJDBC41-1.2.12.1017.jar' to classpath.
scala> :require /home/hadoop/spark-avro_2.11-3.2.0.jar
Added '/home/hadoop/spark-avro_2.11-3.2.0.jar' to classpath.
scala> val read_data = (spark.read
| .format("com.databricks.spark.redshift")
| .option("url", "jdbc:redshift://redshifthost/schema?user=admin&password=password")
| .option("query", "SELECT * FROM schema.table LIMIT 1")
| .option("tempdir", tempS3Dir)
| .option("forward_spark_s3_credentials",true)
| .load())
read_data: org.apache.spark.sql.DataFrame = [aid: int, uid: int ... 3 more fields]
scala> read_data.count()
java.lang.ClassCastException: не может назначить экземпляр scala.collection.immutable.List $ SerializationProxy для поля org.apache.spark.rdd.RDD.org $ apache $ spark $ rdd $ RDD $$ dependencies_ типа scala.collection.Seq в экземпляре org.apache.spark.rdd.MapPartitionsRDDв java.io.ObjectStreamClass $ FieldReflector.setObjFieldValues (ObjectStreamClass.java:2133) в java.io.ObjectStreamClass.setObjFieldValues (ObjectStreamClass.java:1305) в java.io.ObjectInputStreamFlive.jfio.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.javajectoFid.ream.Fid.Of.Of.OffDefault.Of) в java.(ObjectInputStream.java:2245) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1535) в java.io.ObjectInputStream.base.O4) в scala.collection.immutable.List $ SerializationProxy.readObject (List.scala: 479) в sun.reflect.NativeMethodAccessorImpl.invoke0 (собственный метод) в sun.reflect.NativeMethodAccessorImpl.invoke (NativeMetmpl.jav).отражать.readSerialData (ObjectInputStream.java:2136) при java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1535) в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.Java: 2245) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java:1535) в java.InputStream.Oject.Offject.Offject.ObDject.Offject.Offject.Offject.ObDject.Offject.ObDject.Offject.Offject.ObDject.OffDject.Of.java: 2245) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0.jpg ObjectInput.put15:35: ObjectInputв java.io.ObjectInputStream.readObject (ObjectInputStream.java:422) в scala.collection.immutable.List $ SerializationProxy.readObject (List.scala: 479) в sun.reflect.NativeMethodAccessorImpl.invoke0 (собственный метод sun).NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62) в sun.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) в java.lang.reflect.Metho.java.jj.invokeReadObject (ObjectStreamClass.java:1058) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2136) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream) jjio.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2245) в java.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.read_jectO27.java.20)(ObjectInputStream.java:1535) в java.io.ObjectInputStream.defaultReadFields (ObjectInputStream.java:2245) вjava.io.ObjectInputStream.readSerialData (ObjectInputStream.java:2169) в java.io.ObjectInputStream.readOrdinaryObject (ObjectInputStream.java:2027) в java.io.ObjectInputStream.readObject0 (ObjectInputStream.java.15).readObject (ObjectInputStream.java:422) в org.apache.spark.serializer.JavaDeserializationStream.readObject (JavaSerializer.scala: 75) в org.apache.spark.serializer.JavaSerializerInstance.deserialize (JavaSerializer.scala: 114) илиapache.spark.scheduler.ShuffleMapTask.runTask (ShuffleMapTask.scala: 85) в org.apache.spark.scheduler.ShuffleMapTask.runTask (ShuffleMapTask.scala: 53) в org.apk.scala: 99) в org.apache.spark.executor.Executor $ TaskRunner.run (Executor.scala: 322) в java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1149) в java.util.concurrent.ThutorP$ Worker.run (ThreadPoolExecutor.java:624) в java.lang.Thread.run (Thread.java:748)
Отслеживание стека драйверов: в org.apache.spark.scheduler.DAGScheduler.org $ apache $ spark $ планировщик $ DAGScheduler $$ failJobAndIndependentStages (DAGScheduler.scala: 1505) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ abortStage $ 1.apply:1493) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ abortStage $ 1.apply (DAGScheduler.scala: 1492) в scala.collection.mutable.ResizableArray $ class.foreach (ResizableArray.scala: 59) в scala.collection..mutable.ArrayBuffer.foreach (ArrayBuffer.scala: 48) в org.apache.spark.scheduler.DAGScheduler.abortStage (DAGScheduler.scala: 1492) в org.apache.spark.scheduler.DAGScheduler $$ anonfun HandlyTas.(DAGScheduler.scala: 803) в org.apache.spark.scheduler.DAGScheduler $$ anonfun $ handleTaskSetFailed $ 1.apply (DAGScheduler.scala: 803) в scala.Option.foreach (Option.scala: 257) в org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed (DAGScheduler.scala: 803) в org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive (DAGScheduler.scala: 1720) в org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive (DAGScheduler.scala: 1675) в org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive (DAGScheduler.scala: 1664) в org.apache.spark.rtil. $.EventLoop.scala: 48)
в org.apache.spark.scheduler.DAGScheduler.runJob (DAGScheduler.scala: 629) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 1925) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 1938) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 1951) в org.apache.spark.SparkContext.runJob (SparkContext.scala: 1965 at или 1965) или.spark.rdd.RDD $$ anonfun $ collect $ 1.apply (RDD.scala: 936) в org.apache.spark.rdd.RDDOperationScope $ .withScope (RDDOperationScope.scala: 151) в org.apache.spark.rdd.RDDOperationScope $ .withScope (RDDOperationScope.scala: 112) в org.apache.spark.rdd.RDD.withScope (RDD.scala: 362) в org.apache.spark.rdd.RDD.collect (RDD.scala: 935) вorg.apache.spark.sql.execution.SparkPlan.executeCollect (SparkPlan.scala: 275) в илиg.apache.spark.sql.Dataset $$ anonfun $ org $ apache $ spark $ sql $ Набор данных $$ execute $ 1 $ 1.apply (Dataset.scala: 2386) в org.apache.spark.sql.execution.SQLExecution $.withNewExecutionId (SQLExecution.scala: 57) в org.apache.spark.sql.Dataset.withNewExecutionId (Dataset.scala: 2788) в org.apache.spark.sql.Dataset.org $ apache $ spark $ sql $ Dataset $$ execute$ 1 (Dataset.scala: 2385) в org.apache.spark.sql.Dataset.org $ apache $ spark $ sql $ Dataset $$ collect (Dataset.scala: 2392) в org.apache.spark.sql.Dataset $$anonfun $ count $ 1.apply (Dataset.scala: 2420) в org.apache.spark.sql.Dataset $$ anonfun $ count $ 1.apply (Dataset.scala: 2419) в org.apache.spark.sql.Dataset.withCallback(Dataset.scala: 2801) в org.apache.spark.sql.Dataset.count (Dataset.scala: 2419)