Привет, я новичок в Spark и Scala.Я пытался запустить скомпилированный sber-файл uber jar с помощью spark-submit.
package com.spark.scala
import org.apache.spark._
import org.apache.spark.SparkContext._
import org.apache.log4j._
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import org.apache.hadoop.fs.s3a.S3AFileSystem
import com.amazonaws.auth.AWSCredentialsProvider
import com.amazonaws.auth.AWSCredentialsProvider._
import org.apache.hadoop.fs.s3native.NativeS3FileSystem
/** Compute the average number of friends by age in a social network. */
object AwsSample {
/** Our main function where the action happens */
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("AwsSample")
new SparkConf().setAppName("AwsSample") //.setMaster("spark://**.**.**.**:7077")
val sc = new SparkContext(conf)
System.setProperty("com.amazonaws.services.s3.enableV4", "true")
val keyId = "**********"
val accessKey = "****************"
val endPoint= "s3.*some-region*.amazonaws.com"
val hadoopConf = sc.hadoopConfiguration
hadoopConf.set("fs.s3.awsAccessKeyId", keyId)
hadoopConf.set("fs.s3n.awsAccessKeyId", keyId)
hadoopConf.set("fs.s3a.access.key", keyId)
hadoopConf.set("fs.s3.awsSecretAccessKey", accessKey)
hadoopConf.set("fs.s3n.awsSecretAccessKey", accessKey)
hadoopConf.set("fs.s3a.secret.key", accessKey)
hadoopConf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
hadoopConf.set("fs.s3n.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
hadoopConf.set("fs.s3a.endpoint", endPoint)
val sqlContext = new SQLContext(sc)
var poll = sqlContext.read.parquet("s3a://**/***/**/**.snappy")
println(poll.count())
}
}
Я использую эту команду для выполнения jar-файла sbt, когда пакеты и сборка завершены ...
spark-submit --class com.spark.scala.AwsSample --master
spark://****:**** --deploy-mode cluster awssample_2.11-1.0.jar
Запуск его локально - это не проблема.Это работает нормально .. Ошибка, когда я пытаюсь запустить его на кластере, как в коде .. Это ошибка, с которой я сталкиваюсь
<code> Running Spark using the REST application submission protocol.
Exception in thread "main" org.apache.spark.deploy.rest.SubmitRestProtocolException: Malformed response received from server
at org.apache.spark.deploy.rest.RestSubmissionClient.readResponse(RestSubmissionClient.scala:269)
at org.apache.spark.deploy.rest.RestSubmissionClient.org$apache$spark$deploy$rest$RestSubmissionClient$$postJson(RestSubmissionClient.scala:225)
at org.apache.spark.deploy.rest.RestSubmissionClient$$anonfun$createSubmission$3.apply(RestSubmissionClient.scala:90)
at org.apache.spark.deploy.rest.RestSubmissionClient$$anonfun$createSubmission$3.apply(RestSubmissionClient.scala:86)
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at org.apache.spark.deploy.rest.RestSubmissionClient.createSubmission(RestSubmissionClient.scala:86)
at org.apache.spark.deploy.rest.RestSubmissionClientApp.run(RestSubmissionClient.scala:429)
at org.apache.spark.deploy.rest.RestSubmissionClientApp.start(RestSubmissionClient.scala:441)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:879)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:197)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:216)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:136)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: com.fasterxml.jackson.core.JsonParseException: Unexpected character ('<' (code 60)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')
at [Source: <html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>Error 404 Not Found</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /v1/submissions/create. Reason:
<pre> Not Found
Powered by Jetty: // ;строка: 1, столбец: 2] в com.fasterxml.jackson.core.JsonParser._constructError (JsonParser.java:1581) в com.fasterxml.jackson.core.base.ParserMinimalBase._reportError (ParserMinimalBase.java:533) в com.fasterxml.jackson.core.base.ParserMinimalBase._reportUnexpectedChar (ParserMinimalBase.java:462) в com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue (ReaderBasedJson.f6..ReaderBasedJsonParser.nextToken (ReaderBasedJsonParser.java:689) в com.fasterxml.jackson.databind.ObjectMapper._initForReading (ObjectMapper.java:3776) в com.fasterxml.jackson.databind.Oreadjectap.a.com.fasterxml.jackson.databind.ObjectMapper.readValue (ObjectMapper.java:2726) в org.json4s.jackson.JsonMethods $ class.parse (JsonMethods.scala: 20) в org.json4s.jackson.JsonMethods $ .parse.scala: 50) в org.apache.spark.deploy.rest.SubmitRestProtocolMessage $ .parseAction (SubmitRestProtocolMessage.scala: 112) в org.apache.spark.deploy.rest.SubmitRestProtocolMessage $ .fromJson (SubmitRestProtocolMessage.scala: 130) в org.apache.spark.deploy.rest.RestSubmissionClient $$ anonfun $ 1.apply (RestSubmissionClient.scala: 248 at).spark.deploy.rest.RestSubmissionClient $$ anonfun $ 1.apply (RestSubmissionClient.scala: 235) в scala.concurrent.impl.Future $ PromiseCompletingRunnable.liftedTree1 $ 1 (Future.scala: 24) в scala.concurrent $implPromiseCompletingRunnable.run (Future.scala: 24) в scala.concurrent.impl.ExecutionContextImpl $ AdaptedForkJoinTask.exec (ExecutionContextImpl.scala: 121) в scala.concurrent.forkjoin.ForkJoinTask.jj.forkjoin.ForkJoinPool $ WorkQueue.runTask (ForkJoinPool.java:1339) в scala.concurrent.forkjoin.ForkJoinPool.runWorker (ForkJoinPool.java:1979) в scala.concork.Jork1017 *
Может кто-нибудь помочь мне?