Я пытаюсь использовать сборку sbt в проекте spark.Компиляция sbt и работа с пакетами, но когда я пытаюсь выполнить сборку sbt, я получаю следующую ошибку:
объект spark не входит в пакет org.apache
Я включил библиотеки spark core и spark sqlи иметь sbt-сборку в моем файле плагинов.Почему сборка вызывает эти ошибки?
build.sbt:
name := "redis-record-loader"
scalaVersion := "2.11.8"
val sparkVersion = "2.3.1"
val scalatestVersion = "3.0.3"
val scalatest = "org.scalatest" %% "scalatest" % scalatestVersion
libraryDependencies ++=
Seq(
"com.amazonaws" % "aws-java-sdk-s3" % "1.11.347",
"com.typesafe" % "config" % "1.3.1",
"net.debasishg" %% "redisclient" % "3.0",
"org.slf4j" % "slf4j-log4j12" % "1.7.12",
"org.apache.commons" % "commons-lang3" % "3.0" % "test,it",
"org.apache.hadoop" % "hadoop-aws" % "2.8.1" % Provided,
"org.apache.spark" %% "spark-core" % sparkVersion % Provided,
"org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
"org.mockito" % "mockito-core" % "2.21.0" % Test,
scalatest
)
val integrationTestsKey = "it"
val integrationTestLibs = scalatest % integrationTestsKey
lazy val IntegrationTestConfig = config(integrationTestsKey) extend Test
lazy val root = project.in(file("."))
.configs(IntegrationTestConfig)
.settings(inConfig(IntegrationTestConfig)(Defaults.testSettings): _*)
.settings(libraryDependencies ++= Seq(integrationTestLibs))
test in assembly := Seq(
(test in Test).value,
(test in IntegrationTestConfig).value
)
assemblyMergeStrategy in assembly := {
case PathList("META-INF", xs @ _*) => MergeStrategy.discard
case x => MergeStrategy.first
}
plugins.sbt:
logLevel := Level.Warn
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
полное сообщение об ошибке:
/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:11: object spark is not a member of package org.apache
[error] import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:26: not found: type SparkSession
[error] implicit val spark: SparkSession = SparkSession.builder
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:26: not found: value SparkSession
[error] implicit val spark: SparkSession = SparkSession.builder
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:51: not found: type DataFrame
[error] val testDataframe0: DataFrame = testData0.toDF()
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:51: value toDF is not a member of Seq[(String, String)]
[error] val testDataframe0: DataFrame = testData0.toDF()
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:52: not found: type DataFrame
[error] val testDataframe1: DataFrame = testData1.toDF()
[error] ^
[error] /Users/jones8/Work/redis-record-loader/src/it/scala/com/elsevier/bos/RedisRecordLoaderIntegrationSpec.scala:52: value toDF is not a member of Seq[(String, String)]
[error] val testDataframe1: DataFrame = testData1.toDF()
[error] ^
[error] missing or invalid dependency detected while loading class file 'RedisRecordLoader.class'.
[error] Could not access term spark in package org.apache,
[error] because it (or its dependencies) are missing. Check your build definition for
[error] missing or conflicting dependencies. (Re-run with `-Ylog-classpath` to see the problematic classpath.)
[error] A full rebuild may help if 'RedisRecordLoader.class' was compiled against an incompatible version of org.apache.
[error] missing or invalid dependency detected while loading class file 'RedisRecordLoader.class'.
[error] Could not access type SparkSession in value org.apache.sql,
[error] because it (or its dependencies) are missing. Check your build definition for
[error] missing or conflicting dependencies. (Re-run with `-Ylog-classpath` to see the problematic classpath.)
[error] A full rebuild may help if 'RedisRecordLoader.class' was compiled against an incompatible version of org.apache.sql.
[error] 9 errors found