Искра Исключение: java.lang.NoSuchMethodError: org.apache.http.conn.ssl.SSLConnectionSocketFactory - PullRequest
Новая
       71

Искра Исключение: java.lang.NoSuchMethodError: org.apache.http.conn.ssl.SSLConnectionSocketFactory

0 голосов
/ 23 мая 2018

Когда я запускаю свой код в локальном коде, он работает нормально.Однако, когда я запускаю его в кластере, кажется, что в моем файле Jar отсутствует какая-либо зависимость: 

18/05/23 10:46:02 ERROR ApplicationMaster: User class threw exception: java.lang.NoSuchMethodError: org.apache.http.conn.ssl.SSLConnectionSocketFactory.<init>(Ljavax/net/ssl/SSLContext;Ljavax/net/ssl/HostnameVerifier;)V
java.lang.NoSuchMethodError: org.apache.http.conn.ssl.SSLConnectionSocketFactory.<init>(Ljavax/net/ssl/SSLContext;Ljavax/net/ssl/HostnameVerifier;)V
        at com.amazonaws.http.conn.ssl.SdkTLSSocketFactory.<init>(SdkTLSSocketFactory.java:56)
        at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.getPreferredSocketFactory(ApacheConnectionManagerFactory.java:92)
        at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.create(ApacheConnectionManagerFactory.java:65)
        at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.create(ApacheConnectionManagerFactory.java:58)

Это файл SBT: 

scalaVersion := "2.11.8"
val sparkVersion = "2.2.0"
val mahoutVersion = "0.13.1"
libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
  "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
  "org.sedis" %% "sedis" % "1.2.2",
  "org.scalactic" %% "scalactic" % "3.0.0",
  "org.scalatest" %% "scalatest" % "3.0.0" % "test",
  "com.github.nscala-time" %% "nscala-time" % "2.14.0",
  "com.github.scopt" %% "scopt" % "3.3.0"
)
unmanagedSourceDirectories in Compile += baseDirectory.value / "lib"
resolvers += "typesafe repo" at " http://repo.typesafe.com/typesafe/releases/"
resolvers += Resolver.mavenLocal
assemblyMergeStrategy in assembly := {
  case "plugin.properties" => MergeStrategy.discard
  //case PathList("META-INF", xs @ _*) => MergeStrategy.discard
  case PathList("org", "joda", "time", "base", "BaseDateTime.class") => MergeStrategy.first
  case PathList(ps @ _*) if ps.last endsWith "package-info.class" => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", "WrapDynaClass.class") => MergeStrategy.first
  case PathList("javax", "inject", xs @ _*) => MergeStrategy.first
  case PathList("org", "aopalliance", "aop", xs @ _*) => MergeStrategy.first
  case PathList("org", "aopalliance", "intercept", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") => MergeStrategy.first
  case PathList("org", "apache", "commons", "collections", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", "locale", "converters", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", "locale", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", "converters", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", xs @ _*) => MergeStrategy.first
  case PathList("assets", "org", "apache", "commons", "math3", "exception", "util", "LocalizedFormats_fr.properties") => MergeStrategy.first
  case PathList("com", "google", "common", "annotations", "Beta.class") => MergeStrategy.first
  case PathList("com", "google", "common", "base", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "cache", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "collect", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "eventbus", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "hash", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "io", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "math", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "net", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "primitives", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "reflect", xs @ _*) => MergeStrategy.first
  case PathList("com", "google", "common", "util", "concurrent", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "differentiation", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "function", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "integration", "gauss", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "interpolation", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "polynomials", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "analysis", "solvers", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "complex", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "dfp", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "distribution", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "exception", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "exception", "util", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "filter", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "fitting", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "fraction", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "genetics", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "geometry", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "geometry", "euclidean", "oned", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "geometry", "euclidean", "threed", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "geometry", "euclidean", "twod", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "geometry", "partitioning", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "linear", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ml", "clustering", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ml", "distance", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ode", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ode", "events", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ode", "nonstiff", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "ode", "sampling", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "linear", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "nonlinear", "scalar", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "nonlinear", "scalar", "gradient", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "nonlinear", "scalar", "noderiv", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "nonlinear", "vector", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optim", "univariate", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", "direct", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", "fitting", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", "general", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", "linear", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "optimization", "univariate", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "primes", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "random", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "special", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "clustering", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "correlation", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "descriptive", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "descriptive", "moment", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "descriptive", "summary", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "inference", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "ranking", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "stat", "regression", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "transform", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "math3", "util", xs @ _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "logging", xs @ _*) => MergeStrategy.first
  case PathList("overview.html") => MergeStrategy.first
  case PathList("mime.types") => MergeStrategy.first
  case x =>
    val oldStrategy = (assemblyMergeStrategy in assembly).value
    oldStrategy(x)
}

Что не так в моемJar?В IntelliJ IDEA я выбрал опцию проверки класса и вставил SSLConnectionSocketFactory.Этот класс был найден и на него ссылается httpclient-4.3.6.jar

1 Ответ

0 голосов
/ 23 мая 2018

Spark поставляется со многими библиотеками, и вы, вероятно, конфликтуете с одной из них.

Я думаю, у меня была похожая проблема с пакетом io.netty.netty-all.В итоге мы обновили этот пакет на сервере до более поздней версии, но это было потому, что мы создавали интеграцию для spark, которая была развернута на узлах.

Вы можете попробовать развернуть свое искровое приложение с этими параметрами 

--conf 'spark.driver.extraJavaOptions=-verbose:class'
--conf 'spark.executor.extraJavaOptions=-verbose:class'

, которые позволяют увидеть путь к классу.Я почти уверен, что вы увидите там конфликтную посылку.Я бы начал проверять, какая версия netty загружена, так как я почти уверен, что это зависит от httpclient.

...