Не удается инициировать контекст H2O для взаимодействия со Spark в R - PullRequest
Новая
       31

Не удается инициировать контекст H2O для взаимодействия со Spark в R

2 голосов
/ 25 июня 2019

При выполнении следующего фрагмента кода возникает ошибка преобразования mtcars_tbl в h2o dataframe 

mtcars_h2o <- as_h2o_frame(sc, mtcars_tbl, strict_version_check = FALSE)  

Error: java.lang.AbstractMethodError
    at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:99)
    at org.apache.spark.h2o.H2OContext$.initializeLogIfNecessary(H2OContext.scala:360)
    at org.apache.spark.internal.Logging$class.log(Logging.scala:46)
    at org.apache.spark.h2o.H2OContext$.log(H2OContext.scala:360)
    at org.apache.spark.internal.Logging$class.logWarning(Logging.scala:66)
    at org.apache.spark.h2o.H2OContext$.logWarning(H2OContext.scala:360)
    at org.apache.spark.h2o.H2OContext$.getOrCreate(H2OContext.scala:430)
    at org.apache.spark.h2o.H2OContext.getOrCreate(H2OContext.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
    at java.lang.reflect.Method.invoke(Unknown Source)
    at sparklyr.Invoke.invoke(invoke.scala:139)
    at sparklyr.StreamHandler.handleMethodCall(stream.scala:123)
    at sparklyr.StreamHandler.read(stream.scala:66)
    at sparklyr.BackendHandler.channelRead0(handler.scala:51)
    at sparklyr.BackendHandler.channelRead0(handler.scala:4)
    at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:138)
    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
    at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
    at java.lang.Thread.run(Unknown Source)

Вот моя сессияInfo () 

R version 3.5.3 (2019-03-11)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows 10 x64 (build 14393)

Matrix products: default

locale:
[1] LC_COLLATE=Spanish_Ecuador.1252  LC_CTYPE=Spanish_Ecuador.1252    LC_MONETARY=Spanish_Ecuador.1252 LC_NUMERIC=C                    
[5] LC_TIME=Spanish_Ecuador.1252    

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
[1] h2o_3.24.0.5         dplyr_0.8.1          sparklyr_1.0.1       rsparkling_2.2.42    RevoUtils_11.0.3     RevoUtilsMath_11.0.0

loaded via a namespace (and not attached):
 [1] Rcpp_1.0.1        compiler_3.5.3    pillar_1.4.1      dbplyr_1.4.2      r2d3_0.2.3        bitops_1.0-6      base64enc_0.1-3  
 [8] tools_3.5.3       digest_0.6.19     packrat_0.5.0     jsonlite_1.6      tibble_2.1.3      pkgconfig_2.0.2   rlang_0.3.4      
[15] cli_1.1.0         DBI_1.0.0         rstudioapi_0.10   yaml_2.2.0        parallel_3.5.3    withr_2.1.2       httr_1.4.0       
[22] generics_0.0.2    htmlwidgets_1.3   askpass_1.1       rprojroot_1.3-2   tidyselect_0.2.5  glue_1.3.1        forge_0.2.0      
[29] R6_2.4.0          sessioninfo_1.1.1 purrr_0.3.2       magrittr_1.5      backports_1.1.4   htmltools_0.3.6   ellipsis_0.2.0   
[36] assertthat_0.2.1  config_0.3        RCurl_1.95-4.12   openssl_1.4       crayon_1.3.4

И полный код 

options(rsparkling.sparklingwater.version = "2.4.99999-157")

library(rsparkling)
library(sparklyr)
library(dplyr)
library(h2o)

sc <- spark_connect(master = "local", spark_home = "C:/spark-2.4.3/")
mtcars_tbl <- copy_to(sc, mtcars, "mtcars")

mtcars_h2o <- as_h2o_frame(sc, mtcars_tbl, strict_version_check = FALSE)

Если я запускаю h2o_context () Я получаю почти такой же результат, как если бы не было соединения между h2o и Spark.

...