Я следовал следующему руководству:
https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.0/bk_spark-component-guide/content/ch_oozie-spark-action.html#spark-config-oozie-spark2
Это позволило мне настроить следующий файл workflow.xml
<workflow-app name="[WF-DEF-NAME]" xmlns="uri:oozie:workflow:0.3">
<start to = "Raw-To-Parquet" />
<action name="Raw-To-Parquet">
<spark xmlns="uri:oozie:spark-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<!--
<prepare>
<delete path="[PATH]"/>
<mkdir path="[PATH]"/>
</prepare>
<job-xml>[SPARK SETTINGS FILE]</job-xml>
<configuration>
<property>
<name>[PROPERTY-NAME]</name>
<value>[PROPERTY-VALUE]</value>
</property>
</configuration>
-->
<master>${master}</master>
<!--
<mode>[SPARK MODE]</mode>
-->
<name>Raw-To-Parquet</name>
<class>org.apache.spark.examples.SparkPi</class>
<jar>${nameNode}/spark-examples_2.11-2.0.2.2.5.6.3-5.jar</jar>
<spark-opts>--conf spark.yarn.jars=spark2/* --conf spark.driver.extraJavaOptions=-Dhdp.version=2.5.6.3-5 --conf spark.executor.extraJavaOptions=-Dhdp.version=2.5.6.3-5</spark-opts>
<!--SPARK_JAVA_OPTS="-Dhdp.verion=xxx"
<arg>[ARG-VALUE]</arg>
<arg>[ARG-VALUE]</arg>
-->
</spark>
<ok to="End"/>
<error to="Fail"/>
</action>
<kill name = "Fail">
<message>Job failed</message>
</kill>
<end name = "End" />
</workflow-app>
Job.properties
master=yarn-cluster
nameNode=wasb://hdi-adam-ak@hdiadamakstore.blob.core.windows.net
jobTracker=hn1-hdi-ad.hcgue2snotaezkuexzoymd0nlh.ax.internal.cloudapp.net:8088
oozie.use.system.libpath=true
oozie.wf.application.path=${nameNode}/project-example/oozie
oozie.libpath=${nameNode}/user/oozie/share/lib
oozie.action.sharelib.for.spark=spark2
Который начинает рабочий процесс, но затем умирает из-за столкновения с банками Джексона.
18/05/02 12:39:12 ERROR ApplicationMaster: User class threw exception: java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.JavaType.isReferenceType()Z
java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.JavaType.isReferenceType()Z
at com.fasterxml.jackson.databind.ser.BasicSerializerFactory.findSerializerByLookup(BasicSerializerFactory.java:302)
at com.fasterxml.jackson.databind.ser.BeanSerializerFactory._createSerializer2(BeanSerializerFactory.java:218)
at com.fasterxml.jackson.databind.ser.BeanSerializerFactory.createSerializer(BeanSerializerFactory.java:153)
at com.fasterxml.jackson.databind.SerializerProvider._createUntypedSerializer(SerializerProvider.java:1203)
at com.fasterxml.jackson.databind.SerializerProvider._createAndCacheUntypedSerializer(SerializerProvider.java:1157)
at com.fasterxml.jackson.databind.SerializerProvider.findValueSerializer(SerializerProvider.java:481)
at com.fasterxml.jackson.databind.SerializerProvider.findTypedValueSerializer(SerializerProvider.java:679)
at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:107)
at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3559)
at com.fasterxml.jackson.databind.ObjectMapper.writeValueAsString(ObjectMapper.java:2927)
at org.apache.spark.rdd.RDDOperationScope.toJson(RDDOperationScope.scala:52)
См. Ниже, чтобы увидеть содержимое моих oozie sharelibs
oozie@hn0-hdi-ad:/quantexa/oozie$ hadoop fs -ls -R /user/oozie/share/lib/lib_20180502121937 | grep jackson
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/distcp/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/distcp/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/distcp/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/hcatalog/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/hcatalog/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 232248 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/hcatalog/jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/hcatalog/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 780664 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/hcatalog/jackson-mapper-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 18336 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive/jackson-jaxrs-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 27084 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive/jackson-xc-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive2/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive2/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/hive2/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/mapreduce-streaming/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/mapreduce-streaming/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/mapreduce-streaming/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/oozie/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/oozie/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/oozie/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 232248 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 18336 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-jaxrs-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 780664 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-mapper-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 27084 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/pig/jackson-xc-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 46983 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-annotations-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 258876 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-core-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 232248 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 1171380 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-databind-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 48418 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-dataformat-cbor-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 18336 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-jaxrs-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 780664 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-mapper-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 41263 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-module-paranamer-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 515604 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-module-scala_2.11-2.6.5.jar
-rw-r--r-- 1 oozie supergroup 27084 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/jackson-xc-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 40341 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/json4s-jackson_2.11-3.2.11.jar
-rw-r--r-- 1 oozie supergroup 1048110 2018-05-02 12:21 /user/oozie/share/lib/lib_20180502121937/spark2/parquet-jackson-1.7.0.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 232248 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-databind-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 780664 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-mapper-asl-1.9.13.jar
-rw-r--r-- 1 oozie supergroup 549415 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/jackson-module-scala_2.10-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 39953 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/json4s-jackson_2.10-3.2.10.jar
-rw-r--r-- 1 oozie supergroup 1048110 2018-05-02 12:20 /user/oozie/share/lib/lib_20180502121937/spark_orig/parquet-jackson-1.7.0.jar
-rw-r--r-- 1 oozie supergroup 38605 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/sqoop/jackson-annotations-2.4.0.jar
-rw-r--r-- 1 oozie supergroup 225302 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/sqoop/jackson-core-2.4.4.jar
-rw-r--r-- 1 oozie supergroup 1076926 2018-05-02 12:19 /user/oozie/share/lib/lib_20180502121937/sqoop/jackson-databind-2.4.4.jar
Затем вы можете увидеть содержимое контейнера, который создает YARN:
root@wn0-hdi-ad:/mnt/resource/hadoop/yarn/local/usercache/oozie/appcache# ll application_1525249303830_0045/container_1525249303830_0045_01_000001/
total 1020
drwx--x--- 3 yarn hadoop 20480 May 2 12:38 ./
drwx--x--- 16 yarn hadoop 4096 May 2 12:40 ../
//removed due to word count
lrwxrwxrwx 1 yarn hadoop 85 May 2 12:38 __app__.jar -> /mnt/resource/hadoop/yarn/local/filecache/261/spark-examples_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 75 May 2 12:38 jackson-annotations-2.4.0.jar -> /mnt/resource/hadoop/yarn/local/filecache/498/jackson-annotations-2.4.0.jar*
lrwxrwxrwx 1 yarn hadoop 75 May 2 12:38 jackson-annotations-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/504/jackson-annotations-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 68 May 2 12:38 jackson-core-2.4.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/293/jackson-core-2.4.4.jar*
lrwxrwxrwx 1 yarn hadoop 68 May 2 12:38 jackson-core-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/304/jackson-core-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 73 May 2 12:38 jackson-core-asl-1.9.13.jar -> /mnt/resource/hadoop/yarn/local/filecache/467/jackson-core-asl-1.9.13.jar*
lrwxrwxrwx 1 yarn hadoop 72 May 2 12:38 jackson-databind-2.4.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/507/jackson-databind-2.4.4.jar*
lrwxrwxrwx 1 yarn hadoop 72 May 2 12:38 jackson-databind-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/514/jackson-databind-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 79 May 2 12:38 jackson-dataformat-cbor-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/357/jackson-dataformat-cbor-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 70 May 2 12:38 jackson-jaxrs-1.9.13.jar -> /mnt/resource/hadoop/yarn/local/filecache/509/jackson-jaxrs-1.9.13.jar*
lrwxrwxrwx 1 yarn hadoop 75 May 2 12:38 jackson-mapper-asl-1.9.13.jar -> /mnt/resource/hadoop/yarn/local/filecache/446/jackson-mapper-asl-1.9.13.jar*
lrwxrwxrwx 1 yarn hadoop 80 May 2 12:38 jackson-module-paranamer-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/373/jackson-module-paranamer-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 jackson-module-scala_2.11-2.6.5.jar -> /mnt/resource/hadoop/yarn/local/filecache/391/jackson-module-scala_2.11-2.6.5.jar*
lrwxrwxrwx 1 yarn hadoop 67 May 2 12:38 jackson-xc-1.9.13.jar -> /mnt/resource/hadoop/yarn/local/filecache/392/jackson-xc-1.9.13.jar*
//removed due to word count
lrwxrwxrwx 1 yarn hadoop 85 May 2 12:38 spark-catalyst_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/296/spark-catalyst_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 82 May 2 12:38 spark-cloud_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/351/spark-cloud_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 __spark_conf__ -> /mnt/resource/hadoop/yarn/local/usercache/oozie/filecache/1630/__spark_conf__.zip/
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 spark-core_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/482/spark-core_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 83 May 2 12:38 spark-graphx_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/295/spark-graphx_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 spark-hive_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/453/spark-hive_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 94 May 2 12:38 spark-hive-thriftserver_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/317/spark-hive-thriftserver_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 85 May 2 12:38 spark-launcher_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/374/spark-launcher_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 82 May 2 12:38 spark-mllib_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/359/spark-mllib_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 88 May 2 12:38 spark-mllib-local_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/291/spark-mllib-local_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 91 May 2 12:38 spark-network-common_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/433/spark-network-common_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 92 May 2 12:38 spark-network-shuffle_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/333/spark-network-shuffle_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 spark-repl_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/425/spark-repl_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 83 May 2 12:38 spark-sketch_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/407/spark-sketch_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 80 May 2 12:38 spark-sql_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/329/spark-sql_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 86 May 2 12:38 spark-streaming_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/452/spark-streaming_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 spark-tags_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/423/spark-tags_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 83 May 2 12:38 spark-unsafe_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/428/spark-unsafe_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 81 May 2 12:38 spark-yarn_2.11-2.0.2.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/478/spark-yarn_2.11-2.0.2.2.5.6.3-5.jar*
lrwxrwxrwx 1 yarn hadoop 66 May 2 12:38 spire_2.11-0.7.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/300/spire_2.11-0.7.4.jar*
lrwxrwxrwx 1 yarn hadoop 73 May 2 12:38 spire-macros_2.11-0.7.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/297/spire-macros_2.11-0.7.4.jar*
lrwxrwxrwx 1 yarn hadoop 59 May 2 12:38 ST4-4.0.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/289/ST4-4.0.4.jar*
lrwxrwxrwx 1 yarn hadoop 64 May 2 12:38 stax-api-1.0.1.jar -> /mnt/resource/hadoop/yarn/local/filecache/458/stax-api-1.0.1.jar*
lrwxrwxrwx 1 yarn hadoop 64 May 2 12:38 stax-api-1.0-2.jar -> /mnt/resource/hadoop/yarn/local/filecache/404/stax-api-1.0-2.jar*
lrwxrwxrwx 1 yarn hadoop 62 May 2 12:38 stream-2.7.0.jar -> /mnt/resource/hadoop/yarn/local/filecache/345/stream-2.7.0.jar*
lrwxrwxrwx 1 yarn hadoop 70 May 2 12:38 stringtemplate-3.2.1.jar -> /mnt/resource/hadoop/yarn/local/filecache/315/stringtemplate-3.2.1.jar*
lrwxrwxrwx 1 yarn hadoop 65 May 2 12:38 super-csv-2.2.0.jar -> /mnt/resource/hadoop/yarn/local/filecache/469/super-csv-2.2.0.jar*
drwx--x--- 2 yarn hadoop 4096 May 2 12:38 tmp/
lrwxrwxrwx 1 yarn hadoop 73 May 2 12:38 univocity-parsers-2.1.1.jar -> /mnt/resource/hadoop/yarn/local/filecache/416/univocity-parsers-2.1.1.jar*
lrwxrwxrwx 1 yarn hadoop 76 May 2 12:38 validation-api-1.1.0.Final.jar -> /mnt/resource/hadoop/yarn/local/filecache/500/validation-api-1.1.0.Final.jar*
lrwxrwxrwx 1 yarn hadoop 71 May 2 12:38 xbean-asm5-shaded-4.4.jar -> /mnt/resource/hadoop/yarn/local/filecache/493/xbean-asm5-shaded-4.4.jar*
lrwxrwxrwx 1 yarn hadoop 66 May 2 12:38 xercesImpl-2.9.1.jar -> /mnt/resource/hadoop/yarn/local/filecache/476/xercesImpl-2.9.1.jar*
lrwxrwxrwx 1 yarn hadoop 61 May 2 12:38 xmlenc-0.52.jar -> /mnt/resource/hadoop/yarn/local/filecache/491/xmlenc-0.52.jar*
lrwxrwxrwx 1 yarn hadoop 56 May 2 12:38 xz-1.0.jar -> /mnt/resource/hadoop/yarn/local/filecache/456/xz-1.0.jar*
lrwxrwxrwx 1 yarn hadoop 75 May 2 12:38 zookeeper-3.4.6.2.5.6.3-5.jar -> /mnt/resource/hadoop/yarn/local/filecache/449/zookeeper-3.4.6.2.5.6.3-5.jar*
Так что из вышесказанного кажется, что независимо от
oozie.action.sharelib.for.spark = spark2
в свойствах Job.pro YARN / Oozie загружает в контейнер все банки, включая старую версию Джексона. Я устанавливаю --conf spark.yarn.jars = spark2 / * на саму работу с искрой.
Так что я думаю, что Узи порождает работу по уменьшению карты со всей банкой сиськи. Затем это задание порождает новый контейнер для действия Spark, в котором содержатся все банки, вызвавшие столкновение. Мне нужен контейнер с искрой, чтобы в него входили только банки с искрой.