Я сталкиваюсь с ошибкой в hiverunner, когда я пытался выполнить свои модульные тесты, это мой тестовый класс:
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.StandaloneHiveRunner;
import com.klarna.hiverunner.annotations.HiveSQL;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import java.util.Arrays;
import java.util.List;
@RunWith(StandaloneHiveRunner.class)
public class MiniClusterHiveTest {
@Rule
public TestName name = new TestName();
@HiveSQL(files = {})
private HiveShell shell;
@Before
public void createDatabaseAndTable() {
shell.execute("CREATE DATABASE source_db");
shell.execute("CREATE TABLE source_db.survey (" +
"submittedTime STRING," +
"age STRING," +
"gender STRING," +
"country STRING," +
"state STRING," +
"self_employed STRING," +
"family_history STRING," +
"treatment STRING," +
"work_interfere STRING," +
"no_employees STRING," +
"remote_work STRING," +
"tech_company STRING," +
"benefits STRING," +
"care_options STRING," +
"wellness_program STRING," +
"seek_help STRING," +
"anonymity STRING," +
"leave STRING," +
"mental_health_consequence STRING," +
"phys_health_consequence STRING," +
"coworkers STRING," +
"supervisor STRING," +
"mental_health_interview STRING," +
"phys_health_interview STRING," +
"mental_vs_physical STRING," +
"obs_consequence STRING," +
"comments STRING) " +
"ROW FORMAT DELIMITED " +
"FIELDS TERMINATED BY ',' " +
"STORED AS TEXTFILE " +
"LOCATION '/home/datasets/'");
}
@Before
public void createORCTable() {
shell.execute("CREATE TABLE source_db.survey2 (" +
"submittedTime STRING," +
"age STRING," +
"gender STRING," +
"country STRING," +
"state STRING," +
"self_employed STRING," +
"family_history STRING," +
"treatment STRING," +
"work_interfere STRING," +
"no_employees STRING," +
"remote_work STRING," +
"tech_company STRING," +
"benefits STRING," +
"care_options STRING," +
"wellness_program STRING," +
"seek_help STRING," +
"anonymity STRING," +
"leave STRING," +
"mental_health_consequence STRING," +
"phys_health_consequence STRING," +
"coworkers STRING," +
"supervisor STRING," +
"mental_health_interview STRING," +
"phys_health_interview STRING," +
"mental_vs_physical STRING," +
"obs_consequence STRING," +
"comments STRING) " +
"ROW FORMAT DELIMITED " +
"FIELDS TERMINATED BY ',' " +
"STORED AS ORC tblproperties (\"orc.compress\"=\"ZLIB\"); ");
}
@Before
public void createParquetTable() {
shell.execute("CREATE TABLE source_db.survey3 " +
"STORED AS PARQUET TBLPROPERTIES (\"parquet.compression\"=\"SNAPPY\")\n" +
" AS SELECT * FROM source_db.survey;");
}
/**
* We use temporary table survey to load the orc table survey2
*/
@Test
public void loadOrcSurvey2Table() {
shell.execute("INSERT INTO TABLE source_db.survey2 SELECT * from source_db.survey");
}
@Ignore("Use for simple test without external file")
public void simpleInit() {
shell.execute("CREATE DATABASE source_db");
shell.execute("CREATE TABLE source_db.test (" +
"a STRING," +
"b STRING ) ");
}
@Ignore
public void simpleInsertionDataIntoTable() {
shell.insertInto("source_db", "test")
.withAllColumns()
.addRow("bim", "bap")
.commit();
printResult(shell.executeStatement("select * from source_db.test"));
}
@Test
public void executeQuery() {
List<Object[]> result = shell.executeStatement("select * from source_db.survey where age=37");
List<Object[]> result2 = shell.executeStatement("select * from source_db.survey where age=12");
Assert.assertEquals(43L, result.size());
Assert.assertEquals(0L, result2.size());
}
@Test
public void insertDataIntoTable() {
shell.insertInto("source_db", "survey")
.withAllColumns()
.addRow("2019-03-01 09:29:31",
17,
"Male",
"France",
"IL",
"NA",
"No",
"Yes",
"Often",
"6-25",
"No",
"Yes",
"Yes",
"Not sure",
"No",
"Yes",
"Yes",
"Somewhat easy",
"No",
"No",
"Some of them",
"Yes",
"No",
"Maybe",
"Yes",
"No",
"NA"
)
.commit();
printResult(shell.executeStatement("select * from source_db.survey where age=17"));
}
private void printResult(List<Object[]> result) {
System.out.println(String.format("Result from %s:", name.getMethodName()));
result.stream().map(Arrays::asList).forEach(System.out::println);
}
}
Этот тестовый класс работает хорошо, пока я недобавление этой зависимости к pom.xml :
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
Это pom.xml, когда работает тестовый класс:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.finaxys</groupId>
<artifactId>MiniClusterTestBigData</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<java.version>1.8</java.version>
<hadoop.version>2.7.3</hadoop.version>
<spark.version>2.3.0</spark.version>
<junit.version>4.12</junit.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
<dependencies>
<!--hive dependencies-->
<dependency>
<groupId>com.klarna</groupId>
<artifactId>hiverunner</artifactId>
<version>4.1.0</version>
</dependency>
<!--hadoop dependencies-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!--Spark dependencies-->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
</dependencies>
</project>
И это ошибка, которую я получаюкогда я выполняю тестовый класс с зависимостью spark-hive:
java.lang.IllegalStateException: не удалось создать HiveServer: ошибка применения политики авторизации в конфигурации куста: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Невозможно создать экземпляр org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient
У кого-то была похожая проблема, и она объяснила в этой теме:
https://github.com/klarna/HiveRunner/issues/66
Я пытался добавить пакет, как объяснено, но проблема все еще не решена:
org.apache.hadoop.hive.metastore.RetryMetaStoreClient
Do yты знаешь как это исправить?