Я пытаюсь подключить куст, используя hive-jdbc и spark local в двух интерфейсах.Но интерфейс jdbc получает ошибку через 1,2 дня:
- Аутентификация с помощью Kerberos вначале прошла успешно, и я могу использовать оба интерфейса для успешного запроса данных улья.
- Несколько мгновений спустя, интерфейс jdbc завершился неудачно (ОШИБКА опубликована ниже).
- После выполнения spark spark интерфейс jdbc сразу восстанавливается.
Все происходило в одной и той же SpringVot JVM.Похоже, что срок действия билета Kerberos истек.
Версия Jars:
<spark.version>2.3.0.cloudera4</spark.version>
<hive.version>1.1.0-cdh5.15.1</hive.version>
<hadoop.version>2.6.0-cdh5.15.1</hadoop.version>
Приложение Springboot:
@SpringBootApplication
@EnableScheduling
public class Application {
@PostConstruct
void started() {
Logger LOGGER = LoggerFactory.getLogger(Application.class);
System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
Configuration configuration = new Configuration();
configuration.set("hadoop.security.authentication" , "Kerberos" );
UserGroupInformation.setConfiguration(configuration);
try {
UserGroupInformation.loginUserFromKeytab("xxx@USER", "/etc/datasource.keytab");
} catch (IOException e) {
LOGGER.error("Auth failed. ", e);
}
LOGGER.info("authentication with kerberos successful");
}
Интерфейс Hive jdbc:
Connection conn = null;
ResultSet rs = null;
PreparedStatement ps = null;
try {
conn = DriverManager.getConnection(CONNECTION_URL);
ps = conn.prepareStatement("show databases");
rs = ps.executeQuery();
while (rs.next()) {
System.out.println(rs.getString(1));
}
} catch (Exception e) {
e.printStackTrace();
} finally {
conn.close();
}
Spark SQL:
SparkSession spark = SparkSession
.builder()
.setMaster("local")
.appName("Java Spark Hive Example")
.enableHiveSupport()
.getOrCreate();
spark.sql("show databases").show();
spark.sql("select * from balldb.ods_addcrest limit 10").show();
ОШИБКА :
2019-03-01 02:08:39.450 INFO 1 --- [pool-3-thread-5] org.apache.hive.jdbc.Utils : Supplied authorities: bigdata-01:36003
2019-03-01 02:08:39.451 INFO 1 --- [pool-3-thread-5] org.apache.hive.jdbc.Utils : Resolved authority: bigdata-01:36003
2019-03-01 02:08:39.456 ERROR 1 --- [pool-3-thread-5] o.a.thrift.transport.TSaslTransport : SASL negotiation failure
javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[na:1.8.0_191]
at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94) [hive-exec-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:271) ~[hive-exec-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) [hive-exec-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) [hive-shims-common-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) [hive-shims-common-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at java.security.AccessController.doPrivileged(Native Method) [na:1.8.0_191]
at javax.security.auth.Subject.doAs(Subject.java:422) [na:1.8.0_191]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924) [hadoop-common-2.6.0-cdh5.15.1.jar!/:na]
at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) [hive-shims-common-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.hive.jdbc.HiveConnection.openTransport(HiveConnection.java:203) [hive-jdbc-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.hive.jdbc.HiveConnection.<init>(HiveConnection.java:168) [hive-jdbc-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at org.apache.hive.jdbc.HiveDriver.connect(HiveDriver.java:105) [hive-jdbc-1.1.0-cdh5.15.1.jar!/:1.1.0-cdh5.15.1]
at java.sql.DriverManager.getConnection(DriverManager.java:664) [na:1.8.0_191]
at java.sql.DriverManager.getConnection(DriverManager.java:247) [na:1.8.0_191]
at com.task.HiveDSTask.init(HiveDSTask.java:83) [classes!/:na]
at com.task.HiveDSTask.call(HiveDSTask.java:95) [classes!/:na]
at com.task.HiveDSTask.call(HiveDSTask.java:22) [classes!/:na]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_191]
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) ~[na:1.8.0_191]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[na:1.8.0_191]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) ~[na:1.8.0_191]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[na:1.8.0_191]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[na:1.8.0_191]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[na:1.8.0_191]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[na:1.8.0_191]
... 21 common frames omitted
/ etc / krb5.conf:
[libdefaults]
default_realm = bigdata-02
dns_lookup_kdc = false
dns_lookup_realm = false
ticket_lifetime = 86400
renew_lifetime = 604800
forwardable = true
default_tgs_enctypes = aes256-cts arcfour-hmac des3-hmac-sha1 des-cbc-crc des
default_tkt_enctypes = aes256-cts arcfour-hmac des3-hmac-sha1 des-cbc-crc des
permitted_enctypes = aes256-cts arcfour-hmac des3-hmac-sha1 des-cbc-crc des
udp_preference_limit = 1
kdc_timeout = 3000
[realms]
BIGDATA-02 = {
kdc = bigdata-02
admin_server = bigdata-02
default_domain = bigdata-02
kdc = bigdata-03
}
[domain_realm]
bigdata-02 = BIGDATA-02
Я поражен здесь долгое время, Пожалуйста, помогите мне в этой ошибке.Спасибо.