Попытка выполнить запросы с помощью zeppelin в кластере Mesos с использованием Cassandra в качестве источника данных.
Мне удалось запустить его в среде DEV, но при передаче той же конфигурации и пакетов в prod он не работает.
Я использую приведенный ниже код для переноса структуры таблицы, и этот код проходит:
import org.apache.spark.sql.SparkSession
val sparkSession = SparkSession.builder().getOrCreate()
val state = sparkSession.read.format("org.apache.spark.sql.cassandra").options(Map("keyspace" -> "someKeySpace", "table" -> "someTable")).load() state.createOrReplaceTempView("state")
В следующем абзаце выполняется запрос:
%sql
select 1 from state
задача просто зависает, работает с 0% и ничего не происходит, при попытке отладки я получаю в журнале интерпретатора следующее: я выделил достаточно ресурсов CPU / Mem, и весь трафик NW включен master / slave
DEBUG [2018-10-29 10:25:43,915] ({dispatcher-event-loop-15} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:44,915] ({dispatcher-event-loop-0} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:45,915] ({dispatcher-event-loop-2} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:46,553] ({Thread-436} Logging.scala[logDebug]:58) - Received 2 resource offers.
DEBUG [2018-10-29 10:25:46,553] ({Thread-436} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22973 with attributes: Map() mem: 32218.0 cpu: 2.9 port: List((1025,2180), (
2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,27059), (27061,32000))
DEBUG [2018-10-29 10:25:46,553] ({Thread-436} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22974 with attributes: Map() mem: 50650.0 cpu: 10.9 port: List((1025,2180),
(2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,14175), (14177,32000))
DEBUG [2018-10-29 10:25:46,915] ({dispatcher-event-loop-1} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:47,915] ({dispatcher-event-loop-4} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:48,232] ({Thread-437} Logging.scala[logDebug]:58) - Received 1 resource offers.
DEBUG [2018-10-29 10:25:48,232] ({Thread-437} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22977 with attributes: Map() mem: 37338.0 cpu: 4.9 port: List((1025,2180), (
2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,8999), (9002,9041), (9043,9159), (9161,9599), (9601,20592), (20597,32000))
DEBUG [2018-10-29 10:25:48,915] ({dispatcher-event-loop-3} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:49,915] ({dispatcher-event-loop-6} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:50,915] ({dispatcher-event-loop-5} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:51,915] ({dispatcher-event-loop-7} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:52,241] ({Thread-438} Logging.scala[logDebug]:58) - Received 2 resource offers.
DEBUG [2018-10-29 10:25:52,241] ({Thread-438} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22979 with attributes: Map() mem: 32218.0 cpu: 2.9 port: List((1025,2180), (
2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,27059), (27061,32000))
DEBUG [2018-10-29 10:25:52,241] ({Thread-438} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22980 with attributes: Map() mem: 50650.0 cpu: 10.9 port: List((1025,2180),
(2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,14175), (14177,32000))
DEBUG [2018-10-29 10:25:52,915] ({dispatcher-event-loop-8} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:53,243] ({Thread-439} Logging.scala[logDebug]:58) - Received 1 resource offers.
DEBUG [2018-10-29 10:25:53,243] ({Thread-439} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22982 with attributes: Map() mem: 37338.0 cpu: 4.9 port: List((1025,2180), (
2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,8999), (9002,9041), (9043,9159), (9161,9599), (9601,20592), (20597,32000))
DEBUG [2018-10-29 10:25:53,915] ({dispatcher-event-loop-9} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:54,915] ({dispatcher-event-loop-10} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:55,915] ({dispatcher-event-loop-11} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:56,915] ({dispatcher-event-loop-12} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
DEBUG [2018-10-29 10:25:57,251] ({Thread-440} Logging.scala[logDebug]:58) - Received 2 resource offers.
DEBUG [2018-10-29 10:25:57,252] ({Thread-440} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22986 with attributes: Map() mem: 32218.0 cpu: 2.9 port: List((1025,2180), (
2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,27059), (27061,32000))
DEBUG [2018-10-29 10:25:57,252] ({Thread-440} Logging.scala[logDebug]:58) - Declining offer: 7ed1dd50-6efe-422d-913c-4835a3db28f4-O22987 with attributes: Map() mem: 50650.0 cpu: 10.9 port: List((1025,2180),
(2182,3887), (3889,5049), (5052,6999), (7002,7198), (7200,8079), (8082,8180), (8182,9000), (9002,9041), (9043,9159), (9161,9599), (9601,14175), (14177,32000))
DEBUG [2018-10-29 10:25:57,915] ({dispatcher-event-loop-13} Logging.scala[logDebug]:58) - parentName: default, name: TaskSet_0.0, runningTasks: 0
WARN [2018-10-29 10:25:58,182] ({Timer-0} Logging.scala[logWarning]:66) - Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resource