Я использую Hive2.1.1 и Atlas2.0.0.
Я могу найти происхождение таблицы, но не нашел происхождение уровня столбца
подробная информация показана ниже
Для отладки,
Iпопробуйте сначала Restful, ничего о происхождении столбца.
[root@cent1 bin]# curl -X GET -u admin:admin http://cent1:21000/api/atlas/v2/lineage/27c81b16-b422-4479-84b9-2d643b5dba48
{"baseEntityGuid":"27c81b16-b422-4479-84b9-2d643b5dba48","lineageDirection":"BOTH","lineageDepth":3,"guidEntityMap":{},"relations":[]}
Я печатаю сообщение Кафки из ATLAS_HOOK темы ниже:
{
"version":{
"version":"1.0.0",
"versionParts":[
1
]
},
"msgCompressionKind":"NONE",
"msgSplitIdx":1,
"msgSplitCount":1,
"msgSourceIP":"192.168.10.128",
"msgCreatedBy":"root",
"msgCreationTime":1572687183793,
"message":{
"type":"ENTITY_CREATE_V2",
"user":"root",
"entities":{
"referredEntities":{
"-53640556650491":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes.bar@primary",
"name":"bar",
"comment":null,
"position":1,
"type":"string",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes@primary"
}
}
},
"guid":"-53640556650491",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650492":{
"typeName":"hive_table",
"attributes":{
"owner":"root",
"temporary":false,
"lastAccessTime":1572687176000,
"qualifiedName":"atlas1.pokes_create3@primary",
"columns":[
{
"guid":"-53640556650494",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3.foo@primary"
}
},
{
"guid":"-53640556650495",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3.bar@primary"
}
}
],
"tableType":"MANAGED_TABLE",
"sd":{
"guid":"-53640556650493",
"typeName":"hive_storagedesc",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3@primary_storage"
}
},
"createTime":1572687176000,
"name":"pokes_create3",
"comment":null,
"partitionKeys":[
],
"parameters":{
"totalSize":"5812",
"numRows":"500",
"rawDataSize":"5312",
"COLUMN_STATS_ACCURATE":"{"BASIC_STATS":"true"}",
"numFiles":"1",
"transient_lastDdlTime":"1572687178"
},
"db":{
"guid":"-53640556650487",
"typeName":"hive_db",
"uniqueAttributes":{
"qualifiedName":"atlas1@primary"
}
},
"retention":0
},
"guid":"-53640556650492",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650490":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes.foo@primary",
"name":"foo",
"comment":null,
"position":0,
"type":"int",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes@primary"
}
}
},
"guid":"-53640556650490",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650495":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes_create3.bar@primary",
"name":"bar",
"comment":null,
"position":1,
"type":"string",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3@primary"
}
}
},
"guid":"-53640556650495",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650493":{
"typeName":"hive_storagedesc",
"attributes":{
"qualifiedName":"atlas1.pokes_create3@primary_storage",
"storedAsSubDirectories":false,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes_create3",
"compressed":false,
"inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
"parameters":{
},
"outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3@primary"
}
},
"serdeInfo":{
"typeName":"hive_serde",
"attributes":{
"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"name":null,
"parameters":{
"serialization.format":"1"
}
}
},
"numBuckets":-1
},
"guid":"-53640556650493",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650494":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes_create3.foo@primary",
"name":"foo",
"comment":null,
"position":0,
"type":"int",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3@primary"
}
}
},
"guid":"-53640556650494",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650488":{
"typeName":"hive_table",
"attributes":{
"owner":"root",
"temporary":false,
"lastAccessTime":1572663197000,
"qualifiedName":"atlas1.pokes@primary",
"columns":[
{
"guid":"-53640556650490",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes.foo@primary"
}
},
{
"guid":"-53640556650491",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes.bar@primary"
}
}
],
"tableType":"MANAGED_TABLE",
"sd":{
"guid":"-53640556650489",
"typeName":"hive_storagedesc",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes@primary_storage"
}
},
"createTime":1572663197000,
"name":"pokes",
"comment":null,
"partitionKeys":[
],
"parameters":{
"transient_lastDdlTime":"1572663225",
"totalSize":"5812",
"numRows":"0",
"rawDataSize":"0",
"numFiles":"1"
},
"db":{
"guid":"-53640556650487",
"typeName":"hive_db",
"uniqueAttributes":{
"qualifiedName":"atlas1@primary"
}
},
"retention":0
},
"guid":"-53640556650488",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650489":{
"typeName":"hive_storagedesc",
"attributes":{
"qualifiedName":"atlas1.pokes@primary_storage",
"storedAsSubDirectories":false,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes",
"compressed":false,
"inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
"parameters":{
},
"outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes@primary"
}
},
"serdeInfo":{
"typeName":"hive_serde",
"attributes":{
"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"name":null,
"parameters":{
"serialization.format":"1"
}
}
},
"numBuckets":-1
},
"guid":"-53640556650489",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650487":{
"typeName":"hive_db",
"attributes":{
"owner":"root",
"ownerType":"USER",
"qualifiedName":"atlas1@primary",
"clusterName":"primary",
"name":"atlas1",
"description":null,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db",
"parameters":{
}
},
"guid":"-53640556650487",
"provenanceType":0,
"version":0,
"proxy":false
}
},
"entities":[
{
"typeName":"hive_process",
"attributes":{
"outputs":[
{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3@primary"
}
}
],
"recentQueries":[
"create table pokes_create3 as select foo,bar from pokes"
],
"qualifiedName":"atlas1.pokes_create3@primary:1572687176000",
"inputs":[
{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes@primary"
}
}
],
"name":"create table pokes_create3 as select foo,bar from pokes",
"queryText":"create table pokes_create3 as select foo,bar from pokes",
"operationType":"CREATETABLE_AS_SELECT",
"startTime":1572686979104,
"queryPlan":"Not Supported",
"endTime":1572687183754,
"userName":"root",
"queryId":"root_20191102172939_5de2901c-49e1-473c-ad2f-7e5f5b3251ee"
},
"guid":"-53640556650496",
"provenanceType":0,
"version":0,
"proxy":false
}
]
}
}
}
Где находится происхождение столбца? Как я могу отладить это дальше?