Я работаю над некоторым кодом, и мне нужно его понять, в коде есть функция, подобная приведенной ниже:
def DQChecks(self,DBname, TabName,Target_pk, datafm,dfdqcontroldata,dftrailorrec):
print "datafm DQ Checks method starts.."
dfdqcontroldataschema = dfdqcontroldata[0]
dfdqcontroldatarules = dfdqcontroldata[1]
dfdqcontroldatamap = dfdqcontroldata[2]
dfdqcontroldataschema.createOrReplaceTempView("premtarget")
dfdqcontroldatarules.createOrReplaceTempView("Master_Rules")
dfdqcontroldatamap.createOrReplaceTempView("Additional_Rules_Mapping")
#Dollor value check
if not(dftrailorrec.rdd.isEmpty()):
print "inn...."
dftrailorrec.createOrReplaceTempView("fnltrailor")
#dfdollorchktrailor = self.spark.sql("""SELECT premium,premiumgst,premiumstampduty,premiumesl,premiumterrorismlevy,premiumearthquakelevy,agencygrosscommission,gstoncommission,commissionamount FROM fnltrailor""")
dfdollorchktrailor = self.spark.sql("""SELECT basepremiumtotal,premiumgsttotal,premiumstampdutytotal,premiumesltotal,sumpremiumterrorism,premiumearthquakelevytotal,agencygrosscommissiontotal,gstoncommissiontotal,sumintermediarycommission FROM fnltrailor""")
#print 'dfdollorchktrailor'
#dfdollorchktrailor.show(100,False)
DQ_Query = """select
master.rule_type,master.rule_description,master.rejection_type,generic.Primary_Key,master.module_name ,
generic.table_name,generic.field_name,generic.dq_rule_id,generic.additional,generic.source_file_lob
from Master_Rules AS master
join (select source_file_lob,Primary_Key,table_name,field_name,additional,dq_rule_id
from Additional_Rules_Mapping LATERAL VIEW explode(rule_id) ep AS dq_rule_id) AS generic
ON (master.rule_id=generic.dq_rule_id and generic.table_name ='"""+TabName+"""')"""
if "-" in TabName:
TabName_actl, Wrng = TabName.split("-")
else:
TabName_actl = TabName
print 'first collect'
column_names_trgt = self.spark.sql("desc premtarget").select("col_name","data_type").collect()
Extraction_DF = self.spark.sql(DQ_Query)
validations = Extraction_DF.filter("rule_type !='Rejection' and rule_type !='Warning'").groupBy("field_name")\
.agg(sort_array(F.collect_list("module_name")).alias("module_name"))
unionList = []
datatypes = globals().copy()
datatypes.update(locals())
Эта функция должна возвращать четыре искровых кадра данных, но я не могу видеть, где эта функция возвращает кадры данных? Меня это немного смущает, так как я привык возвращать операторы для возврата выходных данных.