Мы можем использовать groupBy и union для достижения этой цели.
val x = Seq((1, 1,2,4),(1, 1,2,4),(1, 1,3,5)).toDF("col1", "col2", "col3", "col4")
val y = x.groupBy("col1", "col2","col3").
agg(count(col("col3")).alias("cnt")).
withColumn("col_name", lit("col3")).
select(col("col1"), col("col2"), col("col_name"), col("col3").alias("col_value"), col("cnt"))
val z = x.groupBy("col1", "col2","col4").
agg(count(col("col4")).alias("cnt")).
withColumn("col_name", lit("col4")).
select(col("col1"), col("col2"), col("col_name"), col("col4").alias("col_value"), col("cnt"))
y.union(z).show()