У меня есть фрагмент кода JavaScript (который является UDF), мне нужна помощь в преобразовании кода JavaScript в код Spark Scala. Схема и пример данных приведены ниже. Часть кода JavaScript преобразует current_sample_data в данные required_output, как показано ниже.
current.printSchema:
root
|-- VIN: string (nullable = true)
|-- TT: long (nullable = true)
|-- MSG_TYPE: string (nullable = true)
|-- ABS: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- E: long (nullable = true)
| | |-- V: double (nullable = true)
|-- ADA: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- E: long (nullable = true)
| | |-- V: double (nullable = true)
|-- ADW: array (nullable = true)
| |-- element: struct (containsNull = true)
| | |-- E: long (nullable = true)
| | |-- V: double (nullable = true)
current_sample_data:
+---------------- +---+---------+-----------------------------------------------------+--------------------------+
|vin |tt |msg_type |ada |adw |
+-----------------+---+---------+-----------------------------------------------------+--------------------------+
|FU7XXXXXXXXXXXXXX|0 |SIGNAL |[{"E":15XXXXXXXX,"V":2, {"E":15XXXXXXXX,"V":1}] |null |
|FU7XXXXXXXXXXXXXX|0 |SIGNAL |null |[{"E":15XXXXXXXX,"V":3}] |
|FU7XXXXXXXXXXXXXX|0 |SIGNAL |null |[{"E":15XXXXXXXX,"V":4.1}]|
+-----------------+---+---------+--------------------------+------------------------------------------------------
код javaScript:
function main(arg) {
// temporary variables for Json
var outputArray = [];
var returnJson = {};
// Header of the payload
var vin = arg.VIN
var truckType = arg.TT
//Array of required Signals
var arrayRequiredSignals = new Array(
{ name: "ADA", value: arg.ADA },
{ name: "ADW", value: arg.ADW }
);
//Tranform the Required Signal
arrayRequiredSignals.forEach(
function (element) {
transformSignalArray(element.value, outputArray, element.name);
});
//Build the final JSON Doc
returnJson = { "VIN": vin, "VEHICLETYPE": truckType, "SIGNALS": outputArray };
//Return the final JSON
return returnJson;
}
//Input Parameters: Input Array, output Array, Signal Name
function transformSignalArray(array, outputArray, signalName) {
//Check if input array has any record, if not exit the function
if (!isSignalExists(array))
return;
//variable to store one document (temporary)
var tempJson = {};
array.forEach(
function (element) {
//Extract epochtime and value
var epoch = Math.floor(element.E / 1000);
var value = Math.round(element.V * 10) / 10;
//Create the temp json and push this to output array
tempJson = { "SN": signalName, "ST": epoch, "SV": value }
outputArray.push(tempJson);
}
);
}
//Function to check if Signal Exists in the payload
function isSignalExists(array) {
return (Array.isArray(array) == 1)
}
обязательные выходные данные: