Конвертировать код JavaScript в код Spark Scala - PullRequest
0 голосов
/ 12 октября 2019

У меня есть фрагмент кода JavaScript (который является UDF), мне нужна помощь в преобразовании кода JavaScript в код Spark Scala. Схема и пример данных приведены ниже. Часть кода JavaScript преобразует current_sample_data в данные required_output, как показано ниже.

current.printSchema:

root
|-- VIN: string (nullable = true)
|-- TT: long (nullable = true)
|-- MSG_TYPE: string (nullable = true)
|-- ABS: array (nullable = true)
|    |-- element: struct (containsNull = true)
|    |    |-- E: long (nullable = true)
|    |    |-- V: double (nullable = true)
|-- ADA: array (nullable = true)
|    |-- element: struct (containsNull = true)
|    |    |-- E: long (nullable = true)
|    |    |-- V: double (nullable = true)
|-- ADW: array (nullable = true)
|    |-- element: struct (containsNull = true)
|    |    |-- E: long (nullable = true)
|    |    |-- V: double (nullable = true)

current_sample_data:
           +---------------- +---+---------+-----------------------------------------------------+--------------------------+
           |vin              |tt |msg_type |ada                                                  |adw                       |                      
           +-----------------+---+---------+-----------------------------------------------------+--------------------------+
           |FU7XXXXXXXXXXXXXX|0  |SIGNAL   |[{"E":15XXXXXXXX,"V":2, {"E":15XXXXXXXX,"V":1}]      |null                      |                          
           |FU7XXXXXXXXXXXXXX|0  |SIGNAL   |null                                                 |[{"E":15XXXXXXXX,"V":3}]  |                      
           |FU7XXXXXXXXXXXXXX|0  |SIGNAL   |null                                                 |[{"E":15XXXXXXXX,"V":4.1}]|
           +-----------------+---+---------+--------------------------+------------------------------------------------------

код javaScript:

function main(arg) {
    // temporary variables for Json
    var outputArray = [];
    var returnJson = {};

    // Header of the payload
    var vin = arg.VIN
    var truckType = arg.TT

    //Array of required Signals
    var arrayRequiredSignals = new Array(
          { name: "ADA", value: arg.ADA },
          { name: "ADW", value: arg.ADW }
        );

    //Tranform the Required Signal
    arrayRequiredSignals.forEach(
        function (element) {
            transformSignalArray(element.value, outputArray, element.name);
        });


    //Build the final JSON Doc
    returnJson = { "VIN": vin, "VEHICLETYPE": truckType, "SIGNALS": outputArray };

    //Return the final JSON
    return returnJson;
}

//Input Parameters: Input Array, output Array, Signal Name
function transformSignalArray(array, outputArray, signalName) {

    //Check if input array has any record, if not exit the function
    if (!isSignalExists(array))
        return;

    //variable to store one document (temporary)
    var tempJson = {};

    array.forEach(
        function (element) {
            //Extract epochtime and value
            var epoch = Math.floor(element.E / 1000);
            var value = Math.round(element.V * 10) / 10;

            //Create the temp json and push this to output array
            tempJson = { "SN": signalName, "ST": epoch, "SV": value }

            outputArray.push(tempJson);
        }
    );
}

//Function to check if Signal Exists in the payload
function isSignalExists(array) {
    return (Array.isArray(array) == 1)
}

обязательные выходные данные: enter image description here

...