У меня проблема с flatten / json_normalize функциями. В нем есть вложенный json с 6 "квитанциями", но при выравнивании этого json мне дается только 1 строка с 1 квитанцией, что также является последним, мне нужны все 6 в моем pandas фрейме данных.
[
{
"_index": "packets-2020-02-03",
"_type": "receipts_file",
"_score": null,
"_source": {
"layers": {
"frame": {
"frame.encap_type": "25",
"frame.time": "Feb 3, 2019 00:17:14.004011000 MSK",
"frame.offset_shift": "0.000000000",
"frame.time_epoch": "2575325034.004011000",
"frame.time_delta": "0.002843000",
"frame.time_delta_displayed": "0.002843000",
"frame.time_relative": "0.002852000",
"frame.number": "4",
"frame.len": "1294",
"frame.cap_len": "1294",
"frame.marked": "0",
"frame.ignored": "0",
"frame.protocols": "several"
},
"receipts": {
"receipts.command_length": "238",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47207",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "117",
"receipts.receipt": "29831",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "47912"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "98982"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "00"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "23080"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "29849"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "949BB6DE"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
"receipts": {
"receipts.command_length": "241",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47208",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "117",
"receipts.receipt": "98341",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "38220"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "93813"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "00"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "98381"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "77371"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "6DED391C"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
"receipts": {
"receipts.command_length": "238",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47209",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "117",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "38717"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "37788"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "74818"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "77812"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "39999"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "273A872F"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
"receipts": {
"receipts.command_length": "242",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47210",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "118",
"receipts.receipt": "69322",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "83881"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "73188"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "00"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "78881"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "74388"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "949C60DF"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
"receipts": {
"receipts.command_length": "238",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47211",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "117",
"receipts.receipt": "12281",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "12727"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "18828"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "00"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "38218"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "47718"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "949BD094"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
"receipts": {
"receipts.command_length": "25",
"receipts.command_id": "0x80000004",
"receipts.command_status": "0x00000000",
"receipts.sequence_number": "35572",
"receipts.receipt_id": "949C23B8"
}
}
}
}
]
Я пытался использовать этот код:
import json
import pandas as pd
from flatten_json import flatten
i_file_name = 'example.json'
with open(i_file_name) as fd:
json_data = json.load(fd)
json_data = (flatten(d, '.') for d in json_data)
df = pd.DataFrame(json_data)
df.head()
и
import pandas as pd
i_file_name = 'example.json'
df = pd.read_json(i_file_name)
df = pd.json_normalize(df['_source'])
df.head()
Они дают мне тот же результат: только 1 строка, а не 6. Я пытался установить record_path и meta с json_normalize, но я не мог понять, как это сделать. Я немного новичок в разборе json и не могу найти здесь подобных проблем. Я знаю, что мне нужно установить правильные ключи, но я не знаю, как
РЕДАКТИРОВАТЬ:
К сожалению, StackOverflow имеет ограниченную поддержку таблиц в вопросах, поэтому я просто попробую чтобы объяснить мой ожидаемый результат.
Сейчас я получаю только одну строку с этими столбцами:
- _index
- _type
- _score
- _source.layers.frame. *
- _source.source.receipts. *
где * означает, что на одном уровне есть несколько столбцов
квитанций. * Содержат всего 5 столбцов:
- длина_команды
- идентификатор_команды
- номер_команды
- номер_последовательности
- квитанция
и эта 1 строка, которую я получаю, содержит значения для этих столбцов из последних записей «квитанций»:
"receipts": {
"receipts.command_length": "25",
"receipts.command_id": "0x80000004",
"receipts.command_status": "0x00000000",
"receipts.sequence_number": "35572",
"receipts.receipt_id": "949C23B8"
}
, но есть и другие «квитанции» уровня записи, такие как:
"receipts": {
"receipts.command_length": "238",
"receipts.command_id": "0x00000005",
"receipts.sequence_number": "47207",
"receipts.data_coding": "0x00000000",
"receipts.data_coding_tree": {
"receipts.rps": "0x00000000",
"Receipt Type 1 Data Coding": {
"receipts.rps.rc_coding_group": "0x00000000",
"receipts.rps.text_compression": "0",
"receipts.rps.class_present": "0",
"receipts.rps.charset": "0x00000000"
},
"Receipt Type 2 Data Coding": {
"receipts.rps.rpk._coding_group": "0x00000000",
"receipts.rps.rpk._language": "0x00000000"
}
},
"receipts.rc_default_receipt_id": "0",
"receipts.rc_length": "117",
"receipts.receipt": "29831",
"receipts.opt_params": {
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003002",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "47912"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003001",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "98982"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003004",
"receipts.opt_param_len": "1",
"receipts.vendor_op": "00"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003000",
"receipts.opt_param_len": "4",
"receipts.vendor_op": "23080"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00003003",
"receipts.opt_param_len": "10",
"receipts.vendor_op": "29849"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x0000001e",
"receipts.opt_param_len": "9",
"receipts.receipted_receipt_id": "949BB6DE"
},
"receipts.opt_param": {
"receipts.opt_param_tag": "0x00000427",
"receipts.opt_param_len": "1",
"receipts.receipt_state": "2"
}
}
},
, которые я хочу видеть также в виде строк в pandas dataframe. Таким образом, текущая строка, которую я получаю, должна быть шестой в этом месте.
Я вроде понял, что мой json как-то сломан, поскольку в нем 6 разных ключей с одинаковым именем (чеки), но, возможно, я могу разобрать его по-другому, чтобы я мог импортировать его в Pandas правильно