Разбор логов на json Python

Question

Народ,

Я пытаюсь преобразовать файл журнала в формат json.

Логов у меня много, есть один из них Как я могу это разобрать?

03:02:03.113 [info]  ext_ref = BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}

В этот тип json или любой другой формат (но я думаю, что json лучше всего)

{
   "time":"03:02:03.113",
   "class_req":"info",
   "ext_ref":"BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ",
   "reqid":"1253166",
   "type":"INREQ",
   "channel":"BANK24AOS",
   "sid":"msid_1591844511335516_KRRNBSLH2FS",
   "duration":"703.991",
   "req_uri":"marketcredit/order/state",
   "login":"77012221122",
   "req_type":"cl_req",
   "req_headers":{
      "accept-encoding":"gzip",
      "connection":"close",
      "host":"test-mobileapp-api.bank.kz",
      "user-agent":"okhttp/4.4.1",
      "x-forwarded-for":"212.154.169.134",
      "x-real-ip":"212.154.169.134"
   },
   "req_body":{
      "$sid":"msid_1591844511335516_KRRNBSLH2FS",
      "$sid":"msid_1591844511335516_KRRNBSLH2FS",
      "app":"bank",
      "app_version":"2.3.2",
      "channel":"aos",
      "colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv",
      "colvir_commercial_id":"-1",
      "colvir_id":"000120.335980",
      "openway_commercial_id":"6247520",
      "openway_id":"6196360",
      "$lang":"ru",
      "ekb_id":"923243",
      "inn":"990830221722",
      "login":"77012221122",
      "bank24_id":"262"
   },
   "resp_body":{
      "task_id":"",
      "status":"success",
      "data":{
         "state":"init",
         "applications":[
            {
               "status":"init",
               "id":"123db561-34a3-4a8d-9fa7-03ed6377b44f",
               "name":"Sulpak",
               "amount":101000,
               "items":[
                  {
                     "name":"Switch CISCO x24",
                     "price":100000,
                     "count":1,
                     "amount":100000
                  }
               ]
            }
         ],
         "segment":{
            "range":{
               "min":6,
               "max":36,
               "step":1
            },
            "payment_day":{
               "max":28,
               "min":1
            }
         }
      }
   }
}

Я пытаюсь чтобы разделить первый весь текст, но там я столкнулся с другой проблемой - сопоставить ключи со значениями в зависимости от знака '='. Также могут быть ключи с пустыми значениями. Например: type = INREQ channel = sid = duration = 1.333 (чтобы узнать, что есть пустое значение, вам нужно обратить внимание на количество пробелов. Обычно между предыдущим значением и следующим ключом есть пробел 1 ). Итак, этот пример должен выглядеть так:

 {
   "type":"INREQ",
   "channel":"",
   "sid":"",
   "duration":"1.333"
 }

Вперед!

Answer 1

Здесь можно указать повторяющийся ключ около "$ sid": "msid_1591844511335516_KRRNBSLH2FS"

import re
text = """03:02:03.113 [info]  ext_ref =  reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}"""
index1 = text.index('[')
index2 = text.index(']')

new_text = 'time = '+ text[:index1-1] + ' class_req = ' + text[index1+1:index2] + text[index2+2:]

lst = re.findall(r'\S+? =  |\S+? = \{.*?\} |\S+? = \{.*?\}$|\S+? = \S+? ', new_text)

res = {}
for item in lst:
    key, equal, value = item.partition('=')
    key, value = key.strip(), value.strip()
    if value.startswith('{'):
        try:
            value = json.loads(value)
        except:
            print(value)
    res[key] = value

Answer 2

регулировку можно попробовать в python.

вот что я пишу, это работает для вашей проблемы.

для удобства я удалил строку перед « ext_ref ... », вы можете напрямую обрезать необработанную строку .

import re
import json
string = 'ext_ref = BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}'


position = re.search("req_headers",string) # position of req_headers
resp_body_pos = re.search("resp_body",string)

resp_body = string[resp_body_pos.span()[0]:]
res1 = {}
res1.setdefault(resp_body.split("=")[0],resp_body.split("=")[1])
print(res1)

before = string[:position.span()[0]]
after = string[position.span()[0]:resp_body_pos.span()[0]] # handle req_body seperately

res2 = re.findall("(\S+) = (\S+)",before)
print(res2)
res3 = re.findall("(\S+) = ({.*?})",after)
print(res3)


#res1  type: dict{'resp_body':'...'}         content in resp_body  
#res2  type: list[(),()..]  content before req_head  
#res3  type: list[(),()..]  the rest content  

и теперь вы можете делать с данными то, что хотите (например, преобразовать их в json соответственно)

Надеюсь, это будет полезно