Трудно загружать логи cloudflare из Amazon s3 в Elastic Search через python - PullRequest
Новая
       22

Трудно загружать логи cloudflare из Amazon s3 в Elastic Search через python

0 голосов
/ 06 ноября 2019

Я настраиваю способ загрузки своих журналов Cloudflare из amazon s3 в amazonastic search через python, но я получаю сообщение об ошибке при каждой попытке загрузки. Я вменял xxxxxxxx в целях конфиденциальности. Я чувствую, что ошибка в этих коробках. Я пытаюсь удалить ящики из журналов, но безуспешно ... {} [] 

import boto3
import re
import requests
from requests_aws4auth import AWS4Auth
import json
import ast

region = 'xxxxxxxxxx' # e.g. us-west-1
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)

host = ' xxxxxxxxxx' # the Amazon ES domain, including https://
index = ' xxxxxxxxxx'
es_type = ' xxxxxxxxxx'
url = host + '/' + index + '/' + es_type

headers = { "Content-Type": "application/json" }

s3 = boto3.client('s3')

# Lambda execution starts here
def handler(event, context):
    for record in event['Records']:

        # Get the bucket name and key for the new file
        bucket = record['s3']['bucket']['name']
        key = record['s3']['object']['key']

        # Get, read, and split the file into lines
        obj = s3.get_object(Bucket=bucket, Key=key)
        body = obj['Body'].read().decode('utf-8_sig')
        lines = body.splitlines()

        for line in lines:
            pre_document = line.replace('true',"true").replace('false',"false").replace('\{\}',"\"{}\"")

            document = ast.literal_eval(pre_document)

            r = requests.post(url, auth=awsauth, json=document, headers=headers)

ЖУРНАЛ ДАННЫХ 

{"ClientRequestPath":"/","ClientRequestReferer":"","CacheTieredFill":false,"FirewallMatchesActions":[],"FirewallMatchesSources":[],"FirewallMatchesRuleIDs":[],
"RequestHeaders":{},"ResponseHeaders":{},"SecurityLevel":"med","WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"",
"WAFRuleMessage":"","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":false,"WorkerSubrequestCount":0,"ZoneID":145825900}

Пока я продолжаюполучить эту ошибку? 

[ERROR] ValueError: malformed node or string: <_ast.Name object at 0x7fb1a9020290> Traceback (most recent call last):   File "/var/task/sample.py", line 51, in handler     document = ast.literal_eval(pre_document)   File "/var/lang/lib/python3.7/ast.py", line 91, in literal_eval     return _convert(node_or_string)   File "/var/lang/lib/python3.7/ast.py", line 79, in _convert     map(_convert, node
[ERROR] ValueError: malformed node or string: <_ast.Name object at 0x7fb1a9020290>
...