Я получил гигантскую строку для анализа в json, я сузил ошибку до этого небольшого случая, возьмите:
t = "b'{\"t\":\"a\"}'"
, затем вызовите:
t = str(t)
json.loads(t)
и этовыдает следующую ошибку:
Traceback (последний вызов был последним): файл "json_dump.py", строка 11, в файле json.loads (t) "/usr/lib64/python3.6/json / init .py ", строка 354, в загрузках возвращает _default_decoder.decode (s) файл" /usr/lib64/python3.6/json/decoder.py ", строка 339, в decode obj,end = self.raw_decode (s, idx = _w (s, 0) .end ()) Файл "/usr/lib64/python3.6/json/decoder.py", строка 357, в raw_decode вызывать JSONDecodeError ("Ожидаемое значение"", s, err.value) из None json.decoder.JSONDecodeError: Ожидаемое значение: строка 1, столбец 1 (символ 0)
мне кажется, что b означает его в ascii. Насколько я могу судить, все остальное правильно экранировано. Я не могу понять, почему json lib имеет проблему здесь.
Для дальнейшего контекста, вот код, выполняемый в AWS:
import base64
import json
import zlib
import aws_encryption_sdk
from aws_encryption_sdk.internal.crypto import WrappingKey
from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider
from aws_encryption_sdk.identifiers import WrappingAlgorithm, EncryptionKeyType
import boto3
REGION_NAME = '<region>' # us-east-1
RESOURCE_ID = '<external-resource-id>' # cluster-ABCD123456
STREAM_NAME = 'aws-rds-das-' + RESOURCE_ID # aws-rds-das-cluster-ABCD123456
class MyRawMasterKeyProvider(RawMasterKeyProvider):
provider_id = "BC"
def __new__(cls, *args, **kwargs):
obj = super(RawMasterKeyProvider, cls).__new__(cls)
return obj
def __init__(self, plain_key):
RawMasterKeyProvider.__init__(self)
self.wrapping_key = WrappingKey(wrapping_algorithm=WrappingAlgorithm.AES_256_GCM_IV12_TAG16_NO_PADDING,
wrapping_key=plain_key, wrapping_key_type=EncryptionKeyType.SYMMETRIC)
def _get_raw_key(self, key_id):
return self.wrapping_key
def decrypt_payload(payload, data_key):
my_key_provider = MyRawMasterKeyProvider(data_key)
my_key_provider.add_master_key("DataKey")
decrypted_plaintext, header = aws_encryption_sdk.decrypt(
source=payload,
materials_manager=aws_encryption_sdk.DefaultCryptoMaterialsManager(master_key_provider=my_key_provider))
return decrypted_plaintext
def decrypt_decompress(payload, key):
decrypted = decrypt_payload(payload, key)
return zlib.decompress(decrypted, zlib.MAX_WBITS + 1)
def main():
session = boto3.session.Session()
kms = session.client('kms', region_name=REGION_NAME)
kinesis = session.client('kinesis', region_name=REGION_NAME)
response = kinesis.describe_stream(StreamName=STREAM_NAME)
shard_iters = []
for shard in response['StreamDescription']['Shards']:
shard_iter_response = kinesis.get_shard_iterator(StreamName=STREAM_NAME, ShardId=shard['ShardId'],
ShardIteratorType='LATEST')
shard_iters.append(shard_iter_response['ShardIterator'])
while len(shard_iters) > 0:
next_shard_iters = []
for shard_iter in shard_iters:
response = kinesis.get_records(ShardIterator=shard_iter, Limit=10000)
for record in response['Records']:
record_data = record['Data']
record_data = json.loads(record_data)
payload_decoded = base64.b64decode(record_data['databaseActivityEvents'])
data_key_decoded = base64.b64decode(record_data['key'])
data_key_decrypt_result = kms.decrypt(CiphertextBlob=data_key_decoded,
EncryptionContext={'aws:rds:dbc-id': RESOURCE_ID})
print decrypt_decompress(payload_decoded, data_key_decrypt_result['Plaintext'])
if 'NextShardIterator' in response:
next_shard_iters.append(response['NextShardIterator'])
shard_iters = next_shard_iters
if __name__ == '__main__':
main()
, а вот ошибка, возвращаемая лямбда-выражением:
{
"errorMessage": "b'{\"type\":\"DatabaseActivityMonitoringRecord\",\"clusterId\":\"cluster-AB5DDGRPE66FSNLZ3ZVTQLKVCY\",\"instanceId\":\"db-56OMGOMNHB73GQDRZJ7PUCYC5Y\",\"databaseActivityEventList\":[{\"logTime\":\"2019-10-07 04:36:48.067795+00\",\"statementId\":20,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"UPDATE\",\"objectName\":\"public.option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.34.89\",\"remotePort\":\"38864\",\"sessionId\":\"5d9ac0ce.2de2\",\"rowCount\":1,\"commandText\":\"SELECT 1 FROM ONLY \\\\\"public\\\\\".\\\\\"option\\\\\" x WHERE \\\\\"id\\\\\" OPERATOR(pg_catalog.=) $1 FOR KEY SHARE OF x\",\"paramList\":[\"913287\"],\"pid\":11746,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"WRITE\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.145962+00\",\"statementId\":53,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.34.162\",\"remotePort\":\"39320\",\"sessionId\":\"5d9ac0c7.2d52\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Driver door bin\",\"1\"],\"pid\":11602,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.147649+00\",\"statementId\":29,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.37.127\",\"remotePort\":\"54256\",\"sessionId\":\"5d9ac09b.2926\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM option\\\\n\\\\tWHERE option_id=uuid($1::TEXT)\\\\n\",\"paramList\":[\"893a662f-fb09-4d1e-b182-8f0581f64ad9\"],\"pid\":10534,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.17333+00\",\"statementId\":84,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.27.4\",\"remotePort\":\"45034\",\"sessionId\":\"5d9ac0af.2aa8\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Child-Safety Locks\",\"49\"],\"pid\":10920,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.177203+00\",\"statementId\":86,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.27.4\",\"remotePort\":\"45042\",\"sessionId\":\"5d9ac0af.2aac\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Passenger Air Bag\",\"49\"],\"pid\":10924,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.1793+00\",\"statementId\":201,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.39.59\",\"remotePort\":\"60488\",\"sessionId\":\"5d9ac071.2620\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Cruise Control\",\"1\"],\"pid\":9760,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.181394+00\",\"statementId\":32,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.37.127\",\"remotePort\":\"54258\",\"sessionId\":\"5d9ac09b.2927\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM option\\\\n\\\\tWHERE option_id=uuid($1::TEXT)\\\\n\",\"paramList\":[\"89d25110-8655-4bc7-8c70-661f7f884be6\"],\"pid\":10535,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.195299+00\",\"statementId\":37,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.34.162\",\"remotePort\":\"39318\",\"sessionId\":\"5d9ac0c7.2d51\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Front Center Armrest\",\"1\"],\"pid\":11601,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.2033+00\",\"statementId\":80,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.34.162\",\"remotePort\":\"45148\",\"sessionId\":\"5d9ac0b1.2ae9\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Illuminated Visor Mirror\",\"49\"],\"pid\":10985,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"},{\"logTime\":\"2019-10-07 04:36:48.2033+00\",\"statementId\":85,\"substatementId\":1,\"objectType\":\"TABLE\",\"command\":\"SELECT\",\"objectName\":\"public.dealer_option\",\"databaseName\":\"metal_api_prototype\",\"dbUserName\":\"metal_api_prototype\",\"remoteHost\":\"10.180.34.54\",\"remotePort\":\"46688\",\"sessionId\":\"5d9ac0b0.2ae1\",\"rowCount\":1,\"commandText\":\"SELECT *\\\\n\\\\tFROM dealer_option\\\\n\\\\tWHERE name=$1\\\\n\\\\tAND dealer_option_category_id=$2\\\\n\",\"paramList\":[\"Rear Head Air Bag\",\"49\"],\"pid\":10977,\"clientApplication\":\"\",\"exitCode\":null,\"class\":\"READ\",\"serverVersion\":\"2.3.5\",\"serverType\":\"PostgreSQL\",\"serviceName\":\"Amazon Aurora PostgreSQL-Compatible edition\",\"serverHost\":\"10.180.24.201\",\"netProtocol\":\"TCP\",\"dbProtocol\":\"Postgres 3.0\",\"type\":\"record\"}]}' is not JSON serializable",
"errorType": "TypeError",
"stackTrace": [
[
"/var/lang/lib/python3.6/json/__init__.py",
238,
"dumps",
"**kw).encode(obj)"
],
[
"/var/lang/lib/python3.6/json/encoder.py",
199,
"encode",
"chunks = self.iterencode(o, _one_shot=True)"
],
[
"/var/lang/lib/python3.6/json/encoder.py",
257,
"iterencode",
"return _iterencode(o, 0)"
],
[
"/var/runtime/awslambda/bootstrap.py",
134,
"decimal_serializer",
"raise TypeError(repr(o) + \" is not JSON serializable\")"
]
]
}