Elasticsearch попытался разобрать field [null] как объект, но нашел конкретное значение - PullRequest
0 голосов
/ 08 октября 2018

Как я могу изменить сопоставление, чтобы устранить эту ошибку, впервые для эластичного стека

`[logstash.outputs.elasticsearch] Не удалось проиндексировать событие для Elasticsearch.{: status => 400,: action => ["index", {: _id => nil,: _index => "cloudtrail-2018.10.08",: _type => "doc",: _routing => nil},#],: response => {"index" => {"_ index" => "cloudtrail-2018.10.08", "_type" => "doc", "_id" => "CkkQVWYBeGi09oGfr-kl", "status"=> 400, "error" => {"type" => "mapper_parsing_exception", "reason" => "объектное сопоставление для [requestParameters.filter] попыталось проанализировать поле [null] как объект, но нашло конкретное значение"}}}}

`А вот карта моих aws-ов, я в основном пытаюсь проглотить журналы cloudtrail в s3, используя logstash, чтобы упростить поиск и kibana, но в данный момент все не работает.нужна дополнительная помощь и понимание, почему не все данные отображаются в кибане

{
  "cloudtrail-2018.09.26": {
    "mappings": {
      "_default_": {
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword"
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          }
        }
      },
      "doc": {
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword"
          },
          "additionalEventData": {
            "properties": {
              "LoginTo": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MFAUsed": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MfaType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MobileVersion": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "SamlProviderArn": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleArn": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleInputParameters": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "managedRuleIdentifier": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "notificationJobType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "service": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "vpcEndpointId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "apiVersion": {
            "type": "date"
          },
          "awsRegion": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "errorCode": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "errorMessage": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventID": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventName": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventSource": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventType": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventVersion": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          },
          "managementEvent": {
            "type": "boolean"
          },
          "readOnly": {
            "type": "boolean"
          },
          "recipientAccountId": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "requestID": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "requestParameters": {
            "properties": {
              "DescribeHostsRequest": {
                "properties": {
                  "Filter": {
                    "properties": {
                      "Name": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "Value": {
                        "properties": {
                          "content": {
                            "type": "text",
                            "norms": false,
                            "fields": {
                              "keyword": {
                                "type": "keyword",
                                "ignore_above": 256
                              }
                            }
                          },
                          "tag": {
                            "type": "long"
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  },
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeInstanceCreditSpecificationsRequest": {
                "properties": {
                  "InstanceId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeLaunchTemplatesRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeNatGatewaysRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  },
                  "NatGatewayId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeSecurityGroupReferencesRequest": {
                "properties": {
                  "GroupId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeStaleSecurityGroupsRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  },
                  "VpcId": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  }
                }
              },
              "DescribeVolumesModificationsRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeVpcClassicLinkDnsSupportRequest": {
                "properties": {
                  "VpcIds": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeVpcEndpointsRequest": {
                "type": "object"
              },
              "aRN": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "accountAttributeNameSet": {
                "properties": {
                  "items": {
                    "properties": {
                      "attributeName": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      }
                    }
                  }
                }
              },
              "accountId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "acl": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "actionPrefix": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentStatus": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentVersion": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "allocationId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "allocationIdsSet": {
                "properties": {
                  "items": {
                    "properties": {
                      "allocationId": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      }
                    }
                  }
                }
              },
              "allowUnauthenticatedIdentities": {
                "type": "boolean"
              },
              "applicationName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "associationFilterList": {
                "properties": {
                  "key": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  },
                  "value": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  }
                }
              },
              "associationId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "assumeRolePolicyDocument": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attachment": {
                "properties": {
                  "attachmentId": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  },
                  "deleteOnTermination": {
                    "type": "boolean"
                  }
                }
              },
              "attribute": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attributeNames": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attributeType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "autoCreateApplication": {
                "type": "boolean"
              },
              "autoScalingGroupName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "autoScalingGroupNames": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "availabilityZone": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }

Заранее спасибо.

1 Ответ

0 голосов
/ 19 июля 2019

Извините, немного поздно.Я столкнулся с той же проблемой, и вот что я обнаружил: вы не можете изменить отображение начиная с ES 6.4.Вы должны удалить существующий индекс и воссоздать его.Это то, что я сделал, и это сработало отлично

https://www.elastic.co/guide/en/elasticsearch/reference/6.4/indices-delete-mapping.html

...