В настоящее время у нас есть запрос на агрегацию медленного эластичного поиска.мы разбиваем данные на несколько индексов, каждый из которых содержит данные за одну неделю, а каждый индекс содержит 5 сегментов.
Запрос агрегации эластичного поиска выглядит следующим образом:
{"profile": true, "size":0,"aggregations":{"whereAggr":{"filters":{"filters":{"combineCondition":{"bool":{"must":[{"bool":{"must":[{"range":{"createdAt":{"from":1569409138454,"to":1569495539454,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"clientId.keyword":{"value":"95ed418f-d255-4847-a432-b7326a8e73da","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}},"other_bucket":false,"other_bucket_key":"_other_"},"aggregations":{"payoutAmount1Day":{"sum":{"field":"payerAmountUsd"}}}}}}
Ответ на запрос с информацией профиля:
{
"took": 519,
"timed_out": false,
"_shards": {
"total": 240,
"successful": 240,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 462286,
"max_score": 0,
"hits": []
},
"aggregations": {
"whereAggr": {
"buckets": {
"combineCondition": {
"doc_count": 5,
"payoutAmount1Day": {
"value": 2802.9300000000003
}
}
}
}
},
"profile": {
"shards": [
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][0]",
"searches": [
{
"query": [
{
"type": "MatchAllDocsQuery",
"description": "*:*",
"time_in_nanos": 1619010,
"breakdown": {
"score": 0,
"build_scorer_count": 4,
"match_count": 0,
"create_weight": 3600,
"next_doc": 5487,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 4,
"score_count": 0,
"build_scorer": 1609914,
"advance": 0,
"advance_count": 0
}
},
{
"type": "BooleanQuery",
"description": "+createdAt:[1569409138454 TO 1569495539454] +clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 1409164,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 65798,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 1343363,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "IndexOrDocValuesQuery",
"description": "createdAt:[1569409138454 TO 1569495539454]",
"time_in_nanos": 14347,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 1123,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 13221,
"advance": 0,
"advance_count": 0
}
},
{
"type": "TermQuery",
"description": "clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 20745,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 1278,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 19464,
"advance": 0,
"advance_count": 0
}
}
]
}
],
"rewrite_time": 444841,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 1906710,
"children": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time_in_nanos": 1592503,
"children": [
{
"name": "EarlyTerminatingCollector",
"reason": "search_count",
"time_in_nanos": 41650
},
{
"name": "ProfilingAggregator: [whereAggr]",
"reason": "aggregation",
"time_in_nanos": 1530021
}
]
}
]
}
]
}
],
"aggregations": [
{
"type": "FiltersAggregator",
"description": "whereAggr",
"time_in_nanos": 25579,
"breakdown": {
"reduce": 0,
"build_aggregation": 2764,
"build_aggregation_count": 1,
"initialize": 21355,
"initialize_count": 1,
"reduce_count": 0,
"collect": 1456,
"collect_count": 2
},
"children": [
{
"type": "SumAggregator",
"description": "payoutAmount1Day",
"time_in_nanos": 1432,
"breakdown": {
"reduce": 0,
"build_aggregation": 591,
"build_aggregation_count": 1,
"initialize": 839,
"initialize_count": 1,
"reduce_count": 0,
"collect": 0,
"collect_count": 0
}
}
]
}
]
},
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][3]",
"searches": [
{
"query": [
{
"type": "MatchAllDocsQuery",
"description": "*:*",
"time_in_nanos": 7322,
"breakdown": {
"score": 0,
"build_scorer_count": 6,
"match_count": 0,
"create_weight": 1909,
"next_doc": 665,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 6,
"score_count": 0,
"build_scorer": 4735,
"advance": 0,
"advance_count": 0
}
},
{
"type": "BooleanQuery",
"description": "+createdAt:[1569409138454 TO 1569495539454] +clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 58604,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 15711,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 42889,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "IndexOrDocValuesQuery",
"description": "createdAt:[1569409138454 TO 1569495539454]",
"time_in_nanos": 12802,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 1265,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 11533,
"advance": 0,
"advance_count": 0
}
},
{
"type": "TermQuery",
"description": "clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 20187,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 985,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 19198,
"advance": 0,
"advance_count": 0
}
}
]
}
],
"rewrite_time": 6118,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 141778,
"children": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time_in_nanos": 138392,
"children": [
{
"name": "EarlyTerminatingCollector",
"reason": "search_count",
"time_in_nanos": 35895
},
{
"name": "ProfilingAggregator: [whereAggr]",
"reason": "aggregation",
"time_in_nanos": 97311
}
]
}
]
}
]
}
],
"aggregations": [
{
"type": "FiltersAggregator",
"description": "whereAggr",
"time_in_nanos": 27128,
"breakdown": {
"reduce": 0,
"build_aggregation": 17543,
"build_aggregation_count": 1,
"initialize": 9223,
"initialize_count": 1,
"reduce_count": 0,
"collect": 357,
"collect_count": 3
},
"children": [
{
"type": "SumAggregator",
"description": "payoutAmount1Day",
"time_in_nanos": 1527,
"breakdown": {
"reduce": 0,
"build_aggregation": 569,
"build_aggregation_count": 1,
"initialize": 956,
"initialize_count": 1,
"reduce_count": 0,
"collect": 0,
"collect_count": 0
}
}
]
}
]
},
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][4]",
.....
}
}
}
Из результатов наблюдения каждая агрегация по индексу/ осколок не выглядит слишком плохо.однако их слишком много!в запросе задействовано 240 осколков, тогда как запрос пытается агрегировать данные только в течение одного дня.Это будет еще хуже со временем.
Некоторые думали, что я не уверен, имеют ли они смысл: 5 шардов на индекс могут быть излишними, учитывая объем данных, но есть ли способ уменьшить количество шардов, когда индексы уже работают.
Запрос содержит фильтр диапазона дат, поэтому, если мы сможем отфильтровать множество индексов, которые не попадают в диапазон дат, это будет полезно.Или переместит фильтр из части агрегации в справочную часть запроса?
Или есть еще одно хорошее предложение о том, как повысить производительность запроса?