Я использую Postgres 11, и у меня есть две таблицы, mouse_move (12805080 строк) и app_event (2983971 строк), к которым я хочу присоединиться с помощью следующего запроса
SELECT *
FROM mouse_move
left JOIN app_event ON
mouse_move.uuid::text = app_event.uuid::text AND
mouse_move.db_name::text = app_event.db_name::text and
mouse_move.event_timestamp >= app_event.start_timestamp AND mouse_move.event_timestamp <= app_event.end_timestamp
Таблицы имеют многостолбцовые индексы со столбцами, используемыми в запросе. Запрос выполняется очень медленно (22 минуты) по сравнению с более простым внутренним объединением с тем же условием (~ 2 минуты). Я взглянул на объяснение: большая часть времени проводится в соединении слиянием, а строки для материализованного шага планировщиком недооцениваются. Я не эксперт по БД, поэтому я хотел бы знать, есть ли что-то, что может помочь улучшить производительность. Особенно, если добавление какого-либо индекса или статистики может улучшить качество плана. По мере сбора дополнительных данных мне кажется, что этот запрос не будет масштабироваться. Спасибо
"Gather (cost=1001.12..4391658.54 rows=12805080 width=253) (actual time=0.348..1834200.357 rows=12805080 loops=1)"
" Workers Planned: 2"
" Workers Launched: 2"
" Buffers: shared hit=494918 read=437424"
" I/O Timings: read=26662.468"
" -> Merge Left Join (cost=1.12..3110150.54 rows=5335450 width=253) (actual time=1.751..1829903.695 rows=4268360 loops=3)"
" Merge Cond: (((mouse_move.uuid)::text = (app_event.uuid)::text) AND ((mouse_move.db_name)::text = (app_event.db_name)::text))"
" Join Filter: ((mouse_move.event_timestamp >= app_event.start_timestamp) AND (mouse_move.event_timestamp <= app_event.end_timestamp))"
" Rows Removed by Join Filter: 2982395649"
" Buffers: shared hit=494918 read=437424"
" I/O Timings: read=26662.468"
" -> Parallel Index Scan using mouse_move_idx_all_timestamp on mouse_move (cost=0.56..1630981.79 rows=5335450 width=105) (actual time=1.694..14439.293 rows=4268360 loops=3)"
" Buffers: shared hit=162579 read=383277"
" I/O Timings: read=26031.084"
" -> Materialize (cost=0.56..489657.96 rows=2983971 width=148) (actual time=0.015..386371.252 rows=2986813075 loops=3)"
" Buffers: shared hit=332339 read=54147"
" I/O Timings: read=631.385"
" -> Index Scan using app_event_idx_all_timestamp on app_event (cost=0.56..482198.03 rows=2983971 width=148) (actual time=0.011..1660.860 rows=2983971 loops=3)"
" Buffers: shared hit=332339 read=54147"
" I/O Timings: read=631.385"
"Planning Time: 1.019 ms"
"Execution Time: 1835802.069 ms"
[
{
"Plan": {
"Node Type": "Gather",
"Parallel Aware": false,
"Startup Cost": 1001.12,
"Total Cost": 4391658.54,
"Plan Rows": 12805080,
"Plan Width": 253,
"Actual Startup Time": 0.335,
"Actual Total Time": 1799731.441,
"Actual Rows": 12805080,
"Actual Loops": 1,
"Output": [
"mouse_move.db_name",
"mouse_move.uuid",
"mouse_move.event_id",
"mouse_move.event_timestamp",
"mouse_move.duration",
"mouse_move.distance",
"mouse_move.distance_mm",
"mouse_move.distance_on_eyetracker_screen",
"mouse_move.distance_mm_on_eyetracker_screen",
"app_event.db_name",
"app_event.uuid",
"app_event.start_timestamp",
"app_event.end_timestamp",
"app_event.window_process",
"app_event.window_class",
"app_event.window_title",
"app_event.app_view_alias",
"app_event.browser_url",
"app_event.window_x",
"app_event.window_y",
"app_event.window_width",
"app_event.window_height",
"app_event.event_id"
],
"Workers Planned": 2,
"Workers Launched": 2,
"Single Copy": false,
"Shared Hit Blocks": 525437,
"Shared Read Blocks": 403511,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 18881.913,
"I/O Write Time": 0,
"Plans": [
{
"Node Type": "Merge Join",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Join Type": "Left",
"Startup Cost": 1.12,
"Total Cost": 3110150.54,
"Plan Rows": 5335450,
"Plan Width": 253,
"Actual Startup Time": 0.073,
"Actual Total Time": 1795390.268,
"Actual Rows": 4268360,
"Actual Loops": 3,
"Output": [
"mouse_move.db_name",
"mouse_move.uuid",
"mouse_move.event_id",
"mouse_move.event_timestamp",
"mouse_move.duration",
"mouse_move.distance",
"mouse_move.distance_mm",
"mouse_move.distance_on_eyetracker_screen",
"mouse_move.distance_mm_on_eyetracker_screen",
"app_event.db_name",
"app_event.uuid",
"app_event.start_timestamp",
"app_event.end_timestamp",
"app_event.window_process",
"app_event.window_class",
"app_event.window_title",
"app_event.app_view_alias",
"app_event.browser_url",
"app_event.window_x",
"app_event.window_y",
"app_event.window_width",
"app_event.window_height",
"app_event.event_id"
],
"Inner Unique": false,
"Merge Cond": "(((mouse_move.uuid)::text = (app_event.uuid)::text) AND ((mouse_move.db_name)::text = (app_event.db_name)::text))",
"Join Filter": "((mouse_move.event_timestamp >= app_event.start_timestamp) AND (mouse_move.event_timestamp <= app_event.end_timestamp))",
"Rows Removed by Join Filter": 2982395649,
"Shared Hit Blocks": 525437,
"Shared Read Blocks": 403511,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 18881.913,
"I/O Write Time": 0,
"Workers": [
{
"Worker Number": 0,
"Actual Startup Time": 0.083,
"Actual Total Time": 1795519.049,
"Actual Rows": 4261870,
"Actual Loops": 1,
"Shared Hit Blocks": 173782,
"Shared Read Blocks": 136197,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 6433.326,
"I/O Write Time": 0
},
{
"Worker Number": 1,
"Actual Startup Time": 0.109,
"Actual Total Time": 1795409.518,
"Actual Rows": 4262034,
"Actual Loops": 1,
"Shared Hit Blocks": 175020,
"Shared Read Blocks": 134574,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 6360.393,
"I/O Write Time": 0
}
],
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": true,
"Scan Direction": "Forward",
"Index Name": "mouse_move_idx_all_timestamp",
"Relation Name": "mouse_move",
"Schema": "public",
"Alias": "mouse_move",
"Startup Cost": 0.56,
"Total Cost": 1630981.79,
"Plan Rows": 5335450,
"Plan Width": 105,
"Actual Startup Time": 0.007,
"Actual Total Time": 10972.966,
"Actual Rows": 4268360,
"Actual Loops": 3,
"Output": [
"mouse_move.db_name",
"mouse_move.uuid",
"mouse_move.event_id",
"mouse_move.event_timestamp",
"mouse_move.duration",
"mouse_move.distance",
"mouse_move.distance_mm",
"mouse_move.distance_on_eyetracker_screen",
"mouse_move.distance_mm_on_eyetracker_screen"
],
"Shared Hit Blocks": 251111,
"Shared Read Blocks": 292452,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 18466.31,
"I/O Write Time": 0,
"Workers": [
{
"Worker Number": 0,
"Actual Startup Time": 0.008,
"Actual Total Time": 11115.945,
"Actual Rows": 4261870,
"Actual Loops": 1,
"Shared Hit Blocks": 83272,
"Shared Read Blocks": 98245,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 6293.593,
"I/O Write Time": 0
},
{
"Worker Number": 1,
"Actual Startup Time": 0.007,
"Actual Total Time": 11074.816,
"Actual Rows": 4262034,
"Actual Loops": 1,
"Shared Hit Blocks": 83899,
"Shared Read Blocks": 97233,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 6217.653,
"I/O Write Time": 0
}
]
},
{
"Node Type": "Materialize",
"Parent Relationship": "Inner",
"Parallel Aware": false,
"Startup Cost": 0.56,
"Total Cost": 489657.96,
"Plan Rows": 2983971,
"Plan Width": 148,
"Actual Startup Time": 0.013,
"Actual Total Time": 379317.533,
"Actual Rows": 2986821333,
"Actual Loops": 3,
"Output": [
"app_event.db_name",
"app_event.uuid",
"app_event.start_timestamp",
"app_event.end_timestamp",
"app_event.window_process",
"app_event.window_class",
"app_event.window_title",
"app_event.app_view_alias",
"app_event.browser_url",
"app_event.window_x",
"app_event.window_y",
"app_event.window_width",
"app_event.window_height",
"app_event.event_id"
],
"Shared Hit Blocks": 274326,
"Shared Read Blocks": 111059,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 415.603,
"I/O Write Time": 0,
"Workers": [
{
"Worker Number": 0,
"Actual Startup Time": 0.017,
"Actual Total Time": 379314.173,
"Actual Rows": 2993043689,
"Actual Loops": 1,
"Shared Hit Blocks": 90510,
"Shared Read Blocks": 37952,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 139.733,
"I/O Write Time": 0
},
{
"Worker Number": 1,
"Actual Startup Time": 0.015,
"Actual Total Time": 380272.5,
"Actual Rows": 2969781288,
"Actual Loops": 1,
"Shared Hit Blocks": 91121,
"Shared Read Blocks": 37341,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 142.74,
"I/O Write Time": 0
}
],
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Scan Direction": "Forward",
"Index Name": "app_event_idx_all_timestamp",
"Relation Name": "app_event",
"Schema": "public",
"Alias": "app_event",
"Startup Cost": 0.56,
"Total Cost": 482198.03,
"Plan Rows": 2983971,
"Plan Width": 148,
"Actual Startup Time": 0.011,
"Actual Total Time": 1656.429,
"Actual Rows": 2983971,
"Actual Loops": 3,
"Output": [
"app_event.db_name",
"app_event.uuid",
"app_event.start_timestamp",
"app_event.end_timestamp",
"app_event.window_process",
"app_event.window_class",
"app_event.window_title",
"app_event.app_view_alias",
"app_event.browser_url",
"app_event.window_x",
"app_event.window_y",
"app_event.window_width",
"app_event.window_height",
"app_event.event_id"
],
"Shared Hit Blocks": 274326,
"Shared Read Blocks": 111059,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 415.603,
"I/O Write Time": 0,
"Workers": [
{
"Worker Number": 0,
"Actual Startup Time": 0.014,
"Actual Total Time": 1590.19,
"Actual Rows": 2983971,
"Actual Loops": 1,
"Shared Hit Blocks": 90510,
"Shared Read Blocks": 37952,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 139.733,
"I/O Write Time": 0
},
{
"Worker Number": 1,
"Actual Startup Time": 0.012,
"Actual Total Time": 1646.447,
"Actual Rows": 2983971,
"Actual Loops": 1,
"Shared Hit Blocks": 91121,
"Shared Read Blocks": 37341,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"I/O Read Time": 142.74,
"I/O Write Time": 0
}
]
}
]
}
]
}
]
},
"Planning Time": 1.012,
"Triggers": [],
"Execution Time": 1801335.068
}
]
ОБНОВЛЕНИЕ После предложения scaisEdge об удалении приведения и изменении индекса: та же производительность. 