From 8b93d6a93ab5910862b6cd4366d2e27c8e5efa6d Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 18:49:06 -0700 Subject: [PATCH 01/12] More Postgres metrics --- .../postgres/postgres.chart.py | 357 +++++++++++++++++- 1 file changed, 356 insertions(+), 1 deletion(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 6cc1c1653512b2..1934d5862ea1ab 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -59,6 +59,14 @@ QUERY_NAME_TX_WRAPAROUND = 'TX_WRAPAROUND' QUERY_NAME_DIFF_LSN = 'DIFF_LSN' QUERY_NAME_WAL_WRITES = 'WAL_WRITES' +QUERY_NAME_PG_STAT_SLRU_COMMITTS = 'PG_STAT_SLRU_COMMITTS' +QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER = 'PG_STAT_SLRU_MULTIXACTMEMBER' +QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET = 'PG_STAT_SLRU_MULTIXACTOFFSET' +QUERY_NAME_PG_STAT_SLRU_NOTIFY = 'PG_STAT_SLRU_NOTIFY' +QUERY_NAME_PG_STAT_SLRU_SERIAL = 'PG_STAT_SLRU_SERIAL' +QUERY_NAME_PG_STAT_SLRU_SUBTRANS = 'PG_STAT_SLRU_SUBTRANS' +QUERY_NAME_PG_STAT_SLRU_XACT = 'PG_STAT_SLRU_XACT' +QUERY_NAME_PG_STAT_SLRU_OTHER = 'PG_STAT_SLRU_OTHER' METRICS = { QUERY_NAME_DATABASE: [ @@ -158,6 +166,78 @@ QUERY_NAME_REPSLOT_FILES: [ 'replslot_wal_keep', 'replslot_files' + ], + QUERY_NAME_PG_STAT_SLRU_COMMITTS: [ + 'committs_blks_zeroed', + 'committs_blks_hit', + 'committs_blks_read', + 'committs_blks_written', + 'committs_blks_exists', + 'committs_flushes', + 'committs_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER: [ + 'multixactmember_blks_zeroed', + 'multixactmember_blks_hit', + 'multixactmember_blks_read', + 'multixactmember_blks_written', + 'multixactmember_blks_exists', + 'multixactmember_flushes', + 'multixactmember_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET: [ + 'multixactoffset_blks_zeroed', + 'multixactoffset_blks_hit', + 'multixactoffset_blks_read', + 'multixactoffset_blks_written', + 'multixactoffset_blks_exists', + 'multixactoffset_flushes', + 'multixactoffset_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_NOTIFY: [ + 'notify_blks_zeroed', + 'notify_blks_hit', + 'notify_blks_read', + 'notify_blks_written', + 'notify_blks_exists', + 'notify_flushes', + 'notify_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_SERIAL: [ + 'serial_blks_zeroed', + 'serial_blks_hit', + 'serial_blks_read', + 'serial_blks_written', + 'serial_blks_exists', + 'serial_flushes', + 'serial_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_SUBTRANS: [ + 'subtrans_blks_zeroed', + 'subtrans_blks_hit', + 'subtrans_blks_read', + 'subtrans_blks_written', + 'subtrans_blks_exists', + 'subtrans_flushes', + 'subtrans_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_XACT: [ + 'xact_blks_zeroed', + 'xact_blks_hit', + 'xact_blks_read', + 'xact_blks_written', + 'xact_blks_exists', + 'xact_flushes', + 'xact_truncates' + ], + QUERY_NAME_PG_STAT_SLRU_OTHER: [ + 'other_blks_zeroed', + 'other_blks_hit', + 'other_blks_read', + 'other_blks_written', + 'other_blks_exists', + 'other_flushes', + 'other_truncates' ] } @@ -744,6 +824,143 @@ """, } +QUERY_PG_STAT_SLRU_COMMITTS = { + DEFAULT: """ +SELECT + blks_zeroed as committs_blks_zeroed, + blks_hit as committs_blks_hit, + blks_read as committs_blks_read, + blks_written as committs_blks_written, + blks_exists as committs_blks_exists, + flushes as committs_flushes, + truncates as committs_truncates +FROM + pg_stat_slru +WHERE + name = 'CommitTs'; +""", +} + +QUERY_PG_STAT_SLRU_MULTIXACTMEMBER = { + DEFAULT: """ +SELECT + blks_zeroed as multixactmember_blks_zeroed, + blks_hit as multixactmember_blks_hit, + blks_read as multixactmember_blks_read, + blks_written as multixactmember_blks_written, + blks_exists as multixactmember_blks_exists, + flushes as multixactmember_flushes, + truncates as multixactmember_truncates +FROM + pg_stat_slru +WHERE + name = 'MultiXactMember'; +""", +} + +QUERY_PG_STAT_SLRU_MULTIXACTOFFSET = { + DEFAULT: """ +SELECT + blks_zeroed as multixactoffset_blks_zeroed, + blks_hit as multixactoffset_blks_hit, + blks_read as multixactoffset_blks_read, + blks_written as multixactoffset_blks_written, + blks_exists as multixactoffset_blks_exists, + flushes as multixactoffset_flushes, + truncates as multixactoffset_truncates +FROM + pg_stat_slru +WHERE + name = 'MultiXactOffset'; +""", +} + +QUERY_PG_STAT_SLRU_NOTIFY = { + DEFAULT: """ +SELECT + blks_zeroed as notify_blks_zeroed, + blks_hit as notify_blks_hit, + blks_read as notify_blks_read, + blks_written as notify_blks_written, + blks_exists as notify_blks_exists, + flushes as notify_flushes, + truncates as notify_truncates +FROM + pg_stat_slru +WHERE + name = 'Notify'; +""", +} + +QUERY_PG_STAT_SLRU_SERIAL = { + DEFAULT: """ +SELECT + blks_zeroed as serial_blks_zeroed, + blks_hit as serial_blks_hit, + blks_read as serial_blks_read, + blks_written as serial_blks_written, + blks_exists as serial_blks_exists, + flushes as serial_flushes, + truncates as serial_truncates +FROM + pg_stat_slru +WHERE + name = 'Serial'; +""", +} + +QUERY_PG_STAT_SLRU_SUBTRANS = { + DEFAULT: """ +SELECT + blks_zeroed as subtrans_blks_zeroed, + blks_hit as subtrans_blks_hit, + blks_read as subtrans_blks_read, + blks_written as subtrans_blks_written, + blks_exists as subtrans_blks_exists, + flushes as subtrans_flushes, + truncates as subtrans_truncates +FROM + pg_stat_slru +WHERE + name = 'Subtrans'; +""", +} + +QUERY_PG_STAT_SLRU_XACT = { + DEFAULT: """ +SELECT + blks_zeroed as xact_blks_zeroed, + blks_hit as xact_blks_hit, + blks_read as xact_blks_read, + blks_written as xact_blks_written, + blks_exists as xact_blks_exists, + flushes as xact_flushes, + truncates as xact_truncates +FROM + pg_stat_slru +WHERE + name = 'Xact'; +""", +} + +QUERY_PG_STAT_SLRU_OTHER = { + DEFAULT: """ +SELECT + blks_zeroed as other_blks_zeroed, + blks_hit as other_blks_hit, + blks_read as other_blks_read, + blks_written as other_blks_written, + blks_exists as other_blks_exists, + flushes as other_flushes, + truncates as other_truncates +FROM + pg_stat_slru +WHERE + name = 'other'; +""", +} + + def query_factory(name, version=NO_VERSION): if name == QUERY_NAME_BACKENDS: return QUERY_BACKEND[DEFAULT] @@ -811,6 +1028,30 @@ def query_factory(name, version=NO_VERSION): if version < 100000: return QUERY_DIFF_LSN[V96] return QUERY_DIFF_LSN[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: + if version > 120000: + return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER: + if version > 120000: + return QUERY_PG_STAT_SLRU_MULTIXACTMEMBER[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET: + if version > 120000: + return QUERY_PG_STAT_SLRU_MULTIXACTOFFSET[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_NOTIFY: + if version > 120000: + return QUERY_PG_STAT_SLRU_NOTIFY[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_SERIAL: + if version > 120000: + return QUERY_PG_STAT_SLRU_SERIAL[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_SUBTRANS: + if version > 120000: + return QUERY_PG_STAT_SLRU_SUBTRANS[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_XACT: + if version > 120000: + return QUERY_PG_STAT_SLRU_XACT[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_OTHER: + if version > 120000: + return QUERY_PG_STAT_SLRU_OTHER[DEFAULT] raise ValueError('unknown query') @@ -847,7 +1088,15 @@ def query_factory(name, version=NO_VERSION): 'autovacuum', 'forced_autovacuum', 'tx_wraparound_oldest_current_xid', - 'tx_wraparound_percent_towards_wraparound' + 'tx_wraparound_percent_towards_wraparound', + 'pg_stat_slru_committs', + 'pg_stat_slru_multixactmember', + 'pg_stat_slru_multixactoffset', + 'pg_stat_slru_notify', + 'pg_stat_slru_serial', + 'pg_stat_slru_subtrans', + 'pg_stat_slru_xact', + 'pg_stat_slru_other' ] CHARTS = { @@ -1075,6 +1324,102 @@ def query_factory(name, version=NO_VERSION): ['replslot_wal_keep', 'wal keeped', 'absolute'], ['replslot_files', 'pg_replslot files', 'absolute'] ] + }, + 'pg_stat_slru_committs': { + 'options': [None, 'pg_stat_slru "CommitTs"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_committs', 'line'], + 'lines': [ + ['committs_blks_zeroed', 'blks zeroed', 'incremental'], + ['committs_blks_hit', 'blks hit', 'incremental'], + ['committs_blks_read', 'blks read', 'incremental'], + ['committs_blks_written', 'blks written', 'incremental'], + ['committs_blks_exists', 'blks exists', 'incremental'], + ['committs_flushes', 'flushes', 'incremental'], + ['committs_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_multixactmember': { + 'options': [None, 'pg_stat_slru "MultiXactMember"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_multixactmember', 'line'], + 'lines': [ + ['multixactmember_blks_zeroed', 'blks zeroed', 'incremental'], + ['multixactmember_blks_hit', 'blks hit', 'incremental'], + ['multixactmember_blks_read', 'blks read', 'incremental'], + ['multixactmember_blks_written', 'blks written', 'incremental'], + ['multixactmember_blks_exists', 'blks exists', 'incremental'], + ['multixactmember_flushes', 'flushes', 'incremental'], + ['multixactmember_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_multixactoffset': { + 'options': [None, 'pg_stat_slru "MultiXactOffset"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_multixactoffset', 'line'], + 'lines': [ + ['multixactoffset_blks_zeroed', 'blks zeroed', 'incremental'], + ['multixactoffset_blks_hit', 'blks hit', 'incremental'], + ['multixactoffset_blks_read', 'blks read', 'incremental'], + ['multixactoffset_blks_written', 'blks written', 'incremental'], + ['multixactoffset_blks_exists', 'blks exists', 'incremental'], + ['multixactoffset_flushes', 'flushes', 'incremental'], + ['multixactoffset_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_notify': { + 'options': [None, 'pg_stat_slru "Notify"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_notify', 'line'], + 'lines': [ + ['notify_blks_zeroed', 'blks zeroed', 'incremental'], + ['notify_blks_hit', 'blks hit', 'incremental'], + ['notify_blks_read', 'blks read', 'incremental'], + ['notify_blks_written', 'blks written', 'incremental'], + ['notify_blks_exists', 'blks exists', 'incremental'], + ['notify_flushes', 'flushes', 'incremental'], + ['notify_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_serial': { + 'options': [None, 'pg_stat_slru "Serial"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_serial', 'line'], + 'lines': [ + ['serial_blks_zeroed', 'blks zeroed', 'incremental'], + ['serial_blks_hit', 'blks hit', 'incremental'], + ['serial_blks_read', 'blks read', 'incremental'], + ['serial_blks_written', 'blks written', 'incremental'], + ['serial_blks_exists', 'blks exists', 'incremental'], + ['serial_flushes', 'flushes', 'incremental'], + ['serial_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_subtrans': { + 'options': [None, 'pg_stat_slru "Subtrans"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_subtrans', 'line'], + 'lines': [ + ['subtrans_blks_zeroed', 'blks zeroed', 'incremental'], + ['subtrans_blks_hit', 'blks hit', 'incremental'], + ['subtrans_blks_read', 'blks read', 'incremental'], + ['subtrans_blks_written', 'blks written', 'incremental'], + ['subtrans_blks_exists', 'blks exists', 'incremental'], + ['subtrans_flushes', 'flushes', 'incremental'], + ['subtrans_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_xact': { + 'options': [None, 'pg_stat_slru "Xact"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_xact', 'line'], + 'lines': [ + ['xact_blks_zeroed', 'blks zeroed', 'incremental'], + ['xact_blks_hit', 'blks hit', 'incremental'], + ['xact_blks_read', 'blks read', 'incremental'], + ['xact_blks_written', 'blks written', 'incremental'], + ['xact_blks_exists', 'blks exists', 'incremental'], + ['xact_flushes', 'flushes', 'incremental'], + ['xact_truncates', 'truncates', 'incremental'] + ] + }, + 'pg_stat_slru_other': { + 'options': [None, 'pg_stat_slru "other"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_other', 'line'], + 'lines': [ + ['other_blks_zeroed', 'blks zeroed', 'incremental'], + ['other_blks_hit', 'blks hit', 'incremental'], + ['other_blks_read', 'blks read', 'incremental'], + ['other_blks_written', 'blks written', 'incremental'], + ['other_blks_exists', 'blks exists', 'incremental'], + ['other_flushes', 'flushes', 'incremental'], + ['other_truncates', 'truncates', 'incremental'] + ] } } @@ -1259,6 +1604,16 @@ def populate_queries(self): self.queries[query_factory(QUERY_NAME_DIFF_LSN, self.server_version)] = METRICS[QUERY_NAME_WAL_WRITES] self.queries[query_factory(QUERY_NAME_STANDBY_DELTA, self.server_version)] = METRICS[QUERY_NAME_STANDBY_DELTA] self.queries[query_factory(QUERY_NAME_BLOCKERS, self.server_version)] = METRICS[QUERY_NAME_BLOCKERS] + self.queries[query_factory(QUERY_NAME_LONG_RUNNING_TX, self.server_version)] = METRICS[QUERY_NAME_LONG_RUNNING_TX] + if self.server_version >= 130000: + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITTS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITTS] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_NOTIFY, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_NOTIFY] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_SERIAL, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_SERIAL] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_SUBTRANS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_SUBTRANS] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_XACT, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_XACT] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_OTHER, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_OTHER] if self.do_index_stats: self.queries[query_factory(QUERY_NAME_INDEX_STATS)] = METRICS[QUERY_NAME_INDEX_STATS] From 153a7f2dbd54ca0795dfec2d1197a908325e708f Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 18:53:19 -0700 Subject: [PATCH 02/12] Fixes: pg 12->13 for SRLU metrics; typos --- .../postgres/postgres.chart.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 1934d5862ea1ab..2d82339e383b4e 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -59,7 +59,8 @@ QUERY_NAME_TX_WRAPAROUND = 'TX_WRAPAROUND' QUERY_NAME_DIFF_LSN = 'DIFF_LSN' QUERY_NAME_WAL_WRITES = 'WAL_WRITES' -QUERY_NAME_PG_STAT_SLRU_COMMITTS = 'PG_STAT_SLRU_COMMITTS' +QUERY_NAME_LONG_RUNNING_TX = 'QUERY_NAME_LONG_RUNNING_TX' +QUERY_NAME_PG_STAT_SLRU_COMMITS = 'PG_STAT_SLRU_COMMITS' QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER = 'PG_STAT_SLRU_MULTIXACTMEMBER' QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET = 'PG_STAT_SLRU_MULTIXACTOFFSET' QUERY_NAME_PG_STAT_SLRU_NOTIFY = 'PG_STAT_SLRU_NOTIFY' @@ -167,7 +168,7 @@ 'replslot_wal_keep', 'replslot_files' ], - QUERY_NAME_PG_STAT_SLRU_COMMITTS: [ + QUERY_NAME_PG_STAT_SLRU_COMMITS: [ 'committs_blks_zeroed', 'committs_blks_hit', 'committs_blks_read', @@ -1028,29 +1029,32 @@ def query_factory(name, version=NO_VERSION): if version < 100000: return QUERY_DIFF_LSN[V96] return QUERY_DIFF_LSN[DEFAULT] - elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: - if version > 120000: + elif name == QUERY_NAME_LONG_RUNNING_TX: + if version > 130000: + return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] + elif name == QUERY_NAME_PG_STAT_SLRU_COMMITS: + if version > 130000: return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_MULTIXACTMEMBER[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_MULTIXACTOFFSET[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_NOTIFY: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_NOTIFY[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_SERIAL: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_SERIAL[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_SUBTRANS: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_SUBTRANS[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_XACT: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_XACT[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_OTHER: - if version > 120000: + if version > 130000: return QUERY_PG_STAT_SLRU_OTHER[DEFAULT] raise ValueError('unknown query') @@ -1606,7 +1610,7 @@ def populate_queries(self): self.queries[query_factory(QUERY_NAME_BLOCKERS, self.server_version)] = METRICS[QUERY_NAME_BLOCKERS] self.queries[query_factory(QUERY_NAME_LONG_RUNNING_TX, self.server_version)] = METRICS[QUERY_NAME_LONG_RUNNING_TX] if self.server_version >= 130000: - self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITTS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITTS] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITS] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_NOTIFY, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_NOTIFY] From 7bf5d51f8cb9a3012bd9e98b67abaaa157bc1dc3 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 18:55:25 -0700 Subject: [PATCH 03/12] Reverting typo edit It was not a typo --- collectors/python.d.plugin/postgres/postgres.chart.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 2d82339e383b4e..f2a618bf5eaa2b 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -60,7 +60,7 @@ QUERY_NAME_DIFF_LSN = 'DIFF_LSN' QUERY_NAME_WAL_WRITES = 'WAL_WRITES' QUERY_NAME_LONG_RUNNING_TX = 'QUERY_NAME_LONG_RUNNING_TX' -QUERY_NAME_PG_STAT_SLRU_COMMITS = 'PG_STAT_SLRU_COMMITS' +QUERY_NAME_PG_STAT_SLRU_COMMITTS = 'PG_STAT_SLRU_COMMITTS' QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER = 'PG_STAT_SLRU_MULTIXACTMEMBER' QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET = 'PG_STAT_SLRU_MULTIXACTOFFSET' QUERY_NAME_PG_STAT_SLRU_NOTIFY = 'PG_STAT_SLRU_NOTIFY' @@ -168,7 +168,7 @@ 'replslot_wal_keep', 'replslot_files' ], - QUERY_NAME_PG_STAT_SLRU_COMMITS: [ + QUERY_NAME_PG_STAT_SLRU_COMMITTS: [ 'committs_blks_zeroed', 'committs_blks_hit', 'committs_blks_read', @@ -1032,7 +1032,7 @@ def query_factory(name, version=NO_VERSION): elif name == QUERY_NAME_LONG_RUNNING_TX: if version > 130000: return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] - elif name == QUERY_NAME_PG_STAT_SLRU_COMMITS: + elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: if version > 130000: return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER: @@ -1610,7 +1610,7 @@ def populate_queries(self): self.queries[query_factory(QUERY_NAME_BLOCKERS, self.server_version)] = METRICS[QUERY_NAME_BLOCKERS] self.queries[query_factory(QUERY_NAME_LONG_RUNNING_TX, self.server_version)] = METRICS[QUERY_NAME_LONG_RUNNING_TX] if self.server_version >= 130000: - self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITS] + self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITTS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITTS] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_NOTIFY, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_NOTIFY] From 47fa5e4d3cf1389d9f8de4399d1c0610498494ff Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 19:13:20 -0700 Subject: [PATCH 04/12] TX age --- .../postgres/postgres.chart.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index f2a618bf5eaa2b..333f8bc21c2a93 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -168,6 +168,9 @@ 'replslot_wal_keep', 'replslot_files' ], + QUERY_NAME_LONG_RUNNING_TX: [ + 'age_in_seconds' + ], QUERY_NAME_PG_STAT_SLRU_COMMITTS: [ 'committs_blks_zeroed', 'committs_blks_hit', @@ -825,6 +828,24 @@ """, } +QUERY_NAME_LONG_RUNNING_TX = { + DEFAULT: """ +SELECT + MAX(EXTRACT(EPOCH FROM (clock_timestamp() - xact_start))) + FILTER (WHERE backend_type = 'client backend') as client_tx_age_in_seconds, + MAX(EXTRACT(EPOCH FROM (clock_timestamp() - xact_start))) + FILTER (WHERE backend_type ~ 'autovacuum') as autovacuum_tx_age_in_seconds, + MAX(EXTRACT(EPOCH FROM (clock_timestamp() - xact_start))) + FILTER (WHERE backend_type !~ 'autovacuum' + AND backend_type <> 'client backend') as other_tx_age_in_seconds +FROM + pg_stat_activity +WHERE + state is distinct from 'idle' + AND backend_xmin IS NOT NULL; +""", +} + QUERY_PG_STAT_SLRU_COMMITTS = { DEFAULT: """ SELECT @@ -1329,6 +1350,14 @@ def query_factory(name, version=NO_VERSION): ['replslot_files', 'pg_replslot files', 'absolute'] ] }, + 'pg_stat_ds': { + 'options': [None, 'Max transaction age', 'seconds', 'db statistics', 'postgres.db_stat_tx_age', 'line'], + 'lines': [ + ['client_tx_age_in_seconds', 'client tx age (s)', 'absolute'], + ['autovacuum_tx_age_in_seconds', 'autovacuum tx age (s)', 'absolute'], + ['other_tx_age_in_seconds', 'other tx age (s)', 'absolute'] + ] + }, 'pg_stat_slru_committs': { 'options': [None, 'pg_stat_slru "CommitTs"', 'count', 'pg_stat_slru', 'postgres.pg_stat_slru_committs', 'line'], 'lines': [ From fa88b78b6c376b652afb17270c17fb8ca7c2e0e6 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 19:33:11 -0700 Subject: [PATCH 05/12] Update collectors/python.d.plugin/postgres/postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 333f8bc21c2a93..d17650d2278e0b 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -169,7 +169,9 @@ 'replslot_files' ], QUERY_NAME_LONG_RUNNING_TX: [ - 'age_in_seconds' + 'client_tx_age_in_seconds', + 'autovacuum_tx_age_in_seconds', + 'other_tx_age_in_seconds' ], QUERY_NAME_PG_STAT_SLRU_COMMITTS: [ 'committs_blks_zeroed', From 1092255b64be3264cc3ccdc67c9ad3e19935627c Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 19:36:06 -0700 Subject: [PATCH 06/12] Update collectors/python.d.plugin/postgres/postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index d17650d2278e0b..7496f2cc00361f 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -1352,7 +1352,7 @@ def query_factory(name, version=NO_VERSION): ['replslot_files', 'pg_replslot files', 'absolute'] ] }, - 'pg_stat_ds': { + 'long_running_tx': { 'options': [None, 'Max transaction age', 'seconds', 'db statistics', 'postgres.db_stat_tx_age', 'line'], 'lines': [ ['client_tx_age_in_seconds', 'client tx age (s)', 'absolute'], From ed4dffbdf15b0bec8a41aca6f38de0716eedc754 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Thu, 26 Aug 2021 19:36:39 -0700 Subject: [PATCH 07/12] Update collectors/python.d.plugin/postgres/postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 7496f2cc00361f..423f5323dbcdce 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -1116,6 +1116,7 @@ def query_factory(name, version=NO_VERSION): 'forced_autovacuum', 'tx_wraparound_oldest_current_xid', 'tx_wraparound_percent_towards_wraparound', + 'long_running_tx', 'pg_stat_slru_committs', 'pg_stat_slru_multixactmember', 'pg_stat_slru_multixactoffset', From 2af2670dff352b92c005a6b9469e2a910195d7aa Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Fri, 27 Aug 2021 11:18:25 -0700 Subject: [PATCH 08/12] Fix > ---> >= --- .../python.d.plugin/postgres/postgres.chart.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 423f5323dbcdce..f12281c77da15d 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -1053,31 +1053,31 @@ def query_factory(name, version=NO_VERSION): return QUERY_DIFF_LSN[V96] return QUERY_DIFF_LSN[DEFAULT] elif name == QUERY_NAME_LONG_RUNNING_TX: - if version > 130000: + if version >= 130000: return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_MULTIXACTMEMBER[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_MULTIXACTOFFSET: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_MULTIXACTOFFSET[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_NOTIFY: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_NOTIFY[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_SERIAL: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_SERIAL[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_SUBTRANS: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_SUBTRANS[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_XACT: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_XACT[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_OTHER: - if version > 130000: + if version >= 130000: return QUERY_PG_STAT_SLRU_OTHER[DEFAULT] raise ValueError('unknown query') From a832e819fa79f83a0479367aafd1e64252321ea5 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Fri, 27 Aug 2021 11:22:59 -0700 Subject: [PATCH 09/12] Update postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index f12281c77da15d..e8321b923cb3bc 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -1053,8 +1053,7 @@ def query_factory(name, version=NO_VERSION): return QUERY_DIFF_LSN[V96] return QUERY_DIFF_LSN[DEFAULT] elif name == QUERY_NAME_LONG_RUNNING_TX: - if version >= 130000: - return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] + return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: if version >= 130000: return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] From b903640ae74ed75241711ddf68834c8f8a9fe82d Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Fri, 27 Aug 2021 11:27:58 -0700 Subject: [PATCH 10/12] Update postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index e8321b923cb3bc..3f2163ab2b4f38 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -830,7 +830,7 @@ """, } -QUERY_NAME_LONG_RUNNING_TX = { +QUERY_LONG_RUNNING_TX = { DEFAULT: """ SELECT MAX(EXTRACT(EPOCH FROM (clock_timestamp() - xact_start))) @@ -1053,7 +1053,7 @@ def query_factory(name, version=NO_VERSION): return QUERY_DIFF_LSN[V96] return QUERY_DIFF_LSN[DEFAULT] elif name == QUERY_NAME_LONG_RUNNING_TX: - return QUERY_NAME_LONG_RUNNING_TX[DEFAULT] + return QUERY_LONG_RUNNING_TX[DEFAULT] elif name == QUERY_NAME_PG_STAT_SLRU_COMMITTS: if version >= 130000: return QUERY_PG_STAT_SLRU_COMMITTS[DEFAULT] From dd29f5bf2a99f2fdd3683754e7aedc5a042b0118 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Fri, 27 Aug 2021 11:33:06 -0700 Subject: [PATCH 11/12] Update postgres.chart.py --- collectors/python.d.plugin/postgres/postgres.chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 3f2163ab2b4f38..3b026aa75cd948 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -1353,7 +1353,7 @@ def query_factory(name, version=NO_VERSION): ] }, 'long_running_tx': { - 'options': [None, 'Max transaction age', 'seconds', 'db statistics', 'postgres.db_stat_tx_age', 'line'], + 'options': [None, 'Max transaction age', 'seconds', 'oldest transactions', 'postgres.db_stat_tx_age', 'line'], 'lines': [ ['client_tx_age_in_seconds', 'client tx age (s)', 'absolute'], ['autovacuum_tx_age_in_seconds', 'autovacuum tx age (s)', 'absolute'], From 159d8aeb84b7c94384715f2f9e514de93cb375a4 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Fri, 8 Oct 2021 20:24:29 +0300 Subject: [PATCH 12/12] Add wait_event_subtrans --- .../postgres/postgres.chart.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/collectors/python.d.plugin/postgres/postgres.chart.py b/collectors/python.d.plugin/postgres/postgres.chart.py index 3b026aa75cd948..bf02ba92441e5c 100644 --- a/collectors/python.d.plugin/postgres/postgres.chart.py +++ b/collectors/python.d.plugin/postgres/postgres.chart.py @@ -68,6 +68,7 @@ QUERY_NAME_PG_STAT_SLRU_SUBTRANS = 'PG_STAT_SLRU_SUBTRANS' QUERY_NAME_PG_STAT_SLRU_XACT = 'PG_STAT_SLRU_XACT' QUERY_NAME_PG_STAT_SLRU_OTHER = 'PG_STAT_SLRU_OTHER' +QUERY_NAME_PG_WAIT_EVENT_SUBTRANS = 'PG_WAIT_EVENT_SUBTRANS' METRICS = { QUERY_NAME_DATABASE: [ @@ -244,6 +245,9 @@ 'other_blks_exists', 'other_flushes', 'other_truncates' + ], + QUERY_NAME_PG_WAIT_EVENT_SUBTRANS: [ + 'subtrans' ] } @@ -984,6 +988,18 @@ """, } +QUERY_PG_WAIT_EVENT_SUBTRANS = { + DEFAULT: """ +SELECT + count(wait_event) as subtrans +FROM + pg_stat_activity +WHERE + wait_event = 'SubtransControlLock' OR + wait_event = 'SubtransSLRU'; +""", +} + def query_factory(name, version=NO_VERSION): if name == QUERY_NAME_BACKENDS: @@ -1078,6 +1094,8 @@ def query_factory(name, version=NO_VERSION): elif name == QUERY_NAME_PG_STAT_SLRU_OTHER: if version >= 130000: return QUERY_PG_STAT_SLRU_OTHER[DEFAULT] + elif name == QUERY_NAME_PG_WAIT_EVENT_SUBTRANS: + return QUERY_PG_WAIT_EVENT_SUBTRANS[DEFAULT] raise ValueError('unknown query') @@ -1123,7 +1141,8 @@ def query_factory(name, version=NO_VERSION): 'pg_stat_slru_serial', 'pg_stat_slru_subtrans', 'pg_stat_slru_xact', - 'pg_stat_slru_other' + 'pg_stat_slru_other', + 'wait_event_subtrans' ] CHARTS = { @@ -1455,6 +1474,12 @@ def query_factory(name, version=NO_VERSION): ['other_flushes', 'flushes', 'incremental'], ['other_truncates', 'truncates', 'incremental'] ] + }, + 'wait_event_subtrans': { + 'options': [None, 'wait_event "SubtransControlLock" or "SubtransSLRU"', 'count', 'wait_event_subtrans', 'postgres.wait_event_subtrans', 'line'], + 'lines': [ + ['subtrans', 'Subtrans', 'absolute'] + ] } } @@ -1640,6 +1665,7 @@ def populate_queries(self): self.queries[query_factory(QUERY_NAME_STANDBY_DELTA, self.server_version)] = METRICS[QUERY_NAME_STANDBY_DELTA] self.queries[query_factory(QUERY_NAME_BLOCKERS, self.server_version)] = METRICS[QUERY_NAME_BLOCKERS] self.queries[query_factory(QUERY_NAME_LONG_RUNNING_TX, self.server_version)] = METRICS[QUERY_NAME_LONG_RUNNING_TX] + self.queries[query_factory(QUERY_NAME_PG_WAIT_EVENT_SUBTRANS)] = METRICS[QUERY_NAME_PG_WAIT_EVENT_SUBTRANS] if self.server_version >= 130000: self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_COMMITTS, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_COMMITTS] self.queries[query_factory(QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER, self.server_version)] = METRICS[QUERY_NAME_PG_STAT_SLRU_MULTIXACTMEMBER]