diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b824dad2a423..b24df6d08417 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,28 +1,34 @@ +# Queue-time parameter. Resolved when the run is created, so it is one of the +# few things a ${{ }} expression can see -- which is what makes it usable as the +# full-CI gate (see the matrix below). +# +# GitHub Actions queues a run with: +# +# POST https://dev.azure.com/it0639/percona-server/_apis/pipelines//runs?api-version=7.1 +# { +# "resources": { "repositories": { "self": { "refName": "refs/pull//merge" } } }, +# "templateParameters": { "fullCI": true } +# } +# +# Omit templateParameters (or pass false) for the five default configs; pass +# true for all 30. It also appears as a checkbox in the Run pipeline dialog. +parameters: +- name: fullCI + displayName: 'Build all configs, not just the default five' + type: boolean + default: false + schedules: -- cron: "0 2 * * *" - displayName: Daily 2:00 AM UTC build +- cron: "0 3 * * *" + displayName: Daily 3:00 AM UTC build branches: include: - trunk -trigger: - branches: - include: - - '*' - exclude: - - trunk - paths: - exclude: - - doc - - build-ps - - man - - mysql-test - - packaging - - policy - - scripts - - support-files +trigger: none pr: + drafts: false branches: include: - '*' @@ -73,15 +79,75 @@ jobs: PARENT_BRANCH: trunk BUILD_PARAMS_TYPE: normal + # Two tiers of coverage: + # + # default the five configs listed first -- built on every pull request. + # extended the remaining 25, each guarded so they are built only when the + # fullCI parameter is set (how GitHub Actions asks for a full run) + # or on the nightly schedule. + # + # The guards are compile-time insertions, so on a default run the extra legs + # are never generated: no agents, and no skipped entries in the run summary. + # + # The gate keys on the parameter rather than on Build.Reason because a run + # created through the REST API reports Build.Reason=Manual -- so testing + # "not a pull request" would make every GitHub-Actions-queued run build all 30. + # Schedule is kept so the nightly still covers everything. + # + # It cannot key on the branch, which is what an earlier attempt did. Measured + # on a pull request from a branch whose name contained "fullci", the + # compile-time values are Build.SourceBranch='refs/pull//merge', + # Build.SourceBranchName='merge' and System.PullRequest.SourceBranch='' -- its + # runtime value is the branch, but expansion happens before that is set. + # + # A runtime condition cannot substitute either: + # job conditions are evaluated before matrix variables are in scope, so a + # per-leg condition lets every leg through. strategy: matrix: - macOS 14 RelWithDebInfo: - imageName: 'macOS-14' + clang-22 RelWithDebInfo [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + UBUNTU_CODE_NAME: noble + Compiler: clang + CompilerVer: 22 + BuildType: RelWithDebInfo + + clang-22 Debug [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + UBUNTU_CODE_NAME: noble Compiler: clang + CompilerVer: 22 + BuildType: Debug + + gcc-16 RelWithDebInfo [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + Compiler: gcc + CompilerVer: 16 BuildType: RelWithDebInfo - # skip for a pull request if branch name doesn't contain "fullci" - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + gcc-16 Debug [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + Compiler: gcc + CompilerVer: 16 + BuildType: Debug + + gcc-16 Debug INVERTED [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + Compiler: gcc + CompilerVer: 16 + BuildType: Debug + BUILD_PARAMS_TYPE: inverted + + # Extended coverage below: built only with fullCI, or on the nightly. + + # macOS 14 + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + macOS 14 RelWithDebInfo: + imageName: 'macOS-14' + Compiler: clang + BuildType: RelWithDebInfo + + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: macOS 14 Debug: imageName: 'macOS-14' Compiler: clang @@ -89,14 +155,7 @@ jobs: # clang-16 and newer compilers - clang-22 RelWithDebInfo [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - UBUNTU_CODE_NAME: noble - Compiler: clang - CompilerVer: 22 - BuildType: RelWithDebInfo - - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-22 RelWithDebInfo INVERTED [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -105,23 +164,7 @@ jobs: BuildType: RelWithDebInfo BUILD_PARAMS_TYPE: inverted - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: - clang-22 Debug [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - UBUNTU_CODE_NAME: noble - Compiler: clang - CompilerVer: 22 - BuildType: Debug - - clang-22 Debug INVERTED [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - UBUNTU_CODE_NAME: noble - Compiler: clang - CompilerVer: 22 - BuildType: Debug - BUILD_PARAMS_TYPE: inverted - - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-21 RelWithDebInfo [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -129,7 +172,7 @@ jobs: CompilerVer: 21 BuildType: RelWithDebInfo - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-21 Debug [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -137,7 +180,7 @@ jobs: CompilerVer: 21 BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-20 RelWithDebInfo [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -145,14 +188,15 @@ jobs: CompilerVer: 20 BuildType: RelWithDebInfo - clang-20 Debug [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - UBUNTU_CODE_NAME: noble - Compiler: clang - CompilerVer: 20 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + clang-20 Debug [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + UBUNTU_CODE_NAME: noble + Compiler: clang + CompilerVer: 20 + BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-19 RelWithDebInfo [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -160,7 +204,7 @@ jobs: CompilerVer: 19 BuildType: RelWithDebInfo - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-19 Debug [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -168,7 +212,7 @@ jobs: CompilerVer: 19 BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-18 RelWithDebInfo [Ubuntu 24.04 Noble]: imageName: 'ubuntu-24.04' UBUNTU_CODE_NAME: noble @@ -176,114 +220,109 @@ jobs: CompilerVer: 18 BuildType: RelWithDebInfo - clang-18 Debug [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - UBUNTU_CODE_NAME: noble - Compiler: clang - CompilerVer: 18 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + clang-18 Debug [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + UBUNTU_CODE_NAME: noble + Compiler: clang + CompilerVer: 18 + BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-17 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: clang CompilerVer: 17 BuildType: RelWithDebInfo - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-17 Debug [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: clang CompilerVer: 17 BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: clang-16 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: clang CompilerVer: 16 BuildType: RelWithDebInfo - clang-16 Debug [Ubuntu 22.04 Jammy]: - imageName: 'ubuntu-22.04' - Compiler: clang - CompilerVer: 16 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + clang-16 Debug [Ubuntu 22.04 Jammy]: + imageName: 'ubuntu-22.04' + Compiler: clang + CompilerVer: 16 + BuildType: Debug # gcc-11 and newer compilers - gcc-16 RelWithDebInfo [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - Compiler: gcc - CompilerVer: 16 - BuildType: RelWithDebInfo - - gcc-16 Debug [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - Compiler: gcc - CompilerVer: 16 - BuildType: Debug - - gcc-15 RelWithDebInfo [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - Compiler: gcc - CompilerVer: 15 - BuildType: RelWithDebInfo + # (gcc-16 RelWithDebInfo and gcc-16 Debug are default configs, listed above) + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + gcc-15 RelWithDebInfo [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + Compiler: gcc + CompilerVer: 15 + BuildType: RelWithDebInfo - gcc-15 Debug [Ubuntu 24.04 Noble]: - imageName: 'ubuntu-24.04' - Compiler: gcc - CompilerVer: 15 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + gcc-15 Debug [Ubuntu 24.04 Noble]: + imageName: 'ubuntu-24.04' + Compiler: gcc + CompilerVer: 15 + BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-14 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc CompilerVer: 14 BuildType: RelWithDebInfo - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-14 Debug [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc CompilerVer: 14 BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-13 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc CompilerVer: 13 BuildType: RelWithDebInfo - gcc-13 Debug [Ubuntu 22.04 Jammy]: - imageName: 'ubuntu-22.04' - Compiler: gcc - CompilerVer: 13 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + gcc-13 Debug [Ubuntu 22.04 Jammy]: + imageName: 'ubuntu-22.04' + Compiler: gcc + CompilerVer: 13 + BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-12 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc CompilerVer: 12 BuildType: RelWithDebInfo - gcc-12 Debug [Ubuntu 22.04 Jammy]: - imageName: 'ubuntu-22.04' - Compiler: gcc - CompilerVer: 12 - BuildType: Debug + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: + gcc-12 Debug [Ubuntu 22.04 Jammy]: + imageName: 'ubuntu-22.04' + Compiler: gcc + CompilerVer: 12 + BuildType: Debug - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-11 RelWithDebInfo [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc CompilerVer: 11 BuildType: RelWithDebInfo - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), contains(variables['Build.SourceBranchName'], 'fullci')) }}: + ${{ if or(parameters.fullCI, eq(variables['Build.Reason'], 'Schedule')) }}: gcc-11 Debug [Ubuntu 22.04 Jammy]: imageName: 'ubuntu-22.04' Compiler: gcc diff --git a/include/my_base.h b/include/my_base.h index 66a4b1e556b7..df03e746f7c6 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -105,10 +105,11 @@ enum ha_key_alg { SEs default algorithm for keys in mysql_prepare_create_table(). */ HA_KEY_ALG_SE_SPECIFIC = 0, - HA_KEY_ALG_BTREE = 1, /* B-tree. */ - HA_KEY_ALG_RTREE = 2, /* R-tree, for spatial searches */ - HA_KEY_ALG_HASH = 3, /* HASH keys (HEAP, NDB). */ - HA_KEY_ALG_FULLTEXT = 4 /* FULLTEXT. */ + HA_KEY_ALG_BTREE = 1, /* B-tree. */ + HA_KEY_ALG_RTREE = 2, /* R-tree, for spatial searches */ + HA_KEY_ALG_HASH = 3, /* HASH keys (HEAP, NDB). */ + HA_KEY_ALG_FULLTEXT = 4, /* FULLTEXT. */ + HA_KEY_ALG_VECTOR = 5, /* VECTOR. */ }; /* Storage media types */ @@ -521,11 +522,14 @@ enum ha_base_keytype { #define HA_USES_COMMENT (1 << 12) /** Key was automatically created to support Foreign Key constraint. */ #define HA_GENERATED_KEY (1 << 13) +/** Vector key (Percona). */ +#define HA_VECTOR (1 << 30) /* The combination of the above can be used for key type comparison. */ #define HA_KEYFLAG_MASK \ (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \ - HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY) + HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY | \ + HA_VECTOR) /** Fulltext index uses [pre]parser */ #define HA_USES_PARSER (1 << 14) diff --git a/mysql-test/suite/innodb/include/flush_dirty_pages_and_stop_flushing.inc b/mysql-test/suite/innodb/include/flush_dirty_pages_and_stop_flushing.inc index 5f5cf92506dd..c2157c40830d 100644 --- a/mysql-test/suite/innodb/include/flush_dirty_pages_and_stop_flushing.inc +++ b/mysql-test/suite/innodb/include/flush_dirty_pages_and_stop_flushing.inc @@ -16,5 +16,6 @@ SET GLOBAL innodb_log_checkpoint_now = ON; # Disable page cleaner threads, because we are going to fill # redo log and we want to collect a group of dirty pages for -# which the redo log records protect changes. +# which the redo log records protect changes. This also pauses the +# per-instance LRU manager threads, which reuse this same debug flag. SET GLOBAL innodb_page_cleaner_disabled_debug = ON; \ No newline at end of file diff --git a/mysql-test/suite/innodb/r/innodb_lru_threads_kill_recovery.result b/mysql-test/suite/innodb/r/innodb_lru_threads_kill_recovery.result new file mode 100644 index 000000000000..23a90998a4c3 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb_lru_threads_kill_recovery.result @@ -0,0 +1,23 @@ +# +# PS-11445: with --innodb-lru-threads=ON, the LRU manager threads +# are created before buf_pool_invalidate() runs on every startup, so +# they are alive throughout crash recovery (including the +# buf_pool_invalidate_instance() pause/resume of run_lru). Restart +# after a crash with a non-trivial amount of redo to apply exercises +# that path; a normal restart afterward exercises the page-cleaner +# coordinator waiting for the LRU managers to stop on shutdown. +# +# restart:--innodb-lru-threads=ON --innodb-buffer-pool-size=8M --innodb-buffer-pool-instances=2 --innodb-lru-scan-depth=100 +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(512)) ENGINE=InnoDB; +# Kill and restart:--innodb-lru-threads=ON --innodb-buffer-pool-size=8M --innodb-buffer-pool-instances=2 --innodb-lru-scan-depth=100 +SELECT COUNT(*) FROM t1; +COUNT(*) +20000 +# Clean shutdown must not hang: the page-cleaner coordinator has to +# observe every LRU manager thread stop. +# restart:--innodb-lru-threads=ON --innodb-buffer-pool-size=8M --innodb-buffer-pool-instances=2 --innodb-lru-scan-depth=100 +SELECT COUNT(*) FROM t1; +COUNT(*) +20000 +DROP TABLE t1; +# restart diff --git a/mysql-test/suite/innodb/r/monitor.result b/mysql-test/suite/innodb/r/monitor.result index 79e7114989f6..3650df0ca786 100644 --- a/mysql-test/suite/innodb/r/monitor.result +++ b/mysql-test/suite/innodb/r/monitor.result @@ -107,6 +107,8 @@ buffer_LRU_search_scanned_per_call disabled buffer_LRU_unzip_search_scanned disabled buffer_LRU_unzip_search_num_scan disabled buffer_LRU_unzip_search_scanned_per_call disabled +buffer_LRU_single_page_flush_count disabled +buffer_LRU_flush_await_count disabled buffer_page_read_index_leaf disabled buffer_page_read_index_non_leaf disabled buffer_page_read_index_ibuf_leaf disabled diff --git a/mysql-test/suite/innodb/t/innodb_lru_threads_kill_recovery.test b/mysql-test/suite/innodb/t/innodb_lru_threads_kill_recovery.test new file mode 100644 index 000000000000..b2e0a586f7b0 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb_lru_threads_kill_recovery.test @@ -0,0 +1,48 @@ +--source include/no_valgrind_without_big.inc + +--echo # +--echo # PS-11445: with --innodb-lru-threads=ON, the LRU manager threads +--echo # are created before buf_pool_invalidate() runs on every startup, so +--echo # they are alive throughout crash recovery (including the +--echo # buf_pool_invalidate_instance() pause/resume of run_lru). Restart +--echo # after a crash with a non-trivial amount of redo to apply exercises +--echo # that path; a normal restart afterward exercises the page-cleaner +--echo # coordinator waiting for the LRU managers to stop on shutdown. +--echo # + +--let $restart_parameters=restart:--innodb-lru-threads=ON --innodb-buffer-pool-size=8M --innodb-buffer-pool-instances=2 --innodb-lru-scan-depth=100 +--source include/restart_mysqld.inc + +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(512)) ENGINE=InnoDB; + +--disable_query_log +DELIMITER |; +CREATE PROCEDURE fill_t1() +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < 20000 DO + INSERT INTO t1 VALUES (i, REPEAT('x', 512)); + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| +CALL fill_t1(); +DROP PROCEDURE fill_t1; +--enable_query_log + +--let $wait_counter= 3000 +--source include/kill_and_restart_mysqld.inc + +SELECT COUNT(*) FROM t1; + +--echo # Clean shutdown must not hang: the page-cleaner coordinator has to +--echo # observe every LRU manager thread stop. +--let $restart_parameters=restart:--innodb-lru-threads=ON --innodb-buffer-pool-size=8M --innodb-buffer-pool-instances=2 --innodb-lru-scan-depth=100 +--source include/restart_mysqld.inc + +SELECT COUNT(*) FROM t1; + +DROP TABLE t1; + +--let $restart_parameters= +--source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb_zip/r/lru_mutex_narrow_stress_debug.result b/mysql-test/suite/innodb_zip/r/lru_mutex_narrow_stress_debug.result new file mode 100644 index 000000000000..81f2172016b8 --- /dev/null +++ b/mysql-test/suite/innodb_zip/r/lru_mutex_narrow_stress_debug.result @@ -0,0 +1,16 @@ +SET GLOBAL DEBUG="+d,buf_lru_free_page_delay_zip_reinsert,buf_page_init_for_read_delay_lru_add"; +CREATE TABLE t1 ( +id INT PRIMARY KEY, +c LONGBLOB +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8; +CALL scan_t1(6); +CALL scan_t1(6); +CALL update_t1(12); +SELECT COUNT(*), MIN(LENGTH(c)) > 0 FROM t1; +COUNT(*) MIN(LENGTH(c)) > 0 +2000 1 +SET GLOBAL DEBUG="-d,buf_lru_free_page_delay_zip_reinsert,buf_page_init_for_read_delay_lru_add"; +DROP PROCEDURE populate_t1; +DROP PROCEDURE scan_t1; +DROP PROCEDURE update_t1; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug-master.opt b/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug-master.opt new file mode 100644 index 000000000000..451844199700 --- /dev/null +++ b/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug-master.opt @@ -0,0 +1,3 @@ +--innodb-buffer-pool-size=24M +--innodb-buffer-pool-chunk-size=2M +--innodb-buffer-pool-instances=1 diff --git a/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug.test b/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug.test new file mode 100644 index 000000000000..8e120feb2013 --- /dev/null +++ b/mysql-test/suite/innodb_zip/t/lru_mutex_narrow_stress_debug.test @@ -0,0 +1,109 @@ +# PS-11141: buf_page_init_for_read() inserts into the page hash without +# holding the LRU list mutex. This test stresses the two windows that the +# narrowed latching opens, with debug sync points widening them: +# +# 1) buf_lru_free_page_delay_zip_reinsert widens the window between the +# page hash delete and the re-insert of the compressed-only descriptor +# in the keep-zip path of buf_LRU_free_page(). The hash cell X-latch is +# held across that window, so a concurrent read of the page being +# unzip-evicted must block on the cell latch instead of inserting a +# second descriptor for the same page id (which would fail +# ut_a(!buf_page_hash_get_low(buf_pool, b->id)) at the re-insert). +# +# 2) buf_page_init_for_read_delay_lru_add widens the window in which a +# page is hash-visible but not yet linked into the LRU list. Threads +# finding such a page via the page hash (buf_page_get_zip() -> +# buf_block_try_discard_uncompressed() -> buf_LRU_free_page(), +# buf_buddy_relocate(), buf_page_make_young_if_needed()) must back off +# from it because it is io-fixed for read; buf_page_can_relocate() +# accepts it (io-fixed, not in the LRU list) without asserting. +# +# The workload keeps a compressed table with externally stored BLOBs +# (read through buf_page_get_zip()) larger than the buffer pool, so that +# unzip-LRU eviction, keep-zip frees, physical re-reads and buddy +# alloc/free churn all run concurrently. + +--source include/have_debug.inc +--source include/have_innodb_16k.inc +--source include/big_test.inc + +SET GLOBAL DEBUG="+d,buf_lru_free_page_delay_zip_reinsert,buf_page_init_for_read_delay_lru_add"; + +CREATE TABLE t1 ( + id INT PRIMARY KEY, + c LONGBLOB +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8; + +--disable_query_log +DELIMITER |; +CREATE PROCEDURE populate_t1() +BEGIN + DECLARE i INT DEFAULT 1; + WHILE i <= 2000 DO + INSERT INTO t1 VALUES (i, REPEAT(CONCAT('row', i, '-'), 2048)); + IF i % 200 = 0 THEN + COMMIT; + END IF; + SET i = i + 1; + END WHILE; + COMMIT; +END| +CREATE PROCEDURE scan_t1(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 1; + DECLARE dummy BIGINT; + WHILE i <= iterations DO + SELECT SUM(LENGTH(c)) INTO dummy FROM t1; + SET i = i + 1; + END WHILE; +END| +CREATE PROCEDURE update_t1(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 1; + WHILE i <= iterations DO + UPDATE t1 SET c = REPEAT(CONCAT('mod', i, '-'), 2048) + WHERE id % 500 = i % 500; + COMMIT; + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| +SET autocommit = 0; +CALL populate_t1(); +SET autocommit = 1; +--enable_query_log + +# Two connections scanning the BLOB column (physical re-reads + +# buf_page_get_zip() -> buf_block_try_discard_uncompressed()), one +# connection updating (creates dirty compressed pages, so the keep-zip +# eviction path sees both ZIP_PAGE and ZIP_DIRTY re-inserts). +--connect(con_scan1, localhost, root,,) +--send CALL scan_t1(6) + +--connect(con_scan2, localhost, root,,) +--send CALL scan_t1(6) + +--connect(con_upd, localhost, root,,) +--send CALL update_t1(12) + +--connection con_scan1 +--reap +--disconnect con_scan1 + +--connection con_scan2 +--reap +--disconnect con_scan2 + +--connection con_upd +--reap +--disconnect con_upd + +--connection default +SELECT COUNT(*), MIN(LENGTH(c)) > 0 FROM t1; + +SET GLOBAL DEBUG="-d,buf_lru_free_page_delay_zip_reinsert,buf_page_init_for_read_delay_lru_add"; + +DROP PROCEDURE populate_t1; +DROP PROCEDURE scan_t1; +DROP PROCEDURE update_t1; +DROP TABLE t1; diff --git a/mysql-test/suite/perfschema/r/dml_setup_threads.result b/mysql-test/suite/perfschema/r/dml_setup_threads.result index 4348f7406098..517723b6dca5 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_threads.result +++ b/mysql-test/suite/perfschema/r/dml_setup_threads.result @@ -1,8 +1,9 @@ select * from performance_schema.setup_threads; select * from performance_schema.setup_threads -order by name limit 13; +order by name limit 14; NAME ENABLED HISTORY PROPERTIES VOLATILITY DOCUMENTATION thread/innodb/buf_dump_thread YES YES singleton 0 NULL +thread/innodb/buf_lru_manager_thread YES YES 0 NULL thread/innodb/buf_pool_create_thread YES YES singleton 0 NULL thread/innodb/buf_resize_thread YES YES singleton 0 NULL thread/innodb/bulk_alloc_thread YES YES singleton 0 NULL diff --git a/mysql-test/suite/perfschema/t/dml_setup_threads.test b/mysql-test/suite/perfschema/t/dml_setup_threads.test index 2d3685a1a661..fd7ace8e1db8 100644 --- a/mysql-test/suite/perfschema/t/dml_setup_threads.test +++ b/mysql-test/suite/perfschema/t/dml_setup_threads.test @@ -12,7 +12,7 @@ select * from performance_schema.setup_threads; --enable_result_log select * from performance_schema.setup_threads - order by name limit 13; + order by name limit 14; --disable_result_log select * from performance_schema.setup_threads diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_populate_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_populate_basic.result index 35dc99586024..e24616fd701f 100644 --- a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_populate_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_populate_basic.result @@ -1,28 +1,28 @@ SET @start_global_value = @@global.innodb_buffer_pool_populate; SELECT @start_global_value; @start_global_value -0 +1 Valid values are 'ON' and 'OFF' select @@global.innodb_buffer_pool_populate in (0, 1); @@global.innodb_buffer_pool_populate in (0, 1) 1 select @@global.innodb_buffer_pool_populate; @@global.innodb_buffer_pool_populate -0 +1 select @@session.innodb_buffer_pool_populate; ERROR HY000: Variable 'innodb_buffer_pool_populate' is a GLOBAL variable show global variables like 'innodb_buffer_pool_populate'; Variable_name Value -innodb_buffer_pool_populate OFF +innodb_buffer_pool_populate ON show session variables like 'innodb_buffer_pool_populate'; Variable_name Value -innodb_buffer_pool_populate OFF +innodb_buffer_pool_populate ON select * from performance_schema.global_variables where variable_name='innodb_buffer_pool_populate'; VARIABLE_NAME VARIABLE_VALUE -innodb_buffer_pool_populate OFF +innodb_buffer_pool_populate ON select * from performance_schema.session_variables where variable_name='innodb_buffer_pool_populate'; VARIABLE_NAME VARIABLE_VALUE -innodb_buffer_pool_populate OFF +innodb_buffer_pool_populate ON set global innodb_buffer_pool_populate='ON'; select @@global.innodb_buffer_pool_populate; @@global.innodb_buffer_pool_populate @@ -59,4 +59,4 @@ ERROR 42000: Variable 'innodb_buffer_pool_populate' can't be set to the value of SET @@global.innodb_buffer_pool_populate = @start_global_value; SELECT @@global.innodb_buffer_pool_populate; @@global.innodb_buffer_pool_populate -0 +1 diff --git a/mysql-test/suite/sys_vars/r/innodb_lru_threads_basic.result b/mysql-test/suite/sys_vars/r/innodb_lru_threads_basic.result new file mode 100644 index 000000000000..f2a17f69b789 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_lru_threads_basic.result @@ -0,0 +1,27 @@ +# +# Basic test for innodb_lru_threads (read-only) +# +SELECT @@GLOBAL.innodb_lru_threads; +@@GLOBAL.innodb_lru_threads +0 +SET GLOBAL innodb_lru_threads = OFF; +ERROR HY000: Variable 'innodb_lru_threads' is a read only variable +SET GLOBAL innodb_lru_threads = ON; +ERROR HY000: Variable 'innodb_lru_threads' is a read only variable +SELECT @@GLOBAL.innodb_lru_threads; +@@GLOBAL.innodb_lru_threads +0 +# +# Startup with innodb_lru_threads=ON +# +# restart:--innodb-lru-threads=ON +SELECT @@GLOBAL.innodb_lru_threads; +@@GLOBAL.innodb_lru_threads +1 +# +# Startup with innodb_lru_threads=OFF +# +# restart +SELECT @@GLOBAL.innodb_lru_threads; +@@GLOBAL.innodb_lru_threads +0 diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result index adbb506eb3a8..77b303242dfa 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result @@ -107,6 +107,8 @@ buffer_LRU_search_scanned_per_call disabled buffer_LRU_unzip_search_scanned disabled buffer_LRU_unzip_search_num_scan disabled buffer_LRU_unzip_search_scanned_per_call disabled +buffer_LRU_single_page_flush_count disabled +buffer_LRU_flush_await_count disabled buffer_page_read_index_leaf disabled buffer_page_read_index_non_leaf disabled buffer_page_read_index_ibuf_leaf disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result index adbb506eb3a8..77b303242dfa 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result @@ -107,6 +107,8 @@ buffer_LRU_search_scanned_per_call disabled buffer_LRU_unzip_search_scanned disabled buffer_LRU_unzip_search_num_scan disabled buffer_LRU_unzip_search_scanned_per_call disabled +buffer_LRU_single_page_flush_count disabled +buffer_LRU_flush_await_count disabled buffer_page_read_index_leaf disabled buffer_page_read_index_non_leaf disabled buffer_page_read_index_ibuf_leaf disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result index adbb506eb3a8..77b303242dfa 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result @@ -107,6 +107,8 @@ buffer_LRU_search_scanned_per_call disabled buffer_LRU_unzip_search_scanned disabled buffer_LRU_unzip_search_num_scan disabled buffer_LRU_unzip_search_scanned_per_call disabled +buffer_LRU_single_page_flush_count disabled +buffer_LRU_flush_await_count disabled buffer_page_read_index_leaf disabled buffer_page_read_index_non_leaf disabled buffer_page_read_index_ibuf_leaf disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result index 88eeaaf46dca..eda84d8e40a1 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result @@ -107,6 +107,8 @@ buffer_LRU_search_scanned_per_call disabled buffer_LRU_unzip_search_scanned disabled buffer_LRU_unzip_search_num_scan disabled buffer_LRU_unzip_search_scanned_per_call disabled +buffer_LRU_single_page_flush_count disabled +buffer_LRU_flush_await_count disabled buffer_page_read_index_leaf disabled buffer_page_read_index_non_leaf disabled buffer_page_read_index_ibuf_leaf disabled diff --git a/mysql-test/suite/sys_vars/t/innodb_lru_threads_basic.test b/mysql-test/suite/sys_vars/t/innodb_lru_threads_basic.test new file mode 100644 index 000000000000..c2de5a7f5c48 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_lru_threads_basic.test @@ -0,0 +1,29 @@ +--echo # +--echo # Basic test for innodb_lru_threads (read-only) +--echo # + +SELECT @@GLOBAL.innodb_lru_threads; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET GLOBAL innodb_lru_threads = OFF; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET GLOBAL innodb_lru_threads = ON; + +SELECT @@GLOBAL.innodb_lru_threads; + +--echo # +--echo # Startup with innodb_lru_threads=ON +--echo # + +--let $restart_parameters=restart:--innodb-lru-threads=ON +--source include/restart_mysqld.inc +SELECT @@GLOBAL.innodb_lru_threads; + +--echo # +--echo # Startup with innodb_lru_threads=OFF +--echo # + +--let $restart_parameters=restart +--source include/restart_mysqld.inc +SELECT @@GLOBAL.innodb_lru_threads; diff --git a/plugin/group_replication/libmysqlgcs/src/bindings/xcom/gcs_xcom_control_interface.cc b/plugin/group_replication/libmysqlgcs/src/bindings/xcom/gcs_xcom_control_interface.cc index 1149c61d0c33..ad709b694e72 100644 --- a/plugin/group_replication/libmysqlgcs/src/bindings/xcom/gcs_xcom_control_interface.cc +++ b/plugin/group_replication/libmysqlgcs/src/bindings/xcom/gcs_xcom_control_interface.cc @@ -593,6 +593,10 @@ bool Gcs_xcom_control::try_send_add_node_request_to_seeds( In this case, we continue the loop and try again using the next peer. */ if (xcom_will_process) add_node_accepted = true; + } else if (connected) { + /* GCS was finalized while we were connecting; close the connection to + free the SSL object and socket so they are not leaked. */ + m_xcom_proxy->xcom_client_close_connection(con); } free_connection(con); diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index e8a5df1a936d..4f759bf41239 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -11112,6 +11112,15 @@ ER_LOG_NAME_NOT_MATCHING_SEC_LOG_PATH_CLIENT # Start of Percona Server 8.4/9.7 error messages to be sent to client # +ER_VECTOR_INDEX_NEEDS_PK + eng "Vector index can only be created in tables with a BIGINT UNSIGNED primary key." + +ER_ONLY_SINGLE_VECTOR_INDEX_ALLOWED + eng "A table can have at most one vector index." + +ER_TABLE_CANT_HANDLE_INDEX + eng "The used table type doesn't support %.20s indexes" + start-error-number 7100 # diff --git a/sql-common/client.cc b/sql-common/client.cc index 29af4ffc11fa..86c521aa1c5c 100644 --- a/sql-common/client.cc +++ b/sql-common/client.cc @@ -7573,6 +7573,11 @@ bool mysql_reconnect(MYSQL *mysql) { } mysql_init(&tmp_mysql); mysql_close_free_options(&tmp_mysql); + /* Guarantee the extension struct exists before the shallow copy so that + mysql_real_connect's ENSURE_EXTENSIONS_PRESENT is a no-op and does not + allocate a new extension that would be leaked if mysql_set_character_set + fails (the error path does memset before mysql_close). */ + ENSURE_EXTENSIONS_PRESENT(&mysql->options); tmp_mysql.options = mysql->options; tmp_mysql.options.my_cnf_file = tmp_mysql.options.my_cnf_group = nullptr; #ifdef MYSQL_SERVER diff --git a/sql/auth/acl_table_user.cc b/sql/auth/acl_table_user.cc index 3fee641916c8..325285af0b3f 100644 --- a/sql/auth/acl_table_user.cc +++ b/sql/auth/acl_table_user.cc @@ -1629,56 +1629,6 @@ bool Acl_table_user_reader::read_plugin_info( user.plugin.str = tmpstr ? tmpstr : ""; user.plugin.length = strlen(user.plugin.str); - /* - In case we are working with 5.6 db layout we need to make server - aware of Password field and that the plugin column can be null. - In case when plugin column is null we use native password plugin - if we can. - */ - if (is_old_db_layout && (user.plugin.length == 0 || - Cached_authentication_plugins::compare_plugin( - PLUGIN_MYSQL_NATIVE_PASSWORD, user.plugin))) { - char *password = get_field( - &m_mem_root, m_table->field[m_table_schema->password_idx()]); - - // We do not support pre 4.1 hashes - plugin_ref native_plugin = - g_cached_authentication_plugins->get_cached_plugin_ref( - PLUGIN_MYSQL_NATIVE_PASSWORD); - if (native_plugin) { - const uint password_len = password ? strlen(password) : 0; - st_mysql_auth *auth = (st_mysql_auth *)plugin_decl(native_plugin)->info; - Auth_plugin_operation_guard op_guard; - if (!op_guard) { - LogErr(WARNING_LEVEL, ER_AUTHCACHE_USER_IGNORED_INVALID_PASSWORD, - user.user ? user.user : "", - user.host.get_host() ? user.host.get_host() : ""); - return true; - } - if (auth->validate_authentication_string(password, password_len) == 0) { - // auth_string takes precedence over password - if (user.credentials[PRIMARY_CRED].m_auth_string.length == 0) { - user.credentials[PRIMARY_CRED].m_auth_string.str = password; - user.credentials[PRIMARY_CRED].m_auth_string.length = password_len; - } - if (user.plugin.length == 0) { - user.plugin.str = Cached_authentication_plugins::get_plugin_name( - PLUGIN_MYSQL_NATIVE_PASSWORD); - user.plugin.length = strlen(user.plugin.str); - } - } else { - if ((user.access & SUPER_ACL) && !super_users_with_empty_plugin && - (user.plugin.length == 0)) - super_users_with_empty_plugin = true; - - LogErr(WARNING_LEVEL, ER_AUTHCACHE_USER_IGNORED_DEPRECATED_PASSWORD, - user.user ? user.user : "", - user.host.get_host() ? user.host.get_host() : ""); - return true; - } - } - } - /* Check if the plugin string is blank or null. If it is, the user will be skipped. diff --git a/sql/create_field.cc b/sql/create_field.cc index 168d5293cd94..ac61abbaeeca 100644 --- a/sql/create_field.cc +++ b/sql/create_field.cc @@ -23,6 +23,7 @@ #include "sql/create_field.h" +#include "field_types.h" #include "m_string.h" #include "mysql/strings/dtoa.h" #include "sql-common/my_decimal.h" @@ -200,9 +201,10 @@ bool Create_field::init( const LEX_CSTRING *fld_comment, const char *fld_change, List *fld_interval_list, const CHARSET_INFO *fld_charset, bool has_explicit_collation, uint fld_geom_type, - const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info, Value_generator *fld_default_val_expr, - LEX_CSTRING fld_masking_policy, std::optional srid, - dd::Column::enum_hidden_type hidden, bool is_array_arg) { + const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info, + Value_generator *fld_default_val_expr, LEX_CSTRING fld_masking_policy, + std::optional srid, dd::Column::enum_hidden_type hidden, + bool is_array_arg) { uint sign_len, allowed_type_modifier = 0; ulong max_field_charlength = MAX_FIELD_CHARLENGTH; @@ -780,7 +782,8 @@ size_t Create_field::key_length() const { case MYSQL_TYPE_JSON: case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_STRING: - case MYSQL_TYPE_VARCHAR: { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VECTOR: { return std::min(max_display_width_in_bytes(), static_cast(MAX_FIELD_BLOBLENGTH)); } @@ -794,10 +797,6 @@ size_t Create_field::key_length() const { } return pack_length() + (max_display_width_in_bytes() & 7 ? 1 : 0); } - /* LCOV_EXCL_START */ - case MYSQL_TYPE_VECTOR: - assert(false); // Key on VECTOR type column is not supported. - /* LCOV_EXCL_STOP */ default: { return pack_length(is_array); } diff --git a/sql/dd/dd_table.cc b/sql/dd/dd_table.cc index 8a14c56e8599..e3fc670c6512 100644 --- a/sql/dd/dd_table.cc +++ b/sql/dd/dd_table.cc @@ -735,6 +735,10 @@ bool fill_dd_columns_from_create_fields(THD *thd, dd::Abstract_table *tab_obj, col_options->set("is_array", true); } + if (field.sql_type == MYSQL_TYPE_VECTOR) { + col_options->set("vector_index", true); + } + // // Write intervals // @@ -825,6 +829,9 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type( case HA_KEY_ALG_FULLTEXT: return dd::Index::IA_FULLTEXT; + + case HA_KEY_ALG_VECTOR: + return dd::Index::IA_SE_SPECIFIC; } /* purecov: begin deadcode */ @@ -836,6 +843,8 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type( } static dd::Index::enum_index_type dd_get_new_index_type(const KEY *key) { + if (key->flags & HA_VECTOR) return dd::Index::IT_MULTIPLE; + if (key->flags & HA_FULLTEXT) return dd::Index::IT_FULLTEXT; if (key->flags & HA_SPATIAL) return dd::Index::IT_SPATIAL; diff --git a/sql/dd/impl/types/column_impl.cc b/sql/dd/impl/types/column_impl.cc index 34cb323def7b..1b6669ed6bc4 100644 --- a/sql/dd/impl/types/column_impl.cc +++ b/sql/dd/impl/types/column_impl.cc @@ -64,11 +64,17 @@ class Sdi_rcontext; class Sdi_wcontext; static const std::set default_valid_option_keys = { - "column_format", "geom_type", - "interval_count", "not_secondary", - "storage", "treat_bit_as_char", "zip_dict_id", - "is_array", "gipk" /* generated implicit primary key column */, - "masking_policy"}; + "column_format", + "geom_type", + "interval_count", + "not_secondary", + "storage", + "treat_bit_as_char", + "zip_dict_id", + "is_array", + "gipk" /* generated implicit primary key column */, + "masking_policy", + "vector_index"}; /////////////////////////////////////////////////////////////////////////// // Column_impl implementation. diff --git a/sql/dd/info_schema/show.cc b/sql/dd/info_schema/show.cc index f37078e4fce8..94cc320d3552 100644 --- a/sql/dd/info_schema/show.cc +++ b/sql/dd/info_schema/show.cc @@ -868,6 +868,50 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd, Select_lex_builder top_query(&pos, thd); + Item *index_type_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_item == nullptr) return nullptr; + + Item *se_specific_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("SE_SPECIFIC"), system_charset_info); + if (se_specific_item == nullptr) return nullptr; + + Item *is_se_specific = + new (thd->mem_root) Item_func_eq(pos, index_type_item, se_specific_item); + if (is_se_specific == nullptr) return nullptr; + + Item *sub_part_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_sub_part.str); + if (sub_part_item == nullptr) return nullptr; + + Item *one_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("1"), system_charset_info); + if (one_item == nullptr) return nullptr; + + Item *is_single_sub_part = + new (thd->mem_root) Item_func_eq(pos, sub_part_item, one_item); + if (is_single_sub_part == nullptr) return nullptr; + + Item *vector_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("VECTOR"), system_charset_info); + if (vector_item == nullptr) return nullptr; + + Item *index_type_else_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_else_item == nullptr) return nullptr; + + Item *index_type_if = new (thd->mem_root) + Item_func_if(pos, is_single_sub_part, vector_item, index_type_else_item); + if (index_type_if == nullptr) return nullptr; + + Item *index_type_default_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_default_item == nullptr) return nullptr; + + Item *index_type_expr = new (thd->mem_root) + Item_func_if(pos, is_se_specific, index_type_if, index_type_default_item); + if (index_type_expr == nullptr) return nullptr; + // SELECT * FROM ... if (top_query.add_select_item(alias_table, alias_table) || top_query.add_select_item(alias_non_unique, alias_non_unique) || @@ -879,7 +923,7 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd, top_query.add_select_item(alias_sub_part, alias_sub_part) || top_query.add_select_item(alias_packed, alias_packed) || top_query.add_select_item(alias_null, alias_null) || - top_query.add_select_item(alias_type, alias_type) || + top_query.add_select_expr(index_type_expr, alias_type) || top_query.add_select_item(alias_comment, alias_comment) || top_query.add_select_item(alias_index_comment, alias_index_comment) || top_query.add_select_item(alias_visible, alias_visible) || diff --git a/sql/dd_table_share.cc b/sql/dd_table_share.cc index c5504726c0a9..56bd874e29f6 100644 --- a/sql/dd_table_share.cc +++ b/sql/dd_table_share.cc @@ -235,6 +235,29 @@ static enum ha_key_alg dd_get_old_index_algorithm_type( return HA_KEY_ALG_SE_SPECIFIC; } +/** + Check whether any visible index element is marked as vector. + + @param[in] idx_obj Index metadata object. + + @return Whether any visible element belongs to a vector column. +*/ +static bool dd_index_has_vector_column(const dd::Index &idx_obj) { + for (const dd::Index_element *idx_elem : idx_obj.elements()) { + if (idx_elem->is_hidden()) continue; + + const dd::Properties &col_options = idx_elem->column().options(); + bool is_vector_column = false; + if (col_options.exists("vector_index") && + !col_options.get("vector_index", &is_vector_column) && + is_vector_column) { + return true; + } + } + + return false; +} + /* Check if the given key_part is suitable to be promoted as part of primary key. @@ -347,6 +370,8 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, share->key_info[key].algorithm == HA_KEY_ALG_FULLTEXT); assert(!(share->key_info[key].flags & HA_SPATIAL) || share->key_info[key].algorithm == HA_KEY_ALG_RTREE); + assert(!(share->key_info[key].flags & HA_VECTOR) || + share->key_info[key].algorithm == HA_KEY_ALG_VECTOR); if (primary_key >= MAX_KEY && (keyinfo->flags & HA_NOSAME)) { /* @@ -1385,6 +1410,8 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, keyinfo->algorithm = dd_get_old_index_algorithm_type(idx_obj->algorithm()); keyinfo->is_algorithm_explicit = idx_obj->is_algorithm_explicit(); + const bool has_vector_column = dd_index_has_vector_column(*idx_obj); + // Visibility keyinfo->is_visible = idx_obj->is_visible(); @@ -1395,6 +1422,11 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, if (!idx_ele->is_hidden()) keyinfo->user_defined_key_parts++; } + if (has_vector_column && keyinfo->user_defined_key_parts == 1) { + keyinfo->algorithm = HA_KEY_ALG_VECTOR; + keyinfo->is_algorithm_explicit = false; + } + // flags switch (idx_obj->type()) { case dd::Index::IT_MULTIPLE: @@ -1416,6 +1448,10 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, break; } + if (has_vector_column && keyinfo->user_defined_key_parts == 1) { + keyinfo->flags |= HA_VECTOR; + } + if (idx_obj->is_generated()) keyinfo->flags |= HA_GENERATED_KEY; /* diff --git a/sql/field.cc b/sql/field.cc index abc98b6f449c..6f969c1c5ddf 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -34,6 +34,7 @@ #include #include "decimal.h" +#include "field_types.h" #include "my_alloc.h" #include "my_byteorder.h" #include "my_compare.h" @@ -75,7 +76,7 @@ #include "sql/mysqld_cs.h" #include "sql/protocol.h" #include "sql/psi_memory_key.h" -#include "sql/spatial.h" // Geometry +#include "sql/spatial.h" // Geometry #include "sql/sql_base.h" #include "sql/sql_class.h" // THD #include "sql/sql_exception_handler.h" // handle_std_exception @@ -1656,6 +1657,7 @@ bool Field::type_can_have_key_part(enum enum_field_types type) { case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_STRING: case MYSQL_TYPE_GEOMETRY: + case MYSQL_TYPE_VECTOR: return true; default: return false; diff --git a/sql/handler.h b/sql/handler.h index cdb1ac488eb4..9a7028e1e1db 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -535,6 +535,7 @@ enum class SelectExecutedIn : bool { kPrimaryEngine, kSecondaryEngine }; ANALYZE TABLE on it */ #define HA_ONLINE_ANALYZE (1LL << 56) +#define HA_CAN_VECTOR (1LL << 57) /* Bits in index_flags(index_number) for what you can do with index. @@ -7055,16 +7056,16 @@ class handler { for details. */ [[nodiscard]] int ha_fast_update(THD *thd, - mem_root_deque &update_fields, - mem_root_deque &update_values, - Item *conds); + mem_root_deque &update_fields, + mem_root_deque &update_values, + Item *conds); /** @brief Offload an upsert to the storage engine. See handler::upsert() for details. */ [[nodiscard]] int ha_upsert(THD *thd, mem_root_deque &update_fields, - mem_root_deque &update_values); + mem_root_deque &update_values); private: /** @@ -7087,11 +7088,11 @@ class handler { handler::ha_update_row(...) does not accept conditions. */ [[nodiscard]] virtual int fast_update(THD *thd [[maybe_unused]], - mem_root_deque &update_fields - [[maybe_unused]], - mem_root_deque &update_values - [[maybe_unused]], - Item *conds [[maybe_unused]]) { + mem_root_deque &update_fields + [[maybe_unused]], + mem_root_deque &update_values + [[maybe_unused]], + Item *conds [[maybe_unused]]) { return ENOTSUP; } @@ -7112,10 +7113,10 @@ class handler { @return an error if the insert should be terminated. */ [[nodiscard]] virtual int upsert(THD *thd [[maybe_unused]], - mem_root_deque &update_fields - [[maybe_unused]], - mem_root_deque &update_values - [[maybe_unused]]) { + mem_root_deque &update_fields + [[maybe_unused]], + mem_root_deque &update_values + [[maybe_unused]]) { return ENOTSUP; } diff --git a/sql/key_spec.h b/sql/key_spec.h index b58e0651b759..47abcff67da2 100644 --- a/sql/key_spec.h +++ b/sql/key_spec.h @@ -43,7 +43,8 @@ enum keytype { KEYTYPE_MULTIPLE = 2, KEYTYPE_FULLTEXT = 4, KEYTYPE_SPATIAL = 8, - KEYTYPE_FOREIGN = 16 + KEYTYPE_FOREIGN = 16, + KEYTYPE_VECTOR = 32, }; enum fk_option { diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 52b03e9f785e..8f699cd6ca77 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2791,6 +2791,8 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, packet->append(STRING_WITH_LEN("FULLTEXT KEY ")); else if (key_info->flags & HA_SPATIAL) packet->append(STRING_WITH_LEN("SPATIAL KEY ")); + else if (key_info->flags & HA_VECTOR) + packet->append(STRING_WITH_LEN("VECTOR KEY ")); else packet->append(STRING_WITH_LEN("KEY ")); @@ -2823,7 +2825,7 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, if (key_part->field && (key_part->length != table->field[key_part->fieldnr - 1]->key_length() && - !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL)))) { + !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)))) { packet->append_parenthesized((long)key_part->length / key_part->field->charset()->mbmaxlen); } @@ -5482,6 +5484,23 @@ static int fill_schema_engines(THD *thd, Table_ref *tables, Item *) { #define TMP_TABLE_KEYS_IS_VISIBLE 14 #define TMP_TABLE_KEYS_EXPRESSION 15 +/** + Detect vector indexes for SHOW INDEX output. + + @param[in] key_info index metadata from TABLE_SHARE + + @return Whether this is a vector index. +*/ +static bool is_show_index_vector_type(const KEY *key_info) { + if (key_info->flags & HA_VECTOR) return true; + + if (key_info->user_defined_key_parts != 1) return false; + + const KEY_PART_INFO *key_part = key_info->key_part; + return key_part != nullptr && key_part->field != nullptr && + key_part->field->real_type() == MYSQL_TYPE_VECTOR; +} + static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, TABLE *table, bool res, LEX_CSTRING, LEX_CSTRING table_name) { @@ -5570,6 +5589,8 @@ static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, // INDEX_TYPE if (key_info->flags & HA_SPATIAL) str = "SPATIAL"; + else if (is_show_index_vector_type(key_info)) + str = "VECTOR"; else { const ha_key_alg key_alg = key_info->algorithm; /* If index algorithm is implicit get SE default. */ diff --git a/sql/sql_table.cc b/sql/sql_table.cc index faaf57304faf..766e11c707d9 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5175,8 +5175,8 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, return true; } - // VECTOR columns cannot be used as keys - if (sql_field->sql_type == MYSQL_TYPE_VECTOR) { + if (sql_field->sql_type == MYSQL_TYPE_VECTOR && + ((key_info->flags & HA_VECTOR) == 0)) { my_error(ER_NON_SCALAR_USED_AS_KEY, MYF(0), column->get_field_name()); return true; } @@ -5227,6 +5227,13 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, data prefix, ignoring column->length). */ column_length = is_blob(sql_field->sql_type); + } else if (key->type == KEYTYPE_VECTOR) { + // VECTOR indexes are only allowed on VECTOR columns. + if (sql_field->sql_type != MYSQL_TYPE_VECTOR) { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + column_length = 1; // Dummy value. } else { switch (sql_field->sql_type) { case MYSQL_TYPE_GEOMETRY: @@ -5833,7 +5840,7 @@ static bool prepare_self_ref_fk_parent_key( for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (hton->foreign_keys_flags & HTON_FKS_NEED_DIFFERENT_PARENT_AND_SUPPORTING_KEYS) { @@ -6036,7 +6043,7 @@ static const KEY *find_fk_supporting_key(handlerton *hton, for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (key->algorithm == HA_KEY_ALG_HASH) { if (hton->foreign_keys_flags & HTON_FKS_WITH_SUPPORTING_HASH_KEYS) { @@ -7686,6 +7693,17 @@ static bool prepare_key( switch (static_cast(key->type)) { case KEYTYPE_MULTIPLE: break; + case KEYTYPE_VECTOR: + if (!(file->ha_table_flags() & HA_CAN_VECTOR)) { + my_error(ER_TABLE_CANT_HANDLE_INDEX, MYF(0), "vector"); + return true; + } + if (key->columns.size() != 1) { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + return true; + } + key_info->flags |= HA_VECTOR; + break; case KEYTYPE_FULLTEXT: if (!(file->ha_table_flags() & HA_CAN_FULLTEXT)) { my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0)); @@ -7743,6 +7761,9 @@ static bool prepare_key( } else if (key_info->flags & HA_FULLTEXT) { assert(!key->key_create_info.is_algorithm_explicit); key_info->algorithm = HA_KEY_ALG_FULLTEXT; + } else if (key_info->flags & HA_VECTOR) { + assert(!key->key_create_info.is_algorithm_explicit); + key_info->algorithm = HA_KEY_ALG_VECTOR; } else { if (key->key_create_info.is_algorithm_explicit) { if (key->key_create_info.algorithm != HA_KEY_ALG_RTREE) { @@ -8644,6 +8665,7 @@ bool mysql_prepare_create_table( uint key_number = 0; bool primary_key = false; + uint vector_key_number = 0; // First prepare non-foreign keys so that they are ready when // we prepare foreign keys. @@ -8660,6 +8682,14 @@ bool mysql_prepare_create_table( primary_key = true; } + if (key->type == KEYTYPE_VECTOR) { + if (vector_key_number) { + my_error(ER_ONLY_SINGLE_VECTOR_INDEX_ALLOWED, MYF(0)); + return true; + } + ++vector_key_number; + } + if (key->type != KEYTYPE_FOREIGN) { if (prepare_key(thd, error_schema_name, error_table_name, create_info, &alter_info->create_list, key, key_info_buffer, key_info, @@ -8678,6 +8708,12 @@ bool mysql_prepare_create_table( } } + // We allow VECTOR keys only with tables with PK + if (!primary_key && vector_key_number) { + my_error(ER_VECTOR_INDEX_NEEDS_PK, MYF(0)); + return true; + } + /* At this point all KEY objects are for indexes are fully constructed. So we can check for duplicate indexes for keys for which it was requested. @@ -8724,6 +8760,26 @@ bool mysql_prepare_create_table( /* Sort keys in optimized order */ std::sort(*key_info_buffer, *key_info_buffer + *key_count, sort_keys()); + // We allow VECTOR indexes only on tables with BIGINT UNSIGNED PKs. + if (vector_key_number) { + assert(primary_key); + const KEY &primary_info = *key_info_buffer[0]; + + if (primary_info.actual_key_parts > 1) { + my_error(ER_VECTOR_INDEX_NEEDS_PK, MYF(0)); + return true; + } + + for (it.rewind(), field_no = 0; (sql_field = it++); field_no++) { + if (field_no >= primary_info.key_part[0].fieldnr) break; + } + assert(sql_field); + if (sql_field->sql_type != MYSQL_TYPE_LONGLONG || !sql_field->is_unsigned) { + my_error(ER_VECTOR_INDEX_NEEDS_PK, MYF(0)); + return true; + } + } + /* Normal keys are done, now prepare foreign keys. @@ -16319,6 +16375,8 @@ bool prepare_fields_and_keys(THD *thd, const dd::Table *src_table, TABLE *table, key_type = KEYTYPE_UNIQUE; } else if (key_info->flags & HA_FULLTEXT) key_type = KEYTYPE_FULLTEXT; + else if (key_info->flags & HA_VECTOR) + key_type = KEYTYPE_VECTOR; else key_type = KEYTYPE_MULTIPLE; diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 3c06ee5adfb1..7a9e0fdf1bd2 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -4655,7 +4655,8 @@ bool btr_validate_index( /* Full Text index are implemented by auxiliary tables, not the B-tree */ - if (dict_index_is_online_ddl(index) || (index->type & DICT_FTS)) { + if (dict_index_is_online_ddl(index) || + ((index->type & DICT_FTS) || dict_index_is_vector(index))) { return (true); } diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index e9ea3deac4b9..6c9422100bda 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1961,12 +1961,13 @@ static bool btr_search_hash_table_validate(ulint part_id) { /* When a block is being freed, buf_LRU_free_page() first removes the block from - buf_pool->page_hash by calling - buf_LRU_block_remove_hashed_page(). - After that, it invokes - buf_LRU_block_remove_hashed() to - remove the block from - btr_search_sys->hash_tables[i]. */ + buf_pool->page_hash and sets it to + BUF_BLOCK_REMOVE_HASH by calling + buf_LRU_block_remove_hashed(). + After that, it removes the block + from btr_search_sys->parts[i] (the + AHI) by calling + btr_search_drop_page_hash_index(). */ ut_a(buf_block_get_state(block) == BUF_BLOCK_REMOVE_HASH); } diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc index 5d52916f1800..70da21ecfa1e 100644 --- a/storage/innobase/buf/buf0buddy.cc +++ b/storage/innobase/buf/buf0buddy.cc @@ -40,6 +40,9 @@ this program; if not, write to the Free Software Foundation, Inc., #include "page0zip.h" +#include "sync0debug.h" +#include "sync0types.h" + /** When freeing a buf we attempt to coalesce by looking at its buddy and deciding whether it is free or not. To ascertain if the buddy is free we look for BUF_BUDDY_STAMP_FREE at BUF_BUDDY_STAMP_OFFSET @@ -416,6 +419,25 @@ static void *buf_buddy_alloc_from(buf_pool_t *buf_pool, void *buf, ulint i, return (buf); } +#ifdef UNIV_DEBUG +/** Asserts that the calling thread holds no buffer pool page hash cell +latch (S or X). The buddy allocator must never be entered while one is +held: buf_buddy_free() may recombine and call buf_buddy_relocate(), +which acquires the cell X-latch of whatever page id is stamped in the +buddy frame - possibly the very cell the caller holds, and in any case +an unordered same-level acquisition (page-hash cell -> zip_free_mutex -> +page-hash cell) which can form a deadlock cycle with other threads. +This rule is what makes it safe for the keep-zip path of +buf_LRU_free_page() to keep a cell X-latch across +buf_LRU_block_remove_hashed() (keep_hash_lock): that path never reaches +the buddy allocator. */ +namespace { +void buf_buddy_no_page_hash_latch_validate() { + ut_ad(sync_check_find(SYNC_BUF_PAGE_HASH) == nullptr); +} +} // namespace +#endif /* UNIV_DEBUG */ + /** Allocate a block. @param[in,out] buf_pool buffer pool instance @param[in] i index of buf_pool->zip_free[] @@ -425,6 +447,7 @@ void *buf_buddy_alloc_low(buf_pool_t *buf_pool, ulint i) { buf_block_t *block; ut_ad(!mutex_own(&buf_pool->zip_mutex)); + ut_d(buf_buddy_no_page_hash_latch_validate()); ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); if (i < BUF_BUDDY_SIZES) { @@ -476,6 +499,7 @@ static bool buf_buddy_relocate(buf_pool_t *buf_pool, void *src, void *dst, ut_ad(mutex_own(&buf_pool->zip_free_mutex)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); + ut_d(buf_buddy_no_page_hash_latch_validate()); ut_ad(!ut_align_offset(src, size)); ut_ad(!ut_align_offset(dst, size)); ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); @@ -611,6 +635,7 @@ void buf_buddy_free_low(buf_pool_t *buf_pool, void *buf, ulint i, buf_buddy_free_t *buddy; ut_ad(!mutex_own(&buf_pool->zip_mutex)); + ut_d(buf_buddy_no_page_hash_latch_validate()); ut_ad(i <= BUF_BUDDY_SIZES); ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 278ea4a1a505..8118e7faeefa 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -58,6 +58,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "log0buf.h" #include "log0chkp.h" #include "page0page.h" +#include "scope_guard.h" #include "sync0rw.h" #include "trx0purge.h" #include "trx0undo.h" @@ -1521,6 +1522,11 @@ static void buf_pool_create(buf_pool_t *buf_pool, ulint buf_pool_size, os_event_set(buf_pool->no_flush[i]); } + buf_pool->run_lru = os_event_create(); + os_event_set(buf_pool->run_lru); + + buf_pool->flushing_allowed = true; + buf_pool->watch = (buf_page_t *)ut::zalloc_withkey( UT_NEW_THIS_FILE_PSI_KEY, sizeof(*buf_pool->watch) * BUF_POOL_WATCH_SIZE); for (i = 0; i < BUF_POOL_WATCH_SIZE; i++) { @@ -1619,6 +1625,8 @@ static void buf_pool_free_instance(buf_pool_t *buf_pool) { os_event_destroy(buf_pool->no_flush[i]); } + os_event_destroy(buf_pool->run_lru); + ut::free(buf_pool->chunks); mutex_exit(&buf_pool->chunks_mutex); mutex_free(&buf_pool->chunks_mutex); @@ -3106,7 +3114,7 @@ the LRU list it resets the value to the tail of the LRU list. buf_page_t *LRUItr::start() { ut_ad(mutex_own(m_mutex)); - if (!m_hp || m_hp->old) { + if (!m_hp || !m_hp->old) { m_hp = UT_LIST_GET_LAST(m_buf_pool->LRU); } @@ -3135,9 +3143,8 @@ bool buf_pool_watch_is_sentinel(const buf_pool_t *buf_pool, } /** Add watch for the given page to be read in. Caller must have -appropriate hash_lock for the bpage and hold the LRU list mutex to avoid a race -condition with buf_LRU_free_page inserting the same page into the page hash. -This function may release the hash_lock and reacquire it. +appropriate hash_lock for the bpage. This function may release the +hash_lock and reacquire it. @param[in] page_id page id @param[in,out] hash_lock hash_lock currently latched @return NULL if watch set, block if the page is in the buffer pool */ @@ -3169,15 +3176,25 @@ static buf_page_t *buf_pool_watch_set(const page_id_t &page_id, of latching. We acquire all the hash_locks. They are needed because we don't want to read any stale information in buf_pool->watch[]. However, it is not in the critical code path - as this function will be called only by the purge thread. */ + as this function will be called only by the purge thread. + + Holding all the hash cell X-latches also stabilizes page hash + membership: every insert into and delete from the page hash happens + under the affected cell's X-latch, including the delete + re-insert + transition of the keep-zip path of buf_LRU_free_page(), which keeps + the cell's X-latch continuously so that the page id is never + observably absent from the hash (see the page_hash membership-change + protocol at its declaration in buf0buf.h). Therefore the LRU list + mutex does not need to be (and is not) taken here: the recheck below + cannot miss a page which is logically in the buffer pool. */ /* To obey latching order first release the hash_lock. */ rw_lock_x_unlock(*hash_lock); - mutex_enter(&buf_pool->LRU_list_mutex); hash_lock_x_all(buf_pool->page_hash); - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* page_hash could have been resized while we did not hold any + hash cell latch. */ *hash_lock = buf_page_hash_lock_get(buf_pool, page_id); /* We have to recheck that the page @@ -3188,7 +3205,6 @@ static buf_page_t *buf_pool_watch_set(const page_id_t &page_id, bpage = buf_page_hash_get_low(buf_pool, page_id); if (bpage) { - mutex_exit(&buf_pool->LRU_list_mutex); hash_unlock_x_all_but(buf_pool->page_hash, *hash_lock); goto page_found; } @@ -3219,8 +3235,6 @@ static buf_page_t *buf_pool_watch_set(const page_id_t &page_id, HASH_INSERT(buf_page_t, hash, buf_pool->page_hash, page_id.hash(), bpage); - mutex_exit(&buf_pool->LRU_list_mutex); - /* Once the sentinel is in the page_hash we can safely release all locks except just the relevant hash_lock */ @@ -3275,7 +3289,9 @@ void buf_pool_watch_unset(const page_id_t &page_id) { rw_lock_t *hash_lock = buf_page_hash_lock_get(buf_pool, page_id); rw_lock_x_lock(hash_lock, UT_LOCATION_HERE); - /* page_hash can be changed. */ + /* A concurrent buffer pool resize can rehash page_hash and remap this + page id to a different shard latch between computing hash_lock and + latching it; re-confirm and re-latch the correct shard. */ hash_lock = buf_page_hash_lock_x_confirm(hash_lock, buf_pool, page_id); /* The page must exist because buf_pool_watch_set() @@ -3302,7 +3318,9 @@ bool buf_pool_watch_occurred(const page_id_t &page_id) { rw_lock_s_lock(hash_lock, UT_LOCATION_HERE); - /* If not own buf_pool_mutex, page_hash can be changed. */ + /* A concurrent buffer pool resize can rehash page_hash and remap this + page id to a different shard latch between computing hash_lock and + latching it; re-confirm and re-latch the correct shard. */ hash_lock = buf_page_hash_lock_s_confirm(hash_lock, buf_pool, page_id); /* The page must exist because buf_pool_watch_set() @@ -3352,6 +3370,26 @@ static void buf_page_make_young_if_needed(buf_page_t *bpage) { ut_ad(bpage->buf_fix_count > 0); ut_a(buf_page_in_file(bpage)); + /* A page whose read IO is still in progress may not yet be linked into + the LRU list: buf_page_init_for_read() makes the page hash-visible before + it links it into the LRU list. Such a page must not be promoted - + buf_LRU_make_block_young() would unlink a node which is not linked, + corrupting the LRU list. + Reading the io-fix snapshot without the block mutex is correct here: the + LRU-add happens-before the read IO is dispatched, which happens-before + io_fix is reset to BUF_IO_NONE at IO completion. Thus observing + !was_io_fix_read() implies the LRU-add has already happened, and the + buf-fix held by our caller keeps the page in the LRU. Skipping the + promotion on a stale BUF_IO_READ snapshot is benign: the page was just + added at the head of the old sublist and a subsequent access will promote + it. + Both orderings this argument depends on (BUF_IO_READ is published before + the page is hash-reachable; the read io-fix is cleared only after the + LRU-add) are asserted at the transitions in buf_page_t::set_io_fix(). */ + if (bpage->was_io_fix_read()) { + return; + } + if (buf_page_peek_if_too_old(bpage)) { buf_page_make_young(bpage); } @@ -3986,7 +4024,9 @@ buf_block_t *Buf_fetch::lookup() { rw_lock_s_lock(m_hash_lock, UT_LOCATION_HERE); - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* A concurrent buffer pool resize can rehash page_hash and remap this + page id to a different shard latch between computing m_hash_lock and + latching it; re-confirm and re-latch the correct shard. */ m_hash_lock = buf_page_hash_lock_s_confirm(m_hash_lock, m_buf_pool, m_page_id); @@ -4036,7 +4076,10 @@ buf_block_t *Buf_fetch::is_on_watch() { rw_lock_x_lock(m_hash_lock, UT_LOCATION_HERE); - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* A concurrent buffer pool resize can rehash page_hash (buf_pool_resize() + changes the number of cells), remapping this page id to a different shard + latch after we computed m_hash_lock but before we latched it. Re-confirm + the shard the page id currently maps to and re-latch it if it moved. */ m_hash_lock = buf_page_hash_lock_x_confirm(m_hash_lock, m_buf_pool, m_page_id); @@ -4101,7 +4144,10 @@ dberr_t Buf_fetch::zip_page_handler(buf_block_t *&fix_block) { mutex_enter(&m_buf_pool->LRU_list_mutex); - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* We hold the LRU list mutex, which blocks a concurrent buffer pool + resize (buf_pool_resize() takes it), so page_hash cannot be rehashed + here: the shard latch for this page id is stable and no _confirm is + needed. */ m_hash_lock = buf_page_hash_lock_get(m_buf_pool, m_page_id); rw_lock_x_lock(m_hash_lock, UT_LOCATION_HERE); @@ -4397,12 +4443,15 @@ dberr_t Buf_fetch::debug_check(buf_block_t *fix_block) { mutex_enter(fix_mutex); if (buf_LRU_free_page(&fix_block->page, true)) { - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* buf_LRU_free_page() released the page hash latch; re-acquire the + shard latch for this page id before re-checking / re-watching it. */ m_hash_lock = buf_page_hash_lock_get(m_buf_pool, m_page_id); rw_lock_x_lock(m_hash_lock, UT_LOCATION_HERE); - /* If not own LRU_list_mutex, page_hash can be changed. */ + /* We held no shard latch across the lines above, so a concurrent + buffer pool resize may have rehashed page_hash and remapped this page + id to a different shard; re-confirm and re-latch the correct shard. */ m_hash_lock = buf_page_hash_lock_x_confirm(m_hash_lock, m_buf_pool, m_page_id); @@ -5135,8 +5184,6 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, data = buf_buddy_alloc(buf_pool, page_size.physical()); } - mutex_enter(&buf_pool->LRU_list_mutex); - hash_lock = buf_page_hash_lock_get(buf_pool, page_id); rw_lock_x_lock(hash_lock, UT_LOCATION_HERE); @@ -5150,8 +5197,6 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, /* The page is already in the buffer pool. */ watch_page = nullptr; - mutex_exit(&buf_pool->LRU_list_mutex); - rw_lock_x_unlock(hash_lock); if (bpage != nullptr) { @@ -5188,39 +5233,81 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, block->mark_for_read_io(); buf_page_set_io_fix(bpage, BUF_IO_READ); - /* The block must be put to the LRU list, to the old blocks */ - buf_LRU_add_block(bpage, true /* to old blocks */); - if (page_size.is_compressed()) { + /* Setting zip.data is still protected by the hash X-latch here: + the page is already in the page hash, but no other thread can look + it up until the latch is released below. */ block->page.zip.data = (page_zip_t *)data; - - /* To maintain the invariant - block->in_unzip_LRU_list - == buf_page_belongs_to_unzip_LRU(&block->page) - we have to add this block to unzip_LRU - after block->page.zip.data is set. */ - ut_ad(buf_page_belongs_to_unzip_LRU(&block->page)); - buf_unzip_LRU_add_block(block, true); } - mutex_exit(&buf_pool->LRU_list_mutex); - - /* We set a pass-type x-lock on the frame because then - the same thread which called for the read operation - (and is running now at this point of code) can wait - for the read to complete by waiting for the x-lock on - the frame; if the x-lock were recursive, the same - thread would illegally get the x-lock before the page - read is completed. The x-lock is cleared by the - io-handler thread. */ - ut_ad(block->latches_initialized); rw_lock_x_lock_gen(&block->lock, BUF_IO_READ, UT_LOCATION_HERE); rw_lock_x_unlock(hash_lock); buf_page_mutex_exit(block); + + /* The page is hash-visible already, but eviction cannot see it + (not on LRU yet) and readers are blocked on the frame X-lock, + so no other thread can race with the add. + + IMPORTANT: we strongly depend here on the fact that there is no + other thread that can try to acquire that frame's S-lock while + holding already the LRU list mutex (it would be deadlock cycle). + For existing use cases, for that thread to exist, the page would + need to be in the LRU list already. This latching rule is documented + at the LRU_list_mutex declaration in buf0buf.h and enforced in debug + builds by rw_lock_assert_wait_allowed() at the rw-lock wait entry + points in sync0rw.cc (it cannot be expressed via latch_level_t + ordering: block->lock is SYNC_LEVEL_VARYING, which LatchDebug + ignores). */ + + /* Widen the hash-visible-not-in-LRU window: the page is already + reachable through the page hash but not yet linked into the LRU list. + Threads finding it via the page hash (buf_page_get_zip() -> + buf_block_try_discard_uncompressed(), buf_buddy_relocate(), + buf_page_make_young_if_needed()) must back off from it, because it is + io-fixed for read. */ + DBUG_EXECUTE_IF( + "buf_page_init_for_read_delay_lru_add", + std::this_thread::sleep_for(std::chrono::microseconds(100));); + + mutex_enter(&buf_pool->LRU_list_mutex); + + /* For a compressed page zip.data was set above, before the page + became reachable through the page hash, so + buf_page_belongs_to_unzip_LRU() already holds and buf_LRU_add_block() + links the block into the unzip_LRU list as well, within this same + critical section: every observer of the LRU list sees the invariant + block->in_unzip_LRU_list == + buf_page_belongs_to_unzip_LRU(&block->page) hold. (This is unlike the + pre-narrowing code, which set zip.data only after buf_LRU_add_block() + and therefore had to add the block to the unzip_LRU list explicitly + afterwards; an explicit second add here would corrupt the list.) */ + buf_LRU_add_block(bpage, true /* to old blocks */); + + ut_ad(!page_size.is_compressed() || block->in_unzip_LRU_list); + + mutex_exit(&buf_pool->LRU_list_mutex); } else { + /* Compressed-only page: a bare BUF_BLOCK_ZIP_PAGE descriptor with no + uncompressed frame (and thus no frame rw-lock). It is initialized and + made hash-visible while the page hash X-latch and zip_mutex (this + descriptor's "block mutex") are held, and is linked into the LRU list + afterwards under a brief LRU_list_mutex hold - the same narrowed + latching order as for the block-backed pages above. + + Setting io_fix = BUF_IO_READ before the descriptor becomes reachable + through the page hash is what makes the hash-visible-but-not-in-LRU + window safe, exactly as for block-backed pages: every path which + could move or free the page based on finding it in the page hash + backs off from a read-io-fixed page (buf_page_make_young_if_needed() + skips it, buf_page_free_stale() bails out, buf_buddy relocation and + Buf_fetch::zip_page_handler() require io_fix == BUF_IO_NONE), and + readers (e.g. buf_page_get_zip()) wait for the read to complete, + which happens-after the LRU-add below, because the read IO is only + dispatched after this function returns. */ + /* Initialize the buf_pool pointer. */ bpage->buf_pool_index = buf_pool_index(buf_pool); @@ -5251,6 +5338,8 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, ut_d(bpage->in_free_list = false); ut_d(bpage->in_LRU_list = false); + buf_page_set_io_fix(bpage, BUF_IO_READ); + ut_d(bpage->in_page_hash = true); if (watch_page != nullptr) { @@ -5271,16 +5360,31 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, rw_lock_x_unlock(hash_lock); - /* The block must be put to the LRU list, to the old blocks. - The zip size is already set into the page zip */ + mutex_exit(&buf_pool->zip_mutex); + + /* Widen the hash-visible-not-in-LRU window, as in the block-backed + branch above. */ + DBUG_EXECUTE_IF( + "buf_page_init_for_read_delay_lru_add", + std::this_thread::sleep_for(std::chrono::microseconds(100));); + + /* The page is hash-visible already (io-fixed for read, see above), + but eviction cannot see it (not on the LRU list yet), so no other + thread can race with the add. The block must be put to the LRU list, + to the old blocks. The zip size is already set into the page zip. */ + mutex_enter(&buf_pool->LRU_list_mutex); +#if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG + /* buf_LRU_insert_zip_clean() requires the zip_mutex; re-acquired + here under the LRU list mutex, which follows the registered + latch_level_t order (SYNC_BUF_LRU_LIST > SYNC_BUF_BLOCK). */ + mutex_enter(&buf_pool->zip_mutex); +#endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ buf_LRU_add_block(bpage, true /* to old blocks */); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG buf_LRU_insert_zip_clean(bpage); + mutex_exit(&buf_pool->zip_mutex); #endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ mutex_exit(&buf_pool->LRU_list_mutex); - buf_page_set_io_fix(bpage, BUF_IO_READ); - - mutex_exit(&buf_pool->zip_mutex); } buf_pool->n_pend_reads.fetch_add(1); @@ -5378,18 +5482,27 @@ buf_block_t *buf_page_create(const page_id_t &page_id, /* Latch the page before releasing hash lock so that concurrent request for this page doesn't see half initialized page. ALTER tablespace for encryption and clone page copy can request page for any page id within tablespace - size limit. */ + size limit. + + The nowait variants must be used and cannot fail: the frame comes from + the free list, so its latch is unlocked, and the block is unreachable by + other threads until the page hash X-latch is released below. This keeps + the LRU_list_mutex latching rule (no waiting for a frame latch under the + LRU list mutex, see the LRU_list_mutex declaration) free of blocking + acquisitions - we hold the LRU list mutex here. */ mtr_memo_type_t mtr_latch_type; + bool latched [[maybe_unused]]; ut_ad(block->latches_initialized); if (rw_latch == RW_X_LATCH) { - rw_lock_x_lock(&block->lock, UT_LOCATION_HERE); + latched = rw_lock_x_lock_nowait(&block->lock, UT_LOCATION_HERE); mtr_latch_type = MTR_MEMO_PAGE_X_FIX; } else { - rw_lock_sx_lock(&block->lock, UT_LOCATION_HERE); + latched = rw_lock_sx_lock_nowait(&block->lock, 0, UT_LOCATION_HERE); mtr_latch_type = MTR_MEMO_PAGE_SX_FIX; } + ut_ad(latched); mtr_memo_push(mtr, block, mtr_latch_type); rw_lock_x_unlock(hash_lock); @@ -5934,6 +6047,30 @@ void buf_page_t::set_io_fix(buf_io_fix io_fix) { take_io_responsibility(); } Latching_rules_helpers::on_transition_to(*this, io_fix); + + if (io_fix == BUF_IO_READ) { + /* BUF_IO_READ may only be stored on a page that is not yet reachable + through the page hash: either it is not in the page hash at all, or + the storing thread still holds the hash cell's X-latch. Threads which + find a page through a page hash lookup therefore can never observe a + pre-read BUF_IO_NONE, which is what allows + buf_page_make_young_if_needed() to test was_io_fix_read() without the + block mutex to detect a page whose LRU-add is still pending. + (This cannot be expressed in buf_io_fix_latching_rules: its latch set + does not include the page hash latches.) */ + ut_ad(!in_page_hash || + buf_page_hash_lock_held_x(buf_pool_from_bpage(this), this)); + } + + if (old_io_fix == BUF_IO_READ && io_fix == BUF_IO_NONE) { + /* The read IO is dispatched only after buf_page_init_for_read() has + linked the page into the LRU list, so by the time the read io-fix is + cleared the page must be in the LRU list. + buf_page_make_young_if_needed() relies on this: observing + io_fix != BUF_IO_READ implies the LRU-add has completed and the page + may be promoted. */ + ut_ad(in_LRU_list); + } #endif this->io_fix.store(io_fix, std::memory_order_relaxed); #ifdef UNIV_DEBUG @@ -6348,18 +6485,24 @@ static void buf_refresh_io_stats(buf_pool_t *buf_pool) { static void buf_pool_invalidate_instance(buf_pool_t *buf_pool) { ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); + /* Pause LRU threads on event. */ + os_event_reset(buf_pool->run_lru); + + /* Prevent new flushes to start (buf_flush_start() checks this flag, + when starting a new flush). */ + mutex_enter(&buf_pool->flush_state_mutex); + buf_pool->flushing_allowed = false; + mutex_exit(&buf_pool->flush_state_mutex); + + auto guard = create_scope_guard([&]() { + mutex_enter(&buf_pool->flush_state_mutex); + buf_pool->flushing_allowed = true; + mutex_exit(&buf_pool->flush_state_mutex); + os_event_set(buf_pool->run_lru); + }); + + /* New flushing has been disallowed; wait for pending flushes. */ for (size_t i = BUF_FLUSH_LRU; i < BUF_FLUSH_N_TYPES; i++) { - /* As this function is called during startup and during redo application - phase during recovery, a flush might be requested either by - recv_writer thread (which is not started yet, or paused by writer_mutex), or - by our own thread (in which case we wait for it to finish initialization). - No new write batch can be in initialization stage at this point. - This also explains why we don't need flush_state_mutex to assert this. */ - ut_ad(!buf_pool->init_flush[i]); - - /* However, it is possible that a write batch that has been posted earlier - is still not complete. For buffer pool invalidation to proceed we must - ensure there is NO write activity happening. */ buf_flush_await_no_flushing(buf_pool, static_cast(i)); } @@ -6409,6 +6552,7 @@ static void buf_pool_validate_instance(buf_pool_t *buf_pool) { ulint n_flush = 0; ulint n_free = 0; ulint n_zip = 0; + ulint n_lru_add_pending = 0; ut_ad(buf_pool); @@ -6457,7 +6601,26 @@ static void buf_pool_validate_instance(buf_pool_t *buf_pool) { } } +#ifdef UNIV_DEBUG + if (!block->page.in_LRU_list) { + /* buf_page_init_for_read() makes the page hash-visible before + linking it into the LRU list. Such a page is still io-fixed for + read. Reading in_LRU_list is stable here: it is only modified + under LRU_list_mutex, which we hold. */ + ut_a(block->page.was_io_fix_read()); + n_lru_add_pending++; + } else { + n_lru++; + } +#else /* UNIV_DEBUG */ + /* Without UNIV_DEBUG there is no in_LRU_list flag; count how many + FILE_PAGE blocks may legitimately be missing from the LRU list so + the length cross-check below can be relaxed by that amount. */ + if (block->page.was_io_fix_read()) { + n_lru_add_pending++; + } n_lru++; +#endif /* UNIV_DEBUG */ break; case BUF_BLOCK_NOT_USED: @@ -6483,10 +6646,10 @@ static void buf_pool_validate_instance(buf_pool_t *buf_pool) { /* All clean blocks should be I/O-unfixed. */ break; case BUF_IO_READ: - /* In buf_LRU_free_page(), we temporarily set - b->io_fix = BUF_IO_READ for a newly allocated - control block in order to prevent - buf_page_get_gen() from decompressing the block. */ + /* A clean compressed-only page can be io-fixed for read only + while its initial read from disk is pending. (buf_LRU_free_page() + pins the re-inserted compressed-only descriptor with + buf_page_set_sticky(), which is BUF_IO_PIN, not BUF_IO_READ.) */ break; default: ut_error; @@ -6558,7 +6721,15 @@ static void buf_pool_validate_instance(buf_pool_t *buf_pool) { << buf_pool->curr_size << " zip " << n_zip << ". Aborting..."; } +#ifdef UNIV_DEBUG + /* Pages whose read IO is in progress and which are not yet linked into + the LRU list were counted into n_lru_add_pending instead of n_lru. */ + (void)n_lru_add_pending; ut_a(UT_LIST_GET_LEN(buf_pool->LRU) == n_lru); +#else /* UNIV_DEBUG */ + ut_a(UT_LIST_GET_LEN(buf_pool->LRU) <= n_lru); + ut_a(n_lru <= UT_LIST_GET_LEN(buf_pool->LRU) + n_lru_add_pending); +#endif /* UNIV_DEBUG */ mutex_exit(&buf_pool->LRU_list_mutex); mutex_exit(&buf_pool->chunks_mutex); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 2060fbd3901b..7cbe7da1e2ca 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -56,6 +56,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "ibuf0ibuf.h" #include "log0buf.h" #include "log0chkp.h" +#include "log0recv.h" #include "log0write.h" #include "my_compiler.h" #include "os0file.h" @@ -95,6 +96,7 @@ lsn_t get_flush_sync_lsn() noexcept { return buf_flush_sync_lsn; } #ifdef UNIV_PFS_THREAD mysql_pfs_key_t page_flush_thread_key; mysql_pfs_key_t page_flush_coordinator_thread_key; +mysql_pfs_key_t buf_lru_manager_thread_key; #endif /* UNIV_PFS_THREAD */ /** Event to synchronise with the flushing. */ @@ -125,7 +127,7 @@ struct page_cleaner_slot_t { protected by page_cleaner_t::mutex if the worker thread got the slot and set to PAGE_CLEANER_STATE_FLUSHING, - n_flushed_lru and n_flushed_list can be + lru_result and n_flushed_list can be updated only by the worker thread */ /* This value is set during state==PAGE_CLEANER_STATE_NONE */ ulint n_pages_requested; @@ -134,9 +136,9 @@ struct page_cleaner_slot_t { /* These values are updated during state==PAGE_CLEANER_STATE_FLUSHING, and committed with state==PAGE_CLEANER_STATE_FINISHED. The consistency is protected by the 'state' */ - ulint n_flushed_lru; - /*!< number of flushed pages - by LRU scan flushing */ + buf_flush_batch_result_t lru_result; + /*!< LRU batch result when dedicated LRU manager + threads are disabled */ ulint n_flushed_list; /*!< number of flushed pages by flush_list flushing */ @@ -224,6 +226,10 @@ static void buf_flush_page_coordinator_thread(); /** Worker thread of page_cleaner. */ static void buf_flush_page_cleaner_thread(); +/** LRU manager thread for performing LRU flushes for buffer pool free +list refill. One thread is created for each buffer pool instance. */ +static void buf_lru_manager_thread(size_t buf_pool_instance); + /** Increases flush_list size in bytes with the page size in inline function */ static inline void incr_flush_list_size_in_bytes( buf_block_t *block, /*!< in: control block */ @@ -1444,9 +1450,9 @@ after a call to this function there will be 'max' blocks in the free list. The caller must hold the LRU list mutex. @param[in] buf_pool buffer pool instance @param[in] max desired number of blocks in the free_list -@return number of blocks moved to the free list. */ -static ulint buf_free_from_unzip_LRU_list_batch(buf_pool_t *buf_pool, - ulint max) { +@return batch result. This path never flushes, so n_flushed is always 0. */ +static buf_flush_batch_result_t buf_free_from_unzip_LRU_list_batch( + buf_pool_t *buf_pool, ulint max) { ulint scanned = 0; ulint count = 0; ulint free_len = UT_LIST_GET_LEN(buf_pool->free); @@ -1481,19 +1487,7 @@ static ulint buf_free_from_unzip_LRU_list_batch(buf_pool_t *buf_pool, ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); - if (count) { - MONITOR_INC_VALUE_CUMULATIVE(MONITOR_LRU_BATCH_EVICT_TOTAL_PAGE, - MONITOR_LRU_BATCH_EVICT_COUNT, - MONITOR_LRU_BATCH_EVICT_PAGES, count); - } - - if (scanned) { - MONITOR_INC_VALUE_CUMULATIVE(MONITOR_LRU_BATCH_SCANNED, - MONITOR_LRU_BATCH_SCANNED_NUM_CALL, - MONITOR_LRU_BATCH_SCANNED_PER_CALL, scanned); - } - - return (count); + return {0, count, scanned}; } /** This utility flushes dirty blocks from the end of the LRU list. @@ -1503,8 +1497,9 @@ it is a best effort attempt and it is not guaranteed that after a call to this function there will be 'max' blocks in the free list. @param[in] buf_pool buffer pool instance @param[in] max desired number for blocks in the free_list -@return number of blocks for which the write request was queued. */ -static ulint buf_flush_LRU_list_batch(buf_pool_t *buf_pool, ulint max) { +@return batch result */ +static buf_flush_batch_result_t buf_flush_LRU_list_batch(buf_pool_t *buf_pool, + ulint max) { buf_page_t *bpage; ulint scanned = 0; ulint evict_count = 0; @@ -1570,42 +1565,36 @@ static ulint buf_flush_LRU_list_batch(buf_pool_t *buf_pool, ulint max) { ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); - if (evict_count) { - MONITOR_INC_VALUE_CUMULATIVE(MONITOR_LRU_BATCH_EVICT_TOTAL_PAGE, - MONITOR_LRU_BATCH_EVICT_COUNT, - MONITOR_LRU_BATCH_EVICT_PAGES, evict_count); - } - - if (scanned) { - MONITOR_INC_VALUE_CUMULATIVE(MONITOR_LRU_BATCH_SCANNED, - MONITOR_LRU_BATCH_SCANNED_NUM_CALL, - MONITOR_LRU_BATCH_SCANNED_PER_CALL, scanned); - } - - return (count); + return {count, evict_count, scanned}; } /** Flush and move pages from LRU or unzip_LRU list to the free list. Whether LRU or unzip_LRU is used depends on the state of the system. @param[in] buf_pool buffer pool instance @param[in] max desired number of blocks in the free_list -@return number of blocks for which either the write request was queued -or in case of unzip_LRU the number of blocks actually moved to the -free list */ -static ulint buf_do_LRU_batch(buf_pool_t *buf_pool, ulint max) { - ulint count = 0; +@return batch result */ +static buf_flush_batch_result_t buf_do_LRU_batch(buf_pool_t *buf_pool, + ulint max) { + buf_flush_batch_result_t result{}; ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); if (buf_LRU_evict_from_unzip_LRU(buf_pool)) { - count += buf_free_from_unzip_LRU_list_batch(buf_pool, max); + const auto unzip_result = buf_free_from_unzip_LRU_list_batch(buf_pool, max); + result.n_flushed += unzip_result.n_flushed; + result.n_evicted += unzip_result.n_evicted; + result.n_scanned += unzip_result.n_scanned; } - if (max > count) { - count += buf_flush_LRU_list_batch(buf_pool, max - count); + const ulint done = result.n_flushed + result.n_evicted; + if (max > done) { + const auto lru_result = buf_flush_LRU_list_batch(buf_pool, max - done); + result.n_flushed += lru_result.n_flushed; + result.n_evicted += lru_result.n_evicted; + result.n_scanned += lru_result.n_scanned; } - return (count); + return result; } /** This utility flushes dirty blocks from the end of the flush_list. @@ -1687,9 +1676,10 @@ not guaranteed that the actual number is that big, though) @param[in] lsn_limit in the case of BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored -@return number of blocks for which the write request was queued */ -static ulint buf_flush_batch(buf_pool_t *buf_pool, buf_flush_t flush_type, - ulint min_n, lsn_t lsn_limit) { +@return batch result. For BUF_FLUSH_LIST n_evicted and n_scanned are 0. */ +static buf_flush_batch_result_t buf_flush_batch(buf_pool_t *buf_pool, + buf_flush_t flush_type, + ulint min_n, lsn_t lsn_limit) { ut_ad(flush_type == BUF_FLUSH_LRU || flush_type == BUF_FLUSH_LIST); #ifdef UNIV_DEBUG @@ -1700,27 +1690,29 @@ static ulint buf_flush_batch(buf_pool_t *buf_pool, buf_flush_t flush_type, } #endif /* UNIV_DEBUG */ - ulint count = 0; + buf_flush_batch_result_t result{}; /* Note: The buffer pool mutexes is released and reacquired within the flush functions. */ switch (flush_type) { case BUF_FLUSH_LRU: mutex_enter(&buf_pool->LRU_list_mutex); - count = buf_do_LRU_batch(buf_pool, min_n); + result = buf_do_LRU_batch(buf_pool, min_n); mutex_exit(&buf_pool->LRU_list_mutex); break; case BUF_FLUSH_LIST: - count = buf_do_flush_list_batch(buf_pool, min_n, lsn_limit); + /* The flush list path only flushes; nothing is evicted here. */ + result.n_flushed = buf_do_flush_list_batch(buf_pool, min_n, lsn_limit); break; default: ut_error; } - DBUG_PRINT("ib_buf", ("flush %u completed, %u pages", unsigned(flush_type), - unsigned(count))); + DBUG_PRINT("ib_buf", ("flush %u completed, %u flushed, %u evicted", + unsigned(flush_type), unsigned(result.n_flushed), + unsigned(result.n_evicted))); - return (count); + return result; } /** Gather the aggregated stats for both flush list and LRU list flushing. @@ -1741,11 +1733,15 @@ static void buf_flush_stats(ulint page_count_flush, ulint page_count_LRU) { @param[in] flush_type BUF_FLUSH_LRU or BUF_FLUSH_LIST */ static bool buf_flush_start(buf_pool_t *buf_pool, buf_flush_t flush_type) { ut_ad(flush_type == BUF_FLUSH_LRU || flush_type == BUF_FLUSH_LIST); + bool started = false; buf_pool->change_flush_state(flush_type, [&]() { /* Can't start a new batch of the same type as one already running - - various synchronization mechanisms/counters would not work. */ - if (!buf_pool->is_flushing(flush_type)) { + various synchronization mechanisms/counters would not work. + + Also, don't start one while buf_pool_invalidate_instance() has closed + the gate (flushing_allowed == false) for the duration of a teardown. */ + if (!buf_pool->is_flushing(flush_type) && buf_pool->flushing_allowed) { buf_pool->init_flush[flush_type] = true; started = true; } @@ -1756,7 +1752,8 @@ static bool buf_flush_start(buf_pool_t *buf_pool, buf_flush_t flush_type) { /** End a buffer flush batch for LRU or flush list @param[in] buf_pool buffer pool instance @param[in] flush_type BUF_FLUSH_LRU or BUF_FLUSH_LIST */ -static void buf_flush_end(buf_pool_t *buf_pool, buf_flush_t flush_type) { +static void buf_flush_end(buf_pool_t *buf_pool, buf_flush_t flush_type, + ulint flushed_page_count) { buf_pool->change_flush_state(flush_type, [&]() { buf_pool->try_LRU_scan = true; buf_pool->init_flush[flush_type] = false; @@ -1764,7 +1761,12 @@ static void buf_flush_end(buf_pool_t *buf_pool, buf_flush_t flush_type) { if (!srv_read_only_mode) { if (dblwr::is_enabled()) { - dblwr::force_flush(flush_type, buf_pool_index(buf_pool)); + /* Nothing was written to the doublewrite buffer when the batch + flushed no pages (e.g. an LRU batch that only evicted clean pages), + so there is no reason to take the dblwr instance mutex. */ + if (flushed_page_count != 0) { + dblwr::force_flush(flush_type, buf_pool_index(buf_pool)); + } } else { buf_flush_sync_datafiles(); } @@ -1786,23 +1788,24 @@ void buf_flush_await_no_flushing(buf_pool_t *buf_pool, buf_flush_t flush_type) { } bool buf_flush_do_batch(buf_pool_t *buf_pool, buf_flush_t type, ulint min_n, - lsn_t lsn_limit, ulint *n_processed) { + lsn_t lsn_limit, buf_flush_batch_result_t *result) { ut_ad(type == BUF_FLUSH_LRU || type == BUF_FLUSH_LIST); - if (n_processed != nullptr) { - *n_processed = 0; + if (result != nullptr) { + *result = {}; } if (!buf_flush_start(buf_pool, type)) { return (false); } - ulint page_count = buf_flush_batch(buf_pool, type, min_n, lsn_limit); + const buf_flush_batch_result_t batch_result = + buf_flush_batch(buf_pool, type, min_n, lsn_limit); - buf_flush_end(buf_pool, type); + buf_flush_end(buf_pool, type, batch_result.n_flushed); - if (n_processed != nullptr) { - *n_processed = page_count; + if (result != nullptr) { + *result = batch_result; } return (true); @@ -1827,12 +1830,12 @@ bool buf_flush_lists(ulint min_n, lsn_t lsn_limit, ulint *n_processed) { /* Flush to lsn_limit in all buffer pool instances */ for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_pool_t *buf_pool; - ulint page_count = 0; + buf_flush_batch_result_t result{}; buf_pool = buf_pool_from_array(i); if (!buf_flush_do_batch(buf_pool, BUF_FLUSH_LIST, min_n, lsn_limit, - &page_count)) { + &result)) { /* We have two choices here. If lsn_limit was specified then skipping an instance of buffer pool means we cannot guarantee that all pages @@ -1848,7 +1851,11 @@ bool buf_flush_lists(ulint min_n, lsn_t lsn_limit, ulint *n_processed) { continue; } - n_flushed += page_count; + /* BUF_FLUSH_LIST never evicts: buf_flush_batch() reaches the LRU + eviction code only for BUF_FLUSH_LRU. */ + ut_ad(result.n_evicted == 0); + ut_ad(result.n_scanned == 0); + n_flushed += result.n_flushed; } if (n_flushed) { @@ -1954,10 +1961,12 @@ Clears up tail of the LRU list of a given buffer pool instance: The depth to which we scan each buffer pool is controlled by dynamic config parameter innodb_LRU_scan_depth. @param buf_pool buffer pool instance -@return total pages flushed */ -static ulint buf_flush_LRU_list(buf_pool_t *buf_pool) { +@param[out] started whether an LRU batch was admitted to start +@return batch result */ +static buf_flush_batch_result_t buf_flush_LRU_list(buf_pool_t *buf_pool, + bool *started) { ulint scan_depth, withdraw_depth; - ulint n_flushed = 0; + buf_flush_batch_result_t result{}; ut_ad(buf_pool); @@ -1972,13 +1981,108 @@ static ulint buf_flush_LRU_list(buf_pool_t *buf_pool) { scan_depth = std::min(static_cast(srv_LRU_scan_depth), scan_depth); } - /* Currently one of page_cleaners is the only thread - that can trigger an LRU flush at the same time. - So, it is not possible that a batch triggered during - last iteration is still running, */ - buf_flush_do_batch(buf_pool, BUF_FLUSH_LRU, scan_depth, 0, &n_flushed); + *started = + buf_flush_do_batch(buf_pool, BUF_FLUSH_LRU, scan_depth, 0, &result); + + return result; +} + +/** Record one LRU pass in the common per-instance statistics accumulator. */ +static void buf_lru_flush_stat_record( + buf_pool_t *buf_pool, const buf_flush_batch_result_t &result, + std::chrono::steady_clock::duration elapsed) { + mutex_enter(&buf_pool->flush_state_mutex); + auto &stat = buf_pool->lru_flush_stat; + + stat.n_flushed_pages += result.n_flushed; + if (result.n_flushed > 0) { + ++stat.n_flush_batches; + stat.max_flushed_pages_per_batch = + std::max(stat.max_flushed_pages_per_batch, + static_cast(result.n_flushed)); + } + + stat.n_evicted_pages += result.n_evicted; + if (result.n_evicted > 0) { + ++stat.n_evict_batches; + stat.max_evicted_pages_per_batch = + std::max(stat.max_evicted_pages_per_batch, + static_cast(result.n_evicted)); + } - return (n_flushed); + stat.n_scanned_pages += result.n_scanned; + if (result.n_scanned > 0) { + ++stat.n_scan_batches; + stat.max_scanned_pages_per_batch = + std::max(stat.max_scanned_pages_per_batch, + static_cast(result.n_scanned)); + } + + const auto elapsed_ms = + std::chrono::duration_cast(elapsed).count(); + ++stat.n_lru_passes; + stat.lru_flush_time_ms += elapsed_ms; + mutex_exit(&buf_pool->flush_state_mutex); +} + +/** Drains per-instance LRU batch counters and updates the accumulator. */ +static void pc_publish_lru_batch_stats() { + buf_pool_t::lru_flush_stat_t lru_stat{}; + for (ulint i = 0; i < srv_buf_pool_instances; i++) { + buf_pool_t *const buf_pool = buf_pool_from_array(i); + mutex_enter(&buf_pool->flush_state_mutex); + auto &instance_stat = buf_pool->lru_flush_stat; + lru_stat.n_flushed_pages += instance_stat.n_flushed_pages; + lru_stat.n_flush_batches += instance_stat.n_flush_batches; + lru_stat.max_flushed_pages_per_batch = + std::max(lru_stat.max_flushed_pages_per_batch, + instance_stat.max_flushed_pages_per_batch); + lru_stat.n_evicted_pages += instance_stat.n_evicted_pages; + lru_stat.n_evict_batches += instance_stat.n_evict_batches; + lru_stat.max_evicted_pages_per_batch = + std::max(lru_stat.max_evicted_pages_per_batch, + instance_stat.max_evicted_pages_per_batch); + lru_stat.n_scanned_pages += instance_stat.n_scanned_pages; + lru_stat.n_scan_batches += instance_stat.n_scan_batches; + lru_stat.max_scanned_pages_per_batch = + std::max(lru_stat.max_scanned_pages_per_batch, + instance_stat.max_scanned_pages_per_batch); + instance_stat.n_flushed_pages = 0; + instance_stat.n_flush_batches = 0; + instance_stat.max_flushed_pages_per_batch = 0; + instance_stat.n_evicted_pages = 0; + instance_stat.n_evict_batches = 0; + instance_stat.max_evicted_pages_per_batch = 0; + instance_stat.n_scanned_pages = 0; + instance_stat.n_scan_batches = 0; + instance_stat.max_scanned_pages_per_batch = 0; + mutex_exit(&buf_pool->flush_state_mutex); + } + + const auto publish_lru_batch_stat = + [](monitor_id_t total_monitor, monitor_id_t count_monitor, + monitor_id_t per_call_monitor, uint64_t n_pages, uint64_t n_batches, + uint64_t max_pages_per_batch) { + if (n_batches > 0 && MONITOR_IS_ON(total_monitor)) { + monitor_inc_value_nocheck(total_monitor, n_pages); + monitor_inc_value_nocheck(count_monitor, n_batches, false); + monitor_set(per_call_monitor, max_pages_per_batch, true, false); + monitor_set(per_call_monitor, n_pages / n_batches, false, false); + } + }; + + publish_lru_batch_stat( + MONITOR_LRU_BATCH_FLUSH_TOTAL_PAGE, MONITOR_LRU_BATCH_FLUSH_COUNT, + MONITOR_LRU_BATCH_FLUSH_PAGES, lru_stat.n_flushed_pages, + lru_stat.n_flush_batches, lru_stat.max_flushed_pages_per_batch); + publish_lru_batch_stat( + MONITOR_LRU_BATCH_EVICT_TOTAL_PAGE, MONITOR_LRU_BATCH_EVICT_COUNT, + MONITOR_LRU_BATCH_EVICT_PAGES, lru_stat.n_evicted_pages, + lru_stat.n_evict_batches, lru_stat.max_evicted_pages_per_batch); + publish_lru_batch_stat( + MONITOR_LRU_BATCH_SCANNED, MONITOR_LRU_BATCH_SCANNED_NUM_CALL, + MONITOR_LRU_BATCH_SCANNED_PER_CALL, lru_stat.n_scanned_pages, + lru_stat.n_scan_batches, lru_stat.max_scanned_pages_per_batch); } namespace Adaptive_flush { @@ -2122,6 +2226,19 @@ void set_average() { mutex_exit(&page_cleaner->mutex); + /* Dedicated LRU managers do not use page-cleaner slots. Consume only their + timing fields here; batch counters are published independently each second. */ + for (ulint i = 0; i < srv_buf_pool_instances; i++) { + buf_pool_t *const buf_pool = buf_pool_from_array(i); + mutex_enter(&buf_pool->flush_state_mutex); + auto &instance_stat = buf_pool->lru_flush_stat; + lru_tm += instance_stat.lru_flush_time_ms; + lru_pass += static_cast(instance_stat.n_lru_passes); + instance_stat.n_lru_passes = 0; + instance_stat.lru_flush_time_ms = 0; + mutex_exit(&buf_pool->flush_state_mutex); + } + /* minimum values are 1, to avoid dividing by zero. */ if (lru_tm < 1) { lru_tm = 1; @@ -2531,6 +2648,15 @@ bool buf_flush_page_cleaner_is_active() { return (srv_thread_is_active(srv_threads.m_page_cleaner_coordinator)); } +/** Returns the count of currently active LRU manager threads. */ +size_t buf_flush_active_lru_managers() { + size_t count = 0; + for (size_t i = 0; i < srv_threads.m_lru_managers_n; ++i) { + count += (srv_thread_is_active(srv_threads.m_lru_managers[i]) ? 1 : 0); + } + return count; +} + void buf_flush_page_cleaner_init() { ut_ad(page_cleaner == nullptr); @@ -2560,6 +2686,17 @@ void buf_flush_page_cleaner_init() { /* Make sure page cleaner is active. */ ut_a(buf_flush_page_cleaner_is_active()); + + /* One LRU manager thread per buf_pool instance when enabled. */ + if (srv_lru_threads_enabled) { + for (size_t i = 0; i < srv_threads.m_lru_managers_n; ++i) { + srv_threads.m_lru_managers[i] = os_thread_create( + buf_lru_manager_thread_key, i, buf_lru_manager_thread, i); + srv_threads.m_lru_managers[i].start(); + } + + ut_a(buf_flush_active_lru_managers() == srv_buf_pool_instances); + } } /** @@ -2572,6 +2709,16 @@ static void buf_flush_page_cleaner_close(void) { srv_threads.m_page_cleaner_workers[i].wait(); } + /* Wait for all LRU manager threads to exit. They observe + srv_shutdown_state > SRV_SHUTDOWN_CLEANUP and break out of their + loops; the run_lru event is always set during normal shutdown so they + do not block. */ + if (srv_lru_threads_enabled) { + for (size_t i = 0; i < srv_threads.m_lru_managers_n; ++i) { + srv_threads.m_lru_managers[i].wait(); + } + } + mutex_destroy(&page_cleaner->mutex); os_event_destroy(page_cleaner->is_finished); @@ -2636,7 +2783,7 @@ static void pc_request(ulint min_n, lsn_t lsn_limit) { Do flush for one slot. @return the number of the slots which has not been treated yet. */ static ulint pc_flush_slot(void) { - std::chrono::steady_clock::duration lru_time; + std::chrono::steady_clock::duration lru_time{}; std::chrono::steady_clock::duration flush_list_time{}; int lru_pass = 0; int list_pass = 0; @@ -2670,39 +2817,59 @@ static ulint pc_flush_slot(void) { } if (!page_cleaner->is_running) { - slot->n_flushed_lru = 0; + slot->lru_result = {}; slot->n_flushed_list = 0; } else { - mutex_exit(&page_cleaner->mutex); + const auto n_pages_requested = slot->n_pages_requested; + const auto requested = page_cleaner->requested; + const auto lsn_limit = page_cleaner->lsn_limit; - const auto lru_start = std::chrono::steady_clock::now(); - - /* Flush pages from end of LRU if required */ - slot->n_flushed_lru = buf_flush_LRU_list(buf_pool); - - lru_time = std::chrono::steady_clock::now() - lru_start; - lru_pass = 1; + mutex_exit(&page_cleaner->mutex); - if (!page_cleaner->is_running) { - slot->n_flushed_list = 0; + /* Flush pages from LRU tail if required. */ + buf_flush_batch_result_t lru_result; + if (!srv_lru_threads_enabled) { + const auto lru_start = std::chrono::steady_clock::now(); + bool started = false; + lru_result = buf_flush_LRU_list(buf_pool, &started); + lru_pass = started ? 1 : 0; + lru_time = std::chrono::steady_clock::now() - lru_start; + buf_lru_flush_stat_record(buf_pool, lru_result, lru_time); } else { - /* Flush pages from flush_list if required */ - if (page_cleaner->requested) { - const auto flush_list_start = std::chrono::steady_clock::now(); - - slot->succeeded_list = buf_flush_do_batch( - buf_pool, BUF_FLUSH_LIST, slot->n_pages_requested, - page_cleaner->lsn_limit, &slot->n_flushed_list); + lru_result = {}; + } - flush_list_time = std::chrono::steady_clock::now() - flush_list_start; - list_pass = 1; - } else { - slot->n_flushed_list = 0; - slot->succeeded_list = true; - } + /* Flush pages from flush_list if required. LRU-tail flushing is the + responsibility of the per-pool buf_lru_manager_thread when + innodb_lru_threads is on. When it is off, the page cleaner performs + LRU flushing (including during recovery via recv_writer). */ + ulint n_flushed_list; + bool succeeded_list; + if (requested) { + const auto flush_list_start = std::chrono::steady_clock::now(); + + buf_flush_batch_result_t result{}; + succeeded_list = buf_flush_do_batch( + buf_pool, BUF_FLUSH_LIST, n_pages_requested, lsn_limit, &result); + /* BUF_FLUSH_LIST never evicts and does not report its scan count + through this result yet. */ + ut_ad(result.n_evicted == 0); + ut_ad(result.n_scanned == 0); + n_flushed_list = result.n_flushed; + + flush_list_time = std::chrono::steady_clock::now() - flush_list_start; + list_pass = 1; + } else { + n_flushed_list = 0; + succeeded_list = true; } + mutex_enter(&page_cleaner->mutex); + slot->lru_result = lru_result; + slot->succeeded_list = succeeded_list; + slot->n_flushed_list = n_flushed_list; } + page_cleaner->n_slots_flushing--; page_cleaner->n_slots_finished++; slot->state = PAGE_CLEANER_STATE_FINISHED; @@ -2729,14 +2896,15 @@ static ulint pc_flush_slot(void) { /** Wait until all flush requests are finished. -@param n_flushed_lru number of pages flushed from the end of the LRU list. +@param lru_result aggregate LRU result from all slots @param n_flushed_list number of pages flushed from the end of the flush_list. @return true if all flush_list flushing batch were success. */ -static bool pc_wait_finished(ulint *n_flushed_lru, ulint *n_flushed_list) { +static bool pc_wait_finished(buf_flush_batch_result_t *lru_result, + ulint *n_flushed_list) { bool all_succeeded = true; - *n_flushed_lru = 0; + *lru_result = {}; *n_flushed_list = 0; os_event_wait(page_cleaner->is_finished); @@ -2752,7 +2920,9 @@ static bool pc_wait_finished(ulint *n_flushed_lru, ulint *n_flushed_list) { ut_ad(slot->state == PAGE_CLEANER_STATE_FINISHED); - *n_flushed_lru += slot->n_flushed_lru; + lru_result->n_flushed += slot->lru_result.n_flushed; + lru_result->n_evicted += slot->lru_result.n_evicted; + lru_result->n_scanned += slot->lru_result.n_scanned; *n_flushed_list += slot->n_flushed_list; all_succeeded &= slot->succeeded_list; @@ -2774,7 +2944,7 @@ static bool pc_wait_finished(ulint *n_flushed_lru, ulint *n_flushed_list) { #ifdef UNIV_LINUX /** -Set priority for page_cleaner threads. +Set priority for page_cleaner and LRU manager threads. @param[in] priority priority intended to set @return true if set as intended */ static bool buf_flush_page_cleaner_set_priority(int priority) { @@ -2784,15 +2954,15 @@ static bool buf_flush_page_cleaner_set_priority(int priority) { #endif /* UNIV_LINUX */ #ifdef UNIV_DEBUG -/** Loop used to disable page cleaner threads. */ +/** Loop used to disable page cleaner and LRU manager threads. */ static void buf_flush_page_cleaner_disabled_loop(void) { - ut_ad(page_cleaner != nullptr); - if (!innodb_page_cleaner_disabled_debug) { /* We return to avoid entering and exiting mutex. */ return; } + ut_ad(page_cleaner != nullptr); + mutex_enter(&page_cleaner->mutex); page_cleaner->n_disabled_debug++; mutex_exit(&page_cleaner->mutex); @@ -2834,7 +3004,7 @@ void buf_flush_page_cleaner_disabled_debug_update(THD *, SYS_VAR *, void *, innodb_page_cleaner_disabled_debug = false; - /* Enable page cleaner threads. */ + /* Enable page cleaner and LRU manager threads. */ while (srv_shutdown_state.load() < SRV_SHUTDOWN_CLEANUP) { mutex_enter(&page_cleaner->mutex); const ulint n = page_cleaner->n_disabled_debug; @@ -2867,9 +3037,19 @@ void buf_flush_page_cleaner_disabled_debug_update(THD *, SYS_VAR *, void *, mutex_enter(&page_cleaner->mutex); - ut_ad(page_cleaner->n_disabled_debug <= srv_n_page_cleaners); - - if (page_cleaner->n_disabled_debug == srv_n_page_cleaners) { + /* With LRU manager threads enabled, convergence also waits for each of + them to reach buf_flush_page_cleaner_disabled_loop(). An LRU manager + parked on os_event_wait(buf_pool->run_lru) (a buf_pool_invalidate_instance() + window) does not reach that loop until run_lru is set again, so this + wait can stall for as long as that window lasts. Both this and the + manager's own adaptive sleep (up to 1s) are transient in practice, but + worth knowing if a debug test using this variable ever hangs here. */ + const auto all_flushing_threads = + srv_n_page_cleaners + + (srv_lru_threads_enabled ? srv_buf_pool_instances : 0); + ut_ad(page_cleaner->n_disabled_debug <= all_flushing_threads); + + if (page_cleaner->n_disabled_debug == all_flushing_threads) { mutex_exit(&page_cleaner->mutex); break; } @@ -2888,6 +3068,7 @@ static void buf_flush_page_coordinator_thread() { ulint n_flushed = 0; ulint last_activity = srv_get_activity_count(); ulint last_pages = 0; + ulint n_evicted = 0; THD *thd = create_internal_thd(); @@ -2899,8 +3080,9 @@ static void buf_flush_page_coordinator_thread() { << buf_flush_page_cleaner_priority; } else { ib::info(ER_IB_MSG_127) << "If the mysqld execution user is authorized," - " page cleaner thread priority can be changed." - " See the man page of setpriority()."; + " page cleaner and LRU manager thread priority" + " can be changed. See the man page of" + " setpriority()."; } #endif /* UNIV_LINUX */ @@ -2917,7 +3099,7 @@ static void buf_flush_page_coordinator_thread() { srv_shutdown_state.load() < SRV_SHUTDOWN_CLEANUP && recv_sys->spaces != nullptr) { /* treat flushing requests during recovery. */ - ulint n_flushed_lru = 0; + buf_flush_batch_result_t lru_result{}; ulint n_flushed_list = 0; os_event_wait(recv_sys->flush_start); @@ -2933,7 +3115,7 @@ static void buf_flush_page_coordinator_thread() { pc_request(0, LSN_MAX); while (pc_flush_slot() > 0) { } - pc_wait_finished(&n_flushed_lru, &n_flushed_list); + pc_wait_finished(&lru_result, &n_flushed_list); break; case BUF_FLUSH_LIST: @@ -2942,7 +3124,7 @@ static void buf_flush_page_coordinator_thread() { pc_request(ULINT_MAX, LSN_MAX); while (pc_flush_slot() > 0) { } - } while (!pc_wait_finished(&n_flushed_lru, &n_flushed_list)); + } while (!pc_wait_finished(&lru_result, &n_flushed_list)); break; default: @@ -2956,7 +3138,6 @@ static void buf_flush_page_coordinator_thread() { os_event_wait(buf_flush_event); ulint ret_sleep = 0; - ulint n_evicted = 0; ulint n_flushed_last = 0; ulint warn_interval = 1; ulint warn_count = 0; @@ -3038,6 +3219,8 @@ static void buf_flush_page_coordinator_thread() { was_server_active = srv_check_activity(last_activity); last_activity = srv_get_activity_count(); + + pc_publish_lru_batch_stats(); } lsn_t lsn_limit; @@ -3110,34 +3293,30 @@ static void buf_flush_page_coordinator_thread() { page_cleaner->flush_pass++; /* Wait for all slots to be finished */ - ulint n_flushed_lru = 0; + buf_flush_batch_result_t lru_result{}; ulint n_flushed_list = 0; - pc_wait_finished(&n_flushed_lru, &n_flushed_list); + pc_wait_finished(&lru_result, &n_flushed_list); + pc_publish_lru_batch_stats(); - if (n_flushed_list > 0 || n_flushed_lru > 0) { - buf_flush_stats(n_flushed_list, n_flushed_lru); + if (n_flushed_list > 0 || lru_result.n_flushed > 0) { + buf_flush_stats(n_flushed_list, lru_result.n_flushed); } if (n_to_flush != 0) { last_pages = n_flushed_list; } - n_evicted += n_flushed_lru; + n_evicted += lru_result.n_flushed; n_flushed_last += n_flushed_list; - n_flushed = n_flushed_lru + n_flushed_list; + n_flushed = lru_result.n_flushed + n_flushed_list; if (is_sync_flush) { MONITOR_INC_VALUE_CUMULATIVE( MONITOR_FLUSH_SYNC_TOTAL_PAGE, MONITOR_FLUSH_SYNC_COUNT, - MONITOR_FLUSH_SYNC_PAGES, n_flushed_lru + n_flushed_list); + MONITOR_FLUSH_SYNC_PAGES, lru_result.n_flushed + n_flushed_list); } else { - if (n_flushed_lru) { - MONITOR_INC_VALUE_CUMULATIVE( - MONITOR_LRU_BATCH_FLUSH_TOTAL_PAGE, MONITOR_LRU_BATCH_FLUSH_COUNT, - MONITOR_LRU_BATCH_FLUSH_PAGES, n_flushed_lru); - } if (n_flushed_list) { MONITOR_INC_VALUE_CUMULATIVE( MONITOR_FLUSH_ADAPTIVE_TOTAL_PAGE, MONITOR_FLUSH_ADAPTIVE_COUNT, @@ -3190,6 +3369,9 @@ static void buf_flush_page_coordinator_thread() { the buffer pool but can't be sure that no new pages are being dirtied until we enter SRV_SHUTDOWN_FLUSH_PHASE phase which is the last phase (meanwhile we visit SRV_SHUTDOWN_MASTER_STOP). + Because the LRU manager thread is also flushing at SRV_SHUTDOWN_CLEANUP + but not SRV_SHUTDOWN_FLUSH_PHASE, we only leave the + SRV_SHUTDOWN_CLEANUP loop when the LRU manager quits. Note, that if we are handling fatal error, we set the state directly to EXIT_THREADS in which case we also might exit the loop @@ -3203,17 +3385,18 @@ static void buf_flush_page_coordinator_thread() { while (pc_flush_slot() > 0) { } - ulint n_flushed_lru = 0; + buf_flush_batch_result_t lru_result{}; ulint n_flushed_list = 0; - pc_wait_finished(&n_flushed_lru, &n_flushed_list); + pc_wait_finished(&lru_result, &n_flushed_list); - n_flushed = n_flushed_lru + n_flushed_list; + n_flushed = n_flushed_list + lru_result.n_flushed; /* We sleep only if there are no pages to flush */ if (n_flushed == 0) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - } while (srv_shutdown_state.load() < SRV_SHUTDOWN_FLUSH_PHASE); + } while (srv_shutdown_state.load() < SRV_SHUTDOWN_FLUSH_PHASE || + buf_flush_active_lru_managers() > 0); /* At this point all threads including the master and the purge thread must have been closed, unless we are handling some error @@ -3248,6 +3431,7 @@ static void buf_flush_page_coordinator_thread() { in the flush_list */ buf_flush_await_no_flushing(nullptr, BUF_FLUSH_LIST); buf_flush_await_no_flushing(nullptr, BUF_FLUSH_LRU); + ut_ad(buf_flush_active_lru_managers() == 0); bool success; bool are_any_read_ios_still_underway; @@ -3268,11 +3452,11 @@ static void buf_flush_page_coordinator_thread() { while (pc_flush_slot() > 0) { } - ulint n_flushed_lru = 0; + buf_flush_batch_result_t lru_result{}; ulint n_flushed_list = 0; - success = pc_wait_finished(&n_flushed_lru, &n_flushed_list); + success = pc_wait_finished(&lru_result, &n_flushed_list); - n_flushed = n_flushed_lru + n_flushed_list; + n_flushed = n_flushed_list + lru_result.n_flushed; buf_flush_await_no_flushing(nullptr, BUF_FLUSH_LIST); buf_flush_await_no_flushing(nullptr, BUF_FLUSH_LRU); @@ -3382,6 +3566,128 @@ void buf_flush_sync_all_buf_pools() { buf_flush_fsync(); } +/** Sleep the LRU manager thread until next_loop_time, unless we are already +past it or shutdown is in the flush phase or later (in which case the manager +runs without sleeping so it can exit promptly). */ +static void buf_lru_manager_sleep_if_needed( + std::chrono::steady_clock::time_point next_loop_time) { + if (srv_shutdown_state.load() >= SRV_SHUTDOWN_FLUSH_PHASE) return; + + const auto cur_time = std::chrono::steady_clock::now(); + + if (next_loop_time > cur_time) { + const auto period = std::chrono::duration_cast( + next_loop_time - cur_time); + + std::this_thread::sleep_for( + std::min(std::chrono::milliseconds{1000L}, period)); + } +} + +/** Adjust the LRU manager thread's per-pool sleep time based on free-list +fullness and the work the last iteration accomplished. Aggressive shrinking +when the free list is near-empty, gradual growth when it's healthy. */ +static void buf_lru_manager_adapt_sleep_time( + const buf_pool_t *buf_pool, size_t lru_n_processed, + std::chrono::milliseconds &lru_sleep_time) { + const auto free_len = UT_LIST_GET_LEN(buf_pool->free); + const auto max_free_len = + std::min(UT_LIST_GET_LEN(buf_pool->LRU), srv_LRU_scan_depth); + + if (free_len < max_free_len / 100 && lru_n_processed) { + /* Free list < 1% and we made progress last time: don't sleep. */ + lru_sleep_time = std::chrono::milliseconds::zero(); + } else if (free_len > max_free_len / 5 || + (free_len < max_free_len / 100 && lru_n_processed == 0)) { + /* Free list > 20%, or near-empty but we made no progress (unusual): + back off a bit. */ + lru_sleep_time += std::chrono::milliseconds{1}; + if (lru_sleep_time > std::chrono::milliseconds{1000}) + lru_sleep_time = std::chrono::milliseconds{1000}; + } else if (free_len < max_free_len / 20 && + lru_sleep_time >= std::chrono::milliseconds{50}) { + /* Free list < 5%: shrink the sleep. */ + lru_sleep_time -= std::chrono::milliseconds{50}; + } + /* Otherwise (5%–20%): no change. */ +} + +/** LRU manager thread. One per buf_pool instance. Periodically calls +buf_flush_LRU_list to keep the free list topped up. The current +buf_LRU_get_free_block continues to do its own scan_and_free / +single_page_flush fall-back as needed; this thread is an additional source +of LRU-tail pressure that lets the user-thread fall-back stay rare on +healthy workloads. + +The thread runs until srv_shutdown_state reaches SRV_SHUTDOWN_FLUSH_PHASE, +mirroring the page cleaner coordinator's pre-flush loop, so that user +threads keep finding free pages throughout every earlier shutdown phase. +The buf_pool's run_lru event is set at startup; it is reset only inside +buf_pool_invalidate_instance() so the manager pauses while the pool is +torn down. */ +static void buf_lru_manager_thread(size_t buf_pool_instance) { +#ifdef UNIV_LINUX + /* linux might be able to set different setting for each thread + worth to try to set high priority for page cleaner threads */ + if (buf_flush_page_cleaner_set_priority(buf_flush_page_cleaner_priority)) { + ib::info() << "lru_manager worker priority: " + << buf_flush_page_cleaner_priority; + } +#endif /* UNIV_LINUX */ + + ut_ad(buf_pool_instance < srv_buf_pool_instances); + + buf_pool_t *const buf_pool = buf_pool_from_array(buf_pool_instance); + + std::chrono::milliseconds lru_sleep_time{1000}; + auto next_loop_time = std::chrono::steady_clock::now() + lru_sleep_time; + /* Seed nonzero so the first adapt iteration treats us as "made progress" + and does not immediately back off. */ + size_t lru_n_processed = 1; + + while (srv_shutdown_state.load() < SRV_SHUTDOWN_FLUSH_PHASE) { + ut_d(buf_flush_page_cleaner_disabled_loop()); + + os_event_wait(buf_pool->run_lru); + + buf_lru_manager_sleep_if_needed(next_loop_time); + + buf_lru_manager_adapt_sleep_time(buf_pool, lru_n_processed, lru_sleep_time); + + next_loop_time = std::chrono::steady_clock::now() + lru_sleep_time; + + /* {n_flushed, n_evicted}: evicting a clean page refills the free list + just as a flush does, so the adaptive-sleep "made progress" signal must + consider both. The flush-specific stats below count flushes only. + buf_flush_LRU_list() (via buf_flush_start()) is a no-op returning a zero + result if buf_pool_invalidate_instance() has cleared flushing_allowed in + the meantime; resetting run_lru only guarantees that the *next* + os_event_wait() above parks, so this check is what stops a thread that + already returned from the wait (or was in the sleep) from starting a + batch after invalidation began. */ + + const auto lru_start = std::chrono::steady_clock::now(); + bool started = false; + const auto result = buf_flush_LRU_list(buf_pool, &started); + const auto lru_time = std::chrono::steady_clock::now() - lru_start; + lru_n_processed = result.n_flushed + result.n_evicted; + + if (!started) { + continue; + } + + buf_lru_flush_stat_record(buf_pool, result, lru_time); + + /* Wait for the batch this iteration kicked off (if any) to finish so + the next iteration sees free pages on the free list. */ + buf_flush_await_no_flushing(buf_pool, BUF_FLUSH_LRU); + + if (result.n_flushed) { + srv_stats.buf_pool_flushed.add(result.n_flushed); + } + } +} + #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /** Functor to validate the flush list. */ diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 040d4f87804b..41835dc81ccb 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -154,13 +154,20 @@ If a compressed page is freed other compressed pages may be relocated. compressed page of an uncompressed page @param[in] ignore_content true if should ignore page content, since it could be not initialized +@param[in] keep_hash_lock true if the hash cell X-latch should be kept + by this function instead of being released; + only allowed when the caller will re-insert + a compressed-only descriptor for this page + id into the page hash (the keep-zip path of + buf_LRU_free_page()), so that the page id is + never observably absent from the page hash @retval true if BUF_BLOCK_FILE_PAGE was removed from page_hash. The caller needs to free the page to the free list @retval false if BUF_BLOCK_ZIP_PAGE was removed from page_hash. In this case the block is already returned to the buddy allocator. */ -[[nodiscard]] static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, - bool zip, - bool ignore_content); +[[nodiscard]] static bool buf_LRU_block_remove_hashed( + buf_page_t *bpage, bool zip, bool ignore_content, + bool keep_hash_lock = false); /** Puts a file page whose has no hash index to the free list. @param[in,out] block Must contain a file page and be in a state @@ -1386,7 +1393,7 @@ we put it to free list to be used. @return the free control block, in state BUF_BLOCK_READY_FOR_USE */ buf_block_t *buf_LRU_get_free_block(buf_pool_t *buf_pool) { buf_block_t *block = nullptr; - bool freed = false; + bool freed = false, no_flush_waited = false; ulint n_iterations = 0; ulint flush_failures = 0; bool started_monitor = false; @@ -1497,15 +1504,6 @@ buf_block_t *buf_LRU_get_free_block(buf_pool_t *buf_pool) { (srv_shutdown_state.load() != SRV_SHUTDOWN_NONE && srv_shutdown_state.load() != SRV_SHUTDOWN_CLEANUP)); } - if (buf_pool->init_flush[BUF_FLUSH_LRU] && dblwr::is_enabled()) { - /* If there is an LRU flush happening in the background then we - wait for it to end instead of trying a single page flush. If, - however, we are not using doublewrite buffer then it is better to - do our own single page flush instead of waiting for LRU flush to - end. */ - buf_flush_await_no_flushing(buf_pool, BUF_FLUSH_LRU); - goto loop; - } os_rmb; @@ -1584,9 +1582,21 @@ buf_block_t *buf_LRU_get_free_block(buf_pool_t *buf_pool) { involved (particularly in case of compressed pages). We can do that in a separate patch sometime in future. */ - if (!buf_flush_single_page_from_LRU(buf_pool)) { - MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT); - ++flush_failures; + if (srv_lru_threads_enabled && buf_pool->init_flush[BUF_FLUSH_LRU] && + dblwr::is_enabled() && buf_pool->n_flush[BUF_FLUSH_SINGLE_PAGE] >= 1 && + !no_flush_waited) { + /* Cap reached: wait for an in-progress LRU flush instead of issuing + our own single-page flush. */ + MONITOR_INC(MONITOR_LRU_FLUSH_AWAIT_COUNT); + buf_flush_await_no_flushing(buf_pool, BUF_FLUSH_LRU); + no_flush_waited = true; + } else { + /* Below the cap: issue our own single-page flush. */ + MONITOR_INC(MONITOR_LRU_SINGLE_PAGE_FLUSH_COUNT); + if (!buf_flush_single_page_from_LRU(buf_pool)) { + MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT); + ++flush_failures; + } } srv_stats.buf_pool_wait_free.add(n_iterations, 1); @@ -1915,16 +1925,22 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) { auto block_mutex = buf_page_get_mutex(bpage); auto hash_lock = buf_page_hash_lock_get(buf_pool, bpage->id); - ut_ad(bpage->in_LRU_list); ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); ut_ad(mutex_own(block_mutex)); - ut_ad(buf_page_in_file(bpage)); if (!buf_page_can_relocate(bpage)) { /* Do not free buffer fixed and I/O-fixed blocks. */ return (false); } + /* These assertions can only be checked after the io-fix check + above, because pages become visible in the page hash table before + they are linked into the LRU list (they are io-fixed for read + until after they are linked into the LRU list, so + buf_page_can_relocate() has returned false for them). */ + ut_ad(bpage->in_LRU_list); + ut_ad(buf_page_in_file(bpage)); + #ifdef UNIV_IBUF_COUNT_DEBUG ut_a(ibuf_count_get(bpage->id) == 0); #endif /* UNIV_IBUF_COUNT_DEBUG */ @@ -1945,10 +1961,17 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) { return (false); } else if (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE) { + /* This holds because of the "else" keyword above, + but we assert it for clarity. */ + ut_ad(!zip && bpage->zip.data != nullptr); b = buf_page_alloc_descriptor(); ut_a(b); } + /* Protect: buf_buddy_free must not be invoked in buf_LRU_block_remove_hashed, + when passing keep_hash_lock = true (b != nullptr). */ + ut_ad(b == nullptr || (!zip && bpage->zip.data != nullptr)); + ut_ad(buf_page_in_file(bpage)); ut_ad(bpage->in_LRU_list); ut_ad(bpage->in_flush_list == is_dirty); @@ -1994,7 +2017,20 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) { ut_ad(rw_lock_own(hash_lock, RW_LOCK_X)); ut_ad(buf_page_can_relocate(bpage)); - if (!buf_LRU_block_remove_hashed(bpage, zip, false)) { + /* When the compressed page is to be kept (b != nullptr), ask + buf_LRU_block_remove_hashed() to keep the hash cell X-latch, so that it + is held continuously from before the HASH_DELETE until after the + compressed-only descriptor is re-inserted below: the page id is never + observably absent from the page hash. Otherwise a concurrent + buf_page_init_for_read() (which inserts into the page hash without + holding the LRU list mutex) could insert a second descriptor for this + page id in the meantime. + + Protect: buf_buddy_free must not be invoked in buf_LRU_block_remove_hashed, + when passing keep_hash_lock = true (b != nullptr). */ + ut_ad(b == nullptr || (!zip && bpage->zip.data != nullptr)); + + if (!buf_LRU_block_remove_hashed(bpage, zip, false, b != nullptr)) { mutex_exit(&buf_pool->LRU_list_mutex); if (b != nullptr) { @@ -2004,9 +2040,11 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) { } ut_ad(!mutex_own(block_mutex)); - /* buf_LRU_block_remove_hashed() releases the hash_lock */ - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X) && - !rw_lock_own(hash_lock, RW_LOCK_S)); + /* buf_LRU_block_remove_hashed() releases the hash_lock, unless it was + asked to keep it for the re-insert of the compressed-only descriptor. */ + ut_ad(b != nullptr ? rw_lock_own(hash_lock, RW_LOCK_X) + : (!rw_lock_own(hash_lock, RW_LOCK_X) && + !rw_lock_own(hash_lock, RW_LOCK_S))); /* We have just freed a BUF_BLOCK_FILE_PAGE. If b != nullptr then it was a compressed page with an uncompressed frame and @@ -2015,10 +2053,24 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) { into the LRU and page_hash (and possibly flush_list). if b == nullptr then it was a regular page that has been freed */ + /* Widen the window between the page hash delete and the re-insert of + the compressed-only descriptor. The hash cell X-latch is held here + (keep_hash_lock), so a concurrent buf_page_init_for_read() of this page + must block on the cell latch instead of inserting a second descriptor + for the same page id into the gap. */ + DBUG_EXECUTE_IF( + "buf_lru_free_page_delay_zip_reinsert", if (b != nullptr) { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + }); + if (b != nullptr) { auto prev_b = UT_LIST_GET_PREV(LRU, b); - rw_lock_x_lock(hash_lock, UT_LOCATION_HERE); + /* The hash cell X-latch has been held continuously since before the + HASH_DELETE in buf_LRU_block_remove_hashed() (see keep_hash_lock), + which is what makes the assertion below provable: no other thread can + have inserted a descriptor for this page id in the meantime. */ + ut_ad(rw_lock_own(hash_lock, RW_LOCK_X)); mutex_enter(block_mutex); @@ -2231,7 +2283,8 @@ the object will be freed. The caller must hold buf_pool->LRU_list_mutex, the buf_page_get_mutex() mutex and the appropriate hash_lock. This function will release the -buf_page_get_mutex() and the hash_lock. +buf_page_get_mutex() and the hash_lock (the latter is kept if keep_hash_lock +is passed). If a compressed page is freed other compressed pages may be relocated. @@ -2242,12 +2295,20 @@ If a compressed page is freed other compressed pages may be relocated. compressed page of an uncompressed page @param[in] ignore_content true if should ignore page content, since it could be not initialized +@param[in] keep_hash_lock true if the hash cell X-latch should be kept + by this function instead of being released; + only allowed when the caller will re-insert + a compressed-only descriptor for this page + id into the page hash (the keep-zip path of + buf_LRU_free_page()), so that the page id is + never observably absent from the page hash @retval true if BUF_BLOCK_FILE_PAGE was removed from page_hash. The caller needs to free the page to the free list @retval false if BUF_BLOCK_ZIP_PAGE was removed from page_hash. In this case the block is already returned to the buddy allocator. */ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, bool zip, - bool ignore_content) { + bool ignore_content, + bool keep_hash_lock) { const buf_page_t *hashed_bpage; buf_pool_t *buf_pool = buf_pool_from_bpage(bpage); rw_lock_t *hash_lock; @@ -2255,6 +2316,16 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, bool zip, ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); ut_ad(mutex_own(buf_page_get_mutex(bpage))); + /* keep_hash_lock is only supported for the keep-zip path of + buf_LRU_free_page(): an uncompressed frame being freed while its + compressed page is kept and will be re-inserted into the page hash. + In particular the buddy allocator must not be invoked while the hash + cell X-latch is kept (buf_buddy_free() may take page hash latches via + buf_buddy_relocate()), which is guaranteed by zip == false. */ + ut_ad(!keep_hash_lock || + (!zip && buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE && + bpage->zip.data != nullptr)); + hash_lock = buf_page_hash_lock_get(buf_pool, bpage->id); ut_ad(rw_lock_own(hash_lock, RW_LOCK_X)); @@ -2432,14 +2503,24 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, bool zip, avoid relocation during the scan. But that is not possible because we are holding LRU list mutex. - 2) Not possible because in buf_page_init_for_read() - we do a look up of page_hash while holding LRU list - mutex and since we are holding LRU list mutex here - and by the time we'll release it in the caller we'd - have inserted the compressed only descriptor in the - page_hash. */ + 2) When a compressed-only descriptor will be re-inserted + for this page id (the keep-zip path of buf_LRU_free_page(), + keep_hash_lock == true), the hash cell X-latch is kept from + before the HASH_DELETE above until after the re-insert in the + caller, so the page id is never observably absent from the + page hash: a concurrent buf_page_init_for_read() blocks on + the hash cell latch and then finds the compressed-only + descriptor. (It cannot be the LRU list mutex which protects + this transition: buf_page_init_for_read() inserts into the + page hash without holding the LRU list mutex.) + When nothing will be re-inserted (keep_hash_lock == false), + the page is leaving the buffer pool for good and a concurrent + thread reading it from the disk afresh is the normal cache + miss path. */ ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); - rw_lock_x_unlock(hash_lock); + if (!keep_hash_lock) { + rw_lock_x_unlock(hash_lock); + } mutex_exit(&((buf_block_t *)bpage)->mutex); if (zip && bpage->zip.data) { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index eb5a666de329..847a408d3bf9 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -88,10 +88,17 @@ ulint buf_read_page_low(dberr_t *err, bool sync, ulint type, ulint mode, sync = true; } - /* The following call will also check if the tablespace does not exist - or is being dropped; if we succeed in initing the page in the buffer - pool for read, then DISCARD cannot proceed until the read has - completed */ + /* buf_page_init_for_read() makes the page hash-visible, io-fixed for + read and linked into the LRU list before we dispatch the read IO below. + This does not stop a concurrent tablespace drop or truncation: + tablespace deletion does not scan the buffer pool (BUF_REMOVE_NONE); it + bumps the space version and relies on the pages becoming stale. If the + tablespace is dropped before the IO is dispatched, fil_io() refuses the + read (DB_TABLESPACE_DELETED, see Fil_shard::do_io()) and the page is + removed by buf_read_page_handle_error() below. If a read completes + against a space which was dropped meanwhile, the page is detected as + stale (buf_page_t::was_stale()) and freed lazily by + buf_page_free_stale(), which waits out the read io-fix. */ bpage = buf_page_init_for_read(mode, page_id, page_size, unzip); ut_a(bpage == nullptr || bpage->get_space()->id == page_id.space()); diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 182bf0051a85..d2c85793e473 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -438,8 +438,8 @@ dberr_t dict_create_index_tree_in_mem(dict_index_t *index, trx_t *trx) { DBUG_EXECUTE_IF("ib_dict_create_index_tree_fail", return (DB_OUT_OF_MEMORY);); - if (index->type == DICT_FTS) { - /* FTS index does not need an index tree */ + if ((index->type & DICT_FTS) || dict_index_is_vector(index)) { + /* FTS nor vector index do not need an index tree */ return (DB_SUCCESS); } diff --git a/storage/innobase/dict/dict0dd.cc b/storage/innobase/dict/dict0dd.cc index f77448568965..63798ec908d4 100644 --- a/storage/innobase/dict/dict0dd.cc +++ b/storage/innobase/dict/dict0dd.cc @@ -1979,7 +1979,7 @@ void dd_visit_keys_with_too_long_parts( std::function visitor) { for (uint key_num = 0; key_num < table->s->keys; key_num++) { const KEY &key = table->key_info[key_num]; - if (!(key.flags & (HA_SPATIAL | HA_FULLTEXT))) { + if (!(key.flags & (HA_SPATIAL | HA_FULLTEXT | HA_VECTOR))) { for (unsigned i = 0; i < key.user_defined_key_parts; i++) { const KEY_PART_INFO *key_part = &key.key_part[i]; if (max_part_len < key_part->length) { @@ -2909,7 +2909,7 @@ MY_COMPILER_DIAGNOSTIC_POP() */ static inline uint16_t get_index_prefix_len(const KEY &key, const KEY_PART_INFO *key_part) { - if (key.flags & (HA_SPATIAL | HA_FULLTEXT)) { + if (key.flags & (HA_SPATIAL | HA_FULLTEXT | HA_VECTOR)) { return 0; } @@ -2949,6 +2949,7 @@ template const dict_index_t *dd_find_index( uint key_num) { const KEY &key = form->key_info[key_num]; ulint type = 0; + bool is_vector = false; unsigned n_fields = key.user_defined_key_parts; unsigned n_uniq = n_fields; @@ -2969,6 +2970,10 @@ template const dict_index_t *dd_find_index( ut_ad(!table->is_intrinsic()); type = DICT_FTS; n_uniq = 0; + } else if (key.flags & HA_VECTOR) { + ut_ad(!table->is_intrinsic()); + is_vector = true; + n_uniq = 0; } else if (key_num == form->primary_key) { ut_ad(key.flags & HA_NOSAME); ut_ad(n_uniq > 0); @@ -2977,11 +2982,13 @@ template const dict_index_t *dd_find_index( type = (key.flags & HA_NOSAME) ? DICT_UNIQUE : 0; } - ut_ad(!!(type & DICT_FTS) == (n_uniq == 0)); + ut_ad((!!(type & DICT_FTS) || is_vector) == (n_uniq == 0)); dict_index_t *index = dict_mem_index_create(table->name.m_name, key.name, 0, type, n_fields); + index->is_vector_index = is_vector; + index->n_uniq = n_uniq; DBUG_EXECUTE_IF("ib_create_table_fail_at_create_index", @@ -5206,8 +5213,8 @@ dict_table_t *dd_open_table_one(dd::cache::Dictionary_client *client, } ut_ad(root > 1); - ut_ad(index->type & DICT_FTS || root != FIL_NULL || - dict_table_is_discarded(m_table)); + ut_ad((index->type & DICT_FTS) || dict_index_is_vector(index) || + root != FIL_NULL || dict_table_is_discarded(m_table)); ut_ad(id != 0); index->page = root; index->space = sid; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 6710d5abff14..56344f30c2ee 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -218,6 +218,9 @@ static dict_index_t *dict_index_build_internal_fts( dict_table_t *table, /*!< in: table */ dict_index_t *index); /*!< in: user representation of an FTS index */ +static dict_index_t *dict_index_build_internal_vec(dict_table_t *table, + dict_index_t *index); + /** Removes an index from the dictionary cache. */ static void dict_index_remove_from_cache_low( dict_table_t *table, /*!< in/out: table */ @@ -2217,7 +2220,7 @@ static bool dict_index_too_big_for_tree(const dict_table_t *table, const dict_index_t *new_index) { /* FTS index consists of auxiliary tables, they shall be excluded from index row size check */ - if (new_index->type & DICT_FTS) { + if ((new_index->type & DICT_FTS) || dict_index_is_vector(new_index)) { return (false); } @@ -2446,6 +2449,8 @@ dberr_t dict_index_add_to_cache_w_vcol(dict_table_t *table, dict_index_t *index, if (index->type == DICT_FTS) { new_index = dict_index_build_internal_fts(table, index); + } else if (dict_index_is_vector(index)) { + new_index = dict_index_build_internal_vec(table, index); } else if (index->is_clustered()) { new_index = dict_index_build_internal_clust(table, index); } else { @@ -3284,6 +3289,37 @@ static dict_index_t *dict_index_build_internal_fts( return (new_index); } + +static dict_index_t *dict_index_build_internal_vec( + dict_table_t *table, /*!< in: table */ + dict_index_t *index) /*!< in: user representation of a vector index */ +{ + ut_ad(table && index); + ut_ad(dict_index_is_vector(index)); + ut_ad(!dict_sys_mutex_own()); + ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); + + /* Create a new index */ + auto new_index = + dict_mem_index_create(table->name.m_name, index->name, index->space, + index->type, index->n_fields); + new_index->is_vector_index = index->is_vector_index; + + /* Copy other relevant data from the old index struct to the new + struct: it inherits the values */ + + new_index->n_user_defined_cols = index->n_fields; + + new_index->id = index->id; + + /* Copy fields from index to new_index */ + dict_index_copy(new_index, index, table, 0, index->n_fields); + + new_index->n_uniq = 0; + new_index->cached = true; + + return (new_index); +} /*====================== FOREIGN KEY PROCESSING ========================*/ /** Checks if a table is referenced by foreign keys. @@ -3371,7 +3407,8 @@ NOT NULL */ while (index != nullptr) { if (types_idx != index && !(index->type & DICT_FTS) && - !dict_index_is_spatial(index) && !index->to_be_dropped && + !dict_index_is_vector(index) && !dict_index_is_spatial(index) && + !index->to_be_dropped && (!(index->uncommitted && ((index->online_status == ONLINE_INDEX_ABORTED_DROPPED) || (index->online_status == ONLINE_INDEX_ABORTED)))) && @@ -3628,6 +3665,7 @@ bool dict_index_check_search_tuple( ut_ad(index->page >= FSP_FIRST_INODE_PAGE_NO); ut_ad(dtuple_check_typed(tuple)); ut_ad(!(index->type & DICT_FTS)); + ut_ad(!dict_index_is_vector(index)); return true; } #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e84e74c2bc48..5b972ff07f1c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -888,6 +888,7 @@ static PSI_thread_info all_innodb_threads[] = { PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME), PSI_THREAD_KEY(recv_writer_thread, "ib_recv_write", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME), + PSI_THREAD_KEY(buf_lru_manager_thread, "ib_buf_lru", 0, 0, PSI_DOCUMENT_ME), PSI_THREAD_KEY(srv_error_monitor_thread, "ib_srv_err", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME), PSI_THREAD_KEY(srv_lock_timeout_thread, "ib_srv_lock_to", @@ -3231,7 +3232,7 @@ ha_innobase::ha_innobase(handlerton *hton, TABLE_SHARE *table_arg) HA_ATTACHABLE_TRX_COMPATIBLE | HA_CAN_INDEX_VIRTUAL_GENERATED_COLUMN | HA_DESCENDING_INDEX | HA_MULTI_VALUED_KEY_SUPPORT | HA_BLOB_PARTIAL_UPDATE | HA_SUPPORTS_GEOGRAPHIC_GEOMETRY_COLUMN | - HA_SUPPORTS_DEFAULT_EXPRESSION | HA_ONLINE_ANALYZE), + HA_SUPPORTS_DEFAULT_EXPRESSION | HA_ONLINE_ANALYZE | HA_CAN_VECTOR), m_start_of_scan(), m_stored_select_lock_type(LOCK_NONE_UNSET), m_mysql_has_locked() {} @@ -5750,13 +5751,13 @@ static int innodb_init(void *p) { innobase_hton->lock_hton_log = innobase_lock_hton_log; innobase_hton->unlock_hton_log = innobase_unlock_hton_log; innobase_hton->collect_hton_log_info = innobase_collect_hton_log_info; - innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS | - HTON_SUPPORTS_FOREIGN_KEYS | HTON_SUPPORTS_ATOMIC_DDL | - HTON_CAN_RECREATE | HTON_SUPPORTS_SECONDARY_ENGINE | - HTON_SUPPORTS_TABLE_ENCRYPTION | - HTON_SUPPORTS_GENERATED_INVISIBLE_PK | - HTON_SUPPORTS_BULK_LOAD | HTON_SUPPORTS_SQL_FK | - HTON_SUPPORTS_ONLINE_BACKUPS | HTON_SUPPORTS_COMPRESSED_COLUMNS; + innobase_hton->flags = + HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS | + HTON_SUPPORTS_ATOMIC_DDL | HTON_CAN_RECREATE | + HTON_SUPPORTS_SECONDARY_ENGINE | HTON_SUPPORTS_TABLE_ENCRYPTION | + HTON_SUPPORTS_GENERATED_INVISIBLE_PK | HTON_SUPPORTS_BULK_LOAD | + HTON_SUPPORTS_SQL_FK | HTON_SUPPORTS_ONLINE_BACKUPS | + HTON_SUPPORTS_COMPRESSED_COLUMNS; // TODO(WL9440): to be enabled when distance scan is implemented in innodb. //| HTON_SUPPORTS_DISTANCE_SCAN; @@ -8303,7 +8304,7 @@ int ha_innobase::open(const char *name, int, uint open_flags, dict_table_autoinc_unlock(ib_table); } - /* Set plugin parser for fulltext index */ + /* Set plugin parser for fulltext index / handle vector index. */ for (uint i = 0; i < table->s->keys; i++) { if (table->key_info[i].flags & HA_USES_PARSER) { dict_index_t *index = innobase_get_index(i); @@ -11019,7 +11020,7 @@ int ha_innobase::index_read( : HA_ERR_TABLE_DEF_CHANGED; } - if (index->type & DICT_FTS) { + if ((index->type & DICT_FTS) || dict_index_is_vector(index)) { return HA_ERR_KEY_NOT_FOUND; } @@ -12871,6 +12872,7 @@ inline int create_index( index = dict_mem_index_create(table_name, key->name, 0, ind_type, key->user_defined_key_parts); + index->is_vector_index = (key->flags & HA_VECTOR); innodb_session_t *&priv = thd_to_innodb_session(trx->mysql_thd); dict_table_t *handler = priv->lookup_table_handler(table_name); @@ -12964,6 +12966,7 @@ inline int create_index( } ut_ad(key->flags & HA_FULLTEXT || !(index->type & DICT_FTS)); + ut_ad((key->flags & HA_VECTOR) || !dict_index_is_vector(index)); multi_val_idx = ((index->type & DICT_MULTI_VALUE) == DICT_MULTI_VALUE); @@ -14059,7 +14062,7 @@ bool create_table_info_t::innobase_table_flags() { if (fts_doc_id_index_bad) { goto index_bad; } - } else if (key->flags & HA_SPATIAL) { + } else if (key->flags & (HA_SPATIAL | HA_VECTOR)) { assert(~m_create_info->options & (HA_LEX_CREATE_TMP_TABLE | HA_LEX_CREATE_INTERNAL_TMP_TABLE)); } @@ -15656,16 +15659,51 @@ int innobase_truncate::exec() { template int innobase_truncate::exec(); template int innobase_truncate::exec(); -/** Check if a column is the only column in an index. -@param[in] index data dictionary index -@param[in] column the column to look for -@return whether the column is the only column in the index */ +/** + Check if a column is the only column in an index. + + @param[in] index data dictionary index + @param[in] column the column to look for + + @return Whether the column is the only column in the index. +*/ static bool dd_is_only_column(const dd::Index *index, const dd::Column *column) { return (index->elements().size() == 1 && &(*index->elements().begin())->column() == column); } +/** + Check if an index uses marker-based vector metadata. + + @param[in] index data dictionary index + + @return Whether the index is a vector index. +*/ +static bool dd_is_vector_index(const dd::Index *index) { + if (index->algorithm() != dd::Index::IA_SE_SPECIFIC || + index->type() != dd::Index::IT_MULTIPLE) { + return false; + } + + uint visible_elements = 0; + for (const dd::Index_element *elem : index->elements()) { + if (elem->is_hidden()) continue; + + visible_elements++; + + const dd::Properties &col_options = elem->column().options(); + bool is_vector_column = false; + if (col_options.exists("vector_index") && + !col_options.get("vector_index", &is_vector_column) && + is_vector_column) { + return visible_elements == 1; + } + } + + return false; +} + /** Add hidden columns and indexes to an InnoDB table definition. @param[in,out] dd_table data dictionary cache object @return error number @@ -15693,6 +15731,10 @@ int ha_innobase::get_extra_columns_and_keys(const HA_CREATE_INFO *, fts_doc_id_index = i; } + if (dd_is_vector_index(i)) { + continue; + } + switch (i->algorithm()) { case dd::Index::IA_SE_SPECIFIC: ut_d(ut_error); @@ -18334,7 +18376,8 @@ void ha_innobase::info_low_key(uint flag, const dict_table_t *ib_table) { /* We do not maintain stats for fulltext or spatial indexes. Thus, we can't calculate pct_cached below because we need dict_index_t::stat_n_leaf_pages for that. See dict_stats_should_ignore_index(). */ - if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL)) { + if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL) || + (key->flags & HA_VECTOR)) { pct_cached = IN_MEMORY_ESTIMATE_UNKNOWN; } else { pct_cached = index_pct_cached(index); @@ -18352,7 +18395,8 @@ void ha_innobase::info_low_key(uint flag, const dict_table_t *ib_table) { } for (ulong j = 0; j < key->actual_key_parts; j++) { - if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL)) { + if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL) || + (key->flags & HA_VECTOR)) { /* The record per key does not apply to FTS or Spatial indexes. */ key->set_records_per_key(j, 1.0f); continue; @@ -18678,7 +18722,8 @@ static bool innobase_get_index_column_cardinality( } DEBUG_SYNC(thd, "innodb.after_init_check"); - if (index->type & (DICT_FTS | DICT_SPATIAL)) { + if ((index->type & (DICT_FTS | DICT_SPATIAL)) || + dict_index_is_vector(index)) { /* For these indexes innodb_rec_per_key is fixed as 1.0 */ *cardinality = ib_table->stat_n_rows; @@ -23599,6 +23644,14 @@ static MYSQL_SYSVAR_ULONG(lru_scan_depth, srv_LRU_scan_depth, "How deep to scan LRU to keep it clean", nullptr, nullptr, 1024, 100, UINT32_MAX, 0); +static MYSQL_SYSVAR_BOOL( + lru_threads, srv_lru_threads_enabled, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, + "Enable to use LRU manager threads that flush the LRU tail and refill " + "the free list. There would be one thread for each buffer pool instance. " + "When disabled (the default), page cleaners perform the LRU flushing.", + nullptr, nullptr, false); + static MYSQL_SYSVAR_ULONG(flush_neighbors, srv_flush_neighbors, PLUGIN_VAR_OPCMDARG, "Set to 0 (don't flush neighbors from buffer pool)," @@ -24184,11 +24237,12 @@ static MYSQL_SYSVAR_BOOL( static MYSQL_SYSVAR_BOOL( buffer_pool_populate, srv_buf_pool_populate, PLUGIN_VAR_NOCMDARG, "Enforce page faults for InnoDB buffer pool allocations at allocation time" - " (pre-populate pages). When OFF (default), the pre-population is skipped" - " to reduce startup time and page-faults happen on first page accesses." + " (pre-populate pages). When ON (default), pre-population happens at" + " allocation time. When OFF, the pre-population is skipped to reduce" + " startup time and page-faults happen on first page accesses." " Note: it is recommended to turn on this variable when using large pages" " on systems with multiple NUMA nodes.", - nullptr, nullptr, false); + nullptr, nullptr, true); static MYSQL_SYSVAR_BOOL( api_enable_binlog, ib_binlog_enabled, @@ -24519,6 +24573,7 @@ static SYS_VAR *innobase_system_variables[] = { MYSQL_SYSVAR(buffer_pool_load_abort), MYSQL_SYSVAR(buffer_pool_load_at_startup), MYSQL_SYSVAR(lru_scan_depth), + MYSQL_SYSVAR(lru_threads), MYSQL_SYSVAR(flush_neighbors), MYSQL_SYSVAR(checksum_algorithm), MYSQL_SYSVAR(log_checksums), diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 6c07516e33cf..1a97d40c66d1 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -2349,7 +2349,24 @@ struct buf_pool_t { for all buf_pool_t-s */ BufListMutex chunks_mutex; - /** LRU list mutex */ + /** LRU list mutex. + Latching rule: no thread may WAIT for a block's frame rw-lock + (block->lock) while holding this mutex. buf_page_init_for_read() + acquires this mutex while holding the X-latch on the frame of the page + being read in, so waiting for a frame latch under this mutex would + create a deadlock cycle with that path. Consequently, a frame latch may + be taken under this mutex only with the rw_lock_*_nowait() variants: + flushing does so and handles the failure, and buf_page_create() does so + on a frame taken from the free list, asserting success (its latch is + unlocked and the block is unreachable by other threads while the page + hash X-latch is still held, so the attempt cannot fail). Compressed-only + pages (BUF_BLOCK_ZIP_PAGE descriptors) have no frame and no frame + rw-lock, so the paths handling them add no edge to this rule. + This rule cannot be expressed via latch_level_t ordering, because + block->lock is registered with SYNC_LEVEL_VARYING which LatchDebug + ignores; instead it is enforced in debug builds (with + --innodb-sync-debug) by rw_lock_assert_wait_allowed() at the rw-lock + wait entry points in sync0rw.cc. */ BufListMutex LRU_list_mutex; /** free and withdraw list mutex */ @@ -2406,7 +2423,17 @@ struct buf_pool_t { /** Hash table of buf_page_t or buf_block_t file pages, buf_page_in_file() == true, indexed by (space_id, offset). page_hash is protected by an array of - mutexes. */ + mutexes. + Membership-change protocol: a descriptor for a page id may be inserted + only after verifying the id's absence, with the cell's X-latch held + continuously from that verification until the insert. Conversely, a + remover which will re-insert a descriptor for the same page id (the + keep-zip path of buf_LRU_free_page()) must keep the cell's X-latch held + continuously from the delete until the re-insert, so that the page id is + never observably absent from the hash while the page is still logically + in the buffer pool. Note that buf_page_init_for_read() inserts while + holding only the cell's X-latch (not the LRU list mutex), so the LRU + list mutex does NOT stabilize page hash membership. */ hash_table_t *page_hash; /** Hash table of buf_block_t blocks whose frames are allocated to the zip @@ -2465,6 +2492,49 @@ struct buf_pool_t { running. Protected by flush_state_mutex. */ os_event_t no_flush[BUF_FLUSH_N_TYPES]; + /** Always set at startup so the LRU manager thread does not have to wait. + Reset by buf_pool_invalidate_instance() so the manager pauses while the + buffer pool is being torn down / re-initialised; set again afterwards. */ + os_event_t run_lru; + + /** Run gate for flushes, checked by buf_flush_start() inside + the change_flush_state() critical section that sets init_flush[type]. + True in normal operation; set to false by buf_pool_invalidate_instance() + for the duration of a teardown. Protected by flush_state_mutex. */ + bool flushing_allowed; + + /** Per-instance LRU flush accounting. Written by either this instance's + LRU manager or its page-cleaner slot and read (summed across instances) by + the page cleaner coordinator. pc_publish_lru_batch_stats() publishes the + buffer_LRU_batch_* counters independently, while + Adaptive_flush::set_average() consumes the timing fields. Protected by + flush_state_mutex so each consumer can atomically gather and reset its + fields. */ + struct lru_flush_stat_t { + /** Pages written to disk by LRU batches. */ + uint64_t n_flushed_pages; + /** LRU batches that wrote at least one page. */ + uint64_t n_flush_batches; + /** Largest number of pages written by one LRU batch. */ + uint64_t max_flushed_pages_per_batch; + /** Clean or stale pages evicted by LRU batches. */ + uint64_t n_evicted_pages; + /** LRU batches that evicted at least one page. */ + uint64_t n_evict_batches; + /** Largest number of pages evicted by one LRU batch. */ + uint64_t max_evicted_pages_per_batch; + /** Pages examined by LRU batches. */ + uint64_t n_scanned_pages; + /** LRU batches that examined at least one page. */ + uint64_t n_scan_batches; + /** Largest number of pages examined by one LRU batch. */ + uint64_t max_scanned_pages_per_batch; + /** Number of buf_flush_LRU_list() calls in the interval. */ + uint64_t n_lru_passes; + /** LRU time accumulated in the interval, in milliseconds. */ + uint64_t lru_flush_time_ms; + } lru_flush_stat; + /** A sequence number used to count the number of buffer blocks removed from the end of the LRU list; NOTE that this counter may wrap around at 4 billion! A thread is allowed to read this for heuristic purposes without diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index ca8d7612a137..4a16d156725d 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -512,7 +512,12 @@ static inline bool buf_page_can_relocate( { ut_ad(mutex_own(buf_page_get_mutex(bpage))); ut_ad(buf_page_in_file(bpage)); - ut_ad(bpage->in_LRU_list); + /* A page whose read IO is still in progress may be hash-visible but not + yet linked into the LRU list: buf_page_init_for_read() links it into the + LRU list only after making it hash-visible. Such a page is io-fixed for + read (and not relocatable), which is the only legitimate way to reach + this function for a page which is not in the LRU list. */ + ut_ad(bpage->in_LRU_list || buf_page_get_io_fix(bpage) == BUF_IO_READ); return (buf_page_get_io_fix(bpage) == BUF_IO_NONE && bpage->buf_fix_count == 0); diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index d681350272e1..7f356cf35041 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -35,6 +35,8 @@ this program; if not, write to the Free Software Foundation, Inc., #ifndef buf0flu_h #define buf0flu_h +#include + #include "buf0types.h" #include "log0types.h" #include "univ.i" @@ -47,6 +49,9 @@ this program; if not, write to the Free Software Foundation, Inc., /** Checks if the page_cleaner is in active state. */ bool buf_flush_page_cleaner_is_active(); +/** Returns the count of currently active LRU manager threads. */ +size_t buf_flush_active_lru_managers(); + #ifdef UNIV_DEBUG /** Value of MySQL global variable used to disable page cleaner. */ @@ -107,7 +112,17 @@ buf_flush_batch() and buf_flush_page(). [[nodiscard]] bool buf_flush_page_try(buf_pool_t *buf_pool, buf_block_t *block); #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ -/** Do flushing batch of a given type. +/** Result of a buffer flush batch. */ +struct buf_flush_batch_result_t { + /** Pages for which a write was queued. */ + ulint n_flushed{}; + /** Clean or stale pages moved directly to the free list. */ + ulint n_evicted{}; + /** Pages examined by the batch. Currently reported for BUF_FLUSH_LRU. */ + ulint n_scanned{}; +}; + +/** Do a flush-list batch. NOTE: The calling thread is not allowed to own any latches on pages! @param[in,out] buf_pool buffer pool instance @param[in] type flush type @@ -116,12 +131,12 @@ NOTE: The calling thread is not allowed to own any latches on pages! @param[in] lsn_limit in the case BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored -@param[out] n_processed the number of pages which were processed is -passed back to caller. Ignored if NULL +@param[out] result batch result; n_evicted and n_scanned are 0. +Ignored if NULL @retval true if a batch was queued successfully. @retval false if another batch of same type was already running. */ bool buf_flush_do_batch(buf_pool_t *buf_pool, buf_flush_t type, ulint min_n, - lsn_t lsn_limit, ulint *n_processed); + lsn_t lsn_limit, buf_flush_batch_result_t *result); /** This utility flushes dirty blocks from the end of the flush list of all buffer pool instances. @@ -186,7 +201,7 @@ bool buf_flush_ready_for_replace(const buf_page_t *bpage); #ifdef UNIV_DEBUG struct SYS_VAR; -/** Disables page cleaner threads (coordinator and workers). +/** Disables page cleaner threads (coordinator and workers) and LRU threads. It's used by: SET GLOBAL innodb_page_cleaner_disabled_debug = 1 (0). @param[in] thd thread handle @param[in] var pointer to system variable diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index abb3b0a8d35d..b7e415c43c31 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -134,6 +134,22 @@ static inline ulint dict_index_is_spatial( return (index->type & DICT_SPATIAL); } +/** + Check whether the index is a vector index. + + @param[in] index Index. + + @return Nonzero for vector index, zero for other indexes +*/ +static inline ulint dict_index_is_vector( + const dict_index_t *index) /*!< in: index */ +{ + ut_ad(index); + ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); + + return (index->is_vector()); +} + /** Check whether the index contains a virtual column @param[in] index index @return nonzero for the index has virtual column, zero for other indexes */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 9a083c0c1c74..0eb6f9ba2ad1 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1208,6 +1208,9 @@ struct dict_index_t { bool hidden; #endif /* !UNIV_HOTBACKUP */ + /** true if this is a vector index according to DD metadata */ + bool is_vector_index; + /** list of indexes of the table */ UT_LIST_NODE_T(dict_index_t) indexes; @@ -1337,6 +1340,17 @@ struct dict_index_t { return (type & DICT_MULTI_VALUE); } + /** + Check whether the index is a vector index. + + @return true if the index is a vector index, false otherwise + */ + [[nodiscard]] bool is_vector() const { + ut_ad(magic_n == DICT_INDEX_MAGIC_N); + + return (is_vector_index); + } + /** Returns the minimum data size of an index record. @return minimum data size in bytes */ ulint get_min_size() const { diff --git a/storage/innobase/include/dict0mem.ic b/storage/innobase/include/dict0mem.ic index 65993ac857a0..7b3bb7dbf788 100644 --- a/storage/innobase/include/dict0mem.ic +++ b/storage/innobase/include/dict0mem.ic @@ -76,6 +76,7 @@ static inline void dict_mem_fill_index_struct( index->allow_duplicates = false; index->nulls_equal = false; index->disable_ahi = false; + index->is_vector_index = false; index->last_ins_cur = nullptr; index->last_sel_cur = nullptr; #ifndef UNIV_HOTBACKUP diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index c0d1cc2d7b89..b677faca1479 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -253,6 +253,8 @@ enum monitor_id_t { MONITOR_LRU_UNZIP_SEARCH_SCANNED, MONITOR_LRU_UNZIP_SEARCH_SCANNED_NUM_CALL, MONITOR_LRU_UNZIP_SEARCH_SCANNED_PER_CALL, + MONITOR_LRU_SINGLE_PAGE_FLUSH_COUNT, + MONITOR_LRU_FLUSH_AWAIT_COUNT, /* Buffer Page I/O specific counters. */ MONITOR_MODULE_BUF_PAGE, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 48393092e734..45a08fdb6a85 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -254,6 +254,16 @@ struct Srv_threads { same shared state as m_page_cleaner_coordinator. */ IB_thread *m_page_cleaner_workers; + /** Number of LRU manager threads and size of array below. One per + buf_pool instance. */ + size_t m_lru_managers_n; + + /** LRU manager threads. When innodb_lru_threads is enabled, they are the + sole owners of buf_flush_LRU_list for free-list refill and the page + cleaner only flushes the flush_list; when disabled, no threads are + started and the page cleaner performs LRU flushing. */ + IB_thread *m_lru_managers; + /** Archiver's log archiver (used by Clone). */ IB_thread m_log_archiver; @@ -625,6 +635,8 @@ extern bool srv_validate_tablespace_paths; extern bool srv_use_fdatasync; /** Scan depth for LRU flush batch i.e.: number of blocks scanned*/ extern ulong srv_LRU_scan_depth; +/** Whether per-pool LRU manager threads are enabled (after recovery). */ +extern bool srv_lru_threads_enabled; /** Whether or not to flush neighbors of a block */ extern ulong srv_flush_neighbors; /** Previously requested size. Accesses protected by memory barriers. */ @@ -889,6 +901,7 @@ extern mysql_pfs_key_t page_archiver_thread_key; extern mysql_pfs_key_t buf_pool_create_thread_key; extern mysql_pfs_key_t buf_dump_thread_key; extern mysql_pfs_key_t buf_resize_thread_key; +extern mysql_pfs_key_t buf_lru_manager_thread_key; extern mysql_pfs_key_t clone_ddl_thread_key; extern mysql_pfs_key_t clone_gtid_thread_key; extern mysql_pfs_key_t ddl_thread_key; diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index c282f90db181..7aae2ed4c45b 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -601,6 +601,16 @@ static monitor_info_t innodb_counter_info[] = { MONITOR_LRU_UNZIP_SEARCH_SCANNED, MONITOR_LRU_UNZIP_SEARCH_SCANNED_PER_CALL}, + {"buffer_LRU_single_page_flush_count", "buffer", + "Times a user thread issued its own single-page flush while" + " searching for a free block", + MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_LRU_SINGLE_PAGE_FLUSH_COUNT}, + + {"buffer_LRU_flush_await_count", "buffer", + "Times a user thread waited for an in-progress LRU flush instead of" + " issuing single-page flush", + MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_LRU_FLUSH_AWAIT_COUNT}, + /* ========== Counters for Buffer Page I/O ========== */ {"module_buffer_page", "buffer_page_io", "Buffer Page I/O Module", static_cast(MONITOR_MODULE | MONITOR_GROUP_MODULE), diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 1b0c0d997a0f..56199ac81a36 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -231,7 +231,7 @@ bool srv_buf_pool_lazy_latch_init = false; /** Whether buffer pool allocations (huge and regular pages) are pre-populated (MAP_POPULATE and the explicit prefault step) at allocation time. Exposed as the innodb_buffer_pool_populate system variable. */ -bool srv_buf_pool_populate = false; +bool srv_buf_pool_populate = true; #ifdef UNIV_DEBUG /** Force all user tables to use page compression. */ @@ -460,6 +460,8 @@ bool srv_validate_tablespace_paths = true; bool srv_use_fdatasync = false; /** Scan depth for LRU flush batch i.e.: number of blocks scanned*/ ulong srv_LRU_scan_depth = 1024; +/** Whether per-pool LRU manager threads are enabled (after recovery). */ +bool srv_lru_threads_enabled = false; /** Whether or not to flush neighbors of a block */ ulong srv_flush_neighbors = 1; /** Previously requested size. Accesses protected by memory barriers. */ @@ -1209,6 +1211,12 @@ static void srv_init(void) { UT_NEW_THIS_FILE_PSI_KEY, ut::Count{srv_threads.m_page_cleaner_workers_n}); + /* One LRU manager thread per buf_pool instance. */ + srv_threads.m_lru_managers_n = srv_buf_pool_instances; + + srv_threads.m_lru_managers = ut::new_arr_withkey( + UT_NEW_THIS_FILE_PSI_KEY, ut::Count{srv_threads.m_lru_managers_n}); + srv_sys = static_cast( ut::zalloc_withkey(UT_NEW_THIS_FILE_PSI_KEY, srv_sys_sz)); @@ -1317,6 +1325,18 @@ void srv_free(void) { srv_threads.m_page_cleaner_workers = nullptr; } + if (srv_threads.m_lru_managers != nullptr) { + for (size_t i = 0; i < srv_threads.m_lru_managers_n; ++i) { + srv_threads.m_lru_managers[i] = {}; + } + /* Allocated with ut::new_arr_withkey(), so it must be released + with ut::delete_arr() (which runs the element destructors and uses the + matching array allocator), not ut::free(). This mirrors the + m_page_cleaner_workers / m_purge_workers teardown above. */ + ut::delete_arr(srv_threads.m_lru_managers); + srv_threads.m_lru_managers = nullptr; + } + if (srv_threads.m_purge_workers != nullptr) { for (size_t i = 0; i < srv_threads.m_purge_workers_n; ++i) { srv_threads.m_purge_workers[i] = {}; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 3dc5ada3a046..4fecf9947f16 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2699,7 +2699,9 @@ static void srv_shutdown_page_cleaners() { here to let it complete the flushing of the buffer pools before proceeding further. */ - for (uint32_t count = 0; buf_flush_page_cleaner_is_active(); ++count) { + for (uint32_t count = 0; buf_flush_page_cleaner_is_active() || + buf_flush_active_lru_managers() > 0; + ++count) { if (count >= SHUTDOWN_SLEEP_ROUNDS) { ib::info(ER_IB_MSG_1251); count = 0; @@ -2709,6 +2711,8 @@ static void srv_shutdown_page_cleaners() { std::chrono::microseconds(SHUTDOWN_SLEEP_TIME_US)); } + ut_ad(buf_flush_active_lru_managers() == 0); + ut_ad(buf_pool_pending_io_reads_count() == 0); ut_ad(buf_pool_pending_io_writes_count() == 0); } diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 07ba18a6ce78..151ec5edca7e 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -38,6 +38,7 @@ this program; if not, write to the Free Software Foundation, Inc., *******************************************************/ #include "sync0debug.h" +#include "univ.i" #include #include @@ -684,7 +685,12 @@ const latch_t *LatchDebug::find(const Latches *latches, @param[in] level The level to lookup @return latch if found or NULL */ const latch_t *LatchDebug::find(latch_level_t level) UNIV_NOTHROW { - return (find(thread_latches(), level)); + /* A thread which has not acquired any latch yet has no latch list + allocated (thread_latches() without the create flag returns nullptr) + and thus holds nothing at any level. */ + const Latches *latches = thread_latches(); + + return (latches != nullptr ? find(latches, level) : nullptr); } /** @@ -1269,7 +1275,9 @@ static void sync_latch_meta_init() UNIV_NOTHROW { LATCH_ADD_MUTEX(FTS_PLL_TOKENIZE, SYNC_FTS_TOKENIZE, fts_pll_tokenize_mutex_key); - LATCH_ADD_MUTEX(HASH_TABLE_MUTEX, SYNC_BUF_PAGE_HASH, hash_table_mutex_key); + /* Do not use SYNC_BUF_PAGE_HASH for HASH_TABLE_MUTEX. + @see buf_buddy_no_page_hash_latch_validate(). */ + LATCH_ADD_MUTEX(HASH_TABLE_MUTEX, SYNC_ANY_LATCH, hash_table_mutex_key); LATCH_ADD_MUTEX(IBUF_BITMAP, SYNC_IBUF_BITMAP_MUTEX, ibuf_bitmap_mutex_key); diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 3520334da16a..7227ad123baa 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -307,6 +307,42 @@ rw_lock_t::~rw_lock_t() { ut_d(magic_n = 0); } +#ifdef UNIV_DEBUG +/** Asserts that the calling thread is allowed to start waiting for the +given rw-lock. + +This is the hook for latch-order rules which cannot be expressed through +latch_level_t ordering and which constrain only actual waits, not +non-blocking acquisitions (the rw_lock_*_nowait() variants never reach +this, by design: an acquisition that never waits cannot participate in a +deadlock cycle). It is called at the only spots where an rw-lock +acquisition starts to wait - every unbounded wait, spinning included, +funnels into a sync array reservation. + +Currently there is one such rule, for the buffer block frame locks +(buf_block_t::lock): a thread must not wait for a frame lock while +holding any buffer pool LRU list mutex. buf_page_init_for_read() acquires +buf_pool_t::LRU_list_mutex while holding the X-latch on the frame of the +page being read in, so such a wait would form a deadlock cycle with that +path. The rule cannot use latch levels because buf_block_t::lock is +registered with SYNC_LEVEL_VARYING (B-tree page latching order genuinely +varies), and LatchDebug ignores such latches entirely - both when they +are acquired and when they are held. The code paths which do latch a +frame under the LRU list mutex (flushing, and buf_page_create() on a +block freshly taken from the free list) use the nowait variants, so they +are exempt by construction. + +Note that this check is only effective when LatchDebug is enabled +(--innodb-sync-debug); an actual deadlock occurrence is additionally caught +by the sync array deadlock detector, which does track frame rw-locks. */ +namespace { +void rw_lock_assert_wait_allowed(const rw_lock_t *lock) { + ut_ad(lock->get_id() != LATCH_ID_BUF_BLOCK_LOCK || + sync_check_find(SYNC_BUF_LRU_LIST) == nullptr); +} +} // namespace +#endif /* UNIV_DEBUG */ + void rw_lock_s_lock_spin(rw_lock_t *lock, ulint pass, ut::Location location) { ulint i = 0; /* spin round count */ sync_array_t *sync_arr; @@ -347,6 +383,8 @@ void rw_lock_s_lock_spin(rw_lock_t *lock, ulint pass, ut::Location location) { ++count_os_wait; + ut_d(rw_lock_assert_wait_allowed(lock)); + sync_cell_t *cell; sync_arr = @@ -429,6 +467,8 @@ static inline void rw_lock_x_lock_wait_func(rw_lock_t *lock, } /* If there is still a reader, then go to sleep.*/ + ut_d(rw_lock_assert_wait_allowed(lock)); + sync_cell_t *cell; sync_arr = sync_array_get_and_reserve_cell(lock, RW_LOCK_X_WAIT, @@ -649,6 +689,8 @@ void rw_lock_x_lock_func(rw_lock_t *lock, ulint pass, ut::Location location) { } } + ut_d(rw_lock_assert_wait_allowed(lock)); + sync_cell_t *cell; sync_arr = sync_array_get_and_reserve_cell(lock, RW_LOCK_X, location, &cell); @@ -714,6 +756,8 @@ void rw_lock_sx_lock_func(rw_lock_t *lock, ulint pass, ut::Location location) { } } + ut_d(rw_lock_assert_wait_allowed(lock)); + sync_cell_t *cell; sync_arr = sync_array_get_and_reserve_cell(lock, RW_LOCK_SX, location, &cell); diff --git a/storage/temptable/src/handler.cc b/storage/temptable/src/handler.cc index 7c3e8e0058bd..929e811a6972 100644 --- a/storage/temptable/src/handler.cc +++ b/storage/temptable/src/handler.cc @@ -877,6 +877,7 @@ ulong Handler::index_flags(uint index_no, uint, bool) const { case HA_KEY_ALG_SE_SPECIFIC: case HA_KEY_ALG_RTREE: case HA_KEY_ALG_FULLTEXT: + case HA_KEY_ALG_VECTOR: flags = 0; break; } diff --git a/storage/temptable/src/table.cc b/storage/temptable/src/table.cc index 3c87e1e7728c..ef3504aee96f 100644 --- a/storage/temptable/src/table.cc +++ b/storage/temptable/src/table.cc @@ -290,6 +290,7 @@ void Table::indexes_create() { case HA_KEY_ALG_SE_SPECIFIC: case HA_KEY_ALG_RTREE: case HA_KEY_ALG_FULLTEXT: + case HA_KEY_ALG_VECTOR: DBUG_ABORT(); } }