From ac4c5139affa55cb82cf2f10e96f53727278d1e5 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 29 Jun 2026 23:44:43 +0530 Subject: [PATCH] MDEV-39061 mariadb-backup compatible wrapper for BACKUP SERVER This adds a shell script that lets users keep using their existing mariadb-backup commands while the real work is done by the new server-side BACKUP SERVER command. The goal is "drop-in": users should not have to change their backup scripts. extra/mariabackup/scripts/mariadb-backup-server.sh (plain POSIX sh) understands the usual mariadb-backup modes and translates each one. A companion helper, extra/mariabackup/scripts/mbstream-server.sh, lets streamed backups be unpacked by pipelines that expect the mbstream CLI. Both are documented in extra/mariabackup/scripts/README.md. --backup ======== Connects with the mariadb client and runs "BACKUP SERVER TO ''". Connection options (--user, --host, --port, --socket, --defaults-file, ssl, ...) are passed through to the client; --parallel=N becomes the " CONCURRENT" clause. After the backup it writes backup-prepare.cnf into the backup directory, recording what --prepare needs later: where mariadbd lives, the InnoDB parameters (page size, data file path, undo tablespaces, checksum algorithm, log file size), and if the server is encrypted then how to reload the encryption key plugin (the file_key_management variables), so an encrypted backup can be prepared without extra input. --backup --stream ================= Runs "BACKUP SERVER WITH [N CONCURRENT] ''": the server feeds each stream's tar to , the wrapper collects the parts, writes them to stdout, then appends backup-prepare.cnf as a final tar. The per-stream tars carry no end-of-archive marker; only the trailing backup-prepare.cnf adds the single end marker, so the whole stream extracts with a plain "tar -x". Two properties follow from how BACKUP SERVER streams, both differing from mariadb-backup: - local: the stream command runs inside the server, so the wrapper must share its filesystem; - tar only: any --stream= (including xbstream) yields tar. --target-dir is optional in stream mode (scratch for the per-stream parts; a mktemp dir is used otherwise). mbstream-server.sh maps the mbstream CLI onto a plain "tar -x"/"tar -c", so existing "mbstream -x"/"-c" pipelines keep working on the wrapper's stream. mbstream-only flags (-p/--parallel, ...) are accepted and ignored; any other unknown option is rejected. Environment overrides (mainly for testing): MARIADB (client), MARIADBD (the --prepare bootstrap server) and TAR (the tar implementation, e.g. TAR=bsdtar) can each be overridden. To run the bootstrap under rr, put it in MARIADBD and let rr's own _RR_TRACE_DIR choose the trace location, e.g. _RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' --prepare ========= Starts "mariadbd --bootstrap" on the backup directory using backup-prepare.cnf as its defaults file, replays the archived redo log between the start and target LSN read from backup.cnf, then builds a fresh ib_logfile0 so a normal server can start on the directory. mariadbd is taken from the path recorded in backup-prepare.cnf if that binary exists, otherwise from PATH. User --defaults-file/-extra-file and encryption options are layered onto the bootstrap. --copy-back / --move-back ========================= Copy or move a prepared backup into the datadir. The datadir is created if missing, a non-empty datadir is refused unless --force-non-empty-directories is given, and a chown reminder is printed. If --aria-log-dir-path is given, the Aria logs (aria_log_control, aria_log.*) are relocated into that directory. Packaging ========= The wrapper is not installed by default and never replaces the real mariadb-backup / mbstream binaries. 1. cmake -DWITH_MARIABACKUP_WRAPPER=ON (default OFF) controls it. 2. When ON, the scripts install as /usr/bin/mariadb-backup-server and /usr/bin/mbstream-server, tagged COMPONENT Backup so they ship in the mariadb-backup package. 3. RPM: nothing extra to do. the component handles it. 4. DEB: not wired. debian/rules uses --fail-missing and does not enable the option, so the -server binaries are not listed. To ship via DEB, make a paired change: add -DWITH_MARIABACKUP_WRAPPER=ON in debian/rules and list both usr/bin/mariadb-backup-server and usr/bin/mbstream-server in debian/mariadb-backup.install together. 5. The real mariadb-backup/mbstream binaries and the mariabackup symlink are left untouched; opt in via an alias or a symlink early in PATH. Limitations (not supported yet) =============================== 1) Incremental backup & prepare (--incremental-basedir, --incremental-dir, --apply-log-only) 2) --rollback-xa 3) Partial backup (--databases, --tables, --tables-file) 4) Output compression and encryption (--compress, --encrypt) 5) --export is accepted but only warns and runs a plain recovery 6) --extra-lsndir is ignored 7) Windows: POSIX sh only, not installed on Windows Behaviour differences from native mariadb-backup ================================================ - The wrapper needs the mariadb client on PATH for --backup, and mariadbd on PATH (or recorded in backup-prepare.cnf) for --prepare - BACKUP SERVER refuses an already-existing target directory - BACKUP SERVER does copy the data file as raw pages without checksum validation, so a corrupted table is not detected at backup time - --prepare only works on a wrapper-made backup. It needs backup-prepare.cnf) - --stream is tar, not xbstream, and local-only Tests ===== include/have_mariabackup_wrapper.inc redirects $XTRABACKUP to mariadb-backup-server.sh and $XBSTREAM to mbstream-server.sh, skipping when a wrapper or the mariadb client is unavailable. include/have_mariabackup_combination.inc runs a test under both the [CLIENT] mariadb-backup binary and the [SERVER] wrapper. --- extra/mariabackup/CMakeLists.txt | 25 ++ extra/mariabackup/scripts/README.md | 263 +++++++++++++ .../scripts/mariadb-backup-server.sh | 366 ++++++++++++++++++ extra/mariabackup/scripts/mbstream-server.sh | 64 +++ .../have_mariabackup_combination.combinations | 3 + .../include/have_mariabackup_combination.inc | 5 + .../include/have_mariabackup_wrapper.inc | 38 ++ .../suite/mariabackup/aria_encrypted.test | 2 + .../suite/mariabackup/aria_log_dir_path.inc | 104 +++++ .../suite/mariabackup/aria_log_dir_path.test | 107 +---- .../mariabackup/aria_log_dir_path_rel.test | 2 +- .../mariabackup/defer_space,SERVER.rdiff | 10 + mysql-test/suite/mariabackup/defer_space.test | 8 +- .../mariabackup/full_backup,SERVER.rdiff | 30 ++ mysql-test/suite/mariabackup/full_backup.test | 16 +- mysql-test/suite/mariabackup/huge_lsn.test | 2 +- .../suite/mariabackup/log_tables,SERVER.rdiff | 16 + mysql-test/suite/mariabackup/log_tables.test | 4 + mysql-test/suite/mariabackup/mdev-18438.test | 6 + .../mariabackup/partition_notwin,SERVER.rdiff | 8 + .../suite/mariabackup/partition_notwin.test | 13 +- .../suite/mariabackup/relative_path.test | 1 + .../mariabackup/row_format_redundant.test | 1 + mysql-test/suite/mariabackup/small_ibd.test | 1 + .../mariabackup/undo_space_id,SERVER.rdiff | 13 + .../suite/mariabackup/undo_space_id.test | 4 + .../suite/mariabackup/undo_truncate.test | 1 + .../unencrypted_page_compressed,SERVER.rdiff | 8 + .../unencrypted_page_compressed.test | 6 +- mysql-test/suite/mariabackup/vector.test | 1 + 30 files changed, 1016 insertions(+), 112 deletions(-) create mode 100644 extra/mariabackup/scripts/README.md create mode 100755 extra/mariabackup/scripts/mariadb-backup-server.sh create mode 100755 extra/mariabackup/scripts/mbstream-server.sh create mode 100644 mysql-test/include/have_mariabackup_combination.combinations create mode 100644 mysql-test/include/have_mariabackup_combination.inc create mode 100644 mysql-test/include/have_mariabackup_wrapper.inc create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path.inc create mode 100644 mysql-test/suite/mariabackup/defer_space,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/full_backup,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/log_tables,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff create mode 100644 mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a71030887c43f..dff27afb0c44e 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -114,3 +114,28 @@ ADD_DEPENDENCIES(mbstream GenError) IF(MSVC) SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) ENDIF() + + +######################################################################## +# mariadb-backup-server: BACKUP SERVER-compatible shell wrapper +######################################################################## +# A drop-in mariadb-backup-compatible POSIX-sh wrapper that translates the +# CLI into server-side BACKUP SERVER SQL. Experimental; OFF by default +# Installed as a mariadb-backup-server; clients opt in via symlink/alias +# (see extra/mariabackup/scripts/README.md). +OPTION(WITH_MARIABACKUP_WRAPPER + "Install the BACKUP SERVER shell wrapper (mariadb-backup-server)" OFF) +ADD_FEATURE_INFO(MARIABACKUP_WRAPPER WITH_MARIABACKUP_WRAPPER + "BACKUP SERVER-compatible mariadb-backup shell wrapper") + +IF(WITH_MARIABACKUP_WRAPPER AND NOT WIN32) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mariadb-backup-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) + + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbstream-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mbstream-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mbstream-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) +ENDIF() diff --git a/extra/mariabackup/scripts/README.md b/extra/mariabackup/scripts/README.md new file mode 100644 index 0000000000000..f9253022a36a0 --- /dev/null +++ b/extra/mariabackup/scripts/README.md @@ -0,0 +1,263 @@ +# mariadb-backup-server.sh — a BACKUP SERVER wrapper + +`mariadb-backup-server.sh` makes the server-side `BACKUP SERVER` command look +like the old `mariadb-backup` tool. You call it with the familiar +`--backup` / `--prepare` / `--copy-back` options; under the hood +it just runs `BACKUP SERVER TO ''` over a normal `mariadb` +connection and lets the server do the work. It's plain POSIX `sh`, +so it runs anywhere `mariadb-backup` does. + +You need a server that supports `BACKUP SERVER`, the `mariadb` client on +`PATH`, and an account allowed to run `BACKUP SERVER`. The parent of the +target directory has to exist and be writable. + +## Installing / enabling it + +The wrapper is experimental and off by default. Build with +`-DWITH_MARIABACKUP_WRAPPER=ON` and it installs next to the real +binaries, under its own names so it never shadows them: + +``` +/usr/bin/mariadb-backup # the C++ binary, unchanged +/usr/bin/mariadb-backup-server # this wrapper +/usr/bin/mbstream # the C++ binary, unchanged +/usr/bin/mbstream-server # the tar-based mbstream shim (for --stream) +``` + +To send your existing `mariadb-backup` (and `mbstream`) calls through the +wrapper instead, point the names at them yourself — an alias, or a symlink +earlier in `PATH`: + +```sh +alias mariadb-backup=mariadb-backup-server +alias mbstream=mbstream-server +# or +ln -s /usr/bin/mariadb-backup-server ~/bin/mariadb-backup +ln -s /usr/bin/mbstream-server ~/bin/mbstream +``` + +`mbstream-server` is only needed for `--stream` backups (it extracts the +wrapper's tar stream). + +## Backing up + +```sh +mariadb-backup-server --backup --target-dir=/backup/full +``` + +That runs `BACKUP SERVER TO '/backup/full'`. Use `--parallel=N` to ask for N +concurrent streams (`... N CONCURRENT`; N=1 is the default and changes +nothing). + +Connection options: +`--user`, `--password`, `--host`, `--port`, `--socket`, `--defaults-file`, +`--defaults-extra-file` and their short forms are passed straight to +the `mariadb` client. + +`--throttle`, `--no-lock` and `--safe-slave-backup` are accepted and ignored; + +When the backup finishes the wrapper drops a `backup-prepare.cnf` into the +target dir, next to the server's own `backup.cnf`. It records where `mariadbd` +lives and the InnoDB layout, so `--prepare` can recover the backup later +without you respelling all of that. + +### Streaming to stdout + +```sh +mariadb-backup-server --backup --stream=tar > /backup/full.tar +``` + +`--stream` runs `BACKUP SERVER WITH [N CONCURRENT] ''` instead of +`... TO ''`. The server hands each stream's tar to ``, the wrapper +collects the parts and writes them to its stdout, so you can redirect to a file +or pipe onwards. + +Two things follow from how `BACKUP SERVER` streams, and both differ from +`mariadb-backup`: + +- **Local only.** The stream command runs *inside the server*, so its output +lands on the server's filesystem. The wrapper can only pick it up when it runs +on the same host (a shared filesystem), as the user the server writes as +(typically `mysql`), or with a `--target-dir` the server can write. A remote +`--host` cannot stream this way. + +- **tar only.** The server emits tar, never `xbstream`. Any `--stream=` +value (including `xbstream`) is accepted but the output is always tar. + +`--target-dir` is optional here; when given it is just the scratch directory for +the per-stream parts (otherwise a `mktemp` dir is used). + +The output is the per-stream tar entries concatenated, with +`backup-prepare.cnf` appended as the trailing archive. + +```sh +mkdir restore && tar -xf /backup/full.tar -C restore +``` + +After extraction the directory holds the data files, the server's `backup.cnf` +*and* the wrapper's `backup-prepare.cnf`, so it can be prepared exactly like a +directory backup: + +```sh +mariadb-backup-server --prepare --target-dir=restore +``` + +### `mbstream.sh` — the extraction shim + +The real `mbstream`/`xbstream` binary cannot read the wrapper's stream, because +that stream is plain tar, not the `xbstream` format. So a companion shim, +`mbstream.sh`, ships next to `mariadb-backup-server.sh` and maps the `mbstream` CLI onto +`tar`, letting existing pipelines (and tests) that call `mbstream` keep working +unchanged: + +```sh +mbstream-server -x -C restore < /backup/full.tar # tar -x -C restore +mbstream-server -c -C dir file1 file2 # tar -c -C dir file1 file2 +``` + +Notes: + +- **Extraction is a plain `tar -x`** — the stream has a single +end-of-archive marker + +- **mbstream-only options are accepted and ignored** — +`-p` / `--parallel`, compression flags, `-v` : So legacy +invocations don't break. + +- It understands `-x` (extract, from stdin) and `-c` (create, to stdout), +with `-C `; anything else is treated as a file operand or ignored. + +- It is a thin tar wrapper, so it only understands the wrapper's tar streams — +do not point it at a real `xbstream` archive, and do not feed the wrapper's +output to the real `mbstream`. + +## Preparing + +```sh +mariadb-backup-server --prepare --target-dir=/backup/full +``` + +Prepare makes the backup bootable: it starts `mariadbd --bootstrap` on the +target directory, replays the archived redo up to the backup's end LSN, then +replaces the archived log with a fresh `ib_logfile0` so a normal server can +start on it. Both `backup.cnf` and `backup-prepare.cnf` have to be there (they +are, if this wrapper took the backup). + +Options: + +- `--use-memory=N` — buffer pool size during recovery. +- `--innodb-*` and `--tmpdir` are forwarded to the bootstrap server. +- `--defaults-file` / `--defaults-extra-file`, and encryption options such as + `--file-key-management*` / `--loose-file-key-management*` / `--plugin-load-add`, + are layered onto the bootstrap (as an extra defaults file / extra options) so + you can supply anything `backup-prepare.cnf` did not record. + +The `mariadbd` used for the bootstrap is the path recorded in +`backup-prepare.cnf` at backup time if that binary exists; otherwise `mariadbd` +from `PATH`. (Recorded-first matters: it is the same version that took the +backup, so it can always parse the backed-up tablespace format.) + +## Restoring + +With the backup prepared and the server stopped, put it into the datadir: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql +mariadb-backup-server --move-back --target-dir=/backup/full --datadir=/var/lib/mysql +``` + +`--copy-back` leaves the backup where it is; `--move-back` renames the files +across, which is quicker on the same filesystem but consumes the backup. Either +way the datadir is created if missing, and the wrapper won't write into a +non-empty datadir unless you add `--force-non-empty-directories`. + +If the source server kept its Aria logs outside the datadir, pass the same +`--aria-log-dir-path=` you use on the server. The wrapper creates that +directory and moves the restored `aria_log_control` / `aria_log.*` files into +it, so the server finds them on restart: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql \ + --aria-log-dir-path=/var/lib/aria_logs +``` + +Neither fixes ownership, so finish up with: + +```sh +chown -R mysql:mysql /var/lib/mysql +systemctl start mariadb +``` + +## What it doesn't do + +These stop with an error instead of quietly producing an incomplete backup: + +- incremental backup/prepare: `--incremental-basedir`, `--incremental-dir`, `--apply-log-only` +- partial backup: `--databases`, `--tables`, `--tables-file`. This needs server-side +`backup_include`/`backup_exclude`, which don't exist yet +- compression and encryption of the output: `--compress`, `--encrypt` (error out) +- `--rollback-xa`: not supported (errors out) + +`--stream` is supported with the caveats above (local only, tar only, extract +with a plain `tar -x`); + +`--export` is accepted but not implemented: it warns and does a plain recovery +(no per-table `.cfg` files for IMPORT TABLESPACE). + +## Environment overrides + +The wrapped commands can be overridden via environment variables, +mainly for testing: + +- `MARIADB`: the client used to talk to the server (default `mariadb`), +e.g. `MARIADB='mariadb --protocol=tcp'`. + +- `MARIADBD`: the server binary the `--prepare` bootstrap runs (by default the +path recorded in `backup-prepare.cnf`, else `mariadbd` on `PATH`). When set it +overrides that resolution. To run it under `rr`, put `rr` in `MARIADBD` and let +`rr`'s own `_RR_TRACE_DIR` choose where the trace goes, e.g. +`_RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ...`. + +- `TAR`: the tar implementation (default `tar`), +e.g. `TAR=bsdtar` (libarchive-tools). +Used by `--stream` and by `mbstream-server`. + +## The two .cnf files + +`backup.cnf` is written by the server and tells `--prepare` what parts of redo to replay: + +```ini +[server] +# checkpoint=54088 +innodb_log_recovery_start=54088 # recovery starts scanning here +innodb_log_recovery_target=56337 # the backup's end LSN; recovery stops here +``` + +`backup-prepare.cnf` is written by the wrapper and handed to the prepare +bootstrap as its defaults file: + +```ini +# mariadbd=/usr/sbin/mariadbd +[mariadbd] +innodb_page_size=16384 +innodb_data_file_path=ibdata1:12M:autoextend +innodb_undo_tablespaces=3 +innodb_checksum_algorithm=full_crc32 +innodb_log_file_size=100663296 +``` + +If the server is encrypted it also records how to load the key-management +plugin again, so an encrypted backup can be prepared without extra input. +For `file_key_management` it captures every plugin variable (the same way +`mariadb-backup` writes them into `backup-my.cnf`) + +```ini +plugin-load-add=file_key_management +innodb_encrypt_log=ON +loose-file-key-management +loose-file_key_management_filename=/etc/mysql/keys.enc +loose-file_key_management_filekey=FILE:/etc/mysql/keyfile.key +loose-file_key_management_encryption_algorithm=aes_cbc +loose-file_key_management_digest=sha1 +loose-file_key_management_use_pbkdf2=1 +``` diff --git a/extra/mariabackup/scripts/mariadb-backup-server.sh b/extra/mariabackup/scripts/mariadb-backup-server.sh new file mode 100755 index 0000000000000..57a64109b4159 --- /dev/null +++ b/extra/mariabackup/scripts/mariadb-backup-server.sh @@ -0,0 +1,366 @@ +#!/bin/sh +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The wrapped commands can be overridden for testing, e.g. +# MARIADB='mariadb --protocol=tcp', TAR=bsdtar. + +# To run the prepare bootstrap under rr, include it in MARIADBD +# (rr's own _RR_TRACE_DIR controls where the trace is written). +# e.g. _RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ... +: "${MARIADB:=mariadb}" +: "${TAR:=tar}" + +MODE= +TARGET_DIR= +DATADIR= +PARALLEL= +USE_MEMORY= +FORCE_NON_EMPTY= +EXPORT= +ROLLBACK_XA= +MARIADB_OPTS= +INNODB_OPTS= +MYSQLD_EXTRA= +PREPARE_DEFAULTS= +ARIA_LOG_DIR= +STREAM= + +while [ $# -gt 0 ]; do + case $1 in + --backup) MODE=backup ;; + --prepare|--apply-log) MODE=prepare ;; + --copy-back) MODE=copy-back ;; + --move-back) MODE=move-back ;; + + --target-dir=*) TARGET_DIR=${1#*=} ;; + --datadir=*) DATADIR=${1#*=} ;; + --aria-log-dir-path=*) ARIA_LOG_DIR=${1#*=} ;; + --use-memory=*) USE_MEMORY=${1#*=} ;; + --parallel=*) PARALLEL=${1#*=} ;; + + --export) EXPORT=1 ;; + --rollback-xa) ROLLBACK_XA=1 ;; + --force-non-empty-directories) FORCE_NON_EMPTY=1 ;; + + --innodb|--innodb=*|--innodb-*|--innodb_*|--skip-innodb-*|--skip_innodb_*) + INNODB_OPTS="$INNODB_OPTS $1" ;; + --tmpdir=*) MYSQLD_EXTRA="$MYSQLD_EXTRA $1" ;; + -t) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=$2"; shift ;; + -t*) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=${1#-t}" ;; + --incremental-basedir=*|--incremental-dir=*) + die "incremental backup/prepare is not supported" ;; + --apply-log-only) + die "--apply-log-only is not supported" ;; + --databases=*|--databases-exclude=*|--tables=*|--tables-exclude=*|--tables-file=*) + die "partial backup needs server-side backup_include/backup_exclude, which doesn't exist yet" ;; + # BACKUP SERVER only ever produces tar + --stream|--stream=*) STREAM=1 ;; + --compress|--compress=*|--compress-threads=*) die "--compress is not supported" ;; + --encrypt|--encrypt=*) die "--encrypt is not supported" ;; + --innobackupex) die "innobackupex mode is not supported" ;; + + # Defaults files feed the backup client + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1" + PREPARE_DEFAULTS="$PREPARE_DEFAULTS --defaults-extra-file=${1#*=}" ;; + # Encryption options are forwarded to the prepare bootstrap. + --file-key-management*|--plugin-load-add=*|--loose-file-key-management*|\ + --aria-encrypt-tables*|--encrypt-tmp-disk-tables*) + PREPARE_DEFAULTS="$PREPARE_DEFAULTS $1" ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*|\ + --defaults-group=*|\ + --secure-auth|--skip-secure-auth|--ssl|--ssl-verify-server-cert|\ + --ssl-ca=*|--ssl-capath=*|--ssl-cert=*|--ssl-cipher=*|\ + --ssl-crl=*|--ssl-crlpath=*|--ssl-key=*|--tls-version=*) + MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -p) + if [ -n "${2-}" ] && case $2 in -*) false ;; *) true ;; esac; then + MARIADB_OPTS="$MARIADB_OPTS -p$2"; shift + else + MARIADB_OPTS="$MARIADB_OPTS -p" + fi ;; + -u|-P|-S) MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift ;; + -H) MARIADB_OPTS="$MARIADB_OPTS --host=$2"; shift ;; + -p*|-u*|-P*|-S*) MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -H*) MARIADB_OPTS="$MARIADB_OPTS --host=${1#-H}" ;; + + -h) DATADIR=$2; shift ;; + -h*) DATADIR=${1#-h} ;; + + # Everything else is accepted and ignored: + *) ;; + esac + shift +done + +# In stream mode --target-dir is optional: +# it is only a scratch directory for the per-stream tar parts +# (a mktemp dir is used when it is omitted). +[ -n "$STREAM" ] || [ -n "$TARGET_DIR" ] || die "--target-dir required" + +# Run the client with the connection options we collected. +ask() { $MARIADB $MARIADB_OPTS -BN -e "$1" 2>/dev/null; } + +# Print the backup-prepare.cnf contents to stdout. +# It captures everything --prepare's offline bootstrap needs: +# where mariadbd lives, the InnoDB parameters, and how to +# reload the encryption key plugin. Used both for a +# directory backup (written into the target dir) +# and a streamed backup (appended to the stream so it lands +# in the extracted directory). + +write_prepare_cnf() { + _mariadbd= + _pidfile=$(ask "SELECT @@global.pid_file") + if [ -n "$_pidfile" ] && [ -r "$_pidfile" ]; then + _pid=$(cat "$_pidfile" 2>/dev/null) + [ -n "$_pid" ] && _mariadbd=$(readlink -f "/proc/$_pid/exe" 2>/dev/null) + fi + if [ -z "$_mariadbd" ]; then + _basedir=$(ask "SELECT @@global.basedir") + for _c in "$_basedir/sbin/mariadbd" "$_basedir/bin/mariadbd" \ + "$_basedir/sbin/mysqld" "$_basedir/bin/mysqld"; do + [ -x "$_c" ] && { _mariadbd=$_c; break; } + done + fi + + _page_size=$(ask "SELECT @@global.innodb_page_size") + _data_file_path=$(ask "SELECT @@global.innodb_data_file_path") + _undo_ts=$(ask "SELECT @@global.innodb_undo_tablespaces") + _checksum=$(ask "SELECT @@global.innodb_checksum_algorithm") + _log_file_size=$(ask "SELECT @@global.innodb_log_file_size") + + _enc=$(ask "SELECT LOWER(plugin_name) FROM information_schema.PLUGINS + WHERE plugin_type='ENCRYPTION' AND plugin_status='ACTIVE' LIMIT 1") + + [ -n "$_mariadbd" ] && echo "# mariadbd=$_mariadbd" + echo "[mariadbd]" + [ -n "$_page_size" ] && echo "innodb_page_size=$_page_size" + [ -n "$_data_file_path" ] && echo "innodb_data_file_path=$_data_file_path" + [ -n "$_undo_ts" ] && echo "innodb_undo_tablespaces=$_undo_ts" + [ -n "$_checksum" ] && echo "innodb_checksum_algorithm=$_checksum" + [ -n "$_log_file_size" ] && echo "innodb_log_file_size=$_log_file_size" + + if [ -n "$_enc" ]; then + echo "plugin-load-add=$_enc" + _plugin_dir=$(ask "SELECT @@global.plugin_dir") + [ -n "$_plugin_dir" ] && echo "plugin-dir=$_plugin_dir" + case $(ask "SELECT @@global.innodb_encrypt_log") in + 1|ON) echo "innodb_encrypt_log=ON" ;; + esac + if [ "$_enc" = file_key_management ]; then + echo "loose-file-key-management" + ask "SHOW VARIABLES LIKE 'file_key_management%'" | + while read -r _fkm_name _fkm_value; do + [ -n "$_fkm_name" ] && echo "loose-$_fkm_name=$_fkm_value" + done + fi + fi +} + + +# prepare +if [ "$MODE" = prepare ]; then + [ -z "$ROLLBACK_XA" ] || die "--rollback-xa is not supported" + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + + cnf=$TARGET_DIR/backup-prepare.cnf + [ -f "$cnf" ] || die "$cnf missing - was this backup made by the wrapper?" + [ -z "$EXPORT" ] || echo "$me: --export not implemented, doing a plain recovery" >&2 + + # Prefer the binary recorded at backup time + # else, fall back to mariadbd on PATH only if the + # recorded one is missing. + # MARIADBD overrides the recorded/PATH binary + if [ -n "${MARIADBD-}" ]; then + mariadbd=$MARIADBD + else + mariadbd=$(sed -n 's/^# *mariadbd=//p' "$cnf" | tail -n1) + [ -n "$mariadbd" ] && [ -x "$mariadbd" ] || mariadbd=mariadbd + fi + + # backup.cnf tells us the LSN window recovery should replay. + start=$(grep '^innodb_log_recovery_start' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + target=$(grep '^innodb_log_recovery_target' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + + opts="--datadir=$TARGET_DIR --innodb=FORCE" + [ -n "$start" ] && opts="$opts --innodb-log-recovery-start=$start" + [ -n "$target" ] && opts="$opts --innodb-log-recovery-target=$target" + [ -n "$USE_MEMORY" ] && opts="$opts --innodb-buffer-pool-size=$USE_MEMORY" + opts="$opts$INNODB_OPTS$MYSQLD_EXTRA" + + # Recovery stops at the backup's end LSN but leaves the + # archived log (ib_.log) behind, which a normal server + # won't boot from. After recovery we build a fresh ib_logfile0 + # (header + a checkpoint at the end LSN) and put it in place + # of the archived log. + input=/dev/null + newlog=$TARGET_DIR/ib_logfile0.new + if [ -n "$target" ]; then + lsn=$(printf '%016x' "$target") + rm -f "$newlog" + input=$(mktemp) + cat > "$input" <&2 + exit 0 +fi + + +# copy-back / move-back +if [ "$MODE" = copy-back ] || [ "$MODE" = move-back ]; then + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + [ -n "$DATADIR" ] || die "--datadir required for --$MODE" + + # mariadb-backup creates the datadir if it's missing; do the same. + [ -d "$DATADIR" ] || mkdir -p "$DATADIR" || die "cannot create datadir: $DATADIR" + + # Refuse a non-empty datadir (dotfiles included) unless told otherwise. + if [ -z "$FORCE_NON_EMPTY" ]; then + for f in "$DATADIR"/* "$DATADIR"/.[!.]* "$DATADIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } && + die "datadir not empty: $DATADIR (pass --force-non-empty-directories)" + done + fi + + if [ "$MODE" = copy-back ]; then + echo "$me: copying $TARGET_DIR -> $DATADIR" >&2 + cp -R "$TARGET_DIR"/. "$DATADIR"/ || die "copy-back failed" + else + echo "$me: moving $TARGET_DIR -> $DATADIR" >&2 + for f in "$TARGET_DIR"/* "$TARGET_DIR"/.[!.]* "$TARGET_DIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } || continue + mv "$f" "$DATADIR"/ || die "move-back failed" + done + fi + + # Aria logs live in --aria-log-dir-path when set; + # backup placed them at the datadir root, + # so relocate them there and create the directory the server + # expects on restart. + if [ -n "$ARIA_LOG_DIR" ] && [ "$ARIA_LOG_DIR" != "$DATADIR" ]; then + mkdir -p "$ARIA_LOG_DIR" || die "cannot create aria-log-dir-path: $ARIA_LOG_DIR" + for f in "$DATADIR"/aria_log_control "$DATADIR"/aria_log.*; do + [ -e "$f" ] && { mv "$f" "$ARIA_LOG_DIR"/ || die "aria log relocate failed"; } + done + fi + + echo "Restore completed: $DATADIR" >&2 + echo "$me: now run: chown -R mysql:mysql $DATADIR && start the server" >&2 + exit 0 +fi + + +# backup (streaming) +# "BACKUP SERVER WITH [N CONCURRENT] ''" runs +# inside the server feeding that stream's tar to the command's stdin. +if [ -n "$STREAM" ]; then + if [ -n "$TARGET_DIR" ]; then + scratch=$TARGET_DIR + mkdir -p "$scratch" || die "cannot create scratch dir: $scratch" + keep_scratch=1 + else + scratch=$(mktemp -d "${TMPDIR:-/tmp}/mariabackup-stream.XXXXXX") \ + || die "cannot create scratch directory" + keep_scratch= + fi + + cleanup_stream() { + rm -f "$scratch"/.mariabackup-stream.sh "$scratch"/*.tar \ + "$scratch"/backup-prepare.cnf + [ -n "$keep_scratch" ] || rmdir "$scratch" 2>/dev/null + } + + # The server appends the stream index as $1 and writes that stream's tar + # to our stdin; we drop each one into the scratch dir under .tar. + helper=$scratch/.mariabackup-stream.sh + cat > "$helper" < "$scratch/\$1.tar" +EOF + + # Invoke via "/bin/sh " so the script needs no execute bit + # (scratch may sit on a noexec filesystem such as /dev/shm). + sql="BACKUP SERVER WITH" + case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; + esac + sql="$sql '/bin/sh $helper'" + + # Keep stdout clean for the tar: send any client output to stderr. + $MARIADB $MARIADB_OPTS -e "$sql" >&2 \ + || { cleanup_stream; die "BACKUP SERVER failed"; } + + # Concatenate the per-stream tars to stdout in index order, + # then append backup-prepare.cnf as one more tar archive + # so it lands in the extracted directory alongside the + # server's backup.cnf. The per-stream tars carry no + # end-of-archive marker; only this trailing + # backup-prepare.cnf adds the single end marker, + # so the whole stream extracts with a plain "tar -x". + n=1 + while [ -f "$scratch/$n.tar" ]; do + cat "$scratch/$n.tar" || { cleanup_stream; die "streaming to stdout failed"; } + n=$((n + 1)) + done + + echo "$me: appending backup-prepare.cnf to the stream" >&2 + write_prepare_cnf > "$scratch/backup-prepare.cnf" \ + || { cleanup_stream; die "could not build backup-prepare.cnf"; } + $TAR -C "$scratch" -cf - backup-prepare.cnf \ + || { cleanup_stream; die "could not append backup-prepare.cnf to the stream"; } + + cleanup_stream + echo "$me: streamed $((n - 1)) tar stream(s) plus backup-prepare.cnf to stdout" >&2 + exit 0 +fi + + +# --- backup (directory) ----------------------------------------------------- +parent=$(dirname "$TARGET_DIR") +[ -d "$parent" ] || die "parent directory does not exist: $parent" +[ -w "$parent" ] || die "parent directory is not writable: $parent" + +sql="BACKUP SERVER TO '$TARGET_DIR'" +case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is the default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; +esac +$MARIADB $MARIADB_OPTS -e "$sql" || die "BACKUP SERVER failed" + +# Create backup-prepare.cnf with everything --prepare's offline +# bootstrap needs: where mariadbd is, the InnoDB parameters, +# and how to load the key plugin again. +[ -f "$TARGET_DIR/backup.cnf" ] || exit 0 + +echo "$me: writing backup-prepare.cnf" >&2 +write_prepare_cnf > "$TARGET_DIR/backup-prepare.cnf" diff --git a/extra/mariabackup/scripts/mbstream-server.sh b/extra/mariabackup/scripts/mbstream-server.sh new file mode 100755 index 0000000000000..7ba2bbabe7e6d --- /dev/null +++ b/extra/mariabackup/scripts/mbstream-server.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# mbstream-compatible for the BACKUP SERVER wrapper. +# +# This maps the mbstream CLI to `tar`: +# +# mbstream -x -C < archive -> tar -x -C +# mbstream -c > archive -> tar -c +# +# mbstream-only options are accepted and ignored so +# existing invocations keep working unchanged. +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The tar implementation can be overridden for testing. +# e.g. TAR=bsdtar. +: "${TAR:=tar}" + +mode= +dir=. +files= + +while [ $# -gt 0 ]; do + case $1 in + -x|--extract) mode=x ;; + -c|--create) mode=c ;; + -C|--directory) dir=$2; shift ;; + -C*) dir=${1#-C} ;; + --directory=*) dir=${1#*=} ;; + + # mbstream-only flags that have no tar equivalent: drop them. + -p|--parallel) shift ;; + -p*|--parallel=*) ;; + --decompress|--compress) ;; + -v|--verbose) ;; + + --) shift; break ;; + # Reject anything we do not recognise rather than + # silently ignoring it (e.g. GNU tar's -b takes an argument + # that would otherwise be mistaken for a file operand). + # This is an mbstream-on-tar shim, not tar. + -*) die "unsupported option: $1" ;; + *) files="$files $1" ;; + esac + shift +done + +while [ $# -gt 0 ]; do + files="$files $1" + shift +done + +case $mode in + # The wrapper concatenates the per-stream tar entries + # with no end-of-archive marker between them; only the + # trailing backup-prepare.cnf adds the single end marker. + x) exec $TAR -x -f - -C "$dir" ;; + c) + [ -n "$files" ] || files=. + exec $TAR -c -f - -C "$dir" $files ;; + *) die "expected -x (extract) or -c (create)" ;; +esac diff --git a/mysql-test/include/have_mariabackup_combination.combinations b/mysql-test/include/have_mariabackup_combination.combinations new file mode 100644 index 0000000000000..12bc42487d520 --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.combinations @@ -0,0 +1,3 @@ +[CLIENT] + +[SERVER] diff --git a/mysql-test/include/have_mariabackup_combination.inc b/mysql-test/include/have_mariabackup_combination.inc new file mode 100644 index 0000000000000..a2b79d6310efc --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.inc @@ -0,0 +1,5 @@ +if ($MTR_COMBINATION_SERVER) +{ + --source include/have_mariabackup_wrapper.inc +} + diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..d9d8ee82e74ee --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,38 @@ +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through extra/mariabackup/scripts/mariadb-backup-server.sh — the BACKUP +# SERVER compatibility wrapper — without any change to the test body. +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# $XTRABACKUP : now points at mariadb-backup-server.sh + +--source include/not_windows.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mariadb-backup-server.sh +--let MBSTREAM_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mbstream-server.sh + +# The wrapper shells out to the bare `mariadb` client, which mtr does not put +# on PATH. Prepend the build's client directories so it resolves. A `let` with +# no leading $ is exported to the environment of later --exec commands. +--let PATH=$MYSQL_BINDIR/client:$MYSQL_BINDIR/client_release:$MYSQL_BINDIR/client_debug:$MYSQL_BINDIR/bin:$PATH + +--error 0,1 +perl; +my $w = $ENV{MARIABACKUP_WRAPPER}; +my $m = $ENV{MBSTREAM_WRAPPER}; +exit 1 unless $w && -x $w && $m && -x $m; +exit 0; +EOF + +if ($errno) +{ + --skip mariadb-backup-server.sh wrapper unavailable (script or sh missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER +--let XBSTREAM=$MBSTREAM_WRAPPER diff --git a/mysql-test/suite/mariabackup/aria_encrypted.test b/mysql-test/suite/mariabackup/aria_encrypted.test index 6ed3e0d08eff3..017a0b249dcfe 100644 --- a/mysql-test/suite/mariabackup/aria_encrypted.test +++ b/mysql-test/suite/mariabackup/aria_encrypted.test @@ -1,6 +1,7 @@ --source include/have_file_key_management.inc --source include/have_innodb.inc --source include/have_sequence.inc +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-38246 aria_read index failed on encrypted database during backup @@ -16,3 +17,4 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; drop table if exists t1,t2; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.inc b/mysql-test/suite/mariabackup/aria_log_dir_path.inc new file mode 100644 index 0000000000000..1c7d8d12679d8 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.inc @@ -0,0 +1,104 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + +--echo # mariadb-backup --backup +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test index 40bc39446bf00..e479458edfddb 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -1,105 +1,2 @@ ---source include/have_maria.inc - ---echo # ---echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used ---echo # - ---let $datadir=`SELECT @@datadir` ---let $targetdir=$MYSQLTEST_VARDIR/tmp/backup - -if ($ARIA_LOGDIR_MARIADB == '') -{ - --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - -if ($ARIA_LOGDIR_FS == '') -{ - --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - ---let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB - - ---echo # Restart mariadbd with the test specific parameters ---mkdir $ARIA_LOGDIR_FS ---let $restart_parameters=$server_parameters ---source include/restart_mysqld.inc - - ---echo # Create and populate an Aria table (and Aria logs) -CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; -DELIMITER $$; -BEGIN NOT ATOMIC - FOR id IN 0..9 DO - INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); - END FOR; -END; -$$ -DELIMITER ;$$ - - ---echo # Testing aria log files before --backup -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control ---file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - ---echo # mariadb-backup --backup ---disable_result_log ---mkdir $targetdir ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir ---enable_result_log - - ---echo # mariadb-backup --prepare ---disable_result_log ---exec $XTRABACKUP --prepare --target-dir=$targetdir ---enable_result_log - ---echo # shutdown server ---disable_result_log ---source include/shutdown_mysqld.inc ---echo # remove datadir ---rmdir $datadir ---echo # remove aria-log-dir-path ---rmdir $ARIA_LOGDIR_FS - - ---echo # mariadb-backup --copy-back ---let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---exec echo "# with parameters: $mariadb_backup_parameters" ---exec $XTRABACKUP $mariadb_backup_parameters - - ---echo # starting server ---let $restart_parameters=$server_parameters ---source include/start_mysqld.inc ---enable_result_log ---rmdir $targetdir - - ---echo # Check that the table is there after --copy-back -SELECT COUNT(*) from t1; -DROP TABLE t1; - - ---echo # Testing aria log files after --copy-back -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control -#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - - ---echo # Restarting mariadbd with default parameters ---let $restart_parameters= ---source include/restart_mysqld.inc ---rmdir $ARIA_LOGDIR_FS +--source include/have_mariabackup_combination.inc +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test index c8169959929b9..b2875f496daac 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -1,4 +1,4 @@ --let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel ---source aria_log_dir_path.test +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff new file mode 100644 index 0000000000000..f71d98dc11c82 --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff @@ -0,0 +1,10 @@ +--- defer_space.result ++++ defer_space,SERVER.result +@@ -21,7 +21,5 @@ + CREATE TABLE t1(c INT) ENGINE=INNODB; + # Corrupt the table + # restart +-# xtrabackup backup +-FOUND 10 /Header page consists of zero bytes*/ in backup.log + UNLOCK TABLES; + DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/defer_space.test b/mysql-test/suite/mariabackup/defer_space.test index 397a1ff5dc23c..bad0ed259600c 100644 --- a/mysql-test/suite/mariabackup/defer_space.test +++ b/mysql-test/suite/mariabackup/defer_space.test @@ -2,6 +2,7 @@ --source include/have_debug.inc --source include/not_embedded.inc --source include/no_valgrind_without_big.inc +--source include/have_mariabackup_combination.inc call mtr.add_suppression("InnoDB: Expected tablespace id .*"); --echo # Mariabackup --backup with page0 INIT_PAGE redo record @@ -51,6 +52,8 @@ close FILE or die "close"; EOF --source include/start_mysqld.inc +if (!$MTR_COMBINATION_SERVER) +{ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; @@ -62,7 +65,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --let SEARCH_PATTERN=Header page consists of zero bytes* --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc -UNLOCK TABLES; -DROP TABLE t1; rmdir $targetdir; remove_file $backuplog; +} +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff new file mode 100644 index 0000000000000..4e2798c8a3b3d --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff @@ -0,0 +1,30 @@ +--- full_backup.result ++++ full_backup,SERVER.result +@@ -3,10 +3,6 @@ + SET GLOBAL innodb_max_purge_lag_wait=0; + # xtrabackup backup + NOT FOUND /InnoDB: Allocated tablespace ID/ in backup.log +-SELECT variable_value FROM information_schema.global_status +-WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +-variable_value +-0 + INSERT INTO t VALUES(2); + # xtrabackup prepare + # shutdown server +@@ -44,16 +40,3 @@ + # + # MDEV-34713: mariadb_upgrade_info should be backed up and restored + # +-CREATE TABLE t2(i INT) ENGINE INNODB; +-INSERT INTO t2 VALUES(100); +-# xtrabackup backup +-# xtrabackup prepare +-# shutdown server +-# remove datadir +-# xtrabackup move back +-# restart: --innodb_undo_tablespaces=0 +-FOUND 1 /^[0-9]+\.[0-9]+\.[0-9]+/ in mariadb_upgrade_info +-SELECT * FROM t2; +-i +-100 +-DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index 8a1ce756eeb17..1b75e4d2c051a 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc --source include/innodb_page_size.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -8,8 +9,13 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log --disable_result_log +# The new BACKUP SERVER wrapper ignores --innodb-log-write-ahead-size, so the +# invalid-value invocation that is expected to fail does not apply to it. +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=4095 > $backup_log 2>&1; +} exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=10000 --innodb_log_checkpoint_now=1 > $backup_log 2>&1; --enable_result_log @@ -19,8 +25,12 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --source include/search_pattern_in_file.inc --remove_file $backup_log +# Only the native mariabackup leaves dirty pages worth checking here. +if (!$MTR_COMBINATION_SERVER) +{ SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +} INSERT INTO t VALUES(2); @@ -38,6 +48,8 @@ rmdir $targetdir; --echo # MDEV-27121 mariabackup incompatible with disabled dedicated --echo # undo log tablespaces --echo # +# The new BACKUP SERVER wrapper's prepare does not preserve prepared XA +# transactions across backup/restore, so this scenario does not apply to it. call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); call mtr.add_suppression("Found 1 prepared XA transactions"); @@ -72,7 +84,8 @@ rmdir $targetdir; --echo # --echo # MDEV-34713: mariadb_upgrade_info should be backed up and restored --echo # - +if (!$MTR_COMBINATION_SERVER) +{ CREATE TABLE t2(i INT) ENGINE INNODB; INSERT INTO t2 VALUES(100); @@ -106,3 +119,4 @@ SELECT * FROM t2; DROP TABLE t2; --remove_file $_datadir/mariadb_upgrade_info rmdir $targetdir; +} diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 0da6774445722..fe7dec0160f1a 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,6 +1,6 @@ --source include/not_embedded.inc --source include/have_file_key_management.inc - +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" --echo # diff --git a/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff new file mode 100644 index 0000000000000..921b39ff6cd58 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff @@ -0,0 +1,16 @@ +--- mysql-test/suite/mariabackup/log_tables.result ++++ mysql-test/suite/mariabackup/log_tables,SERVER.result +@@ -10,7 +10,6 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-# Insert new row into general_log table after it has been copied on BLOCK_DDL. + # Backup to dir. + # Xtrabackup prepare. + # shutdown server +@@ -22,5 +21,4 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO test.t VALUES (2) + DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/log_tables.test b/mysql-test/suite/mariabackup/log_tables.test index 707057a80e642..41c8581e2659f 100644 --- a/mysql-test/suite/mariabackup/log_tables.test +++ b/mysql-test/suite/mariabackup/log_tables.test @@ -1,6 +1,7 @@ # Test for copying log tables tail --source include/have_aria.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup @@ -21,8 +22,11 @@ SELECT * FROM mysql.general_log WHERE argument LIKE "INSERT INTO %" AND (command_type = "Query" OR command_type = "Execute") ; +if (!$MTR_COMBINATION_SERVER) +{ --echo # Insert new row into general_log table after it has been copied on BLOCK_DDL. --let after_stage_block_ddl=INSERT INTO test.t VALUES (2) +} --echo # Backup to dir. --disable_result_log diff --git a/mysql-test/suite/mariabackup/mdev-18438.test b/mysql-test/suite/mariabackup/mdev-18438.test index a6ec45476ff2c..f05b06b147384 100644 --- a/mysql-test/suite/mariabackup/mdev-18438.test +++ b/mysql-test/suite/mariabackup/mdev-18438.test @@ -1,8 +1,14 @@ +--source include/have_mariabackup_combination.inc let $basedir=$MYSQLTEST_VARDIR/tmp/mdev-18438; mkdir $basedir; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extra-lsndir=$basedir/extra_lsndir --stream=xbstream > $basedir/stream.xb; mkdir $basedir/backup; +# The BACKUP SERVER wrapper ignores --extra-lsndir, so it never creates +# extra_lsndir; only the native tool does. +if (!$MTR_COMBINATION_SERVER) +{ rmdir $basedir/extra_lsndir; +} --disable_result_log exec $XBSTREAM -x -C $basedir/backup < $basedir/stream.xb; --enable_result_log diff --git a/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff new file mode 100644 index 0000000000000..dba5eb260296d --- /dev/null +++ b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff @@ -0,0 +1,8 @@ +--- partition_notwin.result ++++ partition_notwin,SERVER.result +@@ -7,5 +7,4 @@ + ) engine=myisam + partition by hash (id) + partitions 600; +-FOUND 1 /Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/partition_notwin.test b/mysql-test/suite/mariabackup/partition_notwin.test index 10687e19935e6..85bafcfb5b83d 100644 --- a/mysql-test/suite/mariabackup/partition_notwin.test +++ b/mysql-test/suite/mariabackup/partition_notwin.test @@ -1,5 +1,6 @@ source include/not_windows.inc; source include/have_partition.inc; +source include/have_mariabackup_combination.inc; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $log=$MYSQL_TMP_DIR/backup.log; @@ -14,11 +15,21 @@ create table t1 ( partition by hash (id) partitions 600; -error 1; +let $errno = 1; +if ($MTR_COMBINATION_SERVER) +{ +let $errno = 0; +} +--error $errno exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $log 2>&1; + +# The "Too many open files" diagnostic only applies to native mariabackup. +if (!$MTR_COMBINATION_SERVER) +{ let SEARCH_FILE=$log; let SEARCH_PATTERN=Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files; source include/search_pattern_in_file.inc; +} rmdir $targetdir; #remove_file $log; diff --git a/mysql-test/suite/mariabackup/relative_path.test b/mysql-test/suite/mariabackup/relative_path.test index bd25a217e711e..60c287403165c 100644 --- a/mysql-test/suite/mariabackup/relative_path.test +++ b/mysql-test/suite/mariabackup/relative_path.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); diff --git a/mysql-test/suite/mariabackup/row_format_redundant.test b/mysql-test/suite/mariabackup/row_format_redundant.test index 5bae9218d840c..1820fec9ebfaa 100644 --- a/mysql-test/suite/mariabackup/row_format_redundant.test +++ b/mysql-test/suite/mariabackup/row_format_redundant.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test index bb476b8771e98..9076abff6c4cf 100644 --- a/mysql-test/suite/mariabackup/small_ibd.test +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -1,4 +1,5 @@ --source include/innodb_page_size.inc +--source include/have_mariabackup_combination.inc # Check if ibd smaller than page size are skipped # It is possible, due to race conditions that new file diff --git a/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff new file mode 100644 index 0000000000000..d886111b06fb7 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff @@ -0,0 +1,13 @@ +--- undo_space_id.result ++++ undo_space_id,SERVER.result +@@ -11,10 +11,3 @@ + undo001 + undo002 + DROP TABLE t1; +-# +-# MDEV-33980 mariadb-backup --backup is missing +-# retry logic for undo tablespaces +-# +-# xtrabackup backup +-# Display undo log files from target directory +-FOUND 5 /Retrying to read undo tablespace*/ in backup.log diff --git a/mysql-test/suite/mariabackup/undo_space_id.test b/mysql-test/suite/mariabackup/undo_space_id.test index 168740fc528ce..e239189040c89 100644 --- a/mysql-test/suite/mariabackup/undo_space_id.test +++ b/mysql-test/suite/mariabackup/undo_space_id.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --echo # Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4)) @@ -24,6 +25,8 @@ list_files $basedir undo*; DROP TABLE t1; rmdir $basedir; +if (!$MTR_COMBINATION_SERVER) +{ --echo # --echo # MDEV-33980 mariadb-backup --backup is missing --echo # retry logic for undo tablespaces @@ -42,3 +45,4 @@ list_files $basedir undo*; --source include/search_pattern_in_file.inc rmdir $basedir; remove_file $backuplog; +} diff --git a/mysql-test/suite/mariabackup/undo_truncate.test b/mysql-test/suite/mariabackup/undo_truncate.test index a23c9cf64ff6b..901bf73920a41 100644 --- a/mysql-test/suite/mariabackup/undo_truncate.test +++ b/mysql-test/suite/mariabackup/undo_truncate.test @@ -2,6 +2,7 @@ --source include/not_embedded.inc --source include/have_sequence.inc --source include/have_file_key_management.inc +--source include/have_mariabackup_combination.inc SET GLOBAL innodb_undo_log_truncate = 0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff new file mode 100644 index 0000000000000..e785a0601fe94 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff @@ -0,0 +1,8 @@ +--- unencrypted_page_compressed.result ++++ unencrypted_page_compressed,SERVER.result +@@ -8,5 +8,4 @@ + # Corrupt the table + # restart: --skip-innodb-buffer-pool-load-at-startup + # xtrabackup backup +-FOUND 1 /Database page corruption detected.*/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 68f22e69e0325..fc79a0a211e89 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); @@ -39,6 +40,8 @@ echo # xtrabackup backup; --disable_result_log let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log @@ -47,6 +50,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc remove_file $backuplog; +rmdir $targetdir; +} drop table t1; -rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/vector.test b/mysql-test/suite/mariabackup/vector.test index 4f60ddce10e9c..1889d8c412ec3 100644 --- a/mysql-test/suite/mariabackup/vector.test +++ b/mysql-test/suite/mariabackup/vector.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)) engine=innodb; insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')),