diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a71030887c43f..dff27afb0c44e 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -114,3 +114,28 @@ ADD_DEPENDENCIES(mbstream GenError) IF(MSVC) SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) ENDIF() + + +######################################################################## +# mariadb-backup-server: BACKUP SERVER-compatible shell wrapper +######################################################################## +# A drop-in mariadb-backup-compatible POSIX-sh wrapper that translates the +# CLI into server-side BACKUP SERVER SQL. Experimental; OFF by default +# Installed as a mariadb-backup-server; clients opt in via symlink/alias +# (see extra/mariabackup/scripts/README.md). +OPTION(WITH_MARIABACKUP_WRAPPER + "Install the BACKUP SERVER shell wrapper (mariadb-backup-server)" OFF) +ADD_FEATURE_INFO(MARIABACKUP_WRAPPER WITH_MARIABACKUP_WRAPPER + "BACKUP SERVER-compatible mariadb-backup shell wrapper") + +IF(WITH_MARIABACKUP_WRAPPER AND NOT WIN32) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mariadb-backup-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) + + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbstream-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mbstream-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mbstream-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) +ENDIF() diff --git a/extra/mariabackup/scripts/README.md b/extra/mariabackup/scripts/README.md new file mode 100644 index 0000000000000..f9253022a36a0 --- /dev/null +++ b/extra/mariabackup/scripts/README.md @@ -0,0 +1,263 @@ +# mariadb-backup-server.sh — a BACKUP SERVER wrapper + +`mariadb-backup-server.sh` makes the server-side `BACKUP SERVER` command look +like the old `mariadb-backup` tool. You call it with the familiar +`--backup` / `--prepare` / `--copy-back` options; under the hood +it just runs `BACKUP SERVER TO ''` over a normal `mariadb` +connection and lets the server do the work. It's plain POSIX `sh`, +so it runs anywhere `mariadb-backup` does. + +You need a server that supports `BACKUP SERVER`, the `mariadb` client on +`PATH`, and an account allowed to run `BACKUP SERVER`. The parent of the +target directory has to exist and be writable. + +## Installing / enabling it + +The wrapper is experimental and off by default. Build with +`-DWITH_MARIABACKUP_WRAPPER=ON` and it installs next to the real +binaries, under its own names so it never shadows them: + +``` +/usr/bin/mariadb-backup # the C++ binary, unchanged +/usr/bin/mariadb-backup-server # this wrapper +/usr/bin/mbstream # the C++ binary, unchanged +/usr/bin/mbstream-server # the tar-based mbstream shim (for --stream) +``` + +To send your existing `mariadb-backup` (and `mbstream`) calls through the +wrapper instead, point the names at them yourself — an alias, or a symlink +earlier in `PATH`: + +```sh +alias mariadb-backup=mariadb-backup-server +alias mbstream=mbstream-server +# or +ln -s /usr/bin/mariadb-backup-server ~/bin/mariadb-backup +ln -s /usr/bin/mbstream-server ~/bin/mbstream +``` + +`mbstream-server` is only needed for `--stream` backups (it extracts the +wrapper's tar stream). + +## Backing up + +```sh +mariadb-backup-server --backup --target-dir=/backup/full +``` + +That runs `BACKUP SERVER TO '/backup/full'`. Use `--parallel=N` to ask for N +concurrent streams (`... N CONCURRENT`; N=1 is the default and changes +nothing). + +Connection options: +`--user`, `--password`, `--host`, `--port`, `--socket`, `--defaults-file`, +`--defaults-extra-file` and their short forms are passed straight to +the `mariadb` client. + +`--throttle`, `--no-lock` and `--safe-slave-backup` are accepted and ignored; + +When the backup finishes the wrapper drops a `backup-prepare.cnf` into the +target dir, next to the server's own `backup.cnf`. It records where `mariadbd` +lives and the InnoDB layout, so `--prepare` can recover the backup later +without you respelling all of that. + +### Streaming to stdout + +```sh +mariadb-backup-server --backup --stream=tar > /backup/full.tar +``` + +`--stream` runs `BACKUP SERVER WITH [N CONCURRENT] ''` instead of +`... TO ''`. The server hands each stream's tar to ``, the wrapper +collects the parts and writes them to its stdout, so you can redirect to a file +or pipe onwards. + +Two things follow from how `BACKUP SERVER` streams, and both differ from +`mariadb-backup`: + +- **Local only.** The stream command runs *inside the server*, so its output +lands on the server's filesystem. The wrapper can only pick it up when it runs +on the same host (a shared filesystem), as the user the server writes as +(typically `mysql`), or with a `--target-dir` the server can write. A remote +`--host` cannot stream this way. + +- **tar only.** The server emits tar, never `xbstream`. Any `--stream=` +value (including `xbstream`) is accepted but the output is always tar. + +`--target-dir` is optional here; when given it is just the scratch directory for +the per-stream parts (otherwise a `mktemp` dir is used). + +The output is the per-stream tar entries concatenated, with +`backup-prepare.cnf` appended as the trailing archive. + +```sh +mkdir restore && tar -xf /backup/full.tar -C restore +``` + +After extraction the directory holds the data files, the server's `backup.cnf` +*and* the wrapper's `backup-prepare.cnf`, so it can be prepared exactly like a +directory backup: + +```sh +mariadb-backup-server --prepare --target-dir=restore +``` + +### `mbstream.sh` — the extraction shim + +The real `mbstream`/`xbstream` binary cannot read the wrapper's stream, because +that stream is plain tar, not the `xbstream` format. So a companion shim, +`mbstream.sh`, ships next to `mariadb-backup-server.sh` and maps the `mbstream` CLI onto +`tar`, letting existing pipelines (and tests) that call `mbstream` keep working +unchanged: + +```sh +mbstream-server -x -C restore < /backup/full.tar # tar -x -C restore +mbstream-server -c -C dir file1 file2 # tar -c -C dir file1 file2 +``` + +Notes: + +- **Extraction is a plain `tar -x`** — the stream has a single +end-of-archive marker + +- **mbstream-only options are accepted and ignored** — +`-p` / `--parallel`, compression flags, `-v` : So legacy +invocations don't break. + +- It understands `-x` (extract, from stdin) and `-c` (create, to stdout), +with `-C `; anything else is treated as a file operand or ignored. + +- It is a thin tar wrapper, so it only understands the wrapper's tar streams — +do not point it at a real `xbstream` archive, and do not feed the wrapper's +output to the real `mbstream`. + +## Preparing + +```sh +mariadb-backup-server --prepare --target-dir=/backup/full +``` + +Prepare makes the backup bootable: it starts `mariadbd --bootstrap` on the +target directory, replays the archived redo up to the backup's end LSN, then +replaces the archived log with a fresh `ib_logfile0` so a normal server can +start on it. Both `backup.cnf` and `backup-prepare.cnf` have to be there (they +are, if this wrapper took the backup). + +Options: + +- `--use-memory=N` — buffer pool size during recovery. +- `--innodb-*` and `--tmpdir` are forwarded to the bootstrap server. +- `--defaults-file` / `--defaults-extra-file`, and encryption options such as + `--file-key-management*` / `--loose-file-key-management*` / `--plugin-load-add`, + are layered onto the bootstrap (as an extra defaults file / extra options) so + you can supply anything `backup-prepare.cnf` did not record. + +The `mariadbd` used for the bootstrap is the path recorded in +`backup-prepare.cnf` at backup time if that binary exists; otherwise `mariadbd` +from `PATH`. (Recorded-first matters: it is the same version that took the +backup, so it can always parse the backed-up tablespace format.) + +## Restoring + +With the backup prepared and the server stopped, put it into the datadir: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql +mariadb-backup-server --move-back --target-dir=/backup/full --datadir=/var/lib/mysql +``` + +`--copy-back` leaves the backup where it is; `--move-back` renames the files +across, which is quicker on the same filesystem but consumes the backup. Either +way the datadir is created if missing, and the wrapper won't write into a +non-empty datadir unless you add `--force-non-empty-directories`. + +If the source server kept its Aria logs outside the datadir, pass the same +`--aria-log-dir-path=` you use on the server. The wrapper creates that +directory and moves the restored `aria_log_control` / `aria_log.*` files into +it, so the server finds them on restart: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql \ + --aria-log-dir-path=/var/lib/aria_logs +``` + +Neither fixes ownership, so finish up with: + +```sh +chown -R mysql:mysql /var/lib/mysql +systemctl start mariadb +``` + +## What it doesn't do + +These stop with an error instead of quietly producing an incomplete backup: + +- incremental backup/prepare: `--incremental-basedir`, `--incremental-dir`, `--apply-log-only` +- partial backup: `--databases`, `--tables`, `--tables-file`. This needs server-side +`backup_include`/`backup_exclude`, which don't exist yet +- compression and encryption of the output: `--compress`, `--encrypt` (error out) +- `--rollback-xa`: not supported (errors out) + +`--stream` is supported with the caveats above (local only, tar only, extract +with a plain `tar -x`); + +`--export` is accepted but not implemented: it warns and does a plain recovery +(no per-table `.cfg` files for IMPORT TABLESPACE). + +## Environment overrides + +The wrapped commands can be overridden via environment variables, +mainly for testing: + +- `MARIADB`: the client used to talk to the server (default `mariadb`), +e.g. `MARIADB='mariadb --protocol=tcp'`. + +- `MARIADBD`: the server binary the `--prepare` bootstrap runs (by default the +path recorded in `backup-prepare.cnf`, else `mariadbd` on `PATH`). When set it +overrides that resolution. To run it under `rr`, put `rr` in `MARIADBD` and let +`rr`'s own `_RR_TRACE_DIR` choose where the trace goes, e.g. +`_RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ...`. + +- `TAR`: the tar implementation (default `tar`), +e.g. `TAR=bsdtar` (libarchive-tools). +Used by `--stream` and by `mbstream-server`. + +## The two .cnf files + +`backup.cnf` is written by the server and tells `--prepare` what parts of redo to replay: + +```ini +[server] +# checkpoint=54088 +innodb_log_recovery_start=54088 # recovery starts scanning here +innodb_log_recovery_target=56337 # the backup's end LSN; recovery stops here +``` + +`backup-prepare.cnf` is written by the wrapper and handed to the prepare +bootstrap as its defaults file: + +```ini +# mariadbd=/usr/sbin/mariadbd +[mariadbd] +innodb_page_size=16384 +innodb_data_file_path=ibdata1:12M:autoextend +innodb_undo_tablespaces=3 +innodb_checksum_algorithm=full_crc32 +innodb_log_file_size=100663296 +``` + +If the server is encrypted it also records how to load the key-management +plugin again, so an encrypted backup can be prepared without extra input. +For `file_key_management` it captures every plugin variable (the same way +`mariadb-backup` writes them into `backup-my.cnf`) + +```ini +plugin-load-add=file_key_management +innodb_encrypt_log=ON +loose-file-key-management +loose-file_key_management_filename=/etc/mysql/keys.enc +loose-file_key_management_filekey=FILE:/etc/mysql/keyfile.key +loose-file_key_management_encryption_algorithm=aes_cbc +loose-file_key_management_digest=sha1 +loose-file_key_management_use_pbkdf2=1 +``` diff --git a/extra/mariabackup/scripts/mariadb-backup-server.sh b/extra/mariabackup/scripts/mariadb-backup-server.sh new file mode 100755 index 0000000000000..57a64109b4159 --- /dev/null +++ b/extra/mariabackup/scripts/mariadb-backup-server.sh @@ -0,0 +1,366 @@ +#!/bin/sh +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The wrapped commands can be overridden for testing, e.g. +# MARIADB='mariadb --protocol=tcp', TAR=bsdtar. + +# To run the prepare bootstrap under rr, include it in MARIADBD +# (rr's own _RR_TRACE_DIR controls where the trace is written). +# e.g. _RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ... +: "${MARIADB:=mariadb}" +: "${TAR:=tar}" + +MODE= +TARGET_DIR= +DATADIR= +PARALLEL= +USE_MEMORY= +FORCE_NON_EMPTY= +EXPORT= +ROLLBACK_XA= +MARIADB_OPTS= +INNODB_OPTS= +MYSQLD_EXTRA= +PREPARE_DEFAULTS= +ARIA_LOG_DIR= +STREAM= + +while [ $# -gt 0 ]; do + case $1 in + --backup) MODE=backup ;; + --prepare|--apply-log) MODE=prepare ;; + --copy-back) MODE=copy-back ;; + --move-back) MODE=move-back ;; + + --target-dir=*) TARGET_DIR=${1#*=} ;; + --datadir=*) DATADIR=${1#*=} ;; + --aria-log-dir-path=*) ARIA_LOG_DIR=${1#*=} ;; + --use-memory=*) USE_MEMORY=${1#*=} ;; + --parallel=*) PARALLEL=${1#*=} ;; + + --export) EXPORT=1 ;; + --rollback-xa) ROLLBACK_XA=1 ;; + --force-non-empty-directories) FORCE_NON_EMPTY=1 ;; + + --innodb|--innodb=*|--innodb-*|--innodb_*|--skip-innodb-*|--skip_innodb_*) + INNODB_OPTS="$INNODB_OPTS $1" ;; + --tmpdir=*) MYSQLD_EXTRA="$MYSQLD_EXTRA $1" ;; + -t) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=$2"; shift ;; + -t*) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=${1#-t}" ;; + --incremental-basedir=*|--incremental-dir=*) + die "incremental backup/prepare is not supported" ;; + --apply-log-only) + die "--apply-log-only is not supported" ;; + --databases=*|--databases-exclude=*|--tables=*|--tables-exclude=*|--tables-file=*) + die "partial backup needs server-side backup_include/backup_exclude, which doesn't exist yet" ;; + # BACKUP SERVER only ever produces tar + --stream|--stream=*) STREAM=1 ;; + --compress|--compress=*|--compress-threads=*) die "--compress is not supported" ;; + --encrypt|--encrypt=*) die "--encrypt is not supported" ;; + --innobackupex) die "innobackupex mode is not supported" ;; + + # Defaults files feed the backup client + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1" + PREPARE_DEFAULTS="$PREPARE_DEFAULTS --defaults-extra-file=${1#*=}" ;; + # Encryption options are forwarded to the prepare bootstrap. + --file-key-management*|--plugin-load-add=*|--loose-file-key-management*|\ + --aria-encrypt-tables*|--encrypt-tmp-disk-tables*) + PREPARE_DEFAULTS="$PREPARE_DEFAULTS $1" ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*|\ + --defaults-group=*|\ + --secure-auth|--skip-secure-auth|--ssl|--ssl-verify-server-cert|\ + --ssl-ca=*|--ssl-capath=*|--ssl-cert=*|--ssl-cipher=*|\ + --ssl-crl=*|--ssl-crlpath=*|--ssl-key=*|--tls-version=*) + MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -p) + if [ -n "${2-}" ] && case $2 in -*) false ;; *) true ;; esac; then + MARIADB_OPTS="$MARIADB_OPTS -p$2"; shift + else + MARIADB_OPTS="$MARIADB_OPTS -p" + fi ;; + -u|-P|-S) MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift ;; + -H) MARIADB_OPTS="$MARIADB_OPTS --host=$2"; shift ;; + -p*|-u*|-P*|-S*) MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -H*) MARIADB_OPTS="$MARIADB_OPTS --host=${1#-H}" ;; + + -h) DATADIR=$2; shift ;; + -h*) DATADIR=${1#-h} ;; + + # Everything else is accepted and ignored: + *) ;; + esac + shift +done + +# In stream mode --target-dir is optional: +# it is only a scratch directory for the per-stream tar parts +# (a mktemp dir is used when it is omitted). +[ -n "$STREAM" ] || [ -n "$TARGET_DIR" ] || die "--target-dir required" + +# Run the client with the connection options we collected. +ask() { $MARIADB $MARIADB_OPTS -BN -e "$1" 2>/dev/null; } + +# Print the backup-prepare.cnf contents to stdout. +# It captures everything --prepare's offline bootstrap needs: +# where mariadbd lives, the InnoDB parameters, and how to +# reload the encryption key plugin. Used both for a +# directory backup (written into the target dir) +# and a streamed backup (appended to the stream so it lands +# in the extracted directory). + +write_prepare_cnf() { + _mariadbd= + _pidfile=$(ask "SELECT @@global.pid_file") + if [ -n "$_pidfile" ] && [ -r "$_pidfile" ]; then + _pid=$(cat "$_pidfile" 2>/dev/null) + [ -n "$_pid" ] && _mariadbd=$(readlink -f "/proc/$_pid/exe" 2>/dev/null) + fi + if [ -z "$_mariadbd" ]; then + _basedir=$(ask "SELECT @@global.basedir") + for _c in "$_basedir/sbin/mariadbd" "$_basedir/bin/mariadbd" \ + "$_basedir/sbin/mysqld" "$_basedir/bin/mysqld"; do + [ -x "$_c" ] && { _mariadbd=$_c; break; } + done + fi + + _page_size=$(ask "SELECT @@global.innodb_page_size") + _data_file_path=$(ask "SELECT @@global.innodb_data_file_path") + _undo_ts=$(ask "SELECT @@global.innodb_undo_tablespaces") + _checksum=$(ask "SELECT @@global.innodb_checksum_algorithm") + _log_file_size=$(ask "SELECT @@global.innodb_log_file_size") + + _enc=$(ask "SELECT LOWER(plugin_name) FROM information_schema.PLUGINS + WHERE plugin_type='ENCRYPTION' AND plugin_status='ACTIVE' LIMIT 1") + + [ -n "$_mariadbd" ] && echo "# mariadbd=$_mariadbd" + echo "[mariadbd]" + [ -n "$_page_size" ] && echo "innodb_page_size=$_page_size" + [ -n "$_data_file_path" ] && echo "innodb_data_file_path=$_data_file_path" + [ -n "$_undo_ts" ] && echo "innodb_undo_tablespaces=$_undo_ts" + [ -n "$_checksum" ] && echo "innodb_checksum_algorithm=$_checksum" + [ -n "$_log_file_size" ] && echo "innodb_log_file_size=$_log_file_size" + + if [ -n "$_enc" ]; then + echo "plugin-load-add=$_enc" + _plugin_dir=$(ask "SELECT @@global.plugin_dir") + [ -n "$_plugin_dir" ] && echo "plugin-dir=$_plugin_dir" + case $(ask "SELECT @@global.innodb_encrypt_log") in + 1|ON) echo "innodb_encrypt_log=ON" ;; + esac + if [ "$_enc" = file_key_management ]; then + echo "loose-file-key-management" + ask "SHOW VARIABLES LIKE 'file_key_management%'" | + while read -r _fkm_name _fkm_value; do + [ -n "$_fkm_name" ] && echo "loose-$_fkm_name=$_fkm_value" + done + fi + fi +} + + +# prepare +if [ "$MODE" = prepare ]; then + [ -z "$ROLLBACK_XA" ] || die "--rollback-xa is not supported" + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + + cnf=$TARGET_DIR/backup-prepare.cnf + [ -f "$cnf" ] || die "$cnf missing - was this backup made by the wrapper?" + [ -z "$EXPORT" ] || echo "$me: --export not implemented, doing a plain recovery" >&2 + + # Prefer the binary recorded at backup time + # else, fall back to mariadbd on PATH only if the + # recorded one is missing. + # MARIADBD overrides the recorded/PATH binary + if [ -n "${MARIADBD-}" ]; then + mariadbd=$MARIADBD + else + mariadbd=$(sed -n 's/^# *mariadbd=//p' "$cnf" | tail -n1) + [ -n "$mariadbd" ] && [ -x "$mariadbd" ] || mariadbd=mariadbd + fi + + # backup.cnf tells us the LSN window recovery should replay. + start=$(grep '^innodb_log_recovery_start' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + target=$(grep '^innodb_log_recovery_target' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + + opts="--datadir=$TARGET_DIR --innodb=FORCE" + [ -n "$start" ] && opts="$opts --innodb-log-recovery-start=$start" + [ -n "$target" ] && opts="$opts --innodb-log-recovery-target=$target" + [ -n "$USE_MEMORY" ] && opts="$opts --innodb-buffer-pool-size=$USE_MEMORY" + opts="$opts$INNODB_OPTS$MYSQLD_EXTRA" + + # Recovery stops at the backup's end LSN but leaves the + # archived log (ib_.log) behind, which a normal server + # won't boot from. After recovery we build a fresh ib_logfile0 + # (header + a checkpoint at the end LSN) and put it in place + # of the archived log. + input=/dev/null + newlog=$TARGET_DIR/ib_logfile0.new + if [ -n "$target" ]; then + lsn=$(printf '%016x' "$target") + rm -f "$newlog" + input=$(mktemp) + cat > "$input" <&2 + exit 0 +fi + + +# copy-back / move-back +if [ "$MODE" = copy-back ] || [ "$MODE" = move-back ]; then + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + [ -n "$DATADIR" ] || die "--datadir required for --$MODE" + + # mariadb-backup creates the datadir if it's missing; do the same. + [ -d "$DATADIR" ] || mkdir -p "$DATADIR" || die "cannot create datadir: $DATADIR" + + # Refuse a non-empty datadir (dotfiles included) unless told otherwise. + if [ -z "$FORCE_NON_EMPTY" ]; then + for f in "$DATADIR"/* "$DATADIR"/.[!.]* "$DATADIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } && + die "datadir not empty: $DATADIR (pass --force-non-empty-directories)" + done + fi + + if [ "$MODE" = copy-back ]; then + echo "$me: copying $TARGET_DIR -> $DATADIR" >&2 + cp -R "$TARGET_DIR"/. "$DATADIR"/ || die "copy-back failed" + else + echo "$me: moving $TARGET_DIR -> $DATADIR" >&2 + for f in "$TARGET_DIR"/* "$TARGET_DIR"/.[!.]* "$TARGET_DIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } || continue + mv "$f" "$DATADIR"/ || die "move-back failed" + done + fi + + # Aria logs live in --aria-log-dir-path when set; + # backup placed them at the datadir root, + # so relocate them there and create the directory the server + # expects on restart. + if [ -n "$ARIA_LOG_DIR" ] && [ "$ARIA_LOG_DIR" != "$DATADIR" ]; then + mkdir -p "$ARIA_LOG_DIR" || die "cannot create aria-log-dir-path: $ARIA_LOG_DIR" + for f in "$DATADIR"/aria_log_control "$DATADIR"/aria_log.*; do + [ -e "$f" ] && { mv "$f" "$ARIA_LOG_DIR"/ || die "aria log relocate failed"; } + done + fi + + echo "Restore completed: $DATADIR" >&2 + echo "$me: now run: chown -R mysql:mysql $DATADIR && start the server" >&2 + exit 0 +fi + + +# backup (streaming) +# "BACKUP SERVER WITH [N CONCURRENT] ''" runs +# inside the server feeding that stream's tar to the command's stdin. +if [ -n "$STREAM" ]; then + if [ -n "$TARGET_DIR" ]; then + scratch=$TARGET_DIR + mkdir -p "$scratch" || die "cannot create scratch dir: $scratch" + keep_scratch=1 + else + scratch=$(mktemp -d "${TMPDIR:-/tmp}/mariabackup-stream.XXXXXX") \ + || die "cannot create scratch directory" + keep_scratch= + fi + + cleanup_stream() { + rm -f "$scratch"/.mariabackup-stream.sh "$scratch"/*.tar \ + "$scratch"/backup-prepare.cnf + [ -n "$keep_scratch" ] || rmdir "$scratch" 2>/dev/null + } + + # The server appends the stream index as $1 and writes that stream's tar + # to our stdin; we drop each one into the scratch dir under .tar. + helper=$scratch/.mariabackup-stream.sh + cat > "$helper" < "$scratch/\$1.tar" +EOF + + # Invoke via "/bin/sh " so the script needs no execute bit + # (scratch may sit on a noexec filesystem such as /dev/shm). + sql="BACKUP SERVER WITH" + case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; + esac + sql="$sql '/bin/sh $helper'" + + # Keep stdout clean for the tar: send any client output to stderr. + $MARIADB $MARIADB_OPTS -e "$sql" >&2 \ + || { cleanup_stream; die "BACKUP SERVER failed"; } + + # Concatenate the per-stream tars to stdout in index order, + # then append backup-prepare.cnf as one more tar archive + # so it lands in the extracted directory alongside the + # server's backup.cnf. The per-stream tars carry no + # end-of-archive marker; only this trailing + # backup-prepare.cnf adds the single end marker, + # so the whole stream extracts with a plain "tar -x". + n=1 + while [ -f "$scratch/$n.tar" ]; do + cat "$scratch/$n.tar" || { cleanup_stream; die "streaming to stdout failed"; } + n=$((n + 1)) + done + + echo "$me: appending backup-prepare.cnf to the stream" >&2 + write_prepare_cnf > "$scratch/backup-prepare.cnf" \ + || { cleanup_stream; die "could not build backup-prepare.cnf"; } + $TAR -C "$scratch" -cf - backup-prepare.cnf \ + || { cleanup_stream; die "could not append backup-prepare.cnf to the stream"; } + + cleanup_stream + echo "$me: streamed $((n - 1)) tar stream(s) plus backup-prepare.cnf to stdout" >&2 + exit 0 +fi + + +# --- backup (directory) ----------------------------------------------------- +parent=$(dirname "$TARGET_DIR") +[ -d "$parent" ] || die "parent directory does not exist: $parent" +[ -w "$parent" ] || die "parent directory is not writable: $parent" + +sql="BACKUP SERVER TO '$TARGET_DIR'" +case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is the default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; +esac +$MARIADB $MARIADB_OPTS -e "$sql" || die "BACKUP SERVER failed" + +# Create backup-prepare.cnf with everything --prepare's offline +# bootstrap needs: where mariadbd is, the InnoDB parameters, +# and how to load the key plugin again. +[ -f "$TARGET_DIR/backup.cnf" ] || exit 0 + +echo "$me: writing backup-prepare.cnf" >&2 +write_prepare_cnf > "$TARGET_DIR/backup-prepare.cnf" diff --git a/extra/mariabackup/scripts/mbstream-server.sh b/extra/mariabackup/scripts/mbstream-server.sh new file mode 100755 index 0000000000000..7ba2bbabe7e6d --- /dev/null +++ b/extra/mariabackup/scripts/mbstream-server.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# mbstream-compatible for the BACKUP SERVER wrapper. +# +# This maps the mbstream CLI to `tar`: +# +# mbstream -x -C < archive -> tar -x -C +# mbstream -c > archive -> tar -c +# +# mbstream-only options are accepted and ignored so +# existing invocations keep working unchanged. +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The tar implementation can be overridden for testing. +# e.g. TAR=bsdtar. +: "${TAR:=tar}" + +mode= +dir=. +files= + +while [ $# -gt 0 ]; do + case $1 in + -x|--extract) mode=x ;; + -c|--create) mode=c ;; + -C|--directory) dir=$2; shift ;; + -C*) dir=${1#-C} ;; + --directory=*) dir=${1#*=} ;; + + # mbstream-only flags that have no tar equivalent: drop them. + -p|--parallel) shift ;; + -p*|--parallel=*) ;; + --decompress|--compress) ;; + -v|--verbose) ;; + + --) shift; break ;; + # Reject anything we do not recognise rather than + # silently ignoring it (e.g. GNU tar's -b takes an argument + # that would otherwise be mistaken for a file operand). + # This is an mbstream-on-tar shim, not tar. + -*) die "unsupported option: $1" ;; + *) files="$files $1" ;; + esac + shift +done + +while [ $# -gt 0 ]; do + files="$files $1" + shift +done + +case $mode in + # The wrapper concatenates the per-stream tar entries + # with no end-of-archive marker between them; only the + # trailing backup-prepare.cnf adds the single end marker. + x) exec $TAR -x -f - -C "$dir" ;; + c) + [ -n "$files" ] || files=. + exec $TAR -c -f - -C "$dir" $files ;; + *) die "expected -x (extract) or -c (create)" ;; +esac diff --git a/mysql-test/include/have_mariabackup_combination.combinations b/mysql-test/include/have_mariabackup_combination.combinations new file mode 100644 index 0000000000000..12bc42487d520 --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.combinations @@ -0,0 +1,3 @@ +[CLIENT] + +[SERVER] diff --git a/mysql-test/include/have_mariabackup_combination.inc b/mysql-test/include/have_mariabackup_combination.inc new file mode 100644 index 0000000000000..a2b79d6310efc --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.inc @@ -0,0 +1,5 @@ +if ($MTR_COMBINATION_SERVER) +{ + --source include/have_mariabackup_wrapper.inc +} + diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..d9d8ee82e74ee --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,38 @@ +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through extra/mariabackup/scripts/mariadb-backup-server.sh — the BACKUP +# SERVER compatibility wrapper — without any change to the test body. +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# $XTRABACKUP : now points at mariadb-backup-server.sh + +--source include/not_windows.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mariadb-backup-server.sh +--let MBSTREAM_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mbstream-server.sh + +# The wrapper shells out to the bare `mariadb` client, which mtr does not put +# on PATH. Prepend the build's client directories so it resolves. A `let` with +# no leading $ is exported to the environment of later --exec commands. +--let PATH=$MYSQL_BINDIR/client:$MYSQL_BINDIR/client_release:$MYSQL_BINDIR/client_debug:$MYSQL_BINDIR/bin:$PATH + +--error 0,1 +perl; +my $w = $ENV{MARIABACKUP_WRAPPER}; +my $m = $ENV{MBSTREAM_WRAPPER}; +exit 1 unless $w && -x $w && $m && -x $m; +exit 0; +EOF + +if ($errno) +{ + --skip mariadb-backup-server.sh wrapper unavailable (script or sh missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER +--let XBSTREAM=$MBSTREAM_WRAPPER diff --git a/mysql-test/suite/mariabackup/aria_encrypted.test b/mysql-test/suite/mariabackup/aria_encrypted.test index 6ed3e0d08eff3..017a0b249dcfe 100644 --- a/mysql-test/suite/mariabackup/aria_encrypted.test +++ b/mysql-test/suite/mariabackup/aria_encrypted.test @@ -1,6 +1,7 @@ --source include/have_file_key_management.inc --source include/have_innodb.inc --source include/have_sequence.inc +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-38246 aria_read index failed on encrypted database during backup @@ -16,3 +17,4 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; drop table if exists t1,t2; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.inc b/mysql-test/suite/mariabackup/aria_log_dir_path.inc new file mode 100644 index 0000000000000..1c7d8d12679d8 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.inc @@ -0,0 +1,104 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + +--echo # mariadb-backup --backup +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test index 40bc39446bf00..e479458edfddb 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -1,105 +1,2 @@ ---source include/have_maria.inc - ---echo # ---echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used ---echo # - ---let $datadir=`SELECT @@datadir` ---let $targetdir=$MYSQLTEST_VARDIR/tmp/backup - -if ($ARIA_LOGDIR_MARIADB == '') -{ - --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - -if ($ARIA_LOGDIR_FS == '') -{ - --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - ---let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB - - ---echo # Restart mariadbd with the test specific parameters ---mkdir $ARIA_LOGDIR_FS ---let $restart_parameters=$server_parameters ---source include/restart_mysqld.inc - - ---echo # Create and populate an Aria table (and Aria logs) -CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; -DELIMITER $$; -BEGIN NOT ATOMIC - FOR id IN 0..9 DO - INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); - END FOR; -END; -$$ -DELIMITER ;$$ - - ---echo # Testing aria log files before --backup -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control ---file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - ---echo # mariadb-backup --backup ---disable_result_log ---mkdir $targetdir ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir ---enable_result_log - - ---echo # mariadb-backup --prepare ---disable_result_log ---exec $XTRABACKUP --prepare --target-dir=$targetdir ---enable_result_log - ---echo # shutdown server ---disable_result_log ---source include/shutdown_mysqld.inc ---echo # remove datadir ---rmdir $datadir ---echo # remove aria-log-dir-path ---rmdir $ARIA_LOGDIR_FS - - ---echo # mariadb-backup --copy-back ---let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---exec echo "# with parameters: $mariadb_backup_parameters" ---exec $XTRABACKUP $mariadb_backup_parameters - - ---echo # starting server ---let $restart_parameters=$server_parameters ---source include/start_mysqld.inc ---enable_result_log ---rmdir $targetdir - - ---echo # Check that the table is there after --copy-back -SELECT COUNT(*) from t1; -DROP TABLE t1; - - ---echo # Testing aria log files after --copy-back -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control -#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - - ---echo # Restarting mariadbd with default parameters ---let $restart_parameters= ---source include/restart_mysqld.inc ---rmdir $ARIA_LOGDIR_FS +--source include/have_mariabackup_combination.inc +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test index c8169959929b9..b2875f496daac 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -1,4 +1,4 @@ --let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel ---source aria_log_dir_path.test +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff new file mode 100644 index 0000000000000..f71d98dc11c82 --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff @@ -0,0 +1,10 @@ +--- defer_space.result ++++ defer_space,SERVER.result +@@ -21,7 +21,5 @@ + CREATE TABLE t1(c INT) ENGINE=INNODB; + # Corrupt the table + # restart +-# xtrabackup backup +-FOUND 10 /Header page consists of zero bytes*/ in backup.log + UNLOCK TABLES; + DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/defer_space.test b/mysql-test/suite/mariabackup/defer_space.test index 397a1ff5dc23c..bad0ed259600c 100644 --- a/mysql-test/suite/mariabackup/defer_space.test +++ b/mysql-test/suite/mariabackup/defer_space.test @@ -2,6 +2,7 @@ --source include/have_debug.inc --source include/not_embedded.inc --source include/no_valgrind_without_big.inc +--source include/have_mariabackup_combination.inc call mtr.add_suppression("InnoDB: Expected tablespace id .*"); --echo # Mariabackup --backup with page0 INIT_PAGE redo record @@ -51,6 +52,8 @@ close FILE or die "close"; EOF --source include/start_mysqld.inc +if (!$MTR_COMBINATION_SERVER) +{ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; @@ -62,7 +65,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --let SEARCH_PATTERN=Header page consists of zero bytes* --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc -UNLOCK TABLES; -DROP TABLE t1; rmdir $targetdir; remove_file $backuplog; +} +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff new file mode 100644 index 0000000000000..4e2798c8a3b3d --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff @@ -0,0 +1,30 @@ +--- full_backup.result ++++ full_backup,SERVER.result +@@ -3,10 +3,6 @@ + SET GLOBAL innodb_max_purge_lag_wait=0; + # xtrabackup backup + NOT FOUND /InnoDB: Allocated tablespace ID/ in backup.log +-SELECT variable_value FROM information_schema.global_status +-WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +-variable_value +-0 + INSERT INTO t VALUES(2); + # xtrabackup prepare + # shutdown server +@@ -44,16 +40,3 @@ + # + # MDEV-34713: mariadb_upgrade_info should be backed up and restored + # +-CREATE TABLE t2(i INT) ENGINE INNODB; +-INSERT INTO t2 VALUES(100); +-# xtrabackup backup +-# xtrabackup prepare +-# shutdown server +-# remove datadir +-# xtrabackup move back +-# restart: --innodb_undo_tablespaces=0 +-FOUND 1 /^[0-9]+\.[0-9]+\.[0-9]+/ in mariadb_upgrade_info +-SELECT * FROM t2; +-i +-100 +-DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index 8a1ce756eeb17..1b75e4d2c051a 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc --source include/innodb_page_size.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -8,8 +9,13 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log --disable_result_log +# The new BACKUP SERVER wrapper ignores --innodb-log-write-ahead-size, so the +# invalid-value invocation that is expected to fail does not apply to it. +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=4095 > $backup_log 2>&1; +} exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=10000 --innodb_log_checkpoint_now=1 > $backup_log 2>&1; --enable_result_log @@ -19,8 +25,12 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --source include/search_pattern_in_file.inc --remove_file $backup_log +# Only the native mariabackup leaves dirty pages worth checking here. +if (!$MTR_COMBINATION_SERVER) +{ SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +} INSERT INTO t VALUES(2); @@ -38,6 +48,8 @@ rmdir $targetdir; --echo # MDEV-27121 mariabackup incompatible with disabled dedicated --echo # undo log tablespaces --echo # +# The new BACKUP SERVER wrapper's prepare does not preserve prepared XA +# transactions across backup/restore, so this scenario does not apply to it. call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); call mtr.add_suppression("Found 1 prepared XA transactions"); @@ -72,7 +84,8 @@ rmdir $targetdir; --echo # --echo # MDEV-34713: mariadb_upgrade_info should be backed up and restored --echo # - +if (!$MTR_COMBINATION_SERVER) +{ CREATE TABLE t2(i INT) ENGINE INNODB; INSERT INTO t2 VALUES(100); @@ -106,3 +119,4 @@ SELECT * FROM t2; DROP TABLE t2; --remove_file $_datadir/mariadb_upgrade_info rmdir $targetdir; +} diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 0da6774445722..fe7dec0160f1a 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,6 +1,6 @@ --source include/not_embedded.inc --source include/have_file_key_management.inc - +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" --echo # diff --git a/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff new file mode 100644 index 0000000000000..921b39ff6cd58 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff @@ -0,0 +1,16 @@ +--- mysql-test/suite/mariabackup/log_tables.result ++++ mysql-test/suite/mariabackup/log_tables,SERVER.result +@@ -10,7 +10,6 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-# Insert new row into general_log table after it has been copied on BLOCK_DDL. + # Backup to dir. + # Xtrabackup prepare. + # shutdown server +@@ -22,5 +21,4 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO test.t VALUES (2) + DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/log_tables.test b/mysql-test/suite/mariabackup/log_tables.test index 707057a80e642..41c8581e2659f 100644 --- a/mysql-test/suite/mariabackup/log_tables.test +++ b/mysql-test/suite/mariabackup/log_tables.test @@ -1,6 +1,7 @@ # Test for copying log tables tail --source include/have_aria.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup @@ -21,8 +22,11 @@ SELECT * FROM mysql.general_log WHERE argument LIKE "INSERT INTO %" AND (command_type = "Query" OR command_type = "Execute") ; +if (!$MTR_COMBINATION_SERVER) +{ --echo # Insert new row into general_log table after it has been copied on BLOCK_DDL. --let after_stage_block_ddl=INSERT INTO test.t VALUES (2) +} --echo # Backup to dir. --disable_result_log diff --git a/mysql-test/suite/mariabackup/mdev-18438.test b/mysql-test/suite/mariabackup/mdev-18438.test index a6ec45476ff2c..f05b06b147384 100644 --- a/mysql-test/suite/mariabackup/mdev-18438.test +++ b/mysql-test/suite/mariabackup/mdev-18438.test @@ -1,8 +1,14 @@ +--source include/have_mariabackup_combination.inc let $basedir=$MYSQLTEST_VARDIR/tmp/mdev-18438; mkdir $basedir; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extra-lsndir=$basedir/extra_lsndir --stream=xbstream > $basedir/stream.xb; mkdir $basedir/backup; +# The BACKUP SERVER wrapper ignores --extra-lsndir, so it never creates +# extra_lsndir; only the native tool does. +if (!$MTR_COMBINATION_SERVER) +{ rmdir $basedir/extra_lsndir; +} --disable_result_log exec $XBSTREAM -x -C $basedir/backup < $basedir/stream.xb; --enable_result_log diff --git a/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff new file mode 100644 index 0000000000000..dba5eb260296d --- /dev/null +++ b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff @@ -0,0 +1,8 @@ +--- partition_notwin.result ++++ partition_notwin,SERVER.result +@@ -7,5 +7,4 @@ + ) engine=myisam + partition by hash (id) + partitions 600; +-FOUND 1 /Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/partition_notwin.test b/mysql-test/suite/mariabackup/partition_notwin.test index 10687e19935e6..85bafcfb5b83d 100644 --- a/mysql-test/suite/mariabackup/partition_notwin.test +++ b/mysql-test/suite/mariabackup/partition_notwin.test @@ -1,5 +1,6 @@ source include/not_windows.inc; source include/have_partition.inc; +source include/have_mariabackup_combination.inc; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $log=$MYSQL_TMP_DIR/backup.log; @@ -14,11 +15,21 @@ create table t1 ( partition by hash (id) partitions 600; -error 1; +let $errno = 1; +if ($MTR_COMBINATION_SERVER) +{ +let $errno = 0; +} +--error $errno exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $log 2>&1; + +# The "Too many open files" diagnostic only applies to native mariabackup. +if (!$MTR_COMBINATION_SERVER) +{ let SEARCH_FILE=$log; let SEARCH_PATTERN=Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files; source include/search_pattern_in_file.inc; +} rmdir $targetdir; #remove_file $log; diff --git a/mysql-test/suite/mariabackup/relative_path.test b/mysql-test/suite/mariabackup/relative_path.test index bd25a217e711e..60c287403165c 100644 --- a/mysql-test/suite/mariabackup/relative_path.test +++ b/mysql-test/suite/mariabackup/relative_path.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); diff --git a/mysql-test/suite/mariabackup/row_format_redundant.test b/mysql-test/suite/mariabackup/row_format_redundant.test index 5bae9218d840c..1820fec9ebfaa 100644 --- a/mysql-test/suite/mariabackup/row_format_redundant.test +++ b/mysql-test/suite/mariabackup/row_format_redundant.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test index bb476b8771e98..9076abff6c4cf 100644 --- a/mysql-test/suite/mariabackup/small_ibd.test +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -1,4 +1,5 @@ --source include/innodb_page_size.inc +--source include/have_mariabackup_combination.inc # Check if ibd smaller than page size are skipped # It is possible, due to race conditions that new file diff --git a/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff new file mode 100644 index 0000000000000..d886111b06fb7 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff @@ -0,0 +1,13 @@ +--- undo_space_id.result ++++ undo_space_id,SERVER.result +@@ -11,10 +11,3 @@ + undo001 + undo002 + DROP TABLE t1; +-# +-# MDEV-33980 mariadb-backup --backup is missing +-# retry logic for undo tablespaces +-# +-# xtrabackup backup +-# Display undo log files from target directory +-FOUND 5 /Retrying to read undo tablespace*/ in backup.log diff --git a/mysql-test/suite/mariabackup/undo_space_id.test b/mysql-test/suite/mariabackup/undo_space_id.test index 168740fc528ce..e239189040c89 100644 --- a/mysql-test/suite/mariabackup/undo_space_id.test +++ b/mysql-test/suite/mariabackup/undo_space_id.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --echo # Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4)) @@ -24,6 +25,8 @@ list_files $basedir undo*; DROP TABLE t1; rmdir $basedir; +if (!$MTR_COMBINATION_SERVER) +{ --echo # --echo # MDEV-33980 mariadb-backup --backup is missing --echo # retry logic for undo tablespaces @@ -42,3 +45,4 @@ list_files $basedir undo*; --source include/search_pattern_in_file.inc rmdir $basedir; remove_file $backuplog; +} diff --git a/mysql-test/suite/mariabackup/undo_truncate.test b/mysql-test/suite/mariabackup/undo_truncate.test index a23c9cf64ff6b..901bf73920a41 100644 --- a/mysql-test/suite/mariabackup/undo_truncate.test +++ b/mysql-test/suite/mariabackup/undo_truncate.test @@ -2,6 +2,7 @@ --source include/not_embedded.inc --source include/have_sequence.inc --source include/have_file_key_management.inc +--source include/have_mariabackup_combination.inc SET GLOBAL innodb_undo_log_truncate = 0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff new file mode 100644 index 0000000000000..e785a0601fe94 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff @@ -0,0 +1,8 @@ +--- unencrypted_page_compressed.result ++++ unencrypted_page_compressed,SERVER.result +@@ -8,5 +8,4 @@ + # Corrupt the table + # restart: --skip-innodb-buffer-pool-load-at-startup + # xtrabackup backup +-FOUND 1 /Database page corruption detected.*/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 68f22e69e0325..fc79a0a211e89 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); @@ -39,6 +40,8 @@ echo # xtrabackup backup; --disable_result_log let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log @@ -47,6 +50,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc remove_file $backuplog; +rmdir $targetdir; +} drop table t1; -rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/vector.test b/mysql-test/suite/mariabackup/vector.test index 4f60ddce10e9c..1889d8c412ec3 100644 --- a/mysql-test/suite/mariabackup/vector.test +++ b/mysql-test/suite/mariabackup/vector.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)) engine=innodb; insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')),