From da25a00f64fc893bc2923c4fd3368d1b21ba8ddb Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Wed, 10 Jan 2024 09:59:53 -0500 Subject: [PATCH 01/78] HubSpot Edit: Add HubSpot build setup --- .blazar.yaml | 25 + .build-jdk17 | 0 build-scripts/prepare_environment.sh | 97 ++++ hbase-rpm/.blazar.yaml | 30 ++ hbase-rpm/build.sh | 51 ++ hbase-rpm/hbase.spec | 131 +++++ hbase-rpm/sources/hbase.1 | 88 ++++ hbase-rpm/sources/install_hbase.sh | 180 +++++++ hubspot-client-bundles/.blazar.yaml | 24 + hubspot-client-bundles/.build-jdk17 | 0 hubspot-client-bundles/README.md | 59 +++ .../hbase-backup-restore-bundle/.blazar.yaml | 26 + .../hbase-backup-restore-bundle/.build-jdk17 | 0 .../hbase-backup-restore-bundle/pom.xml | 119 +++++ .../hbase-client-bundle/.blazar.yaml | 25 + .../hbase-client-bundle/.build-jdk17 | 0 .../hbase-client-bundle/pom.xml | 127 +++++ .../hbase-mapreduce-bundle/.blazar.yaml | 25 + .../hbase-mapreduce-bundle/.build-jdk17 | 0 .../hbase-mapreduce-bundle/pom.xml | 251 ++++++++++ .../hbase-server-it-bundle/.blazar.yaml | 26 + .../hbase-server-it-bundle/.build-jdk17 | 0 .../hbase-server-it-bundle/pom.xml | 168 +++++++ hubspot-client-bundles/pom.xml | 458 ++++++++++++++++++ 24 files changed, 1910 insertions(+) create mode 100644 .blazar.yaml create mode 100644 .build-jdk17 create mode 100755 build-scripts/prepare_environment.sh create mode 100644 hbase-rpm/.blazar.yaml create mode 100755 hbase-rpm/build.sh create mode 100644 hbase-rpm/hbase.spec create mode 100644 hbase-rpm/sources/hbase.1 create mode 100755 hbase-rpm/sources/install_hbase.sh create mode 100644 hubspot-client-bundles/.blazar.yaml create mode 100644 hubspot-client-bundles/.build-jdk17 create mode 100644 hubspot-client-bundles/README.md create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-client-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-client-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-client-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/pom.xml create mode 100644 hubspot-client-bundles/pom.xml diff --git a/.blazar.yaml b/.blazar.yaml new file mode 100644 index 000000000000..e034ada7508d --- /dev/null +++ b/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java-single-module + +env: + MAVEN_PHASE: "package assembly:single deploy" + HADOOP_DEP_VERSION: "3.3.6-hubspot-SNAPSHOT" + MAVEN_BUILD_ARGS: "-Phadoop-3.0 -Dhadoop.profile=3.0 -Dhadoop-three.version=$HADOOP_DEP_VERSION -Dgpg.skip=true -DskipTests -DdeployAtEnd -pl hbase-assembly -am -T1C" + + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +provides: + - hbase diff --git a/.build-jdk17 b/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/build-scripts/prepare_environment.sh b/build-scripts/prepare_environment.sh new file mode 100755 index 000000000000..65842dcd4d17 --- /dev/null +++ b/build-scripts/prepare_environment.sh @@ -0,0 +1,97 @@ +# +# Generates the appropriate environment vars so that we: +# - build against the right version of hadoop, and properly set up maven +# - generate the correct maven version based on the branches +# - upload RPMs with the correct release based on the branch, and to the right yum repo +# +# Since we need to distribute .blazar.yaml to all sub-modules of the project, we define our constants once +# in this script which can be re-used by every .blazar.yaml. +# +set -ex +printenv + +# We base the expected main branch and resulting maven version for clients on the hbase minor version +# The reason for this is hbase re-branches for each minor release (2.4, 2.5, 2.6, etc). At each re-branch +# the histories diverge. So we'll need to create our own fork of each new minor release branch. +# The convention is a fork named "hubspot-$minorVersion", and the maven coordinates "$minorVersion-hubspot-SNAPSHOT" +MINOR_VERSION="2.6" +MAIN_BRANCH="hubspot-${MINOR_VERSION}" + +# +# Validate inputs from blazar +# + +if [ -z "$WORKSPACE" ]; then + echo "Missing env var \$WORKSPACE" + exit 1 +fi +if [ -z "$GIT_BRANCH" ]; then + echo "Missing env var \$GIT_BRANCH" + exit 1 +fi +if [ -z "$BUILD_COMMAND_RC_FILE" ]; then + echo "Missing env var \$BUILD_COMMAND_RC_FILE" + exit 1 +fi + +# +# Extract current hbase version from root pom.xml +# + +# the pom.xml has an invalid xml namespace, so just remove that so xmllint can parse it. +cat $WORKSPACE/pom.xml | sed '2 s/xmlns=".*"//g' > pom.xml.tmp +HBASE_VERSION=$(echo "cat /project/properties/revision/text()" | xmllint --nocdata --shell pom.xml.tmp | sed '1d;$d') +rm pom.xml.tmp + +# sanity check that we've got some that looks right. it wouldn't be the end of the world if we got it wrong, but +# will help avoid confusion. +if [[ ! "$HBASE_VERSION" =~ 2\.[0-9]+\.[0-9]+ ]]; then + echo "Unexpected HBASE_Version extracted from pom.xml. Got $HBASE_VERSION but expected a string like '2.4.3', with 3 numbers separated by decimals, the first number being 2." + exit 1 +fi + +# +# Generate branch-specific env vars +# We are going to generate the maven version and the RPM release here: +# - For the maven version, we need to special case our main branch +# - For RPM, we want our final version to be: +# main branch: {hbase_version}-hs.{build_number}.el6 +# other branches: {hbase_version}-hs~{branch_name}.{build_number}.el6, where branch_name substitutes underscore for non-alpha-numeric characters +# + +echo "Git branch $GIT_BRANCH. Detecting appropriate version override and RPM release." + +RELEASE="hs" + +if [[ "$GIT_BRANCH" = "$MAIN_BRANCH" ]]; then + SET_VERSION="${MINOR_VERSION}-hubspot-SNAPSHOT" + REPO_NAME="AnyLinuxVersion_hs-hbase" +elif [[ "$GIT_BRANCH" != "hubspot" ]]; then + SET_VERSION="${MINOR_VERSION}-${GIT_BRANCH}-SNAPSHOT" + RELEASE="${RELEASE}~${GIT_BRANCH//[^[:alnum:]]/_}" + REPO_NAME="AnyLinuxVersion_hs-hbase-develop" +else + echo "Invalid git branch $GIT_BRANCH" + exit 1 +fi + +RELEASE="${RELEASE}.${BUILD_NUMBER}" +FULL_BUILD_VERSION="${HBASE_VERSION}-${RELEASE}" + +# SET_VERSION is not the most intuitive name, but it's required for set-maven-versions script +write-build-env-var SET_VERSION "$SET_VERSION" +write-build-env-var HBASE_VERSION "$HBASE_VERSION" +write-build-env-var PKG_RELEASE "$RELEASE" +write-build-env-var FULL_BUILD_VERSION "$FULL_BUILD_VERSION" +write-build-env-var REPO_NAME "$REPO_NAME" +# Adding this value as versioninfo.version ensures we have the same value as would normally +# show up in a non-hubspot hbase build. Otherwise due to set-maven-versions we'd end up +# with 2.6-hubspot-SNAPSHOT which is not very useful as a point of reference. +# Another option would be to pass in our FULL_BUILD_VERSION but that might cause some funniness +# with the expectations in VersionInfo.compareVersion(). +write-build-env-var MAVEN_BUILD_ARGS "$MAVEN_BUILD_ARGS -Dversioninfo.version=$HBASE_VERSION" + +echo "Building HBase version $HBASE_VERSION" +echo "Will deploy to nexus with version $SET_VERSION" +echo "Will create rpm with version $FULL_BUILD_VERSION" +echo "Will run maven with extra args $MAVEN_BUILD_ARGS" diff --git a/hbase-rpm/.blazar.yaml b/hbase-rpm/.blazar.yaml new file mode 100644 index 000000000000..a1bfcb2ae17b --- /dev/null +++ b/hbase-rpm/.blazar.yaml @@ -0,0 +1,30 @@ +buildpack: + name: Buildpack-RPMs + +env: + RPM_BUILD_COMMAND: ./build.sh + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +enableBuildTargets: + - almalinux9_amd64 + +depends: + - hbase + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +stepActivation: + uploadRpms: + branchRegexes: ['.*'] diff --git a/hbase-rpm/build.sh b/hbase-rpm/build.sh new file mode 100755 index 000000000000..b527ca732913 --- /dev/null +++ b/hbase-rpm/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e +set -x + +ROOT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +for iv in HBASE_VERSION SET_VERSION PKG_RELEASE; do + if [[ "X${!iv}" = "X" ]]; then + echo "Must specifiy $iv" + exit 1 + fi +done + +# Setup build dir +BUILD_DIR="${ROOT_DIR}/build" +rm -rf $BUILD_DIR +mkdir -p ${BUILD_DIR}/{SOURCES,SPECS,RPMS} +cp -a $ROOT_DIR/sources/* ${BUILD_DIR}/SOURCES/ +cp $ROOT_DIR/hbase.spec ${BUILD_DIR}/SPECS/ + +# Download bin tar built by hbase-assembly +SOURCES_DIR=$BUILD_DIR/SOURCES +mvn dependency:copy \ + -Dartifact=org.apache.hbase:hbase-assembly:${SET_VERSION}:tar.gz:bin \ + -DoutputDirectory=$SOURCES_DIR \ + -DlocalRepositoryDirectory=$SOURCES_DIR \ + -Dtransitive=false +INPUT_TAR=`ls -d $SOURCES_DIR/hbase-assembly-*.tar.gz` + +if [[ $HBASE_VERSION == *"-SNAPSHOT" ]]; then + # unreleased verion. do i want to denote that in the rpm release somehow? + # it can't be in the version, so strip here + HBASE_VERSION=${HBASE_VERSION//-SNAPSHOT/} +fi + +rpmbuild \ + --define "_topdir $BUILD_DIR" \ + --define "input_tar $INPUT_TAR" \ + --define "hbase_version ${HBASE_VERSION}" \ + --define "maven_version ${SET_VERSION}" \ + --define "release ${PKG_RELEASE}%{?dist}" \ + -bb \ + $BUILD_DIR/SPECS/hbase.spec + +if [[ -d $RPMS_OUTPUT_DIR ]]; then + mkdir -p $RPMS_OUTPUT_DIR + + # Move rpms to output dir for upload + + find ${BUILD_DIR}/RPMS -name "*.rpm" -exec mv {} $RPMS_OUTPUT_DIR/ \; +fi diff --git a/hbase-rpm/hbase.spec b/hbase-rpm/hbase.spec new file mode 100644 index 000000000000..107c92636f06 --- /dev/null +++ b/hbase-rpm/hbase.spec @@ -0,0 +1,131 @@ +# taken from hbase.spec in https://github.com/apache/bigtop/ +# greatly modified to simplify and fix dependencies to work in the hubspot environment + +%define hadoop_major_version 3.2 +%define hbase_major_version 2.4 +%define etc_hbase_conf %{_sysconfdir}/hbase/conf +%define etc_hbase_conf_dist %{etc_hbase_conf}.dist +%define hbase_home /usr/lib/hbase +%define bin_hbase %{hbase_home}/bin +%define lib_hbase %{hbase_home}/lib +%define conf_hbase %{hbase_home}/conf +%define logs_hbase %{hbase_home}/logs +%define pids_hbase %{hbase_home}/pids +%define man_dir %{_mandir} +%define hbase_username hbase +%define hadoop_home /usr/lib/hadoop +%define zookeeper_home /usr/lib/zookeeper + +# FIXME: brp-repack-jars uses unzip to expand jar files +# Unfortunately guice-2.0.jar pulled by ivy contains some files and directories without any read permission +# and make whole process to fail. +# So for now brp-repack-jars is being deactivated until this is fixed. +# See BIGTOP-294 +%define __os_install_post \ + %{_rpmconfigdir}/brp-compress ; \ + %{_rpmconfigdir}/brp-strip-static-archive %{__strip} ; \ + %{_rpmconfigdir}/brp-strip-comment-note %{__strip} %{__objdump} ; \ + /usr/lib/rpm/brp-python-bytecompile ; \ + %{nil} + +%define doc_hbase %{_docdir}/hbase-%{hbase_version} +%global initd_dir %{_sysconfdir}/rc.d/init.d +%define alternatives_cmd alternatives + +# Disable debuginfo package +%define debug_package %{nil} + +# HubSpot: use zstd because it decompresses much faster +%define _binary_payload w19.zstdio +%define _source_payload w19.zstdio + +Name: hbase +Version: %{hbase_version} +Release: %{release} +BuildArch: noarch +Summary: HBase is the Hadoop database. Use it when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. +URL: http://hbase.apache.org/ +Group: Systems/Daemons +Buildroot: %{_topdir}/INSTALL/hbase-%{maven_version} +License: ASL 2.0 +Source0: %{input_tar} +Source1: install_hbase.sh + +Requires: coreutils, /usr/sbin/useradd, /sbin/chkconfig, /sbin/service +Requires: hadoop >= %{hadoop_major_version} + +AutoReq: no + +%description +HBase is an open-source, distributed, column-oriented store modeled after Google' Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, HBase provides Bigtable-like capabilities on top of Hadoop. HBase includes: + + * Convenient base classes for backing Hadoop MapReduce jobs with HBase tables + * Query predicate push down via server side scan and get filters + * Optimizations for real time queries + * A high performance Thrift gateway + * A REST-ful Web service gateway that supports XML, Protobuf, and binary data encoding options + * Cascading source and sink modules + * Extensible jruby-based (JIRB) shell + * Support for exporting metrics via the Hadoop metrics subsystem to files or Ganglia; or via JMX + +%prep +%setup -n hbase-%{maven_version} + +%install +%__rm -rf $RPM_BUILD_ROOT +bash %{SOURCE1} \ + --input-tar=%{SOURCE0} \ + --doc-dir=%{doc_hbase} \ + --conf-dir=%{etc_hbase_conf_dist} \ + --prefix=$RPM_BUILD_ROOT + +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/log/hbase +ln -s %{_localstatedir}/log/hbase %{buildroot}/%{logs_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/run/hbase +ln -s %{_localstatedir}/run/hbase %{buildroot}/%{pids_hbase} + +%__install -d -m 0755 %{buildroot}/%{_localstatedir}/lib/hbase + +%__install -d -m 0755 $RPM_BUILD_ROOT/usr/bin + +# Pull hadoop from its packages +rm -f $RPM_BUILD_ROOT/%{lib_hbase}/{hadoop,slf4j-log4j12-}*.jar + +ln -f -s %{hadoop_home}/client/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-hdfs-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_hbase} +ln -f -s %{hadoop_home}/client/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_hbase} + +%pre +getent group hbase 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hbase +getent passwd hbase 2>&1 > /dev/null || /usr/sbin/useradd -c "HBase" -s /sbin/nologin -g hbase -r -d /var/lib/hbase hbase 2> /dev/null || : + +%post +%{alternatives_cmd} --install %{etc_hbase_conf} %{name}-conf %{etc_hbase_conf_dist} 30 + +%files +%defattr(-,hbase,hbase) +%{logs_hbase} +%{pids_hbase} +%dir %{_localstatedir}/log/hbase +%dir %{_localstatedir}/run/hbase +%dir %{_localstatedir}/lib/hbase + +%defattr(-,root,root) +%{hbase_home} +%{hbase_home}/hbase-*.jar +/usr/bin/hbase +%config(noreplace) %{etc_hbase_conf_dist} + +# files from doc package +%defattr(-,root,root) +%doc %{doc_hbase}/ diff --git a/hbase-rpm/sources/hbase.1 b/hbase-rpm/sources/hbase.1 new file mode 100644 index 000000000000..349218fe1d87 --- /dev/null +++ b/hbase-rpm/sources/hbase.1 @@ -0,0 +1,88 @@ +.\" Licensed to the Apache Software Foundation (ASF) under one or more +.\" contributor license agreements. See the NOTICE file distributed with +.\" this work for additional information regarding copyright ownership. +.\" The ASF licenses this file to You under the Apache License, Version 2.0 +.\" (the "License"); you may not use this file except in compliance with +.\" the License. You may obtain a copy of the License at +.\" +.\" http://www.apache.org/licenses/LICENSE-2.0 +.\" +.\" Unless required by applicable law or agreed to in writing, software +.\" distributed under the License is distributed on an "AS IS" BASIS, +.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.\" See the License for the specific language governing permissions and +.\" limitations under the License. +.\" +.\" Process this file with +.\" groff -man -Tascii hbase.1 +.\" +.TH hbase 1 "October 2010 " Linux "User Manuals" + +.SH NAME +HBase \- HBase is the Hadoop database. + +.SH SYNOPSIS + +.B hbase +\fICOMMAND\fR + +.SH DESCRIPTION + +HBase is the Hadoop database. Use it when you need random, realtime +read/write access to your Big Data. This project's goal is the hosting +of very large tables -- billions of rows X millions of columns -- atop +clusters of commodity hardware. + +HBase is an open-source, distributed, versioned, column-oriented store +modeled after Google's Bigtable: A Distributed Storage System for +Structured Data by Chang et al. Just as Bigtable leverages the +distributed data storage provided by the Google File System, HBase +provides Bigtable-like capabilities on top of Hadoop. + +For more information about HBase, see http://hbase.apache.org. + +\fICOMMAND\fR may be one of the following: + shell run the HBase shell + shell-tests run the HBase shell tests + zkcli run the ZooKeeper shell + master run an HBase HMaster node + regionserver run an HBase HRegionServer node + zookeeper run a Zookeeper server + rest run an HBase REST server + thrift run an HBase Thrift server + avro run an HBase Avro server + migrate upgrade an hbase.rootdir + hbck run the hbase 'fsck' tool + or + CLASSNAME run the class named CLASSNAME + +Most commands print help when invoked w/o parameters or with --help. + +.SH ENVIRONMENT + +.IP JAVA_HOME +The java implementation to use. Overrides JAVA_HOME. + +.IP HBASE_CLASSPATH +Extra Java CLASSPATH entries. + +.IP HBASE_HEAPSIZE +The maximum amount of heap to use, in MB. Default is 1000. + +.IP HBASE_OPTS +Extra Java runtime options. + +.IP HBASE_CONF_DIR +Alternate conf dir. Default is ${HBASE_HOME}/conf. + +.IP HBASE_ROOT_LOGGER +The root appender. Default is INFO,console + +.IP HIVE_OPT +Extra Java runtime options. + +.IP HADOOP_HOME +Optionally, the Hadoop home to run with. + +.SH COPYRIGHT +Copyright (C) 2010 The Apache Software Foundation. All rights reserved. diff --git a/hbase-rpm/sources/install_hbase.sh b/hbase-rpm/sources/install_hbase.sh new file mode 100755 index 000000000000..95265d2100c8 --- /dev/null +++ b/hbase-rpm/sources/install_hbase.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() { + echo " +usage: $0 + Required not-so-options: + --mvn-target-dir=DIR path to the output of the mvn assembly + --prefix=PREFIX path to install into + + Optional options: + --doc-dir=DIR path to install docs into [/usr/share/doc/hbase] + --lib-dir=DIR path to install hbase home [/usr/lib/hbase] + --installed-lib-dir=DIR path where lib-dir will end up on target system + --bin-dir=DIR path to install bins [/usr/bin] + --examples-dir=DIR path to install examples [doc-dir/examples] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'doc-dir:' \ + -l 'lib-dir:' \ + -l 'installed-lib-dir:' \ + -l 'bin-dir:' \ + -l 'examples-dir:' \ + -l 'conf-dir:' \ + -l 'input-tar:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --input-tar) + INPUT_TAR=$2 ; shift 2 + ;; + --doc-dir) + DOC_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --examples-dir) + EXAMPLES_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX INPUT_TAR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +MAN_DIR=${MAN_DIR:-/usr/share/man/man1} +DOC_DIR=${DOC_DIR:-/usr/share/doc/hbase} +LIB_DIR=${LIB_DIR:-/usr/lib/hbase} + +BIN_DIR=${BIN_DIR:-/usr/lib/hbase/bin} +ETC_DIR=${ETC_DIR:-/etc/hbase} +CONF_DIR=${CONF_DIR:-${ETC_DIR}/conf.dist} +THRIFT_DIR=${THRIFT_DIR:-${LIB_DIR}/include/thrift} + +EXTRACT_DIR=extracted +rm -rf $EXTRACT_DIR +mkdir $EXTRACT_DIR + +version_part=$SET_VERSION +if [ -z "$version_part" ]; then + version_part=$HBASE_VERSION +fi + +tar -C $EXTRACT_DIR --strip-components=1 -xzf $INPUT_TAR + +# we do not need the shaded clients in our rpm. they bloat the size and cause classpath issues for hbck2. +rm -rf $EXTRACT_DIR/lib/shaded-clients + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$DOC_DIR +install -d -m 0755 $PREFIX/$BIN_DIR +install -d -m 0755 $PREFIX/$ETC_DIR +install -d -m 0755 $PREFIX/$MAN_DIR +install -d -m 0755 $PREFIX/$THRIFT_DIR + +cp -ra $EXTRACT_DIR/lib/* ${PREFIX}/${LIB_DIR}/lib/ +cp $EXTRACT_DIR/lib/hbase*.jar $PREFIX/$LIB_DIR + +# We do not currently run "mvn site", so do not have a docs dir. +# Only copy contents if dir exists +if [ -n "$(ls -A $EXTRACT_DIR/docs 2>/dev/null)" ]; then + cp -a $EXTRACT_DIR/docs/* $PREFIX/$DOC_DIR + cp $EXTRACT_DIR/*.txt $PREFIX/$DOC_DIR/ +else + echo "Doc generation is currently disabled in our RPM build. If this is an issue, it should be possible to enable them with some work. See https://git.hubteam.com/HubSpot/apache-hbase/blob/hubspot-2/rpm/sources/do-component-build#L17-L24 for details." > $PREFIX/$DOC_DIR/README.txt +fi + +cp -a $EXTRACT_DIR/conf $PREFIX/$CONF_DIR +cp -a $EXTRACT_DIR/bin/* $PREFIX/$BIN_DIR + +# Purge scripts that don't work with packages +for file in rolling-restart.sh graceful_stop.sh local-regionservers.sh \ + master-backup.sh regionservers.sh zookeepers.sh hbase-daemons.sh \ + start-hbase.sh stop-hbase.sh local-master-backup.sh ; do + rm -f $PREFIX/$BIN_DIR/$file +done + + +ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/conf + +# Make a symlink of hbase.jar to hbase-version.jar +pushd `pwd` +cd $PREFIX/$LIB_DIR +for i in `ls hbase*jar | grep -v tests.jar` +do + ln -s $i `echo $i | sed -n 's/\(.*\)\(-[0-9].*\)\(.jar\)/\1\3/p'` +done +popd + +wrapper=$PREFIX/usr/bin/hbase +mkdir -p `dirname $wrapper` +cat > $wrapper < dependencies.sorted` to get a file that can be compared with another such-processed file +4. Make the change you want in the bundle, then `mvn clean install` +5. Re-run steps 2 and 3, outputting to a new file +6. Run `comm -13 first second` to see what might be newly added after your change, or `comm -23` to see what might have been removed +7. If trying to track a specific dependency from the list, go back here and run `mvn dependency:tree -Dincludes=`. This might show you what dependency you need to add an exclusion to + +This ends up being pretty iterative and trial/error, but can eventually get to a jar which has what you want (and doesn't what you don't). diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml new file mode 100644 index 000000000000..9399e5dc0aa4 --- /dev/null +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle + - hbase-mapreduce-bundle +provides: + - hbase-backup-restore-bundle diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-backup-restore-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml b/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml new file mode 100644 index 000000000000..9707d9d8118d --- /dev/null +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-backup-restore-bundle + + + + com.hubspot.hbase + hbase-client-bundle + + + + commons-io + commons-io + + + + + com.hubspot.hbase + hbase-mapreduce-bundle + + + + commons-io + commons-io + + + + + org.apache.hbase + hbase-backup + + + + commons-io + commons-io + + + + org.apache.hbase + * + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + org.apache.hbase:* + + io.opentelemetry:opentelemetry-api + io.opentelemetry:opentelemetry-context + com.google.protobuf:protobuf-java + io.dropwizard.metrics:metrics-core + + + + + org.apache.kerby:* + + krb5-template.conf + krb5_udp-template.conf + ccache.txt + keytab.txt + + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml new file mode 100644 index 000000000000..300be28892e8 --- /dev/null +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles +provides: + - hbase-client-bundle + diff --git a/hubspot-client-bundles/hbase-client-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-client-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-client-bundle/pom.xml b/hubspot-client-bundles/hbase-client-bundle/pom.xml new file mode 100644 index 000000000000..24ce44daf93a --- /dev/null +++ b/hubspot-client-bundles/hbase-client-bundle/pom.xml @@ -0,0 +1,127 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-client-bundle + + + + org.apache.hbase + hbase-openssl + + + + org.apache.hbase + hbase-client + + + + org.apache.hbase + hbase-hadoop-compat + + + org.apache.hbase + hbase-hadoop2-compat + + + + commons-logging + commons-logging + + + org.jruby.joni + joni + + + org.jruby.jcodings + jcodings + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-endpoint + + + * + * + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-logging + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-openssl + + org.apache.hbase.thirdparty:* + + org.apache.hbase:hbase-endpoint + + + + com.google.protobuf:protobuf-java + + io.dropwizard.metrics:metrics-core + + commons-io:commons-io + + + + + org.apache.hbase:hbase-endpoint + + org/apache/hadoop/hbase/client/coprocessor/** + org/apache/hadoop/hbase/protobuf/generated/** + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml new file mode 100644 index 000000000000..5c020e374927 --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -0,0 +1,25 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle +provides: + - hbase-mapreduce-bundle diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-mapreduce-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml new file mode 100644 index 000000000000..233e33750fe1 --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml @@ -0,0 +1,251 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-mapreduce-bundle + + + + + com.hubspot.hbase + hbase-client-bundle + + + + commons-io + commons-io + + + + + + org.apache.hbase + hbase-mapreduce + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-protocol-shaded + + + org.apache.hbase + hbase-logging + + + com.google.protobuf + protobuf-java + + + org.apache.hbase.thirdparty + hbase-shaded-gson + + + org.apache.hbase.thirdparty + hbase-shaded-protobuf + + + org.apache.hbase.thirdparty + hbase-unsafe + + + org.apache.hbase.thirdparty + hbase-shaded-miscellaneous + + + org.apache.hbase.thirdparty + hbase-shaded-netty + + + + commons-logging + commons-logging + + + + commons-io + commons-io + + + com.sun.jersey + * + + + tomcat + jasper-runtime + + + org.mortbay.jetty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-server + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-protocol-shaded + + + org.apache.hbase + hbase-logging + + + com.google.protobuf + protobuf-java + + + org.apache.hbase.thirdparty + hbase-shaded-gson + + + org.apache.hbase.thirdparty + hbase-shaded-protobuf + + + org.apache.hbase.thirdparty + hbase-unsafe + + + org.apache.hbase.thirdparty + hbase-shaded-miscellaneous + + + org.apache.hbase.thirdparty + hbase-shaded-netty + + + + + commons-logging + commons-logging + + + + commons-io + commons-io + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + org.glassfish.hk2.external + jakarta.inject + + + jakarta.ws.rs + jakarta.ws.rs-api + + + + + org.apache.hbase + hbase-compression-zstd + + + org.apache.hbase + * + + + + commons-io + commons-io + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + + org.apache.hbase:* + + org.apache.hbase.thirdparty:* + + + + + + + + + diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml new file mode 100644 index 000000000000..26db8c8066b3 --- /dev/null +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + YUM_REPO_UPLOAD_OVERRIDE_CENTOS_8: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + REPO_NAME: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hbase +provides: + - hbase-server-it-bundle + diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-server-it-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hubspot-client-bundles/hbase-server-it-bundle/pom.xml b/hubspot-client-bundles/hbase-server-it-bundle/pom.xml new file mode 100644 index 000000000000..fa617258f82d --- /dev/null +++ b/hubspot-client-bundles/hbase-server-it-bundle/pom.xml @@ -0,0 +1,168 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-server-it-bundle + + + + org.apache.hbase + hbase-it + test-jar + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-server + test-jar + ${project.version} + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase-testing-util + ${project.version} + + + + commons-logging + commons-logging + + + javax.servlet.jsp + * + + + javax.servlet + * + + + org.glassfish.web + * + + + org.jamon + jamon-runtime + + + io.netty + * + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + org.apache.hbase:* + + junit:junit + + commons-io:commons-io + + org.apache.hbase.thirdparty:* + com.google.protobuf:protobuf-java + + io.opentelemetry:opentelemetry-api + io.opentelemetry:opentelemetry-context + com.google.protobuf:protobuf-java + io.dropwizard.metrics:metrics-core + + + + + org.apache.kerby:* + + krb5-template.conf + krb5_udp-template.conf + ccache.txt + keytab.txt + + + + + + + + + + diff --git a/hubspot-client-bundles/pom.xml b/hubspot-client-bundles/pom.xml new file mode 100644 index 000000000000..0105ffd28c43 --- /dev/null +++ b/hubspot-client-bundles/pom.xml @@ -0,0 +1,458 @@ + + + 4.0.0 + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + pom + Bundled versions of the hbase client + + + hbase-client-bundle + hbase-mapreduce-bundle + hbase-backup-restore-bundle + hbase-server-it-bundle + + + + org.apache.hadoop.hbase.shaded + + 3.6.3-shaded-SNAPSHOT + + 2.6-hubspot-SNAPSHOT + + + + + + + com.hubspot.hbase + hbase-client-bundle + ${project.version} + + + org.apache.hbase + hbase-client + + + + + com.hubspot.hbase + hbase-mapreduce-bundle + ${project.version} + + + com.hubspot.hbase + hbase-backup-restore-bundle + ${project.version} + + + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + org.apache.hbase + hbase-openssl + ${project.version} + + + org.apache.hbase + hbase-compression-zstd + ${project.version} + + + org.apache.hbase + hbase-client + ${project.version} + + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-server + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-mapreduce + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-endpoint + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-backup + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-hadoop2-compat + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + org.apache.hbase + hbase-it + test-jar + ${project.version} + + + javax.activation + javax.activation-api + + + javax.annotation + javax.annotation-api + + + org.slf4j + slf4j-reload4j + + + com.google.code.findbugs + jsr305 + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey.contribs + jersey-guice + + + com.github.pjfanning + jersey-json + + + org.apache.avro + avro + + + org.eclipse.jetty + jetty-client + + + com.google.j2objc + j2objc-annotations + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + create-bundle-with-relocations + + shade + + package + + + true + true + true + true + true + + + com.google.protobuf + ${shade.prefix}.com.google.protobuf + + + com.codahale.metrics + ${shade.prefix}.com.codahale.metrics + + + org.apache.commons.io + ${shade.prefix}.org.apache.commons.io + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + From 78256996d30d73125bdbfb2bef4091ca32deeda2 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Mon, 12 Feb 2024 12:01:54 -0500 Subject: [PATCH 02/78] HubSpot Edit: HBASE-28365: ChaosMonkey batch suspend/resume action assume shell implementation (not yet written upstream) --- .../chaos/actions/RollingBatchSuspendResumeRsAction.java | 4 ++++ .../hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java index 30babcd4d413..f56d23df3c65 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java @@ -122,6 +122,8 @@ public void perform() throws Exception { suspendRs(server); } catch (Shell.ExitCodeException e) { LOG.warn("Problem suspending but presume successful; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem suspending but presume successful", e); } suspendedServers.add(server); break; @@ -131,6 +133,8 @@ public void perform() throws Exception { resumeRs(server); } catch (Shell.ExitCodeException e) { LOG.info("Problem resuming, will retry; code={}", e.getExitCode(), e); + } catch (Exception e) { + LOG.warn("Problem resulting, will retry", e); } resumedServers.add(server); break; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index fb8ab209c3a1..756f0d3846a6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -86,7 +86,6 @@ private static ExecutorService buildMonkeyThreadPool(final int size) { return Executors.newFixedThreadPool(size, new ThreadFactoryBuilder().setDaemon(false) .setNameFormat("ChaosMonkey-%d").setUncaughtExceptionHandler((t, e) -> { LOG.error("Uncaught exception in thread {}", t.getName(), e); - throw new RuntimeException(e); }).build()); } From ccc2716f1bff1e6612e84e3ada1eedaac8b41ec1 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Sat, 17 Feb 2024 11:40:11 -0500 Subject: [PATCH 03/78] HubSpot Edit: Add retries to verify step of ITBLL --- .../test/IntegrationTestBigLinkedList.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index c1854d87c199..2c4dd96eedab 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -1532,9 +1532,20 @@ protected void runVerify(String outputDir, int numReducers, long expectedNumNode Verify verify = new Verify(); verify.setConf(getConf()); - int retCode = verify.run(iterationOutput, numReducers); - if (retCode > 0) { - throw new RuntimeException("Verify.run failed with return code: " + retCode); + + int retries = getConf().getInt("hbase.itbll.verify.retries", 1); + + while (true) { + int retCode = verify.run(iterationOutput, numReducers); + if (retCode > 0) { + if (retries-- > 0) { + LOG.warn("Verify.run failed with return code: {}. Will retry", retries); + } else { + throw new RuntimeException("Verify.run failed with return code: " + retCode); + } + } else { + break; + } } if (!verify.verify(expectedNumNodes)) { From 032cbf8d4e81c01783f862aa78da640b95e8a296 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 2 Feb 2024 09:17:58 -0500 Subject: [PATCH 04/78] HubSpot Edit: Add an hbase-site.xml to our bundles that configures ZStdCodec Co-authored-by: Charles Connell --- .../src/main/resources/hbase-site.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml b/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml new file mode 100644 index 000000000000..629c6f84f30e --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/src/main/resources/hbase-site.xml @@ -0,0 +1,12 @@ + + + + + + hbase.io.compress.zstd.codec + org.apache.hadoop.hbase.io.compress.zstd.ZstdCodec + + From 8c8d387f5072c9984dc4f602f44dfcb7176a1891 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Fri, 19 Apr 2024 10:28:56 -0400 Subject: [PATCH 05/78] HubSpot Edit: Add hdfs stats for local and remote rack bytes read --- .../MetricsRegionServerSource.java | 8 ++++++ .../MetricsRegionServerWrapper.java | 4 +++ .../MetricsRegionServerSourceImpl.java | 4 +++ .../MetricsRegionServerWrapperImpl.java | 25 +++++++++++++++++++ .../MetricsRegionServerWrapperStub.java | 10 ++++++++ 5 files changed, 51 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index c68809a1fddb..c23c222edc54 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -533,6 +533,14 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; String ZEROCOPY_BYTES_READ_DESC = "The number of bytes read through HDFS zero copy"; + String LOCAL_RACK_BYTES_READ = "localRackBytesRead"; + String LOCAL_RACK_BYTES_READ_DESC = + "The number of bytes read from the same rack of the RegionServer, but not the local HDFS DataNode"; + + String REMOTE_RACK_BYTES_READ = "remoteRackBytesRead"; + String REMOTE_RACK_BYTES_READ_DESC = + "The number of bytes read from a different rack from that of the RegionServer"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + "larger than blockingMemStoreSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 10e71d091f59..67d31ffe64c4 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -544,6 +544,10 @@ public interface MetricsRegionServerWrapper { /** Returns Number of bytes read from the local HDFS DataNode. */ long getLocalBytesRead(); + long getLocalRackBytesRead(); + + long getRemoteRackBytesRead(); + /** Returns Number of bytes read locally through HDFS short circuit. */ long getShortCircuitBytesRead(); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index e0429cfb55d1..b42a02d0e659 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -560,6 +560,10 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) .addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead()) .addGauge(Interns.info(LOCAL_BYTES_READ, LOCAL_BYTES_READ_DESC), rsWrap.getLocalBytesRead()) + .addGauge(Interns.info(LOCAL_RACK_BYTES_READ, LOCAL_RACK_BYTES_READ_DESC), + rsWrap.getLocalRackBytesRead()) + .addGauge(Interns.info(REMOTE_RACK_BYTES_READ, REMOTE_RACK_BYTES_READ_DESC), + rsWrap.getRemoteRackBytesRead()) .addGauge(Interns.info(SHORTCIRCUIT_BYTES_READ, SHORTCIRCUIT_BYTES_READ_DESC), rsWrap.getShortCircuitBytesRead()) .addGauge(Interns.info(ZEROCOPY_BYTES_READ, ZEROCOPY_BYTES_READ_DESC), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 2bd396242a17..a256e8827a39 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -29,6 +29,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.GlobalStorageStatistics; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; @@ -1052,6 +1054,29 @@ public long getLocalBytesRead() { return FSDataInputStreamWrapper.getLocalBytesRead(); } + @Override + public long getLocalRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfOneOrTwo"); + } + + @Override + public long getRemoteRackBytesRead() { + return getGlobalStorageStatistic("bytesReadDistanceOfThreeOrFour") + + getGlobalStorageStatistic("bytesReadDistanceOfFiveOrLarger"); + } + + private static long getGlobalStorageStatistic(String name) { + StorageStatistics stats = GlobalStorageStatistics.INSTANCE.get("hdfs"); + if (stats == null) { + return 0; + } + Long val = stats.getLong(name); + if (val == null) { + return 0; + } + return val; + } + @Override public long getShortCircuitBytesRead() { return FSDataInputStreamWrapper.getShortCircuitBytesRead(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 0e77ae89fef2..84654784c58d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -537,6 +537,16 @@ public long getLocalBytesRead() { return 0; } + @Override + public long getLocalRackBytesRead() { + return 0; + } + + @Override + public long getRemoteRackBytesRead() { + return 0; + } + @Override public long getShortCircuitBytesRead() { return 0; From aa64fe85d783f0ffc8d89819993eb55fa7d981a3 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Thu, 18 Apr 2024 08:54:07 -0400 Subject: [PATCH 06/78] HubSpot Edit: Basic healthcheck servlets --- .../apache/hadoop/hbase/master/HMaster.java | 6 + .../master/http/MasterHealthServlet.java | 48 ++++++++ .../hbase/monitoring/HealthCheckServlet.java | 103 ++++++++++++++++++ .../hbase/regionserver/HRegionServer.java | 12 +- .../regionserver/http/RSHealthServlet.java | 95 ++++++++++++++++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index e099760a7a84..84c782aaab72 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -137,6 +137,7 @@ import org.apache.hadoop.hbase.master.cleaner.SnapshotCleanerChore; import org.apache.hadoop.hbase.master.hbck.HbckChore; import org.apache.hadoop.hbase.master.http.MasterDumpServlet; +import org.apache.hadoop.hbase.master.http.MasterHealthServlet; import org.apache.hadoop.hbase.master.http.MasterRedirectServlet; import org.apache.hadoop.hbase.master.http.MasterStatusServlet; import org.apache.hadoop.hbase.master.http.api_v1.ResourceConfigFactory; @@ -776,6 +777,11 @@ protected Class getDumpServlet() { return MasterDumpServlet.class; } + @Override + protected Class getHealthServlet() { + return MasterHealthServlet.class; + } + @Override public MetricsMaster getMasterMetrics() { return metricsMaster; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java new file mode 100644 index 000000000000..99f2f08ac8bd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterHealthServlet.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.http; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Optional; +import javax.servlet.http.HttpServletRequest; +import org.apache.hadoop.hbase.ClusterMetrics; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.monitoring.HealthCheckServlet; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class MasterHealthServlet extends HealthCheckServlet { + + public MasterHealthServlet() { + super(HMaster.MASTER); + } + + @Override + protected Optional check(HMaster master, HttpServletRequest req, Connection conn) + throws IOException { + + if (master.isActiveMaster() && master.isOnline()) { + // this will fail if there is a problem with the active master + conn.getAdmin().getClusterMetrics(EnumSet.of(ClusterMetrics.Option.CLUSTER_ID)); + } + + return Optional.empty(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java new file mode 100644 index 000000000000..8d09089b0c64 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/HealthCheckServlet.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.monitoring; + +import java.io.IOException; +import java.util.Optional; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RpcConnectionRegistry; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public abstract class HealthCheckServlet extends HttpServlet { + + private static final String CLIENT_RPC_TIMEOUT = "healthcheck.hbase.client.rpc.timeout"; + private static final int CLIENT_RPC_TIMEOUT_DEFAULT = 5000; + private static final String CLIENT_RETRIES = "healthcheck.hbase.client.retries"; + private static final int CLIENT_RETRIES_DEFAULT = 2; + private static final String CLIENT_OPERATION_TIMEOUT = + "healthcheck.hbase.client.operation.timeout"; + private static final int CLIENT_OPERATION_TIMEOUT_DEFAULT = 15000; + + private final String serverLookupKey; + + public HealthCheckServlet(String serverLookupKey) { + this.serverLookupKey = serverLookupKey; + } + + @SuppressWarnings("unchecked") + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + T server = (T) getServletContext().getAttribute(serverLookupKey); + try { + check(server, req); + Optional message = check(server, req); + resp.setStatus(200); + resp.getWriter().write(message.orElse("ok")); + } catch (Exception e) { + resp.setStatus(500); + resp.getWriter().write(e.toString()); + } finally { + resp.getWriter().close(); + } + } + + private Optional check(T server, HttpServletRequest req) throws IOException { + if (server == null) { + throw new IOException("Unable to get access to " + serverLookupKey); + } + if (server.isAborted() || server.isStopped() || server.isStopping() || server.isKilled()) { + throw new IOException("The " + serverLookupKey + " is stopping!"); + } + if (!server.getRpcServer().isStarted()) { + throw new IOException("The " + serverLookupKey + "'s RpcServer is not started"); + } + + Configuration conf = new Configuration(server.getConfiguration()); + conf.set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, + RpcConnectionRegistry.class.getName()); + conf.set(RpcConnectionRegistry.BOOTSTRAP_NODES, server.getServerName().getAddress().toString()); + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, + conf.getInt(CLIENT_RPC_TIMEOUT, CLIENT_RPC_TIMEOUT_DEFAULT)); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + conf.getInt(CLIENT_RETRIES, CLIENT_RETRIES_DEFAULT)); + conf.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + conf.getInt(CLIENT_OPERATION_TIMEOUT, CLIENT_OPERATION_TIMEOUT_DEFAULT)); + + try (Connection conn = ConnectionFactory.createConnection(conf)) { + // this will fail if the server is not accepting requests + if (conn.getClusterId() == null) { + throw new IOException("Could not retrieve clusterId from self via rpc"); + } + + return check(server, req, conn); + } + } + + protected abstract Optional check(T server, HttpServletRequest req, Connection conn) + throws IOException; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 810e10f1c56d..2465038468fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -169,6 +169,7 @@ import org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler; import org.apache.hadoop.hbase.regionserver.handler.RegionReplicaFlushHandler; import org.apache.hadoop.hbase.regionserver.http.RSDumpServlet; +import org.apache.hadoop.hbase.regionserver.http.RSHealthServlet; import org.apache.hadoop.hbase.regionserver.http.RSStatusServlet; import org.apache.hadoop.hbase.regionserver.throttle.FlushThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; @@ -383,7 +384,7 @@ public class HRegionServer extends Thread // A state before we go into stopped state. At this stage we're closing user // space regions. - private boolean stopping = false; + private volatile boolean stopping = false; private volatile boolean killed = false; private volatile boolean shutDown = false; @@ -864,6 +865,10 @@ protected Class getDumpServlet() { return RSDumpServlet.class; } + protected Class getHealthServlet() { + return RSHealthServlet.class; + } + /** * Used by {@link RSDumpServlet} to generate debugging information. */ @@ -2472,6 +2477,7 @@ private void putUpWebUI() throws IOException { try { this.infoServer = new InfoServer(getProcessName(), addr, port, false, this.conf); infoServer.addPrivilegedServlet("dump", "/dump", getDumpServlet()); + infoServer.addPrivilegedServlet("health", "/health", getHealthServlet()); configureInfoServer(); this.infoServer.start(); break; @@ -3199,6 +3205,10 @@ public boolean isStopping() { return this.stopping; } + public boolean isKilled() { + return this.killed; + } + @Override public Configuration getConfiguration() { return conf; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java new file mode 100644 index 000000000000..bc0f35193389 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSHealthServlet.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.http; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import javax.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.monitoring.HealthCheckServlet; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class RSHealthServlet extends HealthCheckServlet { + + private final Map regionUnavailableSince = new ConcurrentHashMap<>(); + + public RSHealthServlet() { + super(HRegionServer.REGIONSERVER); + } + + @Override + protected Optional check(HRegionServer regionServer, HttpServletRequest req, + Connection conn) throws IOException { + long maxUnavailableMillis = Optional.ofNullable(req.getParameter("maxUnavailableMillis")) + .filter(StringUtils::isNumeric).map(Long::parseLong).orElse(Long.MAX_VALUE); + + Instant oldestUnavailableSince = Instant.MAX; + String longestUnavailableRegion = null; + int unavailableCount = 0; + + synchronized (regionUnavailableSince) { + Set regionsPreviouslyUnavailable = new HashSet<>(regionUnavailableSince.keySet()); + + for (HRegion region : regionServer.getOnlineRegionsLocalContext()) { + regionsPreviouslyUnavailable.remove(region.getRegionInfo().getEncodedName()); + if (!region.isAvailable()) { + unavailableCount++; + Instant unavailableSince = regionUnavailableSince + .computeIfAbsent(region.getRegionInfo().getEncodedName(), k -> Instant.now()); + + if (unavailableSince.isBefore(oldestUnavailableSince)) { + oldestUnavailableSince = unavailableSince; + longestUnavailableRegion = region.getRegionInfo().getEncodedName(); + } + + } else { + regionUnavailableSince.remove(region.getRegionInfo().getEncodedName()); + } + } + + regionUnavailableSince.keySet().removeAll(regionsPreviouslyUnavailable); + } + + String message = "ok"; + + if (unavailableCount > 0) { + Duration longestUnavailableRegionTime = + Duration.between(oldestUnavailableSince, Instant.now()); + if (longestUnavailableRegionTime.toMillis() > maxUnavailableMillis) { + throw new IOException("Region " + longestUnavailableRegion + + " has been unavailable too long, since " + oldestUnavailableSince); + } + + message += " - unavailableRegions: " + unavailableCount + ", longestUnavailableDuration: " + + longestUnavailableRegionTime + ", longestUnavailableRegion: " + longestUnavailableRegion; + } + + return Optional.of(message); + + } +} From 55f505fe072f217211de59fda39bd05c49472b60 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Thu, 7 Mar 2024 16:57:11 -0500 Subject: [PATCH 07/78] HubSpot Edit: More info when interrupted while waiting on actions --- .../hbase/client/AsyncRequestFutureImpl.java | 63 +++++++++++++++---- .../hbase/client/MultiServerCallable.java | 9 ++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index 32776dde3e65..e52ae6cfac21 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -34,6 +35,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseServerException; import org.apache.hadoop.hbase.HConstants; @@ -52,6 +54,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Strings; + /** * The context, and return value, for a single submit/submitAll call. Note on how this class (one AP * submit) works. Initially, all requests are split into groups by server; request is sent to each @@ -195,7 +199,7 @@ public void run() { try { // setup the callable based on the actions, if we don't have one already from the request if (callable == null) { - callable = createCallable(server, tableName, multiAction); + callable = createCallable(server, tableName, multiAction, numAttempt); } RpcRetryingCaller caller = asyncProcess.createCaller(callable, rpcTimeout); @@ -387,10 +391,8 @@ public AsyncRequestFutureImpl(AsyncProcessTask task, List actions, long } else { this.replicaGetIndices = null; } - this.callsInProgress = !hasAnyReplicaGets - ? null - : Collections - .newSetFromMap(new ConcurrentHashMap()); + this.callsInProgress = + Collections.newSetFromMap(new ConcurrentHashMap()); this.asyncProcess = asyncProcess; this.errorsByServer = createServerErrorTracker(); this.errors = new BatchErrors(); @@ -536,7 +538,12 @@ private HRegionLocation getReplicaLocationOrFail(Action action) { private void manageLocationError(Action action, Exception ex) { String msg = - "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction(); + "Cannot get replica " + action.getReplicaId() + " location for " + action.getAction() + ": "; + if (ex instanceof OperationTimeoutExceededException) { + msg += "Operation timeout exceeded."; + } else { + msg += ex == null ? "null cause" : ex.toString(); + } LOG.error(msg); if (ex == null) { ex = new IOException(msg); @@ -1276,20 +1283,31 @@ private String buildDetailedErrorMsg(String string, int index) { @Override public void waitUntilDone() throws InterruptedIOException { + long startTime = EnvironmentEdgeManager.currentTime(); try { if (this.operationTimeout > 0) { // the worker thread maybe over by some exception without decrement the actionsInProgress, // then the guarantee of operationTimeout will be broken, so we should set cutoff to avoid // stuck here forever - long cutoff = (EnvironmentEdgeManager.currentTime() + this.operationTimeout) * 1000L; + long cutoff = (startTime + this.operationTimeout) * 1000L; if (!waitUntilDone(cutoff)) { - throw new SocketTimeoutException("time out before the actionsInProgress changed to zero"); + String msg = "time out before the actionsInProgress changed to zero, with " + + actionsInProgress.get() + " remaining" + getServersInProgress(); + + throw new SocketTimeoutException(msg); } } else { waitUntilDone(Long.MAX_VALUE); } } catch (InterruptedException iex) { - throw new InterruptedIOException(iex.getMessage()); + long duration = EnvironmentEdgeManager.currentTime() - startTime; + String message = "Interrupted after waiting " + duration + "ms of " + operationTimeout + + "ms operation timeout, with " + actionsInProgress.get() + " remaining" + + getServersInProgress(); + if (!Strings.isNullOrEmpty(iex.getMessage())) { + message += ": " + iex.getMessage(); + } + throw new InterruptedIOException(message); } finally { if (callsInProgress != null) { for (CancellableRegionServerCallable clb : callsInProgress) { @@ -1299,6 +1317,29 @@ public void waitUntilDone() throws InterruptedIOException { } } + private String getServersInProgress() { + if (callsInProgress != null) { + Map serversInProgress = new HashMap<>(callsInProgress.size()); + for (CancellableRegionServerCallable callable : callsInProgress) { + if (callable instanceof MultiServerCallable) { + MultiServerCallable multiServerCallable = (MultiServerCallable) callable; + int numAttempt = multiServerCallable.getNumAttempt(); + serversInProgress.compute(multiServerCallable.getServerName(), + (k, v) -> v == null ? numAttempt : Math.max(v, numAttempt)); + } + } + + if (serversInProgress.size() > 0) { + return " on servers: " + serversInProgress.entrySet().stream() + .sorted(Comparator.> comparingInt(Map.Entry::getValue) + .reversed()) + .map(entry -> entry.getKey() + "(" + entry.getValue() + " attempts)") + .collect(Collectors.joining(", ")); + } + } + return ""; + } + private boolean waitUntilDone(long cutoff) throws InterruptedException { boolean hasWait = cutoff != Long.MAX_VALUE; long lastLog = EnvironmentEdgeManager.currentTime(); @@ -1365,10 +1406,10 @@ private ConnectionImplementation.ServerErrorTracker createServerErrorTracker() { * Create a callable. Isolated to be easily overridden in the tests. */ private MultiServerCallable createCallable(final ServerName server, TableName tableName, - final MultiAction multi) { + final MultiAction multi, int numAttempt) { return new MultiServerCallable(asyncProcess.connection, tableName, server, multi, asyncProcess.rpcFactory.newController(), rpcTimeout, tracker, multi.getPriority(), - requestAttributes); + requestAttributes, numAttempt); } private void updateResult(int index, Object result) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java index 6ba0832b26e5..33933dd5684f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java @@ -48,14 +48,17 @@ @InterfaceAudience.Private class MultiServerCallable extends CancellableRegionServerCallable { private MultiAction multiAction; + private final int numAttempt; private boolean cellBlock; MultiServerCallable(final ClusterConnection connection, final TableName tableName, final ServerName location, final MultiAction multi, RpcController rpcController, int rpcTimeout, - RetryingTimeTracker tracker, int priority, Map requestAttributes) { + RetryingTimeTracker tracker, int priority, Map requestAttributes, + int numAttempt) { super(connection, tableName, null, rpcController, rpcTimeout, tracker, priority, requestAttributes); this.multiAction = multi; + this.numAttempt = numAttempt; // RegionServerCallable has HRegionLocation field, but this is a multi-region request. // Using region info from parent HRegionLocation would be a mistake for this class; so // we will store the server here, and throw if someone tries to obtain location/regioninfo. @@ -63,6 +66,10 @@ class MultiServerCallable extends CancellableRegionServerCallable this.cellBlock = isCellBlock(); } + public int getNumAttempt() { + return numAttempt; + } + public void reset(ServerName location, MultiAction multiAction) { this.location = new HRegionLocation(null, location); this.multiAction = multiAction; From 2a9fd1c33971c9106002085d2ee6b45b0172b0af Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 28 Feb 2025 16:39:02 -0500 Subject: [PATCH 08/78] HubSpot Backport: HBASE-28513 The StochasticLoadBalancer should support discrete evaluations (will be in 2.7) Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../master/balancer/AssignRegionAction.java | 10 + .../hbase/master/balancer/BalanceAction.java | 36 ++- .../master/balancer/BalancerClusterState.java | 91 ++++++- .../master/balancer/BalancerConditionals.java | 213 +++++++++++++++ .../master/balancer/BaseLoadBalancer.java | 12 +- .../balancer/CacheAwareLoadBalancer.java | 2 +- .../master/balancer/CandidateGenerator.java | 2 + .../hbase/master/balancer/CostFunction.java | 7 + .../DistributeReplicasCandidateGenerator.java | 115 ++++++++ .../DistributeReplicasConditional.java | 97 +++++++ .../balancer/FavoredStochasticBalancer.java | 4 +- .../master/balancer/MoveBatchAction.java | 77 ++++++ .../master/balancer/MoveRegionAction.java | 10 + .../balancer/RegionPlanConditional.java | 133 +++++++++ ...gionPlanConditionalCandidateGenerator.java | 113 ++++++++ .../SlopFixingCandidateGenerator.java | 105 +++++++ .../balancer/StochasticLoadBalancer.java | 135 +++++++-- .../master/balancer/SwapRegionsAction.java | 13 + .../master/balancer/replicas/ReplicaKey.java | 55 ++++ .../balancer/replicas/ReplicaKeyCache.java | 93 +++++++ .../BalancerConditionalsTestUtil.java | 221 +++++++++++++++ .../balancer/CandidateGeneratorTestUtil.java | 256 ++++++++++++++++++ .../DistributeReplicasTestConditional.java | 39 +++ .../LoadOnlyFavoredStochasticBalancer.java | 3 +- .../balancer/TestBalancerConditionals.java | 83 ++++++ ...lancingConditionalReplicaDistribution.java | 114 ++++++++ ...eplicaDistributionBalancerConditional.java | 120 ++++++++ ...ochasticLoadBalancerHeterogeneousCost.java | 2 +- 28 files changed, 2120 insertions(+), 41 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveBatchAction.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKey.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKeyCache.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionalsTestUtil.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasTestConditional.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestReplicaDistributionBalancerConditional.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/AssignRegionAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/AssignRegionAction.java index c99ae092d775..8a79b64142e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/AssignRegionAction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/AssignRegionAction.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hbase.master.balancer; +import java.util.List; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + @InterfaceAudience.Private class AssignRegionAction extends BalanceAction { private final int region; @@ -46,6 +50,12 @@ public BalanceAction undoAction() { throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED); } + @Override + List toRegionPlans(BalancerClusterState cluster) { + return ImmutableList + .of(new RegionPlan(cluster.regions[getRegion()], null, cluster.servers[getServer()])); + } + @Override public String toString() { return getType() + ": " + region + ":" + server; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalanceAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalanceAction.java index 56b473ae710c..a65b5253907c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalanceAction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalanceAction.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.master.balancer; +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.yetus.audience.InterfaceAudience; /** @@ -28,11 +31,11 @@ enum Type { ASSIGN_REGION, MOVE_REGION, SWAP_REGIONS, + MOVE_BATCH, NULL, } - static final BalanceAction NULL_ACTION = new BalanceAction(Type.NULL) { - }; + static final BalanceAction NULL_ACTION = new NullBalanceAction(); private final Type type; @@ -43,16 +46,39 @@ enum Type { /** * Returns an Action which would undo this action */ - BalanceAction undoAction() { - return this; - } + abstract BalanceAction undoAction(); + + /** + * Returns the Action represented as RegionPlans + */ + abstract List toRegionPlans(BalancerClusterState cluster); Type getType() { return type; } + long getStepCount() { + return 1; + } + @Override public String toString() { return type + ":"; } + + private static final class NullBalanceAction extends BalanceAction { + private NullBalanceAction() { + super(Type.NULL); + } + + @Override + BalanceAction undoAction() { + return this; + } + + @Override + List toRegionPlans(BalancerClusterState cluster) { + return Collections.emptyList(); + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index b857055fb3ab..67755fc317c6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -26,6 +26,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import org.agrona.collections.Hashing; import org.agrona.collections.Int2IntCounterMap; import org.apache.hadoop.hbase.HDFSBlocksDistribution; @@ -39,6 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Suppliers; + /** * An efficient array based implementation similar to ClusterState for keeping the status of the * cluster in terms of region assignment and distribution. LoadBalancers, such as @@ -123,6 +128,15 @@ class BalancerClusterState { // Maps regionName -> oldServerName -> cache ratio of the region on the old server Map> regionCacheRatioOnOldServerMap; + private final Supplier> shuffledServerIndicesSupplier = + Suppliers.memoizeWithExpiration(() -> { + Collection serverIndices = serversToIndex.values(); + List shuffledServerIndices = new ArrayList<>(serverIndices); + Collections.shuffle(shuffledServerIndices); + return shuffledServerIndices; + }, 5, TimeUnit.SECONDS); + private long stopRequestedAt = Long.MAX_VALUE; + static class DefaultRackManager extends RackManager { @Override public String getRack(ServerName server) { @@ -728,8 +742,25 @@ public void doAction(BalanceAction action) { regionMoved(a.getFromRegion(), a.getFromServer(), a.getToServer()); regionMoved(a.getToRegion(), a.getToServer(), a.getFromServer()); break; + case MOVE_BATCH: + assert action instanceof MoveBatchAction : action.getClass(); + MoveBatchAction mba = (MoveBatchAction) action; + for (int serverIndex : mba.getServerToRegionsToRemove().keySet()) { + Set regionsToRemove = mba.getServerToRegionsToRemove().get(serverIndex); + regionsPerServer[serverIndex] = + removeRegions(regionsPerServer[serverIndex], regionsToRemove); + } + for (int serverIndex : mba.getServerToRegionsToAdd().keySet()) { + Set regionsToAdd = mba.getServerToRegionsToAdd().get(serverIndex); + regionsPerServer[serverIndex] = addRegions(regionsPerServer[serverIndex], regionsToAdd); + } + for (MoveRegionAction moveRegionAction : mba.getMoveActions()) { + regionMoved(moveRegionAction.getRegion(), moveRegionAction.getFromServer(), + moveRegionAction.getToServer()); + } + break; default: - throw new RuntimeException("Uknown action:" + action.getType()); + throw new RuntimeException("Unknown action:" + action.getType()); } } @@ -891,6 +922,52 @@ int[] addRegion(int[] regions, int regionIndex) { return newRegions; } + int[] removeRegions(int[] regions, Set regionIndicesToRemove) { + // Calculate the size of the new regions array + int newSize = regions.length - regionIndicesToRemove.size(); + if (newSize < 0) { + throw new IllegalStateException( + "Region indices mismatch: more regions to remove than in the regions array"); + } + + int[] newRegions = new int[newSize]; + int newIndex = 0; + + // Copy only the regions not in the removal set + for (int region : regions) { + if (!regionIndicesToRemove.contains(region)) { + newRegions[newIndex++] = region; + } + } + + // If the newIndex is smaller than newSize, some regions were missing from the input array + if (newIndex != newSize) { + throw new IllegalStateException("Region indices mismatch: some regions in the removal " + + "set were not found in the regions array"); + } + + return newRegions; + } + + int[] addRegions(int[] regions, Set regionIndicesToAdd) { + int[] newRegions = new int[regions.length + regionIndicesToAdd.size()]; + + // Copy the existing regions to the new array + System.arraycopy(regions, 0, newRegions, 0, regions.length); + + // Add the new regions at the end of the array + int newIndex = regions.length; + for (int regionIndex : regionIndicesToAdd) { + newRegions[newIndex++] = regionIndex; + } + + return newRegions; + } + + List getShuffledServerIndices() { + return shuffledServerIndicesSupplier.get(); + } + int[] addRegionSorted(int[] regions, int regionIndex) { int[] newRegions = new int[regions.length + 1]; int i = 0; @@ -990,6 +1067,18 @@ void setNumMovedRegions(int numMovedRegions) { this.numMovedRegions = numMovedRegions; } + public int getMaxReplicas() { + return maxReplicas; + } + + void setStopRequestedAt(long stopRequestedAt) { + this.stopRequestedAt = stopRequestedAt; + } + + long getStopRequestedAt() { + return stopRequestedAt; + } + @Override public String toString() { StringBuilder desc = new StringBuilder("Cluster={servers=["); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java new file mode 100644 index 000000000000..c44e47996932 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.lang.reflect.Constructor; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKeyCache; +import org.apache.hadoop.hbase.util.ReflectionUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +/** + * Balancer conditionals supplement cost functions in the {@link StochasticLoadBalancer}. Cost + * functions are insufficient and difficult to work with when making discrete decisions; this is + * because they operate on a continuous scale, and each cost function's multiplier affects the + * relative importance of every other cost function. So it is difficult to meaningfully and clearly + * value many aspects of your region distribution via cost functions alone. Conditionals allow you + * to very clearly define discrete rules that your balancer would ideally follow. To clarify, a + * conditional violation will not block a region assignment because we would prefer to have uptime + * than have perfectly intentional balance. But conditionals allow you to, for example, define that + * a region's primary and secondary should not live on the same rack. Another example, conditionals + * make it easy to define that system tables will ideally be isolated on their own RegionServer + * (without needing to manage distinct RegionServer groups). + */ +@InterfaceAudience.Private +final class BalancerConditionals implements Configurable { + + private static final Logger LOG = LoggerFactory.getLogger(BalancerConditionals.class); + + public static final String DISTRIBUTE_REPLICAS_KEY = + "hbase.master.balancer.stochastic.conditionals.distributeReplicas"; + public static final boolean DISTRIBUTE_REPLICAS_DEFAULT = false; + + public static final String ADDITIONAL_CONDITIONALS_KEY = + "hbase.master.balancer.stochastic.additionalConditionals"; + + private Set> conditionalClasses = Collections.emptySet(); + private Set conditionals = Collections.emptySet(); + private Configuration conf; + + static BalancerConditionals create() { + return new BalancerConditionals(); + } + + private BalancerConditionals() { + } + + boolean shouldRunBalancer(BalancerClusterState cluster) { + return isConditionalBalancingEnabled() && conditionals.stream() + .map(RegionPlanConditional::getCandidateGenerators).flatMap(Collection::stream) + .map(generator -> generator.getWeight(cluster)).anyMatch(weight -> weight > 0); + } + + Set> getConditionalClasses() { + return new HashSet<>(conditionalClasses); + } + + Collection getConditionals() { + return conditionals; + } + + boolean isReplicaDistributionEnabled() { + return conditionalClasses.stream() + .anyMatch(DistributeReplicasConditional.class::isAssignableFrom); + } + + boolean shouldSkipSloppyServerEvaluation() { + return isConditionalBalancingEnabled(); + } + + boolean isConditionalBalancingEnabled() { + return !conditionalClasses.isEmpty(); + } + + void clearConditionalWeightCaches() { + conditionals.stream().map(RegionPlanConditional::getCandidateGenerators) + .flatMap(Collection::stream) + .forEach(RegionPlanConditionalCandidateGenerator::clearWeightCache); + } + + void loadClusterState(BalancerClusterState cluster) { + conditionals = conditionalClasses.stream().map(clazz -> createConditional(clazz, cluster)) + .filter(Objects::nonNull).collect(Collectors.toSet()); + } + + /** + * Indicates whether the action is good for our conditional compliance. + * @param cluster The cluster state + * @param action The proposed action + * @return -1 if conditionals improve, 0 if neutral, 1 if conditionals degrade + */ + int getViolationCountChange(BalancerClusterState cluster, BalanceAction action) { + // Cluster is in pre-move state, so figure out the proposed violations + boolean isViolatingPost = isViolating(cluster, action); + cluster.doAction(action); + + // Cluster is in post-move state, so figure out the original violations + BalanceAction undoAction = action.undoAction(); + boolean isViolatingPre = isViolating(cluster, undoAction); + + // Reset cluster + cluster.doAction(undoAction); + + if (isViolatingPre && isViolatingPost) { + return 0; + } else if (!isViolatingPre && isViolatingPost) { + return 1; + } else { + return -1; + } + } + + /** + * Check if the proposed action violates conditionals + * @param cluster The cluster state + * @param action The proposed action + */ + boolean isViolating(BalancerClusterState cluster, BalanceAction action) { + conditionals.forEach(conditional -> conditional.setClusterState(cluster)); + if (conditionals.isEmpty()) { + return false; + } + List regionPlans = action.toRegionPlans(cluster); + for (RegionPlan regionPlan : regionPlans) { + if (isViolating(regionPlan)) { + return true; + } + } + return false; + } + + private boolean isViolating(RegionPlan regionPlan) { + for (RegionPlanConditional conditional : conditionals) { + if (conditional.isViolating(regionPlan)) { + return true; + } + } + return false; + } + + private RegionPlanConditional createConditional(Class clazz, + BalancerClusterState cluster) { + if (cluster == null) { + cluster = new BalancerClusterState(Collections.emptyMap(), null, null, null, null); + } + try { + Constructor ctor = + clazz.getDeclaredConstructor(BalancerConditionals.class, BalancerClusterState.class); + return ReflectionUtils.instantiate(clazz.getName(), ctor, this, cluster); + } catch (NoSuchMethodException e) { + LOG.warn("Cannot find constructor with Configuration and " + + "BalancerClusterState parameters for class '{}': {}", clazz.getName(), e.getMessage()); + } + return null; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + ImmutableSet.Builder> conditionalClasses = + ImmutableSet.builder(); + + boolean distributeReplicas = + conf.getBoolean(DISTRIBUTE_REPLICAS_KEY, DISTRIBUTE_REPLICAS_DEFAULT); + if (distributeReplicas) { + conditionalClasses.add(DistributeReplicasConditional.class); + } + + Class[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY); + for (Class clazz : classes) { + if (!RegionPlanConditional.class.isAssignableFrom(clazz)) { + LOG.warn("Class {} is not a RegionPlanConditional", clazz.getName()); + continue; + } + conditionalClasses.add(clazz.asSubclass(RegionPlanConditional.class)); + } + this.conditionalClasses = conditionalClasses.build(); + ReplicaKeyCache.getInstance().setConf(conf); + loadClusterState(null); + } + + @Override + public Configuration getConf() { + return conf; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 07cd58920860..fac0d82fe013 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -77,6 +77,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer { public static final boolean DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE = false; + public static final String REGIONS_SLOP_KEY = "hbase.regions.slop"; + public static final float REGIONS_SLOP_DEFAULT = 0.2f; + protected static final int MIN_SERVER_BALANCE = 2; private volatile boolean stopped = false; @@ -256,7 +259,9 @@ protected final boolean sloppyRegionServerExist(ClusterLoadState cs) { float average = cs.getLoadAverage(); // for logging int floor = (int) Math.floor(average * (1 - slop)); int ceiling = (int) Math.ceil(average * (1 + slop)); - if (!(cs.getMaxLoad() > ceiling || cs.getMinLoad() < floor)) { + int maxLoad = cs.getMaxLoad(); + int minLoad = cs.getMinLoad(); + if (!(maxLoad > ceiling || minLoad < floor)) { NavigableMap> serversByLoad = cs.getServersByLoad(); if (LOG.isTraceEnabled()) { // If nothing to balance, then don't say anything unless trace-level logging. @@ -549,7 +554,7 @@ public Map> retainAssignment(Map, CandidateGenerator> - createCandidateGenerators() { + createCandidateGenerators(Configuration conf) { Map, CandidateGenerator> candidateGenerators = new HashMap<>(2); candidateGenerators.put(CacheAwareSkewnessCandidateGenerator.class, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java index d9245495e204..642e8162fff9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CandidateGenerator.java @@ -28,6 +28,8 @@ @InterfaceAudience.Private abstract class CandidateGenerator { + protected static final double MAX_WEIGHT = 1.0; + abstract BalanceAction generate(BalancerClusterState cluster); /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java index 1dcd4580b1a6..ee2fc2b6a5e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFunction.java @@ -76,6 +76,13 @@ void postAction(BalanceAction action) { regionMoved(a.getFromRegion(), a.getFromServer(), a.getToServer()); regionMoved(a.getToRegion(), a.getToServer(), a.getFromServer()); break; + case MOVE_BATCH: + MoveBatchAction mba = (MoveBatchAction) action; + for (MoveRegionAction moveRegionAction : mba.getMoveActions()) { + regionMoved(moveRegionAction.getRegion(), moveRegionAction.getFromServer(), + moveRegionAction.getToServer()); + } + break; default: throw new RuntimeException("Uknown action:" + action.getType()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java new file mode 100644 index 000000000000..38fbcc4a0fbc --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.DistributeReplicasConditional.getReplicaKey; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CandidateGenerator to distribute colocated replicas across different servers. + */ +@InterfaceAudience.Private +final class DistributeReplicasCandidateGenerator extends RegionPlanConditionalCandidateGenerator { + + private static final Logger LOG = + LoggerFactory.getLogger(DistributeReplicasCandidateGenerator.class); + private static final int BATCH_SIZE = 100_000; + + DistributeReplicasCandidateGenerator(BalancerConditionals balancerConditionals) { + super(balancerConditionals); + } + + @Override + BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing) { + return generateCandidate(cluster, isWeighing, false); + } + + BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing, + boolean isForced) { + if (cluster.getMaxReplicas() < cluster.numRacks) { + LOG.trace("Skipping replica distribution as there are not enough racks to distribute them."); + return BalanceAction.NULL_ACTION; + } + + // Iterate through shuffled servers to find colocated replicas + boolean foundColocatedReplicas = false; + List moveRegionActions = new ArrayList<>(); + List shuffledServerIndices = cluster.getShuffledServerIndices(); + for (int sourceIndex : shuffledServerIndices) { + if ( + moveRegionActions.size() >= BATCH_SIZE + || EnvironmentEdgeManager.currentTime() > cluster.getStopRequestedAt() + ) { + break; + } + int[] serverRegions = cluster.regionsPerServer[sourceIndex]; + Set replicaKeys = new HashSet<>(serverRegions.length); + for (int regionIndex : serverRegions) { + ReplicaKey replicaKey = getReplicaKey(cluster.regions[regionIndex]); + if (replicaKeys.contains(replicaKey)) { + foundColocatedReplicas = true; + if (isWeighing) { + // If weighing, fast exit with an actionable move + return getAction(sourceIndex, regionIndex, pickOtherRandomServer(cluster, sourceIndex), + -1); + } + // If not weighing, pick a good move + for (int i = 0; i < cluster.numServers; i++) { + // Randomize destination ordering so we aren't overloading one destination + int destinationIndex = pickOtherRandomServer(cluster, sourceIndex); + if (destinationIndex == sourceIndex) { + continue; + } + MoveRegionAction possibleAction = + new MoveRegionAction(regionIndex, sourceIndex, destinationIndex); + if (isForced) { + return possibleAction; + } + if (willBeAccepted(cluster, possibleAction)) { + cluster.doAction(possibleAction); // Update cluster state to reflect move + moveRegionActions.add(possibleAction); + break; + } + } + } else { + replicaKeys.add(replicaKey); + } + } + } + + if (!moveRegionActions.isEmpty()) { + return batchMovesAndResetClusterState(cluster, moveRegionActions); + } + // If no colocated replicas are found, return NULL_ACTION + if (foundColocatedReplicas) { + LOG.warn("Could not find a place to put a colocated replica! We will force a move."); + return generateCandidate(cluster, isWeighing, true); + } + LOG.trace("No colocated replicas found. No balancing action required."); + return BalanceAction.NULL_ACTION; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java new file mode 100644 index 000000000000..2cd27615e5fd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKeyCache; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + +/** + * If enabled, this class will help the balancer ensure that replicas aren't placed on the same + * servers or racks as their primary. Configure this via + * {@link BalancerConditionals#DISTRIBUTE_REPLICAS_KEY} + */ +@InterfaceAudience.Private +public class DistributeReplicasConditional extends RegionPlanConditional { + + private final List candidateGenerators; + + public DistributeReplicasConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(balancerConditionals.getConf(), cluster); + Configuration conf = balancerConditionals.getConf(); + float slop = + conf.getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, BaseLoadBalancer.REGIONS_SLOP_DEFAULT); + this.candidateGenerators = + ImmutableList.of(new DistributeReplicasCandidateGenerator(balancerConditionals), + new SlopFixingCandidateGenerator(balancerConditionals, slop)); + } + + @Override + public ValidationLevel getValidationLevel() { + return ValidationLevel.SERVER_HOST_RACK; + } + + @Override + List getCandidateGenerators() { + return candidateGenerators; + } + + @Override + boolean isViolatingServer(RegionPlan regionPlan, Set serverRegions) { + return checkViolation(regionPlan.getRegionInfo(), getReplicaKey(regionPlan.getRegionInfo()), + serverRegions); + } + + @Override + boolean isViolatingHost(RegionPlan regionPlan, Set hostRegions) { + return checkViolation(regionPlan.getRegionInfo(), getReplicaKey(regionPlan.getRegionInfo()), + hostRegions); + } + + @Override + boolean isViolatingRack(RegionPlan regionPlan, Set rackRegions) { + return checkViolation(regionPlan.getRegionInfo(), getReplicaKey(regionPlan.getRegionInfo()), + rackRegions); + } + + private boolean checkViolation(RegionInfo movingRegion, ReplicaKey movingReplicaKey, + Set destinationRegions) { + for (RegionInfo regionInfo : destinationRegions) { + if (regionInfo.equals(movingRegion)) { + continue; + } + if (getReplicaKey(regionInfo).equals(movingReplicaKey)) { + return true; + } + } + return false; + } + + static ReplicaKey getReplicaKey(RegionInfo regionInfo) { + return ReplicaKeyCache.getInstance().getReplicaKey(regionInfo); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java index db4c7c95b656..98ad3beac8de 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java @@ -81,7 +81,7 @@ public void setFavoredNodesManager(FavoredNodesManager fnm) { @Override protected Map, CandidateGenerator> - createCandidateGenerators() { + createCandidateGenerators(Configuration conf) { Map, CandidateGenerator> fnPickers = new HashMap<>(2); fnPickers.put(FavoredNodeLoadPicker.class, new FavoredNodeLoadPicker()); fnPickers.put(FavoredNodeLocalityPicker.class, new FavoredNodeLocalityPicker()); @@ -90,7 +90,7 @@ public void setFavoredNodesManager(FavoredNodesManager fnm) { /** Returns any candidate generator in random */ @Override - protected CandidateGenerator getRandomGenerator() { + protected CandidateGenerator getRandomGenerator(BalancerClusterState cluster) { Class clazz = shuffledGeneratorClasses.get() .get(ThreadLocalRandom.current().nextInt(candidateGenerators.size())); return candidateGenerators.get(clazz); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveBatchAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveBatchAction.java new file mode 100644 index 000000000000..e7ea3ed15e1d --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveBatchAction.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap; +import org.apache.hbase.thirdparty.com.google.common.collect.Multimaps; + +@InterfaceAudience.Private +public class MoveBatchAction extends BalanceAction { + private final List moveActions; + + MoveBatchAction(List moveActions) { + super(Type.MOVE_BATCH); + this.moveActions = moveActions; + } + + @Override + BalanceAction undoAction() { + List undoMoves = new ArrayList<>(getMoveActions().size()); + for (int i = getMoveActions().size() - 1; i >= 0; i--) { + MoveRegionAction move = getMoveActions().get(i); + undoMoves + .add(new MoveRegionAction(move.getRegion(), move.getToServer(), move.getFromServer())); + } + return new MoveBatchAction(undoMoves); + } + + @Override + List toRegionPlans(BalancerClusterState cluster) { + List mbRegionPlans = new ArrayList<>(getMoveActions().size()); + for (MoveRegionAction moveRegionAction : getMoveActions()) { + mbRegionPlans.add(new RegionPlan(cluster.regions[moveRegionAction.getRegion()], + cluster.servers[moveRegionAction.getFromServer()], + cluster.servers[moveRegionAction.getToServer()])); + } + return mbRegionPlans; + } + + @Override + long getStepCount() { + return moveActions.size(); + } + + public HashMultimap getServerToRegionsToRemove() { + return moveActions.stream().collect(Multimaps.toMultimap(MoveRegionAction::getFromServer, + MoveRegionAction::getRegion, HashMultimap::create)); + } + + public HashMultimap getServerToRegionsToAdd() { + return moveActions.stream().collect(Multimaps.toMultimap(MoveRegionAction::getToServer, + MoveRegionAction::getRegion, HashMultimap::create)); + } + + List getMoveActions() { + return moveActions; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveRegionAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveRegionAction.java index 547c9c5b28e9..9798e9cebe87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveRegionAction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MoveRegionAction.java @@ -17,8 +17,12 @@ */ package org.apache.hadoop.hbase.master.balancer; +import java.util.List; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + @InterfaceAudience.Private class MoveRegionAction extends BalanceAction { private final int region; @@ -49,6 +53,12 @@ public BalanceAction undoAction() { return new MoveRegionAction(region, toServer, fromServer); } + @Override + List toRegionPlans(BalancerClusterState cluster) { + return ImmutableList.of(new RegionPlan(cluster.regions[getRegion()], + cluster.servers[getFromServer()], cluster.servers[getToServer()])); + } + @Override public String toString() { return getType() + ": " + region + ":" + fromServer + " -> " + toServer; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java new file mode 100644 index 000000000000..8de371d341cd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) +@InterfaceStability.Evolving +public abstract class RegionPlanConditional { + private static final Logger LOG = LoggerFactory.getLogger(RegionPlanConditional.class); + private BalancerClusterState cluster; + + RegionPlanConditional(Configuration conf, BalancerClusterState cluster) { + this.cluster = cluster; + } + + public enum ValidationLevel { + /** + * Just check the server. + */ + SERVER, + /** + * Check the server and the host. + */ + SERVER_HOST, + /** + * Check the server, host, and rack. + */ + SERVER_HOST_RACK + } + + void setClusterState(BalancerClusterState cluster) { + this.cluster = cluster; + } + + /** + * Returns a {@link ValidationLevel} that is appropriate for this conditional. + * @return the validation level + */ + abstract ValidationLevel getValidationLevel(); + + /** + * Get the candidate generator(s) for this conditional. This can be useful to provide the balancer + * with hints that will appease your conditional. Your conditionals will be triggered in order. + * @return the candidate generator for this conditional + */ + abstract List getCandidateGenerators(); + + /** + * Check if the conditional is violated by the given region plan. + * @param regionPlan the region plan to check + * @return true if the conditional is violated + */ + boolean isViolating(RegionPlan regionPlan) { + if (regionPlan == null) { + return false; + } + int destinationServerIdx = cluster.serversToIndex.get(regionPlan.getDestination().getAddress()); + + // Check Server + int[] destinationRegionIndices = cluster.regionsPerServer[destinationServerIdx]; + Set serverRegions = Arrays.stream(cluster.regionsPerServer[destinationServerIdx]) + .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + for (int regionIdx : destinationRegionIndices) { + serverRegions.add(cluster.regions[regionIdx]); + } + if (isViolatingServer(regionPlan, serverRegions)) { + return true; + } + + if (getValidationLevel() == ValidationLevel.SERVER) { + return false; + } + + // Check Host + int hostIdx = cluster.serverIndexToHostIndex[destinationServerIdx]; + Set hostRegions = Arrays.stream(cluster.regionsPerHost[hostIdx]) + .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + if (isViolatingHost(regionPlan, hostRegions)) { + return true; + } + + if (getValidationLevel() == ValidationLevel.SERVER_HOST) { + return false; + } + + // Check Rack + int rackIdx = cluster.serverIndexToRackIndex[destinationServerIdx]; + Set rackRegions = Arrays.stream(cluster.regionsPerRack[rackIdx]) + .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + if (isViolatingRack(regionPlan, rackRegions)) { + return true; + } + + return false; + } + + abstract boolean isViolatingServer(RegionPlan regionPlan, Set destinationRegions); + + boolean isViolatingHost(RegionPlan regionPlan, Set destinationRegions) { + return false; + } + + boolean isViolatingRack(RegionPlan regionPlan, Set destinationRegions) { + return false; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java new file mode 100644 index 000000000000..f8274841f729 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.time.Duration; +import java.util.List; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public abstract class RegionPlanConditionalCandidateGenerator extends CandidateGenerator { + + private static final Logger LOG = + LoggerFactory.getLogger(RegionPlanConditionalCandidateGenerator.class); + + private static final Duration WEIGHT_CACHE_TTL = Duration.ofMinutes(1); + private long lastWeighedAt = -1; + private double lastWeight = 0.0; + + private final BalancerConditionals balancerConditionals; + + RegionPlanConditionalCandidateGenerator(BalancerConditionals balancerConditionals) { + this.balancerConditionals = balancerConditionals; + } + + BalancerConditionals getBalancerConditionals() { + return this.balancerConditionals; + } + + /** + * Generates a balancing action to appease the conditional. + * @param cluster Current state of the cluster. + * @param isWeighing Flag indicating if the generator is being used for weighing. + * @return A BalanceAction, or NULL_ACTION if no action is needed. + */ + abstract BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing); + + @Override + BalanceAction generate(BalancerClusterState cluster) { + BalanceAction balanceAction = generateCandidate(cluster, false); + if (!willBeAccepted(cluster, balanceAction)) { + LOG.debug("Generated action is not widely accepted by all conditionals. " + + "Likely we are finding our way out of a deadlock. balanceAction={}", balanceAction); + } + return balanceAction; + } + + MoveBatchAction batchMovesAndResetClusterState(BalancerClusterState cluster, + List moves) { + MoveBatchAction batchAction = new MoveBatchAction(moves); + undoBatchAction(cluster, batchAction); + return batchAction; + } + + boolean willBeAccepted(BalancerClusterState cluster, BalanceAction action) { + BalancerConditionals balancerConditionals = getBalancerConditionals(); + if (balancerConditionals == null) { + return true; + } + return !balancerConditionals.isViolating(cluster, action); + } + + void undoBatchAction(BalancerClusterState cluster, MoveBatchAction batchAction) { + for (int i = batchAction.getMoveActions().size() - 1; i >= 0; i--) { + MoveRegionAction action = batchAction.getMoveActions().get(i); + cluster.doAction(action.undoAction()); + } + } + + void clearWeightCache() { + lastWeighedAt = -1; + } + + double getWeight(BalancerClusterState cluster) { + boolean hasCandidate = false; + + // Candidate generation is expensive, so for re-weighing generators we will cache + // the value for a bit + if (EnvironmentEdgeManager.currentTime() - lastWeighedAt < WEIGHT_CACHE_TTL.toMillis()) { + return lastWeight; + } else { + hasCandidate = generateCandidate(cluster, true) != BalanceAction.NULL_ACTION; + lastWeighedAt = EnvironmentEdgeManager.currentTime(); + } + + if (hasCandidate) { + // If this generator has something to do, then it's important + lastWeight = CandidateGenerator.MAX_WEIGHT; + } else { + lastWeight = 0; + } + return lastWeight; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java new file mode 100644 index 000000000000..070e4903394d --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.ServerName; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A simple candidate generator that attempts to move regions from the most-loaded servers to the + * least-loaded servers. + */ +@InterfaceAudience.Private +final class SlopFixingCandidateGenerator extends RegionPlanConditionalCandidateGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(SlopFixingCandidateGenerator.class); + + private final float slop; + + SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals, float slop) { + super(balancerConditionals); + this.slop = slop; + } + + @Override + BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing) { + ClusterLoadState cs = new ClusterLoadState(cluster.clusterState); + float average = cs.getLoadAverage(); + int ceiling = (int) Math.ceil(average * (1 + slop)); + Set sloppyServerIndices = new HashSet<>(); + for (int i = 0; i < cluster.numServers; i++) { + int regionCount = cluster.regionsPerServer[i].length; + if (regionCount > ceiling) { + sloppyServerIndices.add(i); + } + } + + if (sloppyServerIndices.isEmpty()) { + LOG.trace("No action to take because no sloppy servers exist."); + return BalanceAction.NULL_ACTION; + } + + List moves = new ArrayList<>(); + Set fixedServers = new HashSet<>(); + for (int sourceServer : sloppyServerIndices) { + for (int regionIdx : cluster.regionsPerServer[sourceServer]) { + boolean regionFoundMove = false; + for (ServerAndLoad serverAndLoad : cs.getServersByLoad().keySet()) { + ServerName destinationServer = serverAndLoad.getServerName(); + int destinationServerIdx = cluster.serversToIndex.get(destinationServer.getAddress()); + int regionsOnDestination = cluster.regionsPerServer[destinationServerIdx].length; + if (regionsOnDestination < average) { + MoveRegionAction move = + new MoveRegionAction(regionIdx, sourceServer, destinationServerIdx); + if (willBeAccepted(cluster, move)) { + if (isWeighing) { + // Fast exit for weighing candidate + return move; + } + moves.add(move); + cluster.doAction(move); + regionFoundMove = true; + break; + } + } else { + fixedServers.add(serverAndLoad); + } + } + fixedServers.forEach(s -> cs.getServersByLoad().remove(s)); + fixedServers.clear(); + if (!regionFoundMove) { + LOG.debug("Could not find a destination for region {} from server {}.", regionIdx, + sourceServer); + } + if (cluster.regionsPerServer[sourceServer].length <= ceiling) { + break; + } + } + } + + MoveBatchAction batch = new MoveBatchAction(moves); + undoBatchAction(cluster, batch); + return batch; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index fca4ef952073..42784ea4440d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -54,7 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.base.Suppliers; /** @@ -192,6 +191,8 @@ public enum GeneratorType { return shuffled; }, 5, TimeUnit.SECONDS); + private final BalancerConditionals balancerConditionals = BalancerConditionals.create(); + /** * The constructor that pass a MetricsStochasticBalancer to BaseLoadBalancer to replace its * default MetricsBalancer @@ -244,16 +245,24 @@ Map, CandidateGenerator> getCandidateGenerat } protected Map, CandidateGenerator> - createCandidateGenerators() { - Map, CandidateGenerator> candidateGenerators = - new HashMap<>(5); - candidateGenerators.put(RandomCandidateGenerator.class, new RandomCandidateGenerator()); - candidateGenerators.put(LoadCandidateGenerator.class, new LoadCandidateGenerator()); - candidateGenerators.put(LocalityBasedCandidateGenerator.class, localityCandidateGenerator); - candidateGenerators.put(RegionReplicaCandidateGenerator.class, - new RegionReplicaCandidateGenerator()); - candidateGenerators.put(RegionReplicaRackCandidateGenerator.class, - new RegionReplicaRackCandidateGenerator()); + createCandidateGenerators(Configuration conf) { + balancerConditionals.setConf(conf); + Map, CandidateGenerator> candidateGenerators; + if (balancerConditionals.isReplicaDistributionEnabled()) { + candidateGenerators = new HashMap<>(3); + candidateGenerators.put(RandomCandidateGenerator.class, new RandomCandidateGenerator()); + candidateGenerators.put(LoadCandidateGenerator.class, new LoadCandidateGenerator()); + candidateGenerators.put(LocalityBasedCandidateGenerator.class, localityCandidateGenerator); + } else { + candidateGenerators = new HashMap<>(5); + candidateGenerators.put(RandomCandidateGenerator.class, new RandomCandidateGenerator()); + candidateGenerators.put(LoadCandidateGenerator.class, new LoadCandidateGenerator()); + candidateGenerators.put(LocalityBasedCandidateGenerator.class, localityCandidateGenerator); + candidateGenerators.put(RegionReplicaCandidateGenerator.class, + new RegionReplicaCandidateGenerator()); + candidateGenerators.put(RegionReplicaRackCandidateGenerator.class, + new RegionReplicaRackCandidateGenerator()); + } return candidateGenerators; } @@ -288,7 +297,8 @@ protected void loadConf(Configuration conf) { localityCost = new ServerLocalityCostFunction(conf); rackLocalityCost = new RackLocalityCostFunction(conf); - this.candidateGenerators = createCandidateGenerators(); + balancerConditionals.setConf(conf); + this.candidateGenerators = createCandidateGenerators(conf); regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf); regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf); @@ -377,6 +387,11 @@ void updateMetricsSize(int size) { } private boolean areSomeRegionReplicasColocatedOnHost(BalancerClusterState c) { + if (!c.hasRegionReplicas || balancerConditionals.isReplicaDistributionEnabled()) { + // This check is unnecessary without replicas, or with conditional replica distribution + // The balancer will auto-run if conditional replica distribution candidates are available + return false; + } if (c.numHosts >= c.maxReplicas) { regionReplicaHostCostFunction.prepare(c); double hostCost = Math.abs(regionReplicaHostCostFunction.cost()); @@ -390,6 +405,11 @@ private boolean areSomeRegionReplicasColocatedOnHost(BalancerClusterState c) { } private boolean areSomeRegionReplicasColocatedOnRack(BalancerClusterState c) { + if (!c.hasRegionReplicas || balancerConditionals.isReplicaDistributionEnabled()) { + // This check is unnecessary without replicas, or with conditional replica distribution + // The balancer will auto-run if conditional replica distribution candidates are available + return false; + } if (c.numRacks >= c.maxReplicas) { regionReplicaRackCostFunction.prepare(c); double rackCost = Math.abs(regionReplicaRackCostFunction.cost()); @@ -441,6 +461,11 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { return true; } + if (balancerConditionals.shouldRunBalancer(cluster)) { + LOG.info("Running balancer because conditional candidate generators have important moves"); + return true; + } + double total = 0.0; float localSumMultiplier = 0; // in case this.sumMultiplier is not initialized for (CostFunction c : costFunctions) { @@ -470,14 +495,17 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { } LOG.info( "{} - skipping load balancing because weighted average imbalance={} <= " - + "threshold({}). If you want more aggressive balancing, either lower " + + "threshold({}) and conditionals do not have opinionated move candidates. " + + "If you want more aggressive balancing, either lower " + "hbase.master.balancer.stochastic.minCostNeedBalance from {} or increase the relative " + "multiplier(s) of the specific cost function(s). functionCost={}", isByTable ? "Table specific (" + tableName + ")" : "Cluster wide", total / sumMultiplier, minCostNeedBalance, minCostNeedBalance, functionCost()); } else { - LOG.info("{} - Calculating plan. may take up to {}ms to complete.", - isByTable ? "Table specific (" + tableName + ")" : "Cluster wide", maxRunningTime); + LOG.info( + "{} - Calculating plan. may take up to {}ms to complete. currentCost={}, targetCost={}", + isByTable ? "Table specific (" + tableName + ")" : "Cluster wide", maxRunningTime, total, + minCostNeedBalance); } return !balanced; } @@ -485,7 +513,7 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { @RestrictedApi(explanation = "Should only be called in tests", link = "", allowedOnPath = ".*(/src/test/.*|StochasticLoadBalancer).java") Pair nextAction(BalancerClusterState cluster) { - CandidateGenerator generator = getRandomGenerator(); + CandidateGenerator generator = getRandomGenerator(cluster); return Pair.newPair(generator, generator.generate(cluster)); } @@ -494,8 +522,20 @@ Pair nextAction(BalancerClusterState cluster) * selecting a candidate generator is proportional to the share of cost of all cost functions * among all cost functions that benefit from it. */ - protected CandidateGenerator getRandomGenerator() { - Preconditions.checkState(!candidateGenerators.isEmpty(), "No candidate generators available."); + protected CandidateGenerator getRandomGenerator(BalancerClusterState cluster) { + // Prefer conditional generators if they have moves to make + if (balancerConditionals.isConditionalBalancingEnabled()) { + for (RegionPlanConditional conditional : balancerConditionals.getConditionals()) { + List generators = + conditional.getCandidateGenerators(); + for (RegionPlanConditionalCandidateGenerator generator : generators) { + if (generator.getWeight(cluster) > 0) { + return generator; + } + } + } + } + List> generatorClasses = shuffledGeneratorClasses.get(); List partialSums = new ArrayList<>(generatorClasses.size()); double sum = 0.0; @@ -583,8 +623,12 @@ protected List balanceTable(TableName tableName, rackManager, regionCacheRatioOnOldServerMap); long startTime = EnvironmentEdgeManager.currentTime(); + cluster.setStopRequestedAt(startTime + maxRunningTime); initCosts(cluster); + balancerConditionals.loadClusterState(cluster); + balancerConditionals.clearConditionalWeightCaches(); + float localSumMultiplier = 0; for (CostFunction c : costFunctions) { if (c.isNeeded()) { @@ -632,6 +676,7 @@ protected List balanceTable(TableName tableName, final String initFunctionTotalCosts = totalCostsPerFunc(); // Perform a stochastic walk to see if we can get a good fit. long step; + boolean planImprovedConditionals = false; Map, Long> generatorToStepCount = new HashMap<>(); Map, Long> generatorToApprovedActionCount = new HashMap<>(); for (step = 0; step < computedMaxSteps; step++) { @@ -643,16 +688,53 @@ protected List balanceTable(TableName tableName, continue; } - cluster.doAction(action); + int conditionalViolationsChange = 0; + boolean isViolatingConditionals = false; + boolean moveImprovedConditionals = false; + // Only check conditionals if they are enabled + if (balancerConditionals.isConditionalBalancingEnabled()) { + // Always accept a conditional generator output. Sometimes conditional generators + // may need to make controversial moves in order to break what would otherwise + // be a deadlocked situation. + // Otherwise, for normal moves, evaluate the action. + if (RegionPlanConditionalCandidateGenerator.class.isAssignableFrom(generator.getClass())) { + conditionalViolationsChange = -1; + } else { + conditionalViolationsChange = + balancerConditionals.getViolationCountChange(cluster, action); + isViolatingConditionals = balancerConditionals.isViolating(cluster, action); + } + moveImprovedConditionals = conditionalViolationsChange < 0; + if (moveImprovedConditionals) { + planImprovedConditionals = true; + } + } + + // Change state and evaluate costs + try { + cluster.doAction(action); + } catch (IllegalStateException | ArrayIndexOutOfBoundsException e) { + LOG.warn( + "Generator {} produced invalid action! " + + "Debug your candidate generator as this is likely a bug, " + + "and may cause a balancer deadlock. {}", + generator.getClass().getSimpleName(), action, e); + continue; + } updateCostsAndWeightsWithAction(cluster, action); - generatorToStepCount.merge(generator.getClass(), 1L, Long::sum); + generatorToStepCount.merge(generator.getClass(), action.getStepCount(), Long::sum); newCost = computeCost(cluster, currentCost); - // Should this be kept? - if (newCost < currentCost) { + boolean conditionalsSimilarCostsImproved = + (newCost < currentCost && conditionalViolationsChange == 0 && !isViolatingConditionals); + // Our first priority is to reduce conditional violations + // Our second priority is to reduce balancer cost + // change, regardless of cost change + if (moveImprovedConditionals || conditionalsSimilarCostsImproved) { currentCost = newCost; - generatorToApprovedActionCount.merge(generator.getClass(), 1L, Long::sum); + generatorToApprovedActionCount.merge(generator.getClass(), action.getStepCount(), + Long::sum); // save for JMX curOverallCost = currentCost; @@ -665,7 +747,7 @@ protected List balanceTable(TableName tableName, updateCostsAndWeightsWithAction(cluster, undoAction); } - if (EnvironmentEdgeManager.currentTime() - startTime > maxRunningTime) { + if (EnvironmentEdgeManager.currentTime() > cluster.getStopRequestedAt()) { break; } } @@ -682,7 +764,7 @@ protected List balanceTable(TableName tableName, metricsBalancer.balanceCluster(endTime - startTime); - if (initCost > currentCost) { + if (planImprovedConditionals || (initCost > currentCost)) { updateStochasticCosts(tableName, curOverallCost, curFunctionCosts); plans = createRegionPlans(cluster); LOG.info( @@ -697,7 +779,8 @@ protected List balanceTable(TableName tableName, } LOG.info( "Could not find a better moving plan. Tried {} different configurations in " - + "{} ms, and did not find anything with an imbalance score less than {}", + + "{} ms, and did not find anything with an imbalance score less than {} " + + "and could not improve conditional violations", step, endTime - startTime, initCost / sumMultiplier); return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SwapRegionsAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SwapRegionsAction.java index 6f83d2bc930b..c99de022f038 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SwapRegionsAction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SwapRegionsAction.java @@ -17,8 +17,12 @@ */ package org.apache.hadoop.hbase.master.balancer; +import java.util.List; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + @InterfaceAudience.Private public class SwapRegionsAction extends BalanceAction { private final int fromServer; @@ -55,6 +59,15 @@ public BalanceAction undoAction() { return new SwapRegionsAction(fromServer, toRegion, toServer, fromRegion); } + @Override + List toRegionPlans(BalancerClusterState cluster) { + return ImmutableList.of( + new RegionPlan(cluster.regions[getFromRegion()], cluster.servers[getFromServer()], + cluster.servers[getToServer()]), + new RegionPlan(cluster.regions[getToRegion()], cluster.servers[getToServer()], + cluster.servers[getFromServer()])); + } + @Override public String toString() { return getType() + ": " + fromRegion + ":" + fromServer + " <-> " + toRegion + ":" + toServer; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKey.java new file mode 100644 index 000000000000..f43df965da33 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKey.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer.replicas; + +import java.util.Arrays; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class ReplicaKey { + private final TableName tableName; + private final byte[] start; + private final byte[] stop; + + public ReplicaKey(RegionInfo regionInfo) { + this.tableName = regionInfo.getTable(); + this.start = regionInfo.getStartKey(); + this.stop = regionInfo.getEndKey(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ReplicaKey)) { + return false; + } + ReplicaKey other = (ReplicaKey) o; + return Arrays.equals(this.start, other.start) && Arrays.equals(this.stop, other.stop) + && this.tableName.equals(other.tableName); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(tableName).append(start).append(stop).toHashCode(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKeyCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKeyCache.java new file mode 100644 index 000000000000..a40e5f9a2f2d --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/replicas/ReplicaKeyCache.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer.replicas; + +import java.time.Duration; +import java.util.function.Supplier; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +import org.apache.hbase.thirdparty.com.google.common.base.Suppliers; +import org.apache.hbase.thirdparty.com.google.common.cache.CacheBuilder; +import org.apache.hbase.thirdparty.com.google.common.cache.CacheLoader; +import org.apache.hbase.thirdparty.com.google.common.cache.LoadingCache; + +@InterfaceAudience.Private +public final class ReplicaKeyCache implements Configurable { + /** + * ReplicaKey creation is expensive if you have lots of regions. If your HMaster has adequate + * memory, and you would like balancing to be faster, then you can turn on this flag to cache + * ReplicaKey objects. + */ + public static final String CACHE_REPLICA_KEYS_KEY = + "hbase.replica.distribution.conditional.cacheReplicaKeys"; + public static final boolean CACHE_REPLICA_KEYS_DEFAULT = false; + + /** + * If memory is available, then set this to a value greater than your region count to maximize + * replica distribution performance. + */ + public static final String REPLICA_KEY_CACHE_SIZE_KEY = + "hbase.replica.distribution.conditional.replicaKeyCacheSize"; + public static final int REPLICA_KEY_CACHE_SIZE_DEFAULT = 1000; + + private static final Supplier INSTANCE = Suppliers.memoize(ReplicaKeyCache::new); + + private volatile LoadingCache replicaKeyCache = null; + + private Configuration conf; + + public static ReplicaKeyCache getInstance() { + return INSTANCE.get(); + } + + private ReplicaKeyCache() { + } + + public ReplicaKey getReplicaKey(RegionInfo regionInfo) { + return replicaKeyCache == null + ? new ReplicaKey(regionInfo) + : replicaKeyCache.getUnchecked(regionInfo); + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + boolean cacheKeys = conf.getBoolean(CACHE_REPLICA_KEYS_KEY, CACHE_REPLICA_KEYS_DEFAULT); + if (cacheKeys && replicaKeyCache == null) { + int replicaKeyCacheSize = + conf.getInt(REPLICA_KEY_CACHE_SIZE_KEY, REPLICA_KEY_CACHE_SIZE_DEFAULT); + replicaKeyCache = CacheBuilder.newBuilder().maximumSize(replicaKeyCacheSize) + .expireAfterAccess(Duration.ofMinutes(30)).build(new CacheLoader() { + @Override + public ReplicaKey load(RegionInfo regionInfo) { + return new ReplicaKey(regionInfo); + } + }); + } else if (!cacheKeys) { + replicaKeyCache = null; + } + } + + @Override + public Configuration getConf() { + return conf; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionalsTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionalsTestUtil.java new file mode 100644 index 000000000000..0678cc3b67fb --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionalsTestUtil.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +public final class BalancerConditionalsTestUtil { + + private static final Logger LOG = LoggerFactory.getLogger(BalancerConditionalsTestUtil.class); + + private BalancerConditionalsTestUtil() { + } + + static byte[][] generateSplits(int numRegions) { + byte[][] splitKeys = new byte[numRegions - 1][]; + for (int i = 0; i < numRegions - 1; i++) { + splitKeys[i] = + Bytes.toBytes(String.format("%09d", (i + 1) * (Integer.MAX_VALUE / numRegions))); + } + return splitKeys; + } + + static void printRegionLocations(Connection connection) throws IOException { + Admin admin = connection.getAdmin(); + + // Get all table names in the cluster + Set tableNames = admin.listTableDescriptors().stream() + .map(TableDescriptor::getTableName).collect(Collectors.toSet()); + + // Group regions by server + Map>> serverToRegions = + admin.getClusterMetrics().getLiveServerMetrics().keySet().stream() + .collect(Collectors.toMap(server -> server, server -> { + try { + return listRegionsByTable(connection, server, tableNames); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + + // Pretty print region locations + StringBuilder regionLocationOutput = new StringBuilder(); + regionLocationOutput.append("Pretty printing region locations...\n"); + serverToRegions.forEach((server, tableRegions) -> { + regionLocationOutput.append("Server: " + server.getServerName() + "\n"); + tableRegions.forEach((table, regions) -> { + if (regions.isEmpty()) { + return; + } + regionLocationOutput.append(" Table: " + table.getNameAsString() + "\n"); + regions.forEach(region -> regionLocationOutput + .append(String.format(" Region: %s, start: %s, end: %s, replica: %s\n", + region.getEncodedName(), Bytes.toString(region.getStartKey()), + Bytes.toString(region.getEndKey()), region.getReplicaId()))); + }); + }); + LOG.info(regionLocationOutput.toString()); + } + + private static Map> listRegionsByTable(Connection connection, + ServerName server, Set tableNames) throws IOException { + Admin admin = connection.getAdmin(); + + // Find regions for each table + return tableNames.stream().collect(Collectors.toMap(tableName -> tableName, tableName -> { + List allRegions = null; + try { + allRegions = admin.getRegions(server); + } catch (IOException e) { + throw new RuntimeException(e); + } + return allRegions.stream().filter(region -> region.getTable().equals(tableName)) + .collect(Collectors.toList()); + })); + } + + static void validateReplicaDistribution(Connection connection, TableName tableName, + boolean shouldBeDistributed) { + Map> serverToRegions = null; + try { + serverToRegions = connection.getRegionLocator(tableName).getAllRegionLocations().stream() + .collect(Collectors.groupingBy(location -> location.getServerName(), + Collectors.mapping(location -> location.getRegion(), Collectors.toList()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (shouldBeDistributed) { + // Ensure no server hosts more than one replica of any region + for (Map.Entry> serverAndRegions : serverToRegions.entrySet()) { + List regionInfos = serverAndRegions.getValue(); + Set startKeys = new HashSet<>(); + for (RegionInfo regionInfo : regionInfos) { + // each region should have a distinct start key + assertFalse( + "Each region should have its own start key, " + + "demonstrating it is not a replica of any others on this host", + startKeys.contains(regionInfo.getStartKey())); + startKeys.add(regionInfo.getStartKey()); + } + } + } else { + // Ensure all replicas are on the same server + assertEquals("All regions should share one server", 1, serverToRegions.size()); + } + } + + static void validateRegionLocations(Map> tableToServers, + TableName productTableName, boolean shouldBeBalanced) { + ServerName metaServer = + tableToServers.get(TableName.META_TABLE_NAME).stream().findFirst().get(); + ServerName quotaServer = + tableToServers.get(QuotaUtil.QUOTA_TABLE_NAME).stream().findFirst().get(); + Set productServers = tableToServers.get(productTableName); + + if (shouldBeBalanced) { + for (ServerName server : productServers) { + assertNotEquals("Meta table and product table should not share servers", server, + metaServer); + assertNotEquals("Quota table and product table should not share servers", server, + quotaServer); + } + assertNotEquals("The meta server and quotas server should be different", metaServer, + quotaServer); + } else { + for (ServerName server : productServers) { + assertEquals("Meta table and product table must share servers", server, metaServer); + assertEquals("Quota table and product table must share servers", server, quotaServer); + } + assertEquals("The meta server and quotas server must be the same", metaServer, quotaServer); + } + } + + static Map> getTableToServers(Connection connection, + Set tableNames) { + return tableNames.stream().collect(Collectors.toMap(t -> t, t -> { + try { + return connection.getRegionLocator(t).getAllRegionLocations().stream() + .map(HRegionLocation::getServerName).collect(Collectors.toSet()); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + } + + @FunctionalInterface + interface AssertionRunnable { + void run() throws AssertionError; + } + + static void validateAssertionsWithRetries(HBaseTestingUtility testUtil, + boolean runBalancerOnFailure, AssertionRunnable assertion) { + validateAssertionsWithRetries(testUtil, runBalancerOnFailure, ImmutableSet.of(assertion)); + } + + static void validateAssertionsWithRetries(HBaseTestingUtility testUtil, + boolean runBalancerOnFailure, Set assertions) { + int maxAttempts = 50; + for (int i = 0; i < maxAttempts; i++) { + try { + for (AssertionRunnable assertion : assertions) { + assertion.run(); + } + } catch (AssertionError e) { + if (i == maxAttempts - 1) { + throw e; + } + try { + LOG.warn("Failed to validate region locations. Will retry", e); + Thread.sleep(1000); + BalancerConditionalsTestUtil.printRegionLocations(testUtil.getConnection()); + if (runBalancerOnFailure) { + testUtil.getAdmin().balance(); + } + Thread.sleep(1000); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + } + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java new file mode 100644 index 000000000000..4f6e8f70f305 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MAX_RUNNING_TIME_KEY; +import static org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MIN_COST_NEED_BALANCE_KEY; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class CandidateGeneratorTestUtil { + + private static final Logger LOG = LoggerFactory.getLogger(CandidateGeneratorTestUtil.class); + + private static final MasterServices MOCK_MASTER_SERVICES = mock(MasterServices.class); + + private CandidateGeneratorTestUtil() { + } + + static void runBalancerToExhaustion(Configuration conf, + Map> serverToRegions, + Set> expectations, float targetMaxBalancerCost) { + // Do the full plan. We're testing with a lot of regions + conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + conf.setLong(MAX_RUNNING_TIME_KEY, 15000); + + conf.setFloat(MIN_COST_NEED_BALANCE_KEY, targetMaxBalancerCost); + + BalancerClusterState cluster = createMockBalancerClusterState(serverToRegions); + StochasticLoadBalancer stochasticLoadBalancer = buildStochasticLoadBalancer(cluster, conf); + printClusterDistribution(cluster, 0); + int balancerRuns = 0; + int actionsTaken = 0; + long balancingMillis = 0; + boolean isBalanced = false; + while (!isBalanced) { + balancerRuns++; + if (balancerRuns > 1000) { + throw new RuntimeException("Balancer failed to find balance & meet expectations"); + } + long start = System.currentTimeMillis(); + List regionPlans = + stochasticLoadBalancer.balanceCluster(partitionRegionsByTable(serverToRegions)); + balancingMillis += System.currentTimeMillis() - start; + actionsTaken++; + if (regionPlans != null) { + // Apply all plans to serverToRegions + for (RegionPlan rp : regionPlans) { + ServerName source = rp.getSource(); + ServerName dest = rp.getDestination(); + RegionInfo region = rp.getRegionInfo(); + + // Update serverToRegions + serverToRegions.get(source).remove(region); + serverToRegions.get(dest).add(region); + actionsTaken++; + } + + // Now rebuild cluster and balancer from updated serverToRegions + cluster = createMockBalancerClusterState(serverToRegions); + stochasticLoadBalancer = buildStochasticLoadBalancer(cluster, conf); + } + printClusterDistribution(cluster, actionsTaken); + isBalanced = true; + for (Function condition : expectations) { + // Check if we've met all expectations for the candidate generator + if (!condition.apply(cluster)) { + isBalanced = false; + break; + } + } + if (isBalanced) { // Check if the balancer thinks we're done too + LOG.info("All balancer conditions passed. Checking if balancer thinks it's done."); + if (stochasticLoadBalancer.needsBalance(HConstants.ENSEMBLE_TABLE_NAME, cluster)) { + LOG.info("Balancer would still like to run"); + isBalanced = false; + } else { + LOG.info("Balancer is done"); + } + } + } + LOG.info("Balancing took {}sec", Duration.ofMillis(balancingMillis).toMinutes()); + } + + /** + * Prints the current cluster distribution of regions per table per server + */ + static void printClusterDistribution(BalancerClusterState cluster, long actionsTaken) { + LOG.info("=== Cluster Distribution after {} balancer actions taken ===", actionsTaken); + + for (int i = 0; i < cluster.numServers; i++) { + int[] regions = cluster.regionsPerServer[i]; + int regionCount = (regions == null) ? 0 : regions.length; + + LOG.info("Server {}: {} regions", cluster.servers[i].getServerName(), regionCount); + + if (regionCount > 0) { + Map tableRegionCounts = new HashMap<>(); + + for (int regionIndex : regions) { + RegionInfo regionInfo = cluster.regions[regionIndex]; + TableName tableName = regionInfo.getTable(); + tableRegionCounts.put(tableName, tableRegionCounts.getOrDefault(tableName, 0) + 1); + } + + tableRegionCounts + .forEach((table, count) -> LOG.info(" - Table {}: {} regions", table, count)); + } + } + + LOG.info("==========================================="); + } + + /** + * Partitions the given serverToRegions map by table The tables are derived from the RegionInfo + * objects found in serverToRegions. + * @param serverToRegions The map of servers to their assigned regions. + * @return A map of tables to their server-to-region assignments. + */ + public static Map>> + partitionRegionsByTable(Map> serverToRegions) { + + // First, gather all tables from the regions + Set allTables = new HashSet<>(); + for (List regions : serverToRegions.values()) { + for (RegionInfo region : regions) { + allTables.add(region.getTable()); + } + } + + Map>> tablesToServersToRegions = new HashMap<>(); + + // Initialize each table with all servers mapped to empty lists + for (TableName table : allTables) { + Map> serverMap = new HashMap<>(); + for (ServerName server : serverToRegions.keySet()) { + serverMap.put(server, new ArrayList<>()); + } + tablesToServersToRegions.put(table, serverMap); + } + + // Distribute regions to their respective tables + for (Map.Entry> serverAndRegions : serverToRegions.entrySet()) { + ServerName server = serverAndRegions.getKey(); + List regions = serverAndRegions.getValue(); + + for (RegionInfo region : regions) { + TableName regionTable = region.getTable(); + // Now we know for sure regionTable is in allTables + Map> tableServerMap = + tablesToServersToRegions.get(regionTable); + tableServerMap.get(server).add(region); + } + } + + return tablesToServersToRegions; + } + + static StochasticLoadBalancer buildStochasticLoadBalancer(BalancerClusterState cluster, + Configuration conf) { + StochasticLoadBalancer stochasticLoadBalancer = + new StochasticLoadBalancer(new DummyMetricsStochasticBalancer()); + when(MOCK_MASTER_SERVICES.getConfiguration()).thenReturn(conf); + stochasticLoadBalancer.setMasterServices(MOCK_MASTER_SERVICES); + stochasticLoadBalancer.loadConf(conf); + stochasticLoadBalancer.initCosts(cluster); + return stochasticLoadBalancer; + } + + static BalancerClusterState + createMockBalancerClusterState(Map> serverToRegions) { + return new BalancerClusterState(serverToRegions, null, null, null, null); + } + + /** + * Validates that each replica is isolated from its others. Ensures that no server hosts more than + * one replica of the same region (i.e., regions with identical start and end keys). + * @param cluster The current state of the cluster. + * @return true if all replicas are properly isolated, false otherwise. + */ + static boolean areAllReplicasDistributed(BalancerClusterState cluster) { + // Iterate over each server + for (int[] regionsPerServer : cluster.regionsPerServer) { + if (regionsPerServer == null || regionsPerServer.length == 0) { + continue; // Skip empty servers + } + + Set foundKeys = new HashSet<>(); + for (int regionIndex : regionsPerServer) { + RegionInfo regionInfo = cluster.regions[regionIndex]; + ReplicaKey replicaKey = new ReplicaKey(regionInfo); + if (foundKeys.contains(replicaKey)) { + // Violation: Multiple replicas of the same region on the same server + LOG.warn("Replica isolation violated: one server hosts multiple replicas of key [{}].", + generateRegionKey(regionInfo)); + return false; + } + + foundKeys.add(replicaKey); + } + } + + LOG.info( + "Replica isolation validation passed: No server hosts multiple replicas of the same region."); + return true; + } + + /** + * Generates a unique key for a region based on its start and end keys. This method ensures that + * regions with identical start and end keys have the same key. + * @param regionInfo The RegionInfo object. + * @return A string representing the unique key of the region. + */ + private static String generateRegionKey(RegionInfo regionInfo) { + // Using Base64 encoding for byte arrays to ensure uniqueness and readability + String startKey = Base64.getEncoder().encodeToString(regionInfo.getStartKey()); + String endKey = Base64.getEncoder().encodeToString(regionInfo.getEndKey()); + + return regionInfo.getTable().getNameAsString() + ":" + startKey + ":" + endKey; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasTestConditional.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasTestConditional.java new file mode 100644 index 000000000000..5a8fa2524fe6 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasTestConditional.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.conf.Configuration; + +public class DistributeReplicasTestConditional extends DistributeReplicasConditional { + + static void enableConditionalReplicaDistributionForTest(Configuration conf) { + conf.set(BalancerConditionals.ADDITIONAL_CONDITIONALS_KEY, + DistributeReplicasTestConditional.class.getCanonicalName()); + } + + public DistributeReplicasTestConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(balancerConditionals, cluster); + } + + @Override + public ValidationLevel getValidationLevel() { + // Mini-cluster tests can't validate at host/rack levels + return ValidationLevel.SERVER; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadOnlyFavoredStochasticBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadOnlyFavoredStochasticBalancer.java index d658f7cfa167..dfacad1a747c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadOnlyFavoredStochasticBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadOnlyFavoredStochasticBalancer.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.conf.Configuration; /** * Used for FavoredNode unit tests @@ -27,7 +28,7 @@ public class LoadOnlyFavoredStochasticBalancer extends FavoredStochasticBalancer @Override protected Map, CandidateGenerator> - createCandidateGenerators() { + createCandidateGenerators(Configuration conf) { Map, CandidateGenerator> fnPickers = new HashMap<>(1); fnPickers.put(FavoredNodeLoadPicker.class, new FavoredNodeLoadPicker()); return fnPickers; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java new file mode 100644 index 000000000000..884331f161ac --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ SmallTests.class, MasterTests.class }) +public class TestBalancerConditionals extends BalancerTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBalancerConditionals.class); + + private BalancerConditionals balancerConditionals; + private BalancerClusterState mockCluster; + + @Before + public void setUp() { + balancerConditionals = BalancerConditionals.create(); + mockCluster = mockCluster(new int[] { 0, 1, 2 }); + } + + @Test + public void testDefaultConfiguration() { + Configuration conf = new Configuration(); + balancerConditionals.setConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertEquals("No conditionals should be loaded by default", 0, + balancerConditionals.getConditionalClasses().size()); + } + + @Test + public void testCustomConditionalsViaConfiguration() { + Configuration conf = new Configuration(); + conf.set(BalancerConditionals.ADDITIONAL_CONDITIONALS_KEY, + DistributeReplicasConditional.class.getName()); + + balancerConditionals.setConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("Custom conditionals should be loaded", + balancerConditionals.shouldSkipSloppyServerEvaluation()); + } + + @Test + public void testInvalidCustomConditionalClass() { + Configuration conf = new Configuration(); + conf.set(BalancerConditionals.ADDITIONAL_CONDITIONALS_KEY, "java.lang.String"); + + balancerConditionals.setConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertEquals("Invalid classes should not be loaded as conditionals", 0, + balancerConditionals.getConditionalClasses().size()); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java new file mode 100644 index 000000000000..9e0a6f24e106 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKeyCache; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category({ MediumTests.class, MasterTests.class }) +public class TestLargeClusterBalancingConditionalReplicaDistribution { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLargeClusterBalancingConditionalReplicaDistribution.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestLargeClusterBalancingConditionalReplicaDistribution.class); + + private static final int NUM_SERVERS = 1000; + private static final int NUM_REGIONS = 20_000; + private static final int NUM_REPLICAS = 3; + private static final int NUM_TABLES = 100; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + serverToRegions.put(servers[i], new ArrayList<>()); + } + + // Create primary regions and their replicas + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = getTableName(i); + // Define startKey and endKey for the region + byte[] startKey = Bytes.toBytes(i); + byte[] endKey = Bytes.toBytes(i + 1); + + // Create 3 replicas for each primary region + for (int replicaId = 0; replicaId < NUM_REPLICAS; replicaId++) { + RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey) + .setEndKey(endKey).setReplicaId(replicaId).build(); + allRegions.add(regionInfo); + } + } + + // Assign all regions to one server + for (RegionInfo regionInfo : allRegions) { + serverToRegions.get(servers[0]).add(regionInfo); + } + } + + private static TableName getTableName(int i) { + return TableName.valueOf("userTable" + i % NUM_TABLES); + } + + @Test + public void testReplicaDistribution() { + Configuration conf = new Configuration(); + DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); + conf.setBoolean(ReplicaKeyCache.CACHE_REPLICA_KEYS_KEY, true); + conf.setInt(ReplicaKeyCache.REPLICA_KEY_CACHE_SIZE_KEY, Integer.MAX_VALUE); + conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 30_000); + + // turn off replica cost functions + conf.setLong("hbase.master.balancer.stochastic.regionReplicaRackCostKey", 0); + conf.setLong("hbase.master.balancer.stochastic.regionReplicaHostCostKey", 0); + + runBalancerToExhaustion(conf, serverToRegions, + ImmutableSet.of(CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f); + LOG.info("Meta table and system table regions are successfully isolated, " + + "meanwhile region replicas are appropriately distributed across RegionServers."); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestReplicaDistributionBalancerConditional.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestReplicaDistributionBalancerConditional.java new file mode 100644 index 000000000000..7807b07e74f9 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestReplicaDistributionBalancerConditional.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.BalancerConditionalsTestUtil.validateAssertionsWithRetries; + +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category({ LargeTests.class, MasterTests.class }) +public class TestReplicaDistributionBalancerConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReplicaDistributionBalancerConditional.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestReplicaDistributionBalancerConditional.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final int REPLICAS = 3; + private static final int NUM_SERVERS = REPLICAS; + private static final int REGIONS_PER_SERVER = 5; + + @Before + public void setUp() throws Exception { + DistributeReplicasTestConditional + .enableConditionalReplicaDistributionForTest(TEST_UTIL.getConfiguration()); + TEST_UTIL.getConfiguration() + .setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true); + TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_BALANCER_PERIOD, 1000L); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + + // turn off replica cost functions + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaRackCostKey", 0); + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaHostCostKey", 0); + + TEST_UTIL.startMiniCluster(NUM_SERVERS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testReplicaDistribution() throws Exception { + Connection connection = TEST_UTIL.getConnection(); + Admin admin = connection.getAdmin(); + + // Create a "replicated_table" with region replicas + TableName replicatedTableName = TableName.valueOf("replicated_table"); + TableDescriptor replicatedTableDescriptor = + TableDescriptorBuilder.newBuilder(replicatedTableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build()) + .setRegionReplication(REPLICAS).build(); + admin.createTable(replicatedTableDescriptor, + BalancerConditionalsTestUtil.generateSplits(REGIONS_PER_SERVER * NUM_SERVERS)); + + // Pause the balancer + admin.balancerSwitch(false, true); + + // Collect all region replicas and place them on one RegionServer + List allRegions = admin.getRegions(replicatedTableName); + String targetServer = + TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName().getServerName(); + + for (RegionInfo region : allRegions) { + admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(targetServer)); + } + + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + validateAssertionsWithRetries(TEST_UTIL, false, () -> BalancerConditionalsTestUtil + .validateReplicaDistribution(connection, replicatedTableName, false)); + + // Unpause the balancer and trigger balancing + admin.balancerSwitch(true, true); + admin.balance(); + + validateAssertionsWithRetries(TEST_UTIL, true, () -> BalancerConditionalsTestUtil + .validateReplicaDistribution(connection, replicatedTableName, true)); + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java index 960783a8467e..188efa64dcd5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerHeterogeneousCost.java @@ -254,7 +254,7 @@ static class StochasticLoadTestBalancer extends StochasticLoadBalancer { } @Override - protected CandidateGenerator getRandomGenerator() { + protected CandidateGenerator getRandomGenerator(BalancerClusterState cluster) { return fairRandomCandidateGenerator; } } From fe22899d1cd425600cee5331f03a8856287fb3a9 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Thu, 6 Mar 2025 08:23:14 -0500 Subject: [PATCH 09/78] HubSpot Backport: HBASE-29074 Balancer conditionals should support meta table isolation (will be in 2.7) Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../master/balancer/BalancerClusterState.java | 14 +- .../master/balancer/BalancerConditionals.java | 19 +- .../DistributeReplicasCandidateGenerator.java | 6 +- .../MetaTableIsolationCandidateGenerator.java | 34 ++++ .../MetaTableIsolationConditional.java | 37 ++++ ...gionPlanConditionalCandidateGenerator.java | 5 +- .../SlopFixingCandidateGenerator.java | 16 +- .../balancer/StochasticLoadBalancer.java | 7 +- .../TableIsolationCandidateGenerator.java | 130 +++++++++++++ .../balancer/TableIsolationConditional.java | 83 ++++++++ .../balancer/CandidateGeneratorTestUtil.java | 35 ++++ .../balancer/TestBalancerConditionals.java | 14 +- ...lancingConditionalReplicaDistribution.java | 3 +- ...rgeClusterBalancingMetaTableIsolation.java | 103 ++++++++++ ...gTableIsolationAndReplicaDistribution.java | 122 ++++++++++++ ...MetaTableIsolationBalancerConditional.java | 181 ++++++++++++++++++ 16 files changed, 787 insertions(+), 22 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingMetaTableIsolation.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationBalancerConditional.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index 67755fc317c6..b07287c1ed19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -311,11 +312,16 @@ protected BalancerClusterState(Map> clusterState, regionIndex++; } + if (LOG.isTraceEnabled()) { + for (int i = 0; i < numServers; i++) { + LOG.trace("server {} has {} regions", i, regionsPerServer[i].length); + } + } for (int i = 0; i < serversPerHostList.size(); i++) { serversPerHost[i] = new int[serversPerHostList.get(i).size()]; for (int j = 0; j < serversPerHost[i].length; j++) { serversPerHost[i][j] = serversPerHostList.get(i).get(j); - LOG.debug("server {} is on host {}", serversPerHostList.get(i).get(j), i); + LOG.trace("server {} is on host {}", serversPerHostList.get(i).get(j), i); } if (serversPerHost[i].length > 1) { multiServersPerHost = true; @@ -326,7 +332,7 @@ protected BalancerClusterState(Map> clusterState, serversPerRack[i] = new int[serversPerRackList.get(i).size()]; for (int j = 0; j < serversPerRack[i].length; j++) { serversPerRack[i][j] = serversPerRackList.get(i).get(j); - LOG.info("server {} is on rack {}", serversPerRackList.get(i).get(j), i); + LOG.trace("server {} is on rack {}", serversPerRackList.get(i).get(j), i); } } @@ -1075,8 +1081,8 @@ void setStopRequestedAt(long stopRequestedAt) { this.stopRequestedAt = stopRequestedAt; } - long getStopRequestedAt() { - return stopRequestedAt; + boolean isStopRequested() { + return EnvironmentEdgeManager.currentTime() > stopRequestedAt; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java index c44e47996932..88ceb5a55406 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -58,6 +58,10 @@ final class BalancerConditionals implements Configurable { "hbase.master.balancer.stochastic.conditionals.distributeReplicas"; public static final boolean DISTRIBUTE_REPLICAS_DEFAULT = false; + public static final String ISOLATE_META_TABLE_KEY = + "hbase.master.balancer.stochastic.conditionals.isolateMetaTable"; + public static final boolean ISOLATE_META_TABLE_DEFAULT = false; + public static final String ADDITIONAL_CONDITIONALS_KEY = "hbase.master.balancer.stochastic.additionalConditionals"; @@ -91,8 +95,14 @@ boolean isReplicaDistributionEnabled() { .anyMatch(DistributeReplicasConditional.class::isAssignableFrom); } - boolean shouldSkipSloppyServerEvaluation() { - return isConditionalBalancingEnabled(); + boolean isTableIsolationEnabled() { + return conditionalClasses.contains(MetaTableIsolationConditional.class); + } + + boolean isServerHostingIsolatedTables(BalancerClusterState cluster, int serverIdx) { + return conditionals.stream().filter(TableIsolationConditional.class::isInstance) + .map(TableIsolationConditional.class::cast) + .anyMatch(conditional -> conditional.isServerHostingIsolatedTables(cluster, serverIdx)); } boolean isConditionalBalancingEnabled() { @@ -193,6 +203,11 @@ public void setConf(Configuration conf) { conditionalClasses.add(DistributeReplicasConditional.class); } + boolean isolateMetaTable = conf.getBoolean(ISOLATE_META_TABLE_KEY, ISOLATE_META_TABLE_DEFAULT); + if (isolateMetaTable) { + conditionalClasses.add(MetaTableIsolationConditional.class); + } + Class[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY); for (Class clazz : classes) { if (!RegionPlanConditional.class.isAssignableFrom(clazz)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java index 38fbcc4a0fbc..be7c7871f9c7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasCandidateGenerator.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Set; import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,10 +59,7 @@ BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing List moveRegionActions = new ArrayList<>(); List shuffledServerIndices = cluster.getShuffledServerIndices(); for (int sourceIndex : shuffledServerIndices) { - if ( - moveRegionActions.size() >= BATCH_SIZE - || EnvironmentEdgeManager.currentTime() > cluster.getStopRequestedAt() - ) { + if (moveRegionActions.size() >= BATCH_SIZE || cluster.isStopRequested()) { break; } int[] serverRegions = cluster.regionsPerServer[sourceIndex]; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..5aa041f21d7e --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class MetaTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator { + + MetaTableIsolationCandidateGenerator(BalancerConditionals balancerConditionals) { + super(balancerConditionals); + } + + @Override + boolean shouldBeIsolated(RegionInfo regionInfo) { + return regionInfo.isMetaRegion(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java new file mode 100644 index 000000000000..732693c44f3e --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; + +/** + * If enabled, this class will help the balancer ensure that the meta table lives on its own + * RegionServer. Configure this via {@link BalancerConditionals#ISOLATE_META_TABLE_KEY} + */ +class MetaTableIsolationConditional extends TableIsolationConditional { + + public MetaTableIsolationConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(balancerConditionals, cluster); + } + + @Override + boolean isRegionToIsolate(RegionInfo regionInfo) { + return regionInfo.isMetaRegion(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java index f8274841f729..d28a507ff3fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditionalCandidateGenerator.java @@ -64,8 +64,11 @@ BalanceAction generate(BalancerClusterState cluster) { return balanceAction; } - MoveBatchAction batchMovesAndResetClusterState(BalancerClusterState cluster, + BalanceAction batchMovesAndResetClusterState(BalancerClusterState cluster, List moves) { + if (moves.isEmpty()) { + return BalanceAction.NULL_ACTION; + } MoveBatchAction batchAction = new MoveBatchAction(moves); undoBatchAction(cluster, batchAction); return batchAction; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java index 070e4903394d..b1ea1de8d2b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java @@ -44,6 +44,7 @@ final class SlopFixingCandidateGenerator extends RegionPlanConditionalCandidateG @Override BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing) { + boolean isTableIsolationEnabled = getBalancerConditionals().isTableIsolationEnabled(); ClusterLoadState cs = new ClusterLoadState(cluster.clusterState); float average = cs.getLoadAverage(); int ceiling = (int) Math.ceil(average * (1 + slop)); @@ -63,6 +64,13 @@ BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing List moves = new ArrayList<>(); Set fixedServers = new HashSet<>(); for (int sourceServer : sloppyServerIndices) { + if ( + isTableIsolationEnabled + && getBalancerConditionals().isServerHostingIsolatedTables(cluster, sourceServer) + ) { + // Don't fix sloppiness of servers hosting isolated tables + continue; + } for (int regionIdx : cluster.regionsPerServer[sourceServer]) { boolean regionFoundMove = false; for (ServerAndLoad serverAndLoad : cs.getServersByLoad().keySet()) { @@ -88,8 +96,8 @@ BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing } fixedServers.forEach(s -> cs.getServersByLoad().remove(s)); fixedServers.clear(); - if (!regionFoundMove) { - LOG.debug("Could not find a destination for region {} from server {}.", regionIdx, + if (!regionFoundMove && LOG.isTraceEnabled()) { + LOG.trace("Could not find a destination for region {} from server {}.", regionIdx, sourceServer); } if (cluster.regionsPerServer[sourceServer].length <= ceiling) { @@ -98,8 +106,6 @@ BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing } } - MoveBatchAction batch = new MoveBatchAction(moves); - undoBatchAction(cluster, batch); - return batch; + return batchMovesAndResetClusterState(cluster, moves); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 42784ea4440d..d184cf52e80f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -455,7 +455,10 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { return true; } - if (sloppyRegionServerExist(cs)) { + if ( + // table isolation is inherently incompatible with naive "sloppy server" checks + !balancerConditionals.isTableIsolationEnabled() && sloppyRegionServerExist(cs) + ) { LOG.info("Running balancer because cluster has sloppy server(s)." + " function cost={}", functionCost()); return true; @@ -747,7 +750,7 @@ protected List balanceTable(TableName tableName, updateCostsAndWeightsWithAction(cluster, undoAction); } - if (EnvironmentEdgeManager.currentTime() > cluster.getStopRequestedAt()) { + if (cluster.isStopRequested()) { break; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..ec41033999fa --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public abstract class TableIsolationCandidateGenerator + extends RegionPlanConditionalCandidateGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(TableIsolationCandidateGenerator.class); + + TableIsolationCandidateGenerator(BalancerConditionals balancerConditionals) { + super(balancerConditionals); + } + + abstract boolean shouldBeIsolated(RegionInfo regionInfo); + + @Override + BalanceAction generate(BalancerClusterState cluster) { + return generateCandidate(cluster, false); + } + + BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing) { + if (!getBalancerConditionals().isTableIsolationEnabled()) { + return BalanceAction.NULL_ACTION; + } + + List moves = new ArrayList<>(); + List serverIndicesHoldingIsolatedRegions = new ArrayList<>(); + int isolatedTableMaxReplicaCount = 1; + for (int serverIdx : cluster.getShuffledServerIndices()) { + if (cluster.isStopRequested()) { + break; + } + boolean hasRegionsToIsolate = false; + Set regionsToMove = new HashSet<>(); + + // Move non-target regions away from target regions, + // and track replica counts so we know how many isolated hosts we need + for (int regionIdx : cluster.regionsPerServer[serverIdx]) { + RegionInfo regionInfo = cluster.regions[regionIdx]; + if (shouldBeIsolated(regionInfo)) { + hasRegionsToIsolate = true; + int replicaCount = regionInfo.getReplicaId() + 1; + if (replicaCount > isolatedTableMaxReplicaCount) { + isolatedTableMaxReplicaCount = replicaCount; + } + } else { + regionsToMove.add(regionIdx); + } + } + + if (hasRegionsToIsolate) { + serverIndicesHoldingIsolatedRegions.add(serverIdx); + } + + // Generate non-system regions to move, if applicable + if (hasRegionsToIsolate && !regionsToMove.isEmpty()) { + for (int regionToMove : regionsToMove) { + for (int i = 0; i < cluster.numServers; i++) { + int targetServer = pickOtherRandomServer(cluster, serverIdx); + MoveRegionAction possibleMove = + new MoveRegionAction(regionToMove, serverIdx, targetServer); + if (!getBalancerConditionals().isViolating(cluster, possibleMove)) { + if (isWeighing) { + return possibleMove; + } + cluster.doAction(possibleMove); // Update cluster state to reflect move + moves.add(possibleMove); + break; + } + } + } + } + } + + // Try to consolidate regions on only n servers, where n is the number of replicas + if (serverIndicesHoldingIsolatedRegions.size() > isolatedTableMaxReplicaCount) { + // One target per replica + List targetServerIndices = new ArrayList<>(); + for (int i = 0; i < isolatedTableMaxReplicaCount; i++) { + targetServerIndices.add(serverIndicesHoldingIsolatedRegions.get(i)); + } + // Move all isolated regions from non-targets to targets + for (int i = isolatedTableMaxReplicaCount; i + < serverIndicesHoldingIsolatedRegions.size(); i++) { + int fromServer = serverIndicesHoldingIsolatedRegions.get(i); + for (int regionIdx : cluster.regionsPerServer[fromServer]) { + RegionInfo regionInfo = cluster.regions[regionIdx]; + if (shouldBeIsolated(regionInfo)) { + int targetServer = targetServerIndices.get(i % isolatedTableMaxReplicaCount); + MoveRegionAction possibleMove = + new MoveRegionAction(regionIdx, fromServer, targetServer); + if (!getBalancerConditionals().isViolating(cluster, possibleMove)) { + if (isWeighing) { + return possibleMove; + } + cluster.doAction(possibleMove); // Update cluster state to reflect move + moves.add(possibleMove); + } + } + } + } + } + return batchMovesAndResetClusterState(cluster, moves); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java new file mode 100644 index 000000000000..cd3ce0b6fe18 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionPlan; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + +abstract class TableIsolationConditional extends RegionPlanConditional { + + private final List candidateGenerators; + + TableIsolationConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(balancerConditionals.getConf(), cluster); + + float slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, + BaseLoadBalancer.REGIONS_SLOP_DEFAULT); + this.candidateGenerators = + ImmutableList.of(new MetaTableIsolationCandidateGenerator(balancerConditionals), + new SlopFixingCandidateGenerator(balancerConditionals, slop)); + } + + abstract boolean isRegionToIsolate(RegionInfo regionInfo); + + boolean isServerHostingIsolatedTables(BalancerClusterState cluster, int serverIdx) { + for (int regionIdx : cluster.regionsPerServer[serverIdx]) { + if (isRegionToIsolate(cluster.regions[regionIdx])) { + return true; + } + } + return false; + } + + @Override + ValidationLevel getValidationLevel() { + return ValidationLevel.SERVER; + } + + @Override + List getCandidateGenerators() { + return candidateGenerators; + } + + @Override + public boolean isViolatingServer(RegionPlan regionPlan, Set serverRegions) { + RegionInfo regionBeingMoved = regionPlan.getRegionInfo(); + boolean shouldIsolateMovingRegion = isRegionToIsolate(regionBeingMoved); + for (RegionInfo destinationRegion : serverRegions) { + if (destinationRegion.getEncodedName().equals(regionBeingMoved.getEncodedName())) { + // Skip the region being moved + continue; + } + if (shouldIsolateMovingRegion && !isRegionToIsolate(destinationRegion)) { + // Ensure every destination region is also a region to isolate + return true; + } else if (!shouldIsolateMovingRegion && isRegionToIsolate(destinationRegion)) { + // Ensure no destination region is a region to isolate + return true; + } + } + return false; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java index 4f6e8f70f305..d2a2d432ff05 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -239,6 +239,41 @@ static boolean areAllReplicasDistributed(BalancerClusterState cluster) { return true; } + /** + * Generic method to validate table isolation. + */ + static boolean isTableIsolated(BalancerClusterState cluster, TableName tableName, + String tableType) { + for (int i = 0; i < cluster.numServers; i++) { + int[] regionsOnServer = cluster.regionsPerServer[i]; + if (regionsOnServer == null || regionsOnServer.length == 0) { + continue; // Skip empty servers + } + + boolean hasTargetTableRegion = false; + boolean hasOtherTableRegion = false; + + for (int regionIndex : regionsOnServer) { + RegionInfo regionInfo = cluster.regions[regionIndex]; + if (regionInfo.getTable().equals(tableName)) { + hasTargetTableRegion = true; + } else { + hasOtherTableRegion = true; + } + + // If the target table and any other table are on the same server, isolation is violated + if (hasTargetTableRegion && hasOtherTableRegion) { + LOG.debug( + "Server {} has both {} table regions and other table regions, violating isolation.", + cluster.servers[i].getServerName(), tableType); + return false; + } + } + } + LOG.debug("{} table isolation validation passed.", tableType); + return true; + } + /** * Generates a unique key for a region based on its start and end keys. This method ensures that * regions with identical start and end keys have the same key. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java index 884331f161ac..4dc40cda5481 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java @@ -65,7 +65,7 @@ public void testCustomConditionalsViaConfiguration() { balancerConditionals.loadClusterState(mockCluster); assertTrue("Custom conditionals should be loaded", - balancerConditionals.shouldSkipSloppyServerEvaluation()); + balancerConditionals.isConditionalBalancingEnabled()); } @Test @@ -80,4 +80,16 @@ public void testInvalidCustomConditionalClass() { balancerConditionals.getConditionalClasses().size()); } + @Test + public void testMetaTableIsolationConditionalEnabled() { + Configuration conf = new Configuration(); + conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + + balancerConditionals.setConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("MetaTableIsolationConditional should be active", + balancerConditionals.isTableIsolationEnabled()); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java index 9e0a6f24e106..2522a13819f1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingConditionalReplicaDistribution.java @@ -108,7 +108,6 @@ public void testReplicaDistribution() { runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f); - LOG.info("Meta table and system table regions are successfully isolated, " - + "meanwhile region replicas are appropriately distributed across RegionServers."); + LOG.info("Region replicas are appropriately distributed across RegionServers."); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingMetaTableIsolation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingMetaTableIsolation.java new file mode 100644 index 000000000000..27360f3cd570 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingMetaTableIsolation.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category({ MediumTests.class, MasterTests.class }) +public class TestLargeClusterBalancingMetaTableIsolation { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLargeClusterBalancingMetaTableIsolation.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestLargeClusterBalancingMetaTableIsolation.class); + + private static final TableName NON_META_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 1000; + private static final int NUM_REGIONS = 20_000; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + } + + // Create regions + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = i < 3 ? TableName.META_TABLE_NAME : NON_META_TABLE_NAME; + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build(); + allRegions.add(regionInfo); + } + + // Assign all regions to the first server + serverToRegions.put(servers[0], new ArrayList<>(allRegions)); + for (int i = 1; i < NUM_SERVERS; i++) { + serverToRegions.put(servers[i], new ArrayList<>()); + } + } + + @Test + public void testMetaTableIsolation() { + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated), + 10.0f); + LOG.info("Meta table regions are successfully isolated."); + } + + private boolean isMetaTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, TableName.META_TABLE_NAME, "Meta"); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java new file mode 100644 index 000000000000..5fbddf4878be --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category({ MediumTests.class, MasterTests.class }) +public class TestLargeClusterBalancingTableIsolationAndReplicaDistribution { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule + .forClass(TestLargeClusterBalancingTableIsolationAndReplicaDistribution.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestLargeClusterBalancingTableIsolationAndReplicaDistribution.class); + private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); + private static final TableName NON_ISOLATED_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 1000; + private static final int NUM_REGIONS = 10_000; + private static final int NUM_REPLICAS = 3; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + serverToRegions.put(servers[i], new ArrayList<>()); + } + + // Create primary regions and their replicas + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName; + if (i < 1) { + tableName = TableName.META_TABLE_NAME; + } else if (i < 10) { + tableName = SYSTEM_TABLE_NAME; + } else { + tableName = NON_ISOLATED_TABLE_NAME; + } + + // Define startKey and endKey for the region + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + // Create 3 replicas for each primary region + for (int replicaId = 0; replicaId < NUM_REPLICAS; replicaId++) { + RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey) + .setEndKey(endKey).setReplicaId(replicaId).build(); + allRegions.add(regionInfo); + } + } + + // Assign all regions to one server + for (RegionInfo regionInfo : allRegions) { + serverToRegions.get(servers[0]).add(regionInfo); + } + } + + @Test + public void testTableIsolationAndReplicaDistribution() { + + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); + + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated, + CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f); + LOG.info("Meta table regions are successfully isolated, " + + "and region replicas are appropriately distributed."); + } + + /** + * Validates whether all meta table regions are isolated. + */ + private boolean isMetaTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, TableName.META_TABLE_NAME, "Meta"); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationBalancerConditional.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationBalancerConditional.java new file mode 100644 index 000000000000..d2eb7243ec8e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationBalancerConditional.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category(LargeTests.class) +public class TestMetaTableIsolationBalancerConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaTableIsolationBalancerConditional.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestMetaTableIsolationBalancerConditional.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private static final int NUM_SERVERS = 3; + + @Before + public void setUp() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); // for another table + TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_BALANCER_PERIOD, 1000L); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + + TEST_UTIL.startMiniCluster(NUM_SERVERS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testTableIsolation() throws Exception { + Connection connection = TEST_UTIL.getConnection(); + Admin admin = connection.getAdmin(); + + // Create "product" table with 3 regions + TableName productTableName = TableName.valueOf("product"); + TableDescriptor productTableDescriptor = TableDescriptorBuilder.newBuilder(productTableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build()) + .build(); + admin.createTable(productTableDescriptor, + BalancerConditionalsTestUtil.generateSplits(2 * NUM_SERVERS)); + + Set tablesToBeSeparated = ImmutableSet. builder() + .add(TableName.META_TABLE_NAME).add(QuotaUtil.QUOTA_TABLE_NAME).add(productTableName).build(); + + // Pause the balancer + admin.balancerSwitch(false, true); + + // Move all regions (product, meta, and quotas) to one RegionServer + List allRegions = tablesToBeSeparated.stream().map(t -> { + try { + return admin.getRegions(t); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).flatMap(Collection::stream).collect(Collectors.toList()); + String targetServer = + TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName().getServerName(); + for (RegionInfo region : allRegions) { + admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(targetServer)); + } + + validateRegionLocationsWithRetry(connection, tablesToBeSeparated, productTableName, false, + false); + + // Unpause the balancer and run it + admin.balancerSwitch(true, true); + admin.balance(); + + validateRegionLocationsWithRetry(connection, tablesToBeSeparated, productTableName, true, true); + } + + private static void validateRegionLocationsWithRetry(Connection connection, + Set tableNames, TableName productTableName, boolean areDistributed, + boolean runBalancerOnFailure) throws InterruptedException, IOException { + for (int i = 0; i < 100; i++) { + Map> tableToServers = getTableToServers(connection, tableNames); + try { + validateRegionLocations(tableToServers, productTableName, areDistributed); + } catch (AssertionError e) { + if (i == 99) { + throw e; + } + LOG.warn("Failed to validate region locations. Will retry", e); + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + if (runBalancerOnFailure) { + connection.getAdmin().balance(); + } + Thread.sleep(1000); + } + } + } + + private static void validateRegionLocations(Map> tableToServers, + TableName productTableName, boolean shouldBeBalanced) { + // Validate that the region assignments + ServerName metaServer = + tableToServers.get(TableName.META_TABLE_NAME).stream().findFirst().get(); + ServerName quotaServer = + tableToServers.get(QuotaUtil.QUOTA_TABLE_NAME).stream().findFirst().get(); + Set productServers = tableToServers.get(productTableName); + + if (shouldBeBalanced) { + assertNotEquals("Meta table and quota table should not share a server", metaServer, + quotaServer); + for (ServerName productServer : productServers) { + assertNotEquals("Meta table and product table should not share servers", productServer, + metaServer); + } + } else { + assertEquals("Quota table and product table must share servers", metaServer, quotaServer); + for (ServerName server : productServers) { + assertEquals("Meta table and product table must share servers", server, metaServer); + } + } + } + + private static Map> getTableToServers(Connection connection, + Set tableNames) { + return tableNames.stream().collect(Collectors.toMap(t -> t, t -> { + try { + return connection.getRegionLocator(t).getAllRegionLocations().stream() + .map(HRegionLocation::getServerName).collect(Collectors.toSet()); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + } +} From 902ef003a22886a61930000179b7fce859cd2326 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 7 Mar 2025 08:27:15 -0500 Subject: [PATCH 10/78] HubSpot Backport: HBASE-29075 Balancer conditionals should support system table isolation (will be in 2.7) Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../master/balancer/BalancerConditionals.java | 14 +++ .../DistributeReplicasConditional.java | 6 +- .../MetaTableIsolationConditional.java | 3 +- .../SlopFixingCandidateGenerator.java | 5 +- ...ystemTableIsolationCandidateGenerator.java | 41 +++++++ .../SystemTableIsolationConditional.java | 43 ++++++++ .../balancer/TableIsolationConditional.java | 9 +- .../balancer/CandidateGeneratorTestUtil.java | 9 +- ...eClusterBalancingSystemTableIsolation.java | 104 ++++++++++++++++++ ...gTableIsolationAndReplicaDistribution.java | 28 +++-- 10 files changed, 236 insertions(+), 26 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java index 88ceb5a55406..021a34bce6b9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -62,6 +62,10 @@ final class BalancerConditionals implements Configurable { "hbase.master.balancer.stochastic.conditionals.isolateMetaTable"; public static final boolean ISOLATE_META_TABLE_DEFAULT = false; + public static final String ISOLATE_SYSTEM_TABLES_KEY = + "hbase.master.balancer.stochastic.conditionals.isolateSystemTables"; + public static final boolean ISOLATE_SYSTEM_TABLES_DEFAULT = false; + public static final String ADDITIONAL_CONDITIONALS_KEY = "hbase.master.balancer.stochastic.additionalConditionals"; @@ -96,6 +100,10 @@ boolean isReplicaDistributionEnabled() { } boolean isTableIsolationEnabled() { + return conditionalClasses.stream().anyMatch(TableIsolationConditional.class::isAssignableFrom); + } + + boolean isMetaTableIsolationEnabled() { return conditionalClasses.contains(MetaTableIsolationConditional.class); } @@ -208,6 +216,12 @@ public void setConf(Configuration conf) { conditionalClasses.add(MetaTableIsolationConditional.class); } + boolean isolateSystemTables = + conf.getBoolean(ISOLATE_SYSTEM_TABLES_KEY, ISOLATE_SYSTEM_TABLES_DEFAULT); + if (isolateSystemTables) { + conditionalClasses.add(SystemTableIsolationConditional.class); + } + Class[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY); for (Class clazz : classes) { if (!RegionPlanConditional.class.isAssignableFrom(clazz)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java index 2cd27615e5fd..e99c0e93a159 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.balancer.replicas.ReplicaKey; @@ -41,12 +40,9 @@ public class DistributeReplicasConditional extends RegionPlanConditional { public DistributeReplicasConditional(BalancerConditionals balancerConditionals, BalancerClusterState cluster) { super(balancerConditionals.getConf(), cluster); - Configuration conf = balancerConditionals.getConf(); - float slop = - conf.getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, BaseLoadBalancer.REGIONS_SLOP_DEFAULT); this.candidateGenerators = ImmutableList.of(new DistributeReplicasCandidateGenerator(balancerConditionals), - new SlopFixingCandidateGenerator(balancerConditionals, slop)); + new SlopFixingCandidateGenerator(balancerConditionals)); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java index 732693c44f3e..5617468457c4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java @@ -27,7 +27,8 @@ class MetaTableIsolationConditional extends TableIsolationConditional { public MetaTableIsolationConditional(BalancerConditionals balancerConditionals, BalancerClusterState cluster) { - super(balancerConditionals, cluster); + super(new MetaTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals, + cluster); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java index b1ea1de8d2b0..f78e1573b417 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SlopFixingCandidateGenerator.java @@ -37,9 +37,10 @@ final class SlopFixingCandidateGenerator extends RegionPlanConditionalCandidateG private final float slop; - SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals, float slop) { + SlopFixingCandidateGenerator(BalancerConditionals balancerConditionals) { super(balancerConditionals); - this.slop = slop; + this.slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, + BaseLoadBalancer.REGIONS_SLOP_DEFAULT); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..7ce8ff202965 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class SystemTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator { + + private final BalancerConditionals balancerConditionals; + + SystemTableIsolationCandidateGenerator(BalancerConditionals balancerConditionals) { + super(balancerConditionals); + this.balancerConditionals = balancerConditionals; + } + + @Override + boolean shouldBeIsolated(RegionInfo regionInfo) { + if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) { + // If meta isolation is enabled, we can ignore meta regions here + return false; + } + return regionInfo.getTable().isSystemTable(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java new file mode 100644 index 000000000000..b5734b82faf7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class SystemTableIsolationConditional extends TableIsolationConditional { + + private final BalancerConditionals balancerConditionals; + + SystemTableIsolationConditional(BalancerConditionals balancerConditionals, + BalancerClusterState cluster) { + super(new SystemTableIsolationCandidateGenerator(balancerConditionals), balancerConditionals, + cluster); + this.balancerConditionals = balancerConditionals; + } + + @Override + boolean isRegionToIsolate(RegionInfo regionInfo) { + if (balancerConditionals.isMetaTableIsolationEnabled() && regionInfo.isMetaRegion()) { + // If meta isolation is enabled, we can ignore meta regions here + return false; + } + return regionInfo.getTable().isSystemTable(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java index cd3ce0b6fe18..24a6f519e8d8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationConditional.java @@ -28,15 +28,12 @@ abstract class TableIsolationConditional extends RegionPlanConditional { private final List candidateGenerators; - TableIsolationConditional(BalancerConditionals balancerConditionals, - BalancerClusterState cluster) { + TableIsolationConditional(TableIsolationCandidateGenerator generator, + BalancerConditionals balancerConditionals, BalancerClusterState cluster) { super(balancerConditionals.getConf(), cluster); - float slop = balancerConditionals.getConf().getFloat(BaseLoadBalancer.REGIONS_SLOP_KEY, - BaseLoadBalancer.REGIONS_SLOP_DEFAULT); this.candidateGenerators = - ImmutableList.of(new MetaTableIsolationCandidateGenerator(balancerConditionals), - new SlopFixingCandidateGenerator(balancerConditionals, slop)); + ImmutableList.of(generator, new SlopFixingCandidateGenerator(balancerConditionals)); } abstract boolean isRegionToIsolate(RegionInfo regionInfo); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java index d2a2d432ff05..03bfcce8e150 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -54,9 +54,16 @@ private CandidateGeneratorTestUtil() { static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost) { + runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000); + } + + static void runBalancerToExhaustion(Configuration conf, + Map> serverToRegions, + Set> expectations, float targetMaxBalancerCost, + long maxRunningTime) { // Do the full plan. We're testing with a lot of regions conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); - conf.setLong(MAX_RUNNING_TIME_KEY, 15000); + conf.setLong(MAX_RUNNING_TIME_KEY, maxRunningTime); conf.setFloat(MIN_COST_NEED_BALANCE_KEY, targetMaxBalancerCost); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java new file mode 100644 index 000000000000..ef26c548c209 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingSystemTableIsolation.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category({ MediumTests.class, MasterTests.class }) +public class TestLargeClusterBalancingSystemTableIsolation { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLargeClusterBalancingSystemTableIsolation.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestLargeClusterBalancingSystemTableIsolation.class); + + private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); + private static final TableName NON_SYSTEM_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 1000; + private static final int NUM_REGIONS = 20_000; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + } + + // Create regions + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = i < 3 ? SYSTEM_TABLE_NAME : NON_SYSTEM_TABLE_NAME; + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build(); + allRegions.add(regionInfo); + } + + // Assign all regions to the first server + serverToRegions.put(servers[0], new ArrayList<>(allRegions)); + for (int i = 1; i < NUM_SERVERS; i++) { + serverToRegions.put(servers[i], new ArrayList<>()); + } + } + + @Test + public void testSystemTableIsolation() { + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isSystemTableIsolated), + 10.0f); + LOG.info("Meta table regions are successfully isolated."); + } + + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java index 5fbddf4878be..3a28ae801e4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; @@ -53,8 +54,8 @@ public class TestLargeClusterBalancingTableIsolationAndReplicaDistribution { private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); private static final TableName NON_ISOLATED_TABLE_NAME = TableName.valueOf("userTable"); - private static final int NUM_SERVERS = 1000; - private static final int NUM_REGIONS = 10_000; + private static final int NUM_SERVERS = 500; + private static final int NUM_REGIONS = 2_500; private static final int NUM_REPLICAS = 3; private static final ServerName[] servers = new ServerName[NUM_SERVERS]; @@ -69,7 +70,6 @@ public static void setup() { } // Create primary regions and their replicas - List allRegions = new ArrayList<>(); for (int i = 0; i < NUM_REGIONS; i++) { TableName tableName; if (i < 1) { @@ -86,29 +86,28 @@ public static void setup() { byte[] endKey = new byte[1]; endKey[0] = (byte) (i + 1); + Random random = new Random(); // Create 3 replicas for each primary region for (int replicaId = 0; replicaId < NUM_REPLICAS; replicaId++) { RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey) .setEndKey(endKey).setReplicaId(replicaId).build(); - allRegions.add(regionInfo); + // Assign region to random server + int randomServer = random.nextInt(servers.length); + serverToRegions.get(servers[randomServer]).add(regionInfo); } } - - // Assign all regions to one server - for (RegionInfo regionInfo : allRegions) { - serverToRegions.get(servers[0]).add(regionInfo); - } } @Test public void testTableIsolationAndReplicaDistribution() { - Configuration conf = new Configuration(false); conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated, - CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f); + this::isSystemTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f, + 60_000); LOG.info("Meta table regions are successfully isolated, " + "and region replicas are appropriately distributed."); } @@ -119,4 +118,11 @@ public void testTableIsolationAndReplicaDistribution() { private boolean isMetaTableIsolated(BalancerClusterState cluster) { return isTableIsolated(cluster, TableName.META_TABLE_NAME, "Meta"); } + + /** + * Validates whether all meta table regions are isolated. + */ + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } } From 390407db26bbbae6eb34d49d15f19dbf65979e27 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Mon, 17 Mar 2025 13:21:25 -0400 Subject: [PATCH 11/78] HubSpot Backport: HBASE-29186 RegionPlanConditionals can produce a null pointer (will be in 2.7) Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../balancer/RegionPlanConditional.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java index 8de371d341cd..063f3ba5f726 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.master.balancer; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -85,8 +86,8 @@ boolean isViolating(RegionPlan regionPlan) { // Check Server int[] destinationRegionIndices = cluster.regionsPerServer[destinationServerIdx]; - Set serverRegions = Arrays.stream(cluster.regionsPerServer[destinationServerIdx]) - .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + Set serverRegions = + getRegionsFromIndex(destinationServerIdx, cluster.regionsPerServer); for (int regionIdx : destinationRegionIndices) { serverRegions.add(cluster.regions[regionIdx]); } @@ -100,8 +101,7 @@ boolean isViolating(RegionPlan regionPlan) { // Check Host int hostIdx = cluster.serverIndexToHostIndex[destinationServerIdx]; - Set hostRegions = Arrays.stream(cluster.regionsPerHost[hostIdx]) - .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + Set hostRegions = getRegionsFromIndex(hostIdx, cluster.regionsPerHost); if (isViolatingHost(regionPlan, hostRegions)) { return true; } @@ -112,8 +112,7 @@ boolean isViolating(RegionPlan regionPlan) { // Check Rack int rackIdx = cluster.serverIndexToRackIndex[destinationServerIdx]; - Set rackRegions = Arrays.stream(cluster.regionsPerRack[rackIdx]) - .mapToObj(idx -> cluster.regions[idx]).collect(Collectors.toSet()); + Set rackRegions = getRegionsFromIndex(rackIdx, cluster.regionsPerRack); if (isViolatingRack(regionPlan, rackRegions)) { return true; } @@ -130,4 +129,13 @@ boolean isViolatingHost(RegionPlan regionPlan, Set destinationRegion boolean isViolatingRack(RegionPlan regionPlan, Set destinationRegions) { return false; } + + private Set getRegionsFromIndex(int index, int[][] regionsPerIndex) { + int[] regionIndices = regionsPerIndex[index]; + if (regionIndices == null) { + return Collections.emptySet(); + } + return Arrays.stream(regionIndices).mapToObj(idx -> cluster.regions[idx]) + .collect(Collectors.toSet()); + } } From 7b3d9d178e38b9cc7787bbd943c98e13f45e380e Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 21 Mar 2025 08:05:03 -0400 Subject: [PATCH 12/78] HubSpot Backport: HBASE-29202 Balancer conditionals make balancer actions more likely to be approved (will be in 2.7) Co-authored-by: Ray Mattingly Signed-off-by: Nick Dimiduk --- .../master/balancer/BalancerConditionals.java | 2 +- .../balancer/CandidateGeneratorTestUtil.java | 32 ++++-- ...gTableIsolationAndReplicaDistribution.java | 8 +- .../TestUnattainableBalancerCostGoal.java | 108 ++++++++++++++++++ 4 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java index 021a34bce6b9..b82c68b37da3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -146,7 +146,7 @@ int getViolationCountChange(BalancerClusterState cluster, BalanceAction action) // Reset cluster cluster.doAction(undoAction); - if (isViolatingPre && isViolatingPost) { + if (isViolatingPre == isViolatingPost) { return 0; } else if (!isViolatingPre && isViolatingPost) { return 1; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java index 03bfcce8e150..d2a9d17cdba0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/CandidateGeneratorTestUtil.java @@ -51,16 +51,22 @@ public final class CandidateGeneratorTestUtil { private CandidateGeneratorTestUtil() { } + enum ExhaustionType { + COST_GOAL_ACHIEVED, + NO_MORE_MOVES; + } + static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost) { - runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000); + runBalancerToExhaustion(conf, serverToRegions, expectations, targetMaxBalancerCost, 15000, + ExhaustionType.COST_GOAL_ACHIEVED); } static void runBalancerToExhaustion(Configuration conf, Map> serverToRegions, Set> expectations, float targetMaxBalancerCost, - long maxRunningTime) { + long maxRunningTime, ExhaustionType exhaustionType) { // Do the full plan. We're testing with a lot of regions conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); conf.setLong(MAX_RUNNING_TIME_KEY, maxRunningTime); @@ -76,7 +82,7 @@ static void runBalancerToExhaustion(Configuration conf, boolean isBalanced = false; while (!isBalanced) { balancerRuns++; - if (balancerRuns > 1000) { + if (balancerRuns > 10) { throw new RuntimeException("Balancer failed to find balance & meet expectations"); } long start = System.currentTimeMillis(); @@ -111,16 +117,24 @@ static void runBalancerToExhaustion(Configuration conf, } } if (isBalanced) { // Check if the balancer thinks we're done too - LOG.info("All balancer conditions passed. Checking if balancer thinks it's done."); - if (stochasticLoadBalancer.needsBalance(HConstants.ENSEMBLE_TABLE_NAME, cluster)) { - LOG.info("Balancer would still like to run"); - isBalanced = false; + if (exhaustionType == ExhaustionType.COST_GOAL_ACHIEVED) { + // If we expect to achieve the cost goal, then needsBalance should be false + if (stochasticLoadBalancer.needsBalance(HConstants.ENSEMBLE_TABLE_NAME, cluster)) { + LOG.info("Balancer cost goal is not achieved. needsBalance=true"); + isBalanced = false; + } } else { - LOG.info("Balancer is done"); + // If we anticipate running out of moves, then our last balance run should have produced + // nothing + if (regionPlans != null && !regionPlans.isEmpty()) { + LOG.info("Balancer is not out of moves. regionPlans.size()={}", regionPlans.size()); + isBalanced = false; + } } } } - LOG.info("Balancing took {}sec", Duration.ofMillis(balancingMillis).toMinutes()); + LOG.info("Balancer is done. Balancing took {}sec", + Duration.ofMillis(balancingMillis).toMinutes()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java index 3a28ae801e4e..bc31530f4921 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestLargeClusterBalancingTableIsolationAndReplicaDistribution.java @@ -104,10 +104,10 @@ public void testTableIsolationAndReplicaDistribution() { conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); DistributeReplicasTestConditional.enableConditionalReplicaDistributionForTest(conf); - - runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isMetaTableIsolated, - this::isSystemTableIsolated, CandidateGeneratorTestUtil::areAllReplicasDistributed), 10.0f, - 60_000); + runBalancerToExhaustion(conf, serverToRegions, + ImmutableSet.of(this::isMetaTableIsolated, this::isSystemTableIsolated, + CandidateGeneratorTestUtil::areAllReplicasDistributed), + 10.0f, 60_000, CandidateGeneratorTestUtil.ExhaustionType.COST_GOAL_ACHIEVED); LOG.info("Meta table regions are successfully isolated, " + "and region replicas are appropriately distributed."); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java new file mode 100644 index 000000000000..ffa2b4a78212 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.isTableIsolated; +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.runBalancerToExhaustion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * If your minCostNeedsBalance is set too low, then the balancer should still eventually stop making + * moves as further cost improvements become impossible, and balancer plan calculation becomes + * wasteful. This test ensures that the balancer will not get stuck in a loop of continuously moving + * regions. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestUnattainableBalancerCostGoal { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestUnattainableBalancerCostGoal.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestUnattainableBalancerCostGoal.class); + + private static final TableName SYSTEM_TABLE_NAME = TableName.valueOf("hbase:system"); + private static final TableName NON_SYSTEM_TABLE_NAME = TableName.valueOf("userTable"); + + private static final int NUM_SERVERS = 10; + private static final int NUM_REGIONS = 1000; + private static final float UNACHIEVABLE_COST_GOAL = 0.01f; + + private static final ServerName[] servers = new ServerName[NUM_SERVERS]; + private static final Map> serverToRegions = new HashMap<>(); + + @BeforeClass + public static void setup() { + // Initialize servers + for (int i = 0; i < NUM_SERVERS; i++) { + servers[i] = ServerName.valueOf("server" + i, i, System.currentTimeMillis()); + } + + // Create regions + List allRegions = new ArrayList<>(); + for (int i = 0; i < NUM_REGIONS; i++) { + TableName tableName = i < 3 ? SYSTEM_TABLE_NAME : NON_SYSTEM_TABLE_NAME; + byte[] startKey = new byte[1]; + startKey[0] = (byte) i; + byte[] endKey = new byte[1]; + endKey[0] = (byte) (i + 1); + + RegionInfo regionInfo = + RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey).build(); + allRegions.add(regionInfo); + } + + // Assign all regions to the first server + serverToRegions.put(servers[0], new ArrayList<>(allRegions)); + for (int i = 1; i < NUM_SERVERS; i++) { + serverToRegions.put(servers[i], new ArrayList<>()); + } + } + + @Test + public void testSystemTableIsolation() { + Configuration conf = new Configuration(false); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isSystemTableIsolated), + UNACHIEVABLE_COST_GOAL, 10_000, CandidateGeneratorTestUtil.ExhaustionType.NO_MORE_MOVES); + LOG.info("Meta table regions are successfully isolated."); + } + + private boolean isSystemTableIsolated(BalancerClusterState cluster) { + return isTableIsolated(cluster, SYSTEM_TABLE_NAME, "System"); + } +} From ddb32cad7c62d6726194cf68b08929c212e40e03 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 21 Mar 2025 08:10:01 -0400 Subject: [PATCH 13/78] HubSpot Backport: HBASE-29203 There should be a StorefileSize equivalent to the TableSkewCost (will be in 2.7) Co-authored-by: Ray Mattingly Signed-off-by: Nick Dimiduk --- .../master/balancer/BalancerClusterState.java | 4 + .../balancer/CostFromRegionLoadFunction.java | 2 +- .../balancer/StochasticLoadBalancer.java | 1 + .../StoreFileTableSkewCostFunction.java | 127 ++++++++++ .../balancer/TestStochasticLoadBalancer.java | 1 + .../TestStoreFileTableSkewCostFunction.java | 239 ++++++++++++++++++ 6 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index b07287c1ed19..efba0aee733b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -1085,6 +1085,10 @@ boolean isStopRequested() { return EnvironmentEdgeManager.currentTime() > stopRequestedAt; } + Deque[] getRegionLoads() { + return regionLoads; + } + @Override public String toString() { StringBuilder desc = new StringBuilder("Cluster={servers=["); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java index 199aa10a75fa..bc61ead8da86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/CostFromRegionLoadFunction.java @@ -66,7 +66,7 @@ protected void regionMoved(int region, int oldServer, int newServer) { } @Override - protected final double cost() { + protected double cost() { return cost.cost(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d184cf52e80f..44e5aad3a6b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -274,6 +274,7 @@ protected List createCostFunctions(Configuration conf) { addCostFunction(costFunctions, localityCost); addCostFunction(costFunctions, rackLocalityCost); addCostFunction(costFunctions, new TableSkewCostFunction(conf)); + addCostFunction(costFunctions, new StoreFileTableSkewCostFunction(conf)); addCostFunction(costFunctions, regionReplicaHostCostFunction); addCostFunction(costFunctions, regionReplicaRackCostFunction); addCostFunction(costFunctions, new ReadRequestCostFunction(conf)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java new file mode 100644 index 000000000000..d37f8caa72e1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StoreFileTableSkewCostFunction.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Lightweight cost function that mirrors TableSkewCostFunction but aggregates storefile sizes (in + * MB) per table using the CostFromRegionLoadFunction framework. For each table, it computes a + * per-server aggregated storefile size by summing the average storefile size for each region (if + * there are multiple load metrics, it averages them). The imbalance cost (as computed by + * DoubleArrayCost) is then used to drive the balancer to reduce differences between servers. + */ +@InterfaceAudience.Private +public class StoreFileTableSkewCostFunction extends CostFromRegionLoadFunction { + + private static final String STOREFILE_TABLE_SKEW_COST_KEY = + "hbase.master.balancer.stochastic.storefileTableSkewCost"; + private static final float DEFAULT_STOREFILE_TABLE_SKEW_COST = 35; + + // One DoubleArrayCost instance per table. + private DoubleArrayCost[] costsPerTable; + + public StoreFileTableSkewCostFunction(Configuration conf) { + this.setMultiplier( + conf.getFloat(STOREFILE_TABLE_SKEW_COST_KEY, DEFAULT_STOREFILE_TABLE_SKEW_COST)); + } + + @Override + public void prepare(BalancerClusterState cluster) { + // First, set the cluster state and allocate one DoubleArrayCost per table. + this.cluster = cluster; + costsPerTable = new DoubleArrayCost[cluster.numTables]; + for (int tableIdx = 0; tableIdx < cluster.numTables; tableIdx++) { + costsPerTable[tableIdx] = new DoubleArrayCost(); + costsPerTable[tableIdx].prepare(cluster.numServers); + final int tableIndex = tableIdx; + costsPerTable[tableIdx].applyCostsChange(costs -> { + // For each server, compute the aggregated storefile size for this table. + for (int server = 0; server < cluster.numServers; server++) { + double totalStorefileMB = 0; + // Sum over all regions on this server that belong to the given table. + for (int region : cluster.regionsPerServer[server]) { + if (cluster.regionIndexToTableIndex[region] == tableIndex) { + Collection loads = cluster.getRegionLoads()[region]; + double regionCost = 0; + if (loads != null && !loads.isEmpty()) { + // Average the storefile sizes if there are multiple measurements. + for (BalancerRegionLoad rl : loads) { + regionCost += getCostFromRl(rl); + } + regionCost /= loads.size(); + } + totalStorefileMB += regionCost; + } + } + costs[server] = totalStorefileMB; + } + }); + } + } + + @Override + protected void regionMoved(int region, int oldServer, int newServer) { + // Determine the affected table. + int tableIdx = cluster.regionIndexToTableIndex[region]; + costsPerTable[tableIdx].applyCostsChange(costs -> { + // Recompute for the old server if applicable. + updateStoreFilePerServerPerTableCosts(oldServer, tableIdx, costs); + // Recompute for the new server. + updateStoreFilePerServerPerTableCosts(newServer, tableIdx, costs); + }); + } + + private void updateStoreFilePerServerPerTableCosts(int newServer, int tableIdx, double[] costs) { + if (newServer >= 0) { + double totalStorefileMB = 0; + for (int r : cluster.regionsPerServer[newServer]) { + if (cluster.regionIndexToTableIndex[r] == tableIdx) { + Collection loads = cluster.getRegionLoads()[r]; + double regionCost = 0; + if (loads != null && !loads.isEmpty()) { + for (BalancerRegionLoad rl : loads) { + regionCost += getCostFromRl(rl); + } + regionCost /= loads.size(); + } + totalStorefileMB += regionCost; + } + } + costs[newServer] = totalStorefileMB; + } + } + + @Override + protected double cost() { + double totalCost = 0; + // Sum the imbalance cost over all tables. + for (DoubleArrayCost dac : costsPerTable) { + totalCost += dac.cost(); + } + return totalCost; + } + + @Override + protected double getCostFromRl(BalancerRegionLoad rl) { + // Use storefile size in MB as the metric. + return rl.getStorefileSizeMB(); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index 9dc7dab65621..661380814ad9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -531,6 +531,7 @@ public void testDefaultCostFunctionList() { PrimaryRegionCountSkewCostFunction.class.getSimpleName(), MoveCostFunction.class.getSimpleName(), RackLocalityCostFunction.class.getSimpleName(), TableSkewCostFunction.class.getSimpleName(), + StoreFileTableSkewCostFunction.class.getSimpleName(), RegionReplicaHostCostFunction.class.getSimpleName(), RegionReplicaRackCostFunction.class.getSimpleName(), ReadRequestCostFunction.class.getSimpleName(), WriteRequestCostFunction.class.getSimpleName(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java new file mode 100644 index 000000000000..619a055c6502 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.createMockBalancerClusterState; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Size; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +@Category({ MasterTests.class, SmallTests.class }) +public class TestStoreFileTableSkewCostFunction { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestStoreFileTableSkewCostFunction.class); + + private static final TableName DEFAULT_TABLE = TableName.valueOf("testTable"); + private static final Map REGION_TO_STORE_FILE_SIZE_MB = new HashMap<>(); + + /** + * Tests that a uniform store file distribution (single table) across servers results in zero + * cost. + */ + @Test + public void testUniformDistribution() { + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L); + ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L); + + Map> serverToRegions = new HashMap<>(); + serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect zero cost since all regions (from the same table) are balanced. + assertEquals("Uniform distribution should yield zero cost", 0.0, cost, 1e-6); + } + + /** + * Tests that a skewed store file distribution (single table) results in a positive cost. + */ + @Test + public void testSkewedDistribution() { + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L); + ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L); + + Map> serverToRegions = new HashMap<>(); + // Three servers get regions with 10 store files each, + // while one server gets regions with 30 store files each. + serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10))); + serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(30), createMockRegionInfo(30))); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect a positive cost because the distribution is skewed. + assertTrue("Skewed distribution should yield a positive cost", cost > 0.0); + } + + /** + * Tests that an empty cluster (no servers/regions) is handled gracefully. + */ + @Test + public void testEmptyDistribution() { + Map> serverToRegions = new HashMap<>(); + + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect zero cost when there is no load. + assertEquals("Empty distribution should yield zero cost", 0.0, cost, 1e-6); + } + + /** + * Tests that having multiple tables results in a positive cost when each table's regions are not + * balanced across servers – even if the overall load per server is balanced. + */ + @Test + public void testMultipleTablesDistribution() { + // Two servers. + ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L); + ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L); + + // Define two tables. + TableName table1 = TableName.valueOf("testTable1"); + TableName table2 = TableName.valueOf("testTable2"); + + // For table1, all regions are on server1. + // For table2, all regions are on server2. + Map> serverToRegions = new HashMap<>(); + serverToRegions.put(server1, + Arrays.asList(createMockRegionInfo(table1, 10), createMockRegionInfo(table1, 10))); + serverToRegions.put(server2, + Arrays.asList(createMockRegionInfo(table2, 10), createMockRegionInfo(table2, 10))); + + // Although each server gets 20 MB overall, table1 and table2 are not balanced across servers. + BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions); + DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState); + + StoreFileTableSkewCostFunction costFunction = + new StoreFileTableSkewCostFunction(new Configuration()); + costFunction.prepare(state); + double cost = costFunction.cost(); + + // Expect a positive cost because the skew is computed per table. + assertTrue("Multiple table distribution should yield a positive cost", cost > 0.0); + } + + /** + * Helper method to create a RegionInfo for the default table with the given store file size. + */ + private static RegionInfo createMockRegionInfo(int storeFileSizeMb) { + return createMockRegionInfo(DEFAULT_TABLE, storeFileSizeMb); + } + + /** + * Helper method to create a RegionInfo for a specified table with the given store file size. + */ + private static RegionInfo createMockRegionInfo(TableName table, int storeFileSizeMb) { + long regionId = new Random().nextLong(); + REGION_TO_STORE_FILE_SIZE_MB.put(regionId, storeFileSizeMb); + return RegionInfoBuilder.newBuilder(table).setStartKey(generateRandomByteArray(4)) + .setEndKey(generateRandomByteArray(4)).setReplicaId(0).setRegionId(regionId).build(); + } + + private static byte[] generateRandomByteArray(int n) { + byte[] byteArray = new byte[n]; + new Random().nextBytes(byteArray); + return byteArray; + } + + /** + * A simplified BalancerClusterState which ensures we provide the intended test RegionMetrics data + * when balancing this cluster + */ + private static class DummyBalancerClusterState extends BalancerClusterState { + private final RegionInfo[] testRegions; + + DummyBalancerClusterState(BalancerClusterState bcs) { + super(bcs.clusterState, null, null, null, null); + this.testRegions = bcs.regions; + } + + @Override + Deque[] getRegionLoads() { + @SuppressWarnings("unchecked") + Deque[] loads = new Deque[testRegions.length]; + for (int i = 0; i < testRegions.length; i++) { + Deque dq = new ArrayDeque<>(); + dq.add(new BalancerRegionLoad(createMockRegionMetrics(testRegions[i])) { + }); + loads[i] = dq; + } + return loads; + } + } + + /** + * Creates a mocked RegionMetrics for the given region. + */ + private static RegionMetrics createMockRegionMetrics(RegionInfo regionInfo) { + RegionMetrics regionMetrics = Mockito.mock(RegionMetrics.class); + + // Important + int storeFileSizeMb = REGION_TO_STORE_FILE_SIZE_MB.get(regionInfo.getRegionId()); + when(regionMetrics.getRegionSizeMB()).thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE)); + when(regionMetrics.getStoreFileSize()) + .thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE)); + + // Not important + when(regionMetrics.getReadRequestCount()).thenReturn(0L); + when(regionMetrics.getCpRequestCount()).thenReturn(0L); + when(regionMetrics.getWriteRequestCount()).thenReturn(0L); + when(regionMetrics.getMemStoreSize()).thenReturn(new Size(0, Size.Unit.MEGABYTE)); + when(regionMetrics.getCurrentRegionCachedRatio()).thenReturn(0.0f); + return regionMetrics; + } +} From 9743688ed6f9114e6a7ff71de0691e9f1e743c69 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 21 Mar 2025 08:54:04 -0400 Subject: [PATCH 14/78] HubSpot Edit: I messed up 29202, 29203 backports with incompatibilities. Can squash this, or delete in 2.7 (#167) Co-authored-by: Ray Mattingly --- .../master/balancer/TestStoreFileTableSkewCostFunction.java | 1 - .../master/balancer/TestUnattainableBalancerCostGoal.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java index 619a055c6502..3977ad96dd9a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStoreFileTableSkewCostFunction.java @@ -230,7 +230,6 @@ private static RegionMetrics createMockRegionMetrics(RegionInfo regionInfo) { // Not important when(regionMetrics.getReadRequestCount()).thenReturn(0L); - when(regionMetrics.getCpRequestCount()).thenReturn(0L); when(regionMetrics.getWriteRequestCount()).thenReturn(0L); when(regionMetrics.getMemStoreSize()).thenReturn(new Size(0, Size.Unit.MEGABYTE)); when(regionMetrics.getCurrentRegionCachedRatio()).thenReturn(0.0f); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java index ffa2b4a78212..5e95564b6fee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -97,7 +98,7 @@ public static void setup() { public void testSystemTableIsolation() { Configuration conf = new Configuration(false); conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); - runBalancerToExhaustion(conf, serverToRegions, Set.of(this::isSystemTableIsolated), + runBalancerToExhaustion(conf, serverToRegions, ImmutableSet.of(this::isSystemTableIsolated), UNACHIEVABLE_COST_GOAL, 10_000, CandidateGeneratorTestUtil.ExhaustionType.NO_MORE_MOVES); LOG.info("Meta table regions are successfully isolated."); } From fd9346a85cf6e029ae25e86288c4b497c5016a98 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Tue, 15 Apr 2025 10:11:01 -0400 Subject: [PATCH 15/78] HubSpot Backport: HBASE-29262 StochasticLoadBalancer should use the CostFunction epsilon when evaluating whether a move improved costs (will be in 2.7) Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../hbase/master/balancer/StochasticLoadBalancer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 44e5aad3a6b8..689c65fd6ca4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -730,8 +730,12 @@ protected List balanceTable(TableName tableName, newCost = computeCost(cluster, currentCost); + double costImprovement = currentCost - newCost; + double minimumImprovement = + Math.max(CostFunction.getCostEpsilon(currentCost), CostFunction.getCostEpsilon(newCost)); + boolean costsImproved = costImprovement > minimumImprovement; boolean conditionalsSimilarCostsImproved = - (newCost < currentCost && conditionalViolationsChange == 0 && !isViolatingConditionals); + (costsImproved && conditionalViolationsChange == 0 && !isViolatingConditionals); // Our first priority is to reduce conditional violations // Our second priority is to reduce balancer cost // change, regardless of cost change From f4016f377d715da2b14f6f431a5bb21317bf7ccd Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 9 May 2025 10:49:07 -0400 Subject: [PATCH 16/78] HubSpot Edit: add CLAUDE.md --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..a168d77d5600 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +When in the hubspot-2.6 branch, or sub-branches of that branch, use mvn11 instead of mvn, and use the hadoop-3.0 maven profile + +When in the master branch, or sub-branches of that branch, use mvn17 instead of mvn, and do not use the hadoop-3.0 maven profile. + +Always run the spotless:apply maven target after making changes. From eb12b65f1bd7fea78575ab7de277a325b9f9f055 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Wed, 4 Jun 2025 12:04:18 -0400 Subject: [PATCH 17/78] HubSpot Backport: HBASE-29372: Meta cache clear metrics and logs shouldn't use "UnknownException" (not yet written upstream) Signed-off-by: Duo Zhang Signed-off-by: Ray Mattingly Co-authored-by: Hernan Gelaf-Romer --- .../hbase/client/AsyncRequestFutureImpl.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index e52ae6cfac21..eb041a00f501 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -223,13 +223,14 @@ public void run() { } catch (IOException e) { // The service itself failed . It may be an error coming from the communication // layer, but, as well, a functional error raised by the server. - receiveGlobalFailure(multiAction, server, numAttempt, e, true); + + receiveGlobalFailure(multiAction, server, numAttempt, e); return; } catch (Throwable t) { // This should not happen. Let's log & retry anyway. LOG.error("id=" + asyncProcess.id + ", caught throwable. Unexpected." + " Retrying. Server=" + server + ", tableName=" + tableName, t); - receiveGlobalFailure(multiAction, server, numAttempt, t, true); + receiveGlobalFailure(multiAction, server, numAttempt, t); return; } if (res.type() == AbstractResponse.ResponseType.MULTI) { @@ -606,7 +607,6 @@ private void failIncompleteActionsWithOpTimeout(List actions, */ void sendMultiAction(Map actionsByServer, int numAttempt, List actionsForReplicaThread, boolean reuseThread) { - boolean clearServerCache = true; // Run the last item on the same thread if we are already on a send thread. // We hope most of the time it will be the only item, so we can cut down on threads. int actionsRemaining = actionsByServer.size(); @@ -642,7 +642,6 @@ void sendMultiAction(Map actionsByServer, int numAttemp LOG.warn("id=" + asyncProcess.id + ", task rejected by pool. Unexpected." + " Server=" + server.getServerName(), t); // Do not update cache if exception is from failing to submit action to thread pool - clearServerCache = false; } else { // see #HBASE-14359 for more details LOG.warn("Caught unexpected exception/error: ", t); @@ -650,7 +649,7 @@ void sendMultiAction(Map actionsByServer, int numAttemp asyncProcess.decTaskCounters(multiAction.getRegions(), server); // We're likely to fail again, but this will increment the attempt counter, // so it will finish. - receiveGlobalFailure(multiAction, server, numAttempt, t, clearServerCache); + receiveGlobalFailure(multiAction, server, numAttempt, t); } } } @@ -800,13 +799,24 @@ private void failAll(MultiAction actions, ServerName server, int numAttempt, * @param t the throwable (if any) that caused the resubmit */ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int numAttempt, - Throwable t, boolean clearServerCache) { + Throwable t) { errorsByServer.reportServerError(server); Retry canRetry = errorsByServer.canTryMore(numAttempt) ? Retry.YES : Retry.NO_RETRIES_EXHAUSTED; + boolean clearServerCache; + + if (t instanceof RejectedExecutionException) { + clearServerCache = false; + } else { + clearServerCache = ClientExceptionsUtil.isMetaClearingException(t); + } // Do not update cache if exception is from failing to submit action to thread pool if (clearServerCache) { cleanServerCache(server, t); + + if (LOG.isTraceEnabled()) { + LOG.trace("Cleared meta cache for server {} due to global failure {}", server, t); + } } int failed = 0; @@ -815,12 +825,8 @@ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int for (Map.Entry> e : rsActions.actions.entrySet()) { byte[] regionName = e.getKey(); byte[] row = e.getValue().get(0).getAction().getRow(); - // Do not use the exception for updating cache because it might be coming from - // any of the regions in the MultiAction and do not update cache if exception is - // from failing to submit action to thread pool if (clearServerCache) { - updateCachedLocations(server, regionName, row, - ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + updateCachedLocations(server, regionName, row, t); } for (Action action : e.getValue()) { Retry retry = From 829524745ee33a7430cb72e115bab5d2bf2b48c0 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Fri, 13 Jun 2025 11:10:07 -0400 Subject: [PATCH 18/78] HubSpot Backport: HBASE-29391: QueryMetrics are missing for HTable CheckAndMutate methods (not yet written upstream) Signed-off-by: Duo Zhang Signed-off-by: Nihal Jain Co-authored-by: Hernan Gelaf-Romer --- .../apache/hadoop/hbase/client/HTable.java | 13 +- .../shaded/protobuf/ResponseConverter.java | 8 +- .../hbase/client/TestHTableQueryMetrics.java | 268 ++++++++++++++++++ 3 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHTableQueryMetrics.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 8767257bf5b7..bb916a32c310 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -935,11 +935,20 @@ protected CheckAndMutateResult rpcCall() throws Exception { getLocation().getRegionInfo().getRegionName(), row, family, qualifier, op, value, filter, timeRange, mutation, nonceGroup, nonce, queryMetricsEnabled); MutateResponse response = doMutate(request); + CheckAndMutateResult result; if (response.hasResult()) { - return new CheckAndMutateResult(response.getProcessed(), + result = new CheckAndMutateResult(response.getProcessed(), ProtobufUtil.toResult(response.getResult(), getRpcControllerCellScanner())); + } else { + result = new CheckAndMutateResult(response.getProcessed(), null); } - return new CheckAndMutateResult(response.getProcessed(), null); + + if (response.hasMetrics()) { + QueryMetrics metrics = ProtobufUtil.toQueryMetrics(response.getMetrics()); + result.setMetrics(metrics); + } + + return result; } }; return rpcCallerFactory. newCaller(this.writeRpcTimeoutMs) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java index bd1f4be6f0e9..040317c238e0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ResponseConverter.java @@ -166,8 +166,12 @@ public static org.apache.hadoop.hbase.client.MultiResponse getResults(final Mult if (!r.isEmpty()) { result = r; } - results.add(regionName, roe.getIndex(), - new CheckAndMutateResult(actionResult.getProcessed(), result)); + CheckAndMutateResult camResult = + new CheckAndMutateResult(actionResult.getProcessed(), result); + if (roe.hasMetrics()) { + camResult.setMetrics(ProtobufUtil.toQueryMetrics(roe.getMetrics())); + } + results.add(regionName, roe.getIndex(), camResult); } } else { if (actionResult.hasProcessed()) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHTableQueryMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHTableQueryMetrics.java new file mode 100644 index 000000000000..eedaa7c5989d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHTableQueryMetrics.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServer; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSource; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; +import org.apache.hbase.thirdparty.com.google.common.io.Closeables; + +@Category({ MediumTests.class, ClientTests.class }) +public class TestHTableQueryMetrics { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHTableQueryMetrics.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static final TableName TABLE_NAME = TableName.valueOf("ResultMetrics"); + + private static final byte[] CF = Bytes.toBytes("cf"); + + private static final byte[] CQ = Bytes.toBytes("cq"); + + private static final byte[] VALUE = Bytes.toBytes("value"); + + private static final byte[] ROW_1 = Bytes.toBytes("zzz1"); + private static final byte[] ROW_2 = Bytes.toBytes("zzz2"); + private static final byte[] ROW_3 = Bytes.toBytes("zzz3"); + + private static Connection CONN; + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(3); + // Create 3 rows in the table, with rowkeys starting with "zzz*" so that + // scan are forced to hit all the regions. + try (Table table = UTIL.createMultiRegionTable(TABLE_NAME, CF)) { + table.put(Arrays.asList(new Put(ROW_1).addColumn(CF, CQ, VALUE), + new Put(ROW_2).addColumn(CF, CQ, VALUE), new Put(ROW_3).addColumn(CF, CQ, VALUE))); + } + CONN = ConnectionFactory.createConnection(UTIL.getConfiguration()); + CONN.getAdmin().flush(TABLE_NAME); + } + + @AfterClass + public static void tearDown() throws Exception { + Closeables.close(CONN, true); + UTIL.shutdownMiniCluster(); + } + + @Test + public void itTestsGets() throws Exception { + // Test a single Get + Get g1 = new Get(ROW_1); + g1.setQueryMetricsEnabled(true); + + long bbs = getClusterBlockBytesScanned(); + Result result = CONN.getTable(TABLE_NAME).get(g1); + bbs += result.getMetrics().getBlockBytesScanned(); + Assert.assertNotNull(result.getMetrics()); + Assert.assertEquals(getClusterBlockBytesScanned(), bbs); + + // Test multigets + Get g2 = new Get(ROW_2); + g2.setQueryMetricsEnabled(true); + + Get g3 = new Get(ROW_3); + g3.setQueryMetricsEnabled(true); + + Result[] results = CONN.getTable(TABLE_NAME).get(ImmutableList.of(g1, g2, g3)); + + for (Result r : results) { + Assert.assertNotNull(r.getMetrics()); + bbs += r.getMetrics().getBlockBytesScanned(); + } + + Assert.assertEquals(getClusterBlockBytesScanned(), bbs); + } + + @Test + public void itTestsDefaultGetNoMetrics() throws Exception { + // Test a single Get + Get g1 = new Get(ROW_1); + + Result result = CONN.getTable(TABLE_NAME).get(g1); + Assert.assertNull(result.getMetrics()); + + // Test multigets + Get g2 = new Get(ROW_2); + Get g3 = new Get(ROW_3); + Result[] results = CONN.getTable(TABLE_NAME).get(ImmutableList.of(g1, g2, g3)); + for (Result r : results) { + Assert.assertNull(r.getMetrics()); + } + + } + + @Test + public void itTestsScans() throws IOException { + Scan scan = new Scan(); + scan.setQueryMetricsEnabled(true); + + long bbs = getClusterBlockBytesScanned(); + try (ResultScanner scanner = CONN.getTable(TABLE_NAME).getScanner(scan)) { + for (Result result : scanner) { + Assert.assertNotNull(result.getMetrics()); + bbs += result.getMetrics().getBlockBytesScanned(); + Assert.assertEquals(getClusterBlockBytesScanned(), bbs); + } + } + } + + @Test + public void itTestsDefaultScanNoMetrics() throws IOException { + Scan scan = new Scan(); + + try (ResultScanner scanner = CONN.getTable(TABLE_NAME).getScanner(scan)) { + for (Result result : scanner) { + Assert.assertNull(result.getMetrics()); + } + } + } + + @Test + public void itTestsAtomicOperations() throws Exception { + CheckAndMutate cam = CheckAndMutate.newBuilder(ROW_1).ifEquals(CF, CQ, VALUE) + .queryMetricsEnabled(true).build(new Put(ROW_1).addColumn(CF, CQ, VALUE)); + + long bbs = getClusterBlockBytesScanned(); + CheckAndMutateResult result = CONN.getTable(TABLE_NAME).checkAndMutate(cam); + QueryMetrics metrics = result.getMetrics(); + + Assert.assertNotNull(metrics); + Assert.assertEquals(getClusterBlockBytesScanned(), bbs + metrics.getBlockBytesScanned()); + + cam = CheckAndMutate.newBuilder(ROW_1).ifEquals(CF, CQ, VALUE).queryMetricsEnabled(true) + .build(new RowMutations(ROW_1).add((Mutation) new Put(ROW_1).addColumn(CF, CQ, VALUE))); + + bbs = getClusterBlockBytesScanned(); + result = CONN.getTable(TABLE_NAME).checkAndMutate(cam); + metrics = result.getMetrics(); + + Assert.assertNotNull(metrics); + Assert.assertEquals(getClusterBlockBytesScanned(), bbs + metrics.getBlockBytesScanned()); + + bbs = getClusterBlockBytesScanned(); + List batch = new ArrayList<>(); + batch.add(cam); + batch.add(CheckAndMutate.newBuilder(ROW_2).queryMetricsEnabled(true).ifEquals(CF, CQ, VALUE) + .build(new Put(ROW_2).addColumn(CF, CQ, VALUE))); + batch.add(CheckAndMutate.newBuilder(ROW_3).queryMetricsEnabled(true).ifEquals(CF, CQ, VALUE) + .build(new Put(ROW_3).addColumn(CF, CQ, VALUE))); + + Object[] results = new Object[batch.size()]; + CONN.getTable(TABLE_NAME).batch(batch, results); + long totalBbs = 0; + for (Object r : results) { + CheckAndMutateResult camResult = (CheckAndMutateResult) r; + Assert.assertNotNull(camResult.getMetrics()); + totalBbs += camResult.getMetrics().getBlockBytesScanned(); + } + Assert.assertEquals(getClusterBlockBytesScanned(), bbs + totalBbs); + + bbs = getClusterBlockBytesScanned(); + + // flush to force fetch from disk + CONN.getAdmin().flush(TABLE_NAME); + results = new Object[batch.size()]; + CONN.getTable(TABLE_NAME).batch(batch, results); + + totalBbs = 0; + for (Object r : results) { + CheckAndMutateResult camResult = (CheckAndMutateResult) r; + Assert.assertNotNull(camResult.getMetrics()); + totalBbs += camResult.getMetrics().getBlockBytesScanned(); + } + Assert.assertEquals(getClusterBlockBytesScanned(), bbs + totalBbs); + } + + @Test + public void itTestsDefaultAtomicOperations() throws Exception { + CheckAndMutate cam = CheckAndMutate.newBuilder(ROW_1).ifEquals(CF, CQ, VALUE) + .build(new Put(ROW_1).addColumn(CF, CQ, VALUE)); + + CheckAndMutateResult result = CONN.getTable(TABLE_NAME).checkAndMutate(cam); + QueryMetrics metrics = result.getMetrics(); + + Assert.assertNull(metrics); + + cam = CheckAndMutate.newBuilder(ROW_1).ifEquals(CF, CQ, VALUE) + .build(new RowMutations(ROW_1).add((Mutation) new Put(ROW_1).addColumn(CF, CQ, VALUE))); + + result = CONN.getTable(TABLE_NAME).checkAndMutate(cam); + metrics = result.getMetrics(); + Assert.assertNull(metrics); + + List batch = new ArrayList<>(); + batch.add(cam); + batch.add(CheckAndMutate.newBuilder(ROW_2).ifEquals(CF, CQ, VALUE) + .build(new Put(ROW_2).addColumn(CF, CQ, VALUE))); + batch.add(CheckAndMutate.newBuilder(ROW_3).ifEquals(CF, CQ, VALUE) + .build(new Put(ROW_3).addColumn(CF, CQ, VALUE))); + + Object[] results = new Object[batch.size()]; + CONN.getTable(TABLE_NAME).batch(batch, results); + for (Object r : results) { + Assert.assertNull(((CheckAndMutateResult) r).getMetrics()); + } + + // flush to force fetch from disk + CONN.getAdmin().flush(TABLE_NAME); + results = new Object[batch.size()]; + CONN.getTable(TABLE_NAME).batch(batch, results); + + for (Object r : results) { + Assert.assertNull(((CheckAndMutateResult) r).getMetrics()); + } + } + + private static long getClusterBlockBytesScanned() { + long bbs = 0L; + + for (JVMClusterUtil.RegionServerThread rs : UTIL.getHBaseCluster().getRegionServerThreads()) { + MetricsRegionServer metrics = rs.getRegionServer().getMetrics(); + MetricsRegionServerSourceImpl source = + (MetricsRegionServerSourceImpl) metrics.getMetricsSource(); + + bbs += source.getMetricsRegistry() + .getCounter(MetricsRegionServerSource.BLOCK_BYTES_SCANNED_KEY, 0L).value(); + } + + return bbs; + } +} From f222f7775b60e248aa152423242bf944de1925ad Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Mon, 30 Jun 2025 09:59:03 -0400 Subject: [PATCH 19/78] HubSpot Backport: HBASE-29351 Quotas: adaptive wait intervals (drop in 2.7) Co-authored-by: Ray Mattingly --- .../quotas/FeedbackAdaptiveRateLimiter.java | 285 ++++++++++++++++++ .../hadoop/hbase/quotas/TimeBasedLimiter.java | 25 +- .../TestFeedbackAdaptiveRateLimiter.java | 257 ++++++++++++++++ 3 files changed, 562 insertions(+), 5 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FeedbackAdaptiveRateLimiter.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFeedbackAdaptiveRateLimiter.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FeedbackAdaptiveRateLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FeedbackAdaptiveRateLimiter.java new file mode 100644 index 000000000000..2ff9356003b7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FeedbackAdaptiveRateLimiter.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; + +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicDouble; + +/** + * This rate limiter works much like the FixedIntervalRateLimiter, except that: + *
    + *
  1. it will increase backpressure on multithreaded clients
    2. + *
  2. it will allow over-subscription to hit consistent, full allowance utilization
    3. + *
+ */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class FeedbackAdaptiveRateLimiter extends RateLimiter { + + /** + * Amount to increase the backoff multiplier when contention is detected per refill interval. + */ + public static final String FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_INCREMENT = + "hbase.quota.rate.limiter.feedback.adaptive.backoff.multiplier.increment"; + public static final double DEFAULT_BACKOFF_MULTIPLIER_INCREMENT = 0.0001; + + /** + * Amount to decrease the backoff multiplier when no contention is detected per refill interval. + */ + public static final String FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_DECREMENT = + "hbase.quota.rate.limiter.feedback.adaptive.backoff.multiplier.decrement"; + public static final double DEFAULT_BACKOFF_MULTIPLIER_DECREMENT = 0.0001; + + /** + * Maximum ceiling for the backoff multiplier to avoid unbounded waits. + */ + public static final String FEEDBACK_ADAPTIVE_MAX_BACKOFF_MULTIPLIER = + "hbase.quota.rate.limiter.feedback.adaptive.max.backoff.multiplier"; + public static final double DEFAULT_MAX_BACKOFF_MULTIPLIER = 10.0; + + /** + * Amount to increase the oversubscription proportion when utilization is below target. + */ + public static final String FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_INCREMENT = + "hbase.quota.rate.limiter.feedback.adaptive.oversubscription.increment"; + public static final double DEFAULT_OVERSUBSCRIPTION_INCREMENT = 0.00001; + + /** + * Amount to decrease the oversubscription proportion when utilization exceeds target. + */ + public static final String FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_DECREMENT = + "hbase.quota.rate.limiter.feedback.adaptive.oversubscription.decrement"; + public static final double DEFAULT_OVERSUBSCRIPTION_DECREMENT = 0.00001; + + /** + * Maximum ceiling for oversubscription to prevent unbounded bursts. + */ + public static final String FEEDBACK_ADAPTIVE_MAX_OVERSUBSCRIPTION = + "hbase.quota.rate.limiter.feedback.adaptive.max.oversubscription"; + public static final double DEFAULT_MAX_OVERSUBSCRIPTION = 0.05; + + /** + * Acceptable deviation around full utilization (1.0) for adjusting oversubscription. + */ + public static final String FEEDBACK_ADAPTIVE_UTILIZATION_ERROR_BUDGET = + "hbase.quota.rate.limiter.feedback.adaptive.utilization.error.budget"; + public static final double DEFAULT_UTILIZATION_ERROR_BUDGET = 0.025; + + private static final int WINDOW_TIME_MS = 60_000; + + public static class FeedbackAdaptiveRateLimiterFactory { + + private final long refillInterval; + private final double backoffMultiplierIncrement; + private final double backoffMultiplierDecrement; + private final double maxBackoffMultiplier; + private final double oversubscriptionIncrement; + private final double oversubscriptionDecrement; + private final double maxOversubscription; + private final double utilizationErrorBudget; + + public FeedbackAdaptiveRateLimiterFactory(Configuration conf) { + refillInterval = conf.getLong(FixedIntervalRateLimiter.RATE_LIMITER_REFILL_INTERVAL_MS, + RateLimiter.DEFAULT_TIME_UNIT); + + maxBackoffMultiplier = + conf.getDouble(FEEDBACK_ADAPTIVE_MAX_BACKOFF_MULTIPLIER, DEFAULT_MAX_BACKOFF_MULTIPLIER); + + backoffMultiplierIncrement = conf.getDouble(FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_INCREMENT, + DEFAULT_BACKOFF_MULTIPLIER_INCREMENT); + backoffMultiplierDecrement = conf.getDouble(FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_DECREMENT, + DEFAULT_BACKOFF_MULTIPLIER_DECREMENT); + + oversubscriptionIncrement = conf.getDouble(FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_INCREMENT, + DEFAULT_OVERSUBSCRIPTION_INCREMENT); + oversubscriptionDecrement = conf.getDouble(FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_DECREMENT, + DEFAULT_OVERSUBSCRIPTION_DECREMENT); + + maxOversubscription = + conf.getDouble(FEEDBACK_ADAPTIVE_MAX_OVERSUBSCRIPTION, DEFAULT_MAX_OVERSUBSCRIPTION); + utilizationErrorBudget = conf.getDouble(FEEDBACK_ADAPTIVE_UTILIZATION_ERROR_BUDGET, + DEFAULT_UTILIZATION_ERROR_BUDGET); + } + + public FeedbackAdaptiveRateLimiter create() { + return new FeedbackAdaptiveRateLimiter(refillInterval, backoffMultiplierIncrement, + backoffMultiplierDecrement, maxBackoffMultiplier, oversubscriptionIncrement, + oversubscriptionDecrement, maxOversubscription, utilizationErrorBudget); + } + } + + private long nextRefillTime = -1L; + private final long refillInterval; + private final double backoffMultiplierIncrement; + private final double backoffMultiplierDecrement; + private final double maxBackoffMultiplier; + private final double oversubscriptionIncrement; + private final double oversubscriptionDecrement; + private final double maxOversubscription; + private final double minTargetUtilization; + private final double maxTargetUtilization; + + // Adaptive backoff state + private final AtomicDouble currentBackoffMultiplier = new AtomicDouble(1.0); + private volatile boolean hadContentionThisInterval = false; + + // Over-subscription proportion state + private final AtomicDouble oversubscriptionProportion = new AtomicDouble(0.0); + + // EWMA tracking + private final double emaAlpha; + private volatile double utilizationEma = 0.0; + private final AtomicLong lastIntervalConsumed; + + FeedbackAdaptiveRateLimiter(long refillInterval, double backoffMultiplierIncrement, + double backoffMultiplierDecrement, double maxBackoffMultiplier, + double oversubscriptionIncrement, double oversubscriptionDecrement, double maxOversubscription, + double utilizationErrorBudget) { + super(); + Preconditions.checkArgument(getTimeUnitInMillis() >= refillInterval, String.format( + "Refill interval %s must be ≤ TimeUnit millis %s", refillInterval, getTimeUnitInMillis())); + + Preconditions.checkArgument(backoffMultiplierIncrement > 0.0, + String.format("Backoff multiplier increment %s must be > 0.0", backoffMultiplierIncrement)); + Preconditions.checkArgument(backoffMultiplierDecrement > 0.0, + String.format("Backoff multiplier decrement %s must be > 0.0", backoffMultiplierDecrement)); + Preconditions.checkArgument(maxBackoffMultiplier > 1.0, + String.format("Max backoff multiplier %s must be > 1.0", maxBackoffMultiplier)); + Preconditions.checkArgument(utilizationErrorBudget > 0.0 && utilizationErrorBudget <= 1.0, + String.format("Utilization error budget %s must be between 0.0 and 1.0", + utilizationErrorBudget)); + + this.refillInterval = refillInterval; + this.backoffMultiplierIncrement = backoffMultiplierIncrement; + this.backoffMultiplierDecrement = backoffMultiplierDecrement; + this.maxBackoffMultiplier = maxBackoffMultiplier; + this.oversubscriptionIncrement = oversubscriptionIncrement; + this.oversubscriptionDecrement = oversubscriptionDecrement; + this.maxOversubscription = maxOversubscription; + this.minTargetUtilization = 1.0 - utilizationErrorBudget; + this.maxTargetUtilization = 1.0 + utilizationErrorBudget; + + this.emaAlpha = refillInterval / (double) (WINDOW_TIME_MS + refillInterval); + this.lastIntervalConsumed = new AtomicLong(0); + } + + @Override + public long refill(long limit) { + final long now = EnvironmentEdgeManager.currentTime(); + if (nextRefillTime == -1) { + nextRefillTime = now + refillInterval; + hadContentionThisInterval = false; + return getOversubscribedLimit(limit); + } + if (now < nextRefillTime) { + return 0; + } + long diff = refillInterval + now - nextRefillTime; + long refills = diff / refillInterval; + nextRefillTime = now + refillInterval; + + long intendedUsage = getRefillIntervalAdjustedLimit(limit); + if (intendedUsage > 0) { + long consumed = lastIntervalConsumed.get(); + if (consumed > 0) { + double util = (double) consumed / intendedUsage; + utilizationEma = emaAlpha * util + (1.0 - emaAlpha) * utilizationEma; + } + } + + if (hadContentionThisInterval) { + currentBackoffMultiplier.set(Math + .min(currentBackoffMultiplier.get() + backoffMultiplierIncrement, maxBackoffMultiplier)); + } else { + currentBackoffMultiplier + .set(Math.max(currentBackoffMultiplier.get() - backoffMultiplierDecrement, 1.0)); + } + + double avgUtil = utilizationEma; + if (avgUtil < minTargetUtilization) { + oversubscriptionProportion.set(Math + .min(oversubscriptionProportion.get() + oversubscriptionIncrement, maxOversubscription)); + } else if (avgUtil >= maxTargetUtilization) { + oversubscriptionProportion + .set(Math.max(oversubscriptionProportion.get() - oversubscriptionDecrement, 0.0)); + } + + hadContentionThisInterval = false; + lastIntervalConsumed.set(0); + + long refillAmount = refills * getRefillIntervalAdjustedLimit(limit); + long maxRefill = getOversubscribedLimit(limit); + return Math.min(maxRefill, refillAmount); + } + + private long getOversubscribedLimit(long limit) { + return limit + (long) (limit * oversubscriptionProportion.get()); + } + + @Override + public void consume(long amount) { + super.consume(amount); + lastIntervalConsumed.addAndGet(amount); + } + + @Override + public long getWaitInterval(long limit, long available, long amount) { + limit = getRefillIntervalAdjustedLimit(limit); + if (nextRefillTime == -1) return 0; + + final long now = EnvironmentEdgeManager.currentTime(); + final long refillTime = nextRefillTime; + long diff = amount - available; + if (diff > 0) hadContentionThisInterval = true; + + long nextInterval = refillTime - now; + if (diff <= limit) { + return applyBackoffMultiplier(nextInterval); + } + + long extra = diff / limit; + if (diff % limit == 0) extra--; + long baseWait = nextInterval + (extra * refillInterval); + return applyBackoffMultiplier(baseWait); + } + + private long getRefillIntervalAdjustedLimit(long limit) { + return (long) Math.ceil(refillInterval / (double) getTimeUnitInMillis() * limit); + } + + private long applyBackoffMultiplier(long baseWaitInterval) { + return (long) (baseWaitInterval * currentBackoffMultiplier.get()); + } + + // strictly for testing + @Override + public void setNextRefillTime(long nextRefillTime) { + this.nextRefillTime = nextRefillTime; + } + + @Override + public long getNextRefillTime() { + return this.nextRefillTime; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java index 232471092c29..232ceb894ef6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java @@ -48,11 +48,10 @@ public class TimeBasedLimiter implements QuotaLimiter { private RateLimiter reqHandlerUsageTimeLimiter = null; private TimeBasedLimiter() { - if ( - FixedIntervalRateLimiter.class.getName().equals( - conf.getClass(RateLimiter.QUOTA_RATE_LIMITER_CONF_KEY, AverageIntervalRateLimiter.class) - .getName()) - ) { + String limiterClassName = + conf.getClass(RateLimiter.QUOTA_RATE_LIMITER_CONF_KEY, AverageIntervalRateLimiter.class) + .getName(); + if (FixedIntervalRateLimiter.class.getName().equals(limiterClassName)) { long refillInterval = conf.getLong(FixedIntervalRateLimiter.RATE_LIMITER_REFILL_INTERVAL_MS, RateLimiter.DEFAULT_TIME_UNIT); reqsLimiter = new FixedIntervalRateLimiter(refillInterval); @@ -68,6 +67,22 @@ private TimeBasedLimiter() { atomicReadSizeLimiter = new FixedIntervalRateLimiter(refillInterval); atomicWriteSizeLimiter = new FixedIntervalRateLimiter(refillInterval); reqHandlerUsageTimeLimiter = new FixedIntervalRateLimiter(refillInterval); + } else if (FeedbackAdaptiveRateLimiter.class.getName().equals(limiterClassName)) { + FeedbackAdaptiveRateLimiter.FeedbackAdaptiveRateLimiterFactory feedbackLimiterFactory = + new FeedbackAdaptiveRateLimiter.FeedbackAdaptiveRateLimiterFactory(conf); + reqsLimiter = feedbackLimiterFactory.create(); + reqSizeLimiter = feedbackLimiterFactory.create(); + writeReqsLimiter = feedbackLimiterFactory.create(); + writeSizeLimiter = feedbackLimiterFactory.create(); + readReqsLimiter = feedbackLimiterFactory.create(); + readSizeLimiter = feedbackLimiterFactory.create(); + reqCapacityUnitLimiter = feedbackLimiterFactory.create(); + writeCapacityUnitLimiter = feedbackLimiterFactory.create(); + readCapacityUnitLimiter = feedbackLimiterFactory.create(); + atomicReqLimiter = feedbackLimiterFactory.create(); + atomicReadSizeLimiter = feedbackLimiterFactory.create(); + atomicWriteSizeLimiter = feedbackLimiterFactory.create(); + reqHandlerUsageTimeLimiter = feedbackLimiterFactory.create(); } else { reqsLimiter = new AverageIntervalRateLimiter(); reqSizeLimiter = new AverageIntervalRateLimiter(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFeedbackAdaptiveRateLimiter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFeedbackAdaptiveRateLimiter.java new file mode 100644 index 000000000000..38d97ae16077 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFeedbackAdaptiveRateLimiter.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Verify the behavior of the FeedbackAdaptiveRateLimiter including adaptive backoff multipliers and + * over-subscription functionality. + */ +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestFeedbackAdaptiveRateLimiter { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestFeedbackAdaptiveRateLimiter.class); + + private ManualEnvironmentEdge testEdge; + private FeedbackAdaptiveRateLimiter.FeedbackAdaptiveRateLimiterFactory factory; + + @Before + public void setUp() { + testEdge = new ManualEnvironmentEdge(); + EnvironmentEdgeManager.injectEdge(testEdge); + + Configuration conf = HBaseConfiguration.create(); + // Set refill interval for testing + conf.setLong(FixedIntervalRateLimiter.RATE_LIMITER_REFILL_INTERVAL_MS, 500); + // Configure adaptive parameters for testing - using larger values than defaults for + // observability + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_INCREMENT, 0.1); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_BACKOFF_MULTIPLIER_DECREMENT, + 0.05); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_MAX_BACKOFF_MULTIPLIER, 3.0); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_INCREMENT, 0.01); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_OVERSUBSCRIPTION_DECREMENT, 0.005); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_MAX_OVERSUBSCRIPTION, 0.2); + conf.setDouble(FeedbackAdaptiveRateLimiter.FEEDBACK_ADAPTIVE_UTILIZATION_ERROR_BUDGET, 0.1); + + factory = new FeedbackAdaptiveRateLimiter.FeedbackAdaptiveRateLimiterFactory(conf); + } + + @After + public void tearDown() { + EnvironmentEdgeManager.reset(); + } + + @Test + public void testBasicFunctionality() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + // Initially should work like normal rate limiter + assertEquals(0, limiter.getWaitIntervalMs()); + limiter.consume(5); + assertEquals(0, limiter.getWaitIntervalMs()); + limiter.consume(5); + + // Should need to wait after consuming full limit + assertTrue(limiter.getWaitIntervalMs() > 0); + } + + @Test + public void testAdaptiveBackoffIncreases() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + testEdge.setValue(1000); + + // Test that adaptive backoff functionality works without throwing exceptions + // and that we can create contention scenarios + for (int i = 0; i < 3; i++) { + limiter.refill(10); + limiter.consume(10); + // Create contention by asking for more than available + long waitInterval = limiter.getWaitInterval(10, 0, 1); + + // Basic sanity check - wait interval should be reasonable + assertTrue("Wait interval should be positive, got: " + waitInterval, waitInterval > 0); + + // Advance to next interval + testEdge.setValue(1000 + (i + 1) * 500); + } + + // Test passes if the adaptive backoff mechanism works without errors + assertTrue("Adaptive backoff should work without errors", true); + } + + @Test + public void testAdaptiveBackoffDecreases() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + testEdge.setValue(1000); + + // Test that the backoff decrease mechanism works without errors + // Build up some contention first + for (int i = 0; i < 3; i++) { + limiter.refill(10); + limiter.consume(10); + limiter.getWaitInterval(10, 0, 1); // Create contention + testEdge.setValue(1000 + (i + 1) * 500); + } + + // Run several intervals without contention to decrease backoff + for (int i = 0; i < 10; i++) { + testEdge.setValue(2500 + i * 500); + limiter.refill(10); + if (limiter.getAvailable() > 0) { + limiter.consume(Math.min(5, (int) limiter.getAvailable())); // Consume less than limit + } + } + + // Test passes if the backoff decrease mechanism works without errors + assertTrue("Adaptive backoff decrease should work without errors", true); + } + + @Test + public void testOversubscriptionTracking() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + testEdge.setValue(1000); + + // Initial refill to set up the limiter + long initialRefill = limiter.refill(10); + assertTrue("Initial refill should be positive", initialRefill > 0); + + // Just verify that the over-subscription tracking is working without complex assertions + // This test mainly ensures that the adaptive behavior doesn't break basic functionality + for (int i = 0; i < 5; i++) { + testEdge.setValue(1000 + (i + 1) * 500); // Use 500ms intervals, start from next interval + long refilled = limiter.refill(10); + + if (refilled > 0) { + // Only test when we actually get resources + limiter.consume(Math.min(8, (int) refilled)); + long waitInterval = limiter.getWaitInterval(10, 2, 5); + assertTrue("Wait interval should be reasonable", waitInterval >= 0); + } + } + + // Test passes if no exceptions are thrown + assertTrue("Over-subscription tracking should work without errors", true); + } + + @Test + public void testUtilizationTracking() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + testEdge.setValue(1000); + + // Test various utilization levels to ensure tracking works + for (int i = 0; i < 5; i++) { + testEdge.setValue(1000 + i * 500); // Use 500ms intervals + limiter.refill(10); + + // Vary consumption from 20% to 100% + int consumption = 2 + (i * 2); + limiter.consume(consumption); + } + + // The limiter should have tracked utilization without throwing exceptions + assertTrue("Utilization tracking should work correctly", true); + } + + @Test + public void testConcurrentAccess() throws InterruptedException { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(100, TimeUnit.SECONDS); + + testEdge.setValue(1000); + limiter.refill(100); + + // Simulate concurrent access + Thread[] threads = new Thread[10]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + for (int j = 0; j < 10; j++) { + limiter.consume(1); + limiter.getWaitInterval(100, 50, 1); + } + }); + } + + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + // Should complete without exceptions - basic thread safety verification + assertTrue("Concurrent access should complete successfully", true); + } + + @Test + public void testOverconsumptionBehavior() { + FeedbackAdaptiveRateLimiter limiter = factory.create(); + limiter.set(10, TimeUnit.SECONDS); + + testEdge.setValue(1000); + limiter.refill(10); + + // Over-consume significantly + limiter.consume(20); + + // Should require waiting for multiple intervals (500ms refill interval) + long waitInterval = limiter.getWaitInterval(10, -10, 1); + assertTrue("Should require substantial wait after over-consumption", waitInterval >= 500); + } + + private static class ManualEnvironmentEdge implements EnvironmentEdge { + private long currentTime = 1000; + + public void setValue(long time) { + this.currentTime = time; + } + + @Override + public long currentTime() { + return currentTime; + } + } +} From 1c71c45a12d81b9d5267d7c292f19e323a6ffbc8 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 21 Jul 2025 09:07:30 -0400 Subject: [PATCH 20/78] HubSpot Backport: HBASE-29447: Fix WAL archives cause incremental backup failures (will be in 2.6.4) Signed-off-by: Ray Mattingly Co-authored-by: Ray Mattingly Co-authored-by: Hernan Gelaf-Romer --- .../hbase/mapreduce/WALInputFormat.java | 34 ++++++++++-- .../hbase/mapreduce/TestWALInputFormat.java | 55 ++++++++++++++++++- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 7362f585d319..03d3250f54a9 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -318,7 +318,7 @@ List getSplits(final JobContext context, final String startKey, fina for (Path inputPath : inputPaths) { FileSystem fs = inputPath.getFileSystem(conf); try { - List files = getFiles(fs, inputPath, startTime, endTime); + List files = getFiles(fs, inputPath, startTime, endTime, conf); allFiles.addAll(files); } catch (FileNotFoundException e) { if (ignoreMissing) { @@ -349,11 +349,11 @@ private Path[] getInputPaths(Configuration conf) { * equal to this value else we will filter out the file. If name does not seem to * have a timestamp, we will just return it w/o filtering. */ - private List getFiles(FileSystem fs, Path dir, long startTime, long endTime) - throws IOException { + private List getFiles(FileSystem fs, Path dir, long startTime, long endTime, + Configuration conf) throws IOException { List result = new ArrayList<>(); LOG.debug("Scanning " + dir.toString() + " for WAL files"); - RemoteIterator iter = fs.listLocatedStatus(dir); + RemoteIterator iter = listLocatedFileStatus(fs, dir, conf); if (!iter.hasNext()) { return Collections.emptyList(); } @@ -361,7 +361,7 @@ private List getFiles(FileSystem fs, Path dir, long startTime, long LocatedFileStatus file = iter.next(); if (file.isDirectory()) { // Recurse into sub directories - result.addAll(getFiles(fs, file.getPath(), startTime, endTime)); + result.addAll(getFiles(fs, file.getPath(), startTime, endTime, conf)); } else { addFile(result, file, startTime, endTime); } @@ -396,4 +396,28 @@ public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { return new WALKeyRecordReader(); } + + /** + * Attempts to return the {@link LocatedFileStatus} for the given directory. If the directory does + * not exist, it will check if the directory is an archived log file and try to find it + */ + private static RemoteIterator listLocatedFileStatus(FileSystem fs, Path dir, + Configuration conf) throws IOException { + try { + return fs.listLocatedStatus(dir); + } catch (FileNotFoundException e) { + if (AbstractFSWALProvider.isArchivedLogFile(dir)) { + throw e; + } + + LOG.warn("Log file {} not found, trying to find it in archive directory.", dir); + Path archiveFile = AbstractFSWALProvider.findArchivedLog(dir, conf); + if (archiveFile == null) { + LOG.error("Did not find archive file for {}", dir); + throw e; + } + + return fs.listLocatedStatus(archiveFile); + } + } } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java index 70602a371668..6fdfb2bb8e2d 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALInputFormat.java @@ -21,24 +21,43 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL; import org.apache.hadoop.hbase.testclassification.MapReduceTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; -@Category({ MapReduceTests.class, SmallTests.class }) +@Category({ MapReduceTests.class, MediumTests.class }) public class TestWALInputFormat { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + @ClassRule public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestWALInputFormat.class); + @BeforeClass + public static void setupClass() throws Exception { + TEST_UTIL.startMiniCluster(); + TEST_UTIL.createWALRootDir(); + } + /** * Test the primitive start/end time filtering. */ @@ -74,4 +93,36 @@ public void testAddFile() { WALInputFormat.addFile(lfss, lfs, now, now); assertEquals(8, lfss.size()); } + + @Test + public void testHandlesArchivedWALFiles() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + JobContext ctx = Mockito.mock(JobContext.class); + Mockito.when(ctx.getConfiguration()).thenReturn(conf); + Job job = Job.getInstance(conf); + TableMapReduceUtil.initCredentialsForCluster(job, conf); + Mockito.when(ctx.getCredentials()).thenReturn(job.getCredentials()); + + // Setup WAL file, then archive it + HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0); + AbstractFSWAL wal = (AbstractFSWAL) rs.getWALs().get(0); + Path walPath = wal.getCurrentFileName(); + TEST_UTIL.getConfiguration().set(FileInputFormat.INPUT_DIR, walPath.toString()); + TEST_UTIL.getConfiguration().set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); + + Path rootDir = CommonFSUtils.getWALRootDir(conf); + Path archiveWal = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME); + archiveWal = new Path(archiveWal, walPath.getName()); + TEST_UTIL.getTestFileSystem().delete(walPath, true); + TEST_UTIL.getTestFileSystem().mkdirs(archiveWal.getParent()); + TEST_UTIL.getTestFileSystem().create(archiveWal).close(); + + // Test for that we can read from the archived WAL file + WALInputFormat wif = new WALInputFormat(); + List splits = wif.getSplits(ctx); + assertEquals(1, splits.size()); + WALInputFormat.WALSplit split = (WALInputFormat.WALSplit) splits.get(0); + assertEquals(archiveWal.toString(), split.getLogFileName()); + } + } From e10523fbe29a4b5aa9911eb2d59cfff685199c88 Mon Sep 17 00:00:00 2001 From: Sameer Dawani Date: Mon, 14 Jul 2025 13:13:32 -0400 Subject: [PATCH 21/78] HubSpot Backport: Rack aware incremental backup (not yet proposed upstream) --- .../hadoop/hbase/backup/BackupDriver.java | 6 ++ .../hbase/backup/BackupRestoreConstants.java | 8 ++ .../hbase/backup/impl/BackupCommands.java | 20 ++++ .../impl/IncrementalTableBackupClient.java | 3 + .../mapreduce/MapReduceHFileSplitterJob.java | 9 ++ .../hbase/mapreduce/HFileInputFormat.java | 91 +++++++++++++++++++ .../hbase/mapreduce/WALInputFormat.java | 45 ++++++++- .../hadoop/hbase/mapreduce/WALPlayer.java | 24 +++++ 8 files changed, 202 insertions(+), 4 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java index d55a280b4aa4..c135e7e0dff9 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; @@ -35,6 +37,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; @@ -159,6 +163,8 @@ protected void addOptions() { addOptWithArg(OPTION_PATH, OPTION_PATH_DESC); addOptWithArg(OPTION_KEEP, OPTION_KEEP_DESC); addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC); + addOptWithArg(OPTION_WAL_LOCATION_RESOLVER, OPTION_WAL_LOCATION_RESOLVER_DESC); + addOptWithArg(OPTION_HFILE_LOCATION_RESOLVER, OPTION_HFILE_LOCATION_RESOLVER_DESC); } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index 57454d402173..881e7ccf9353 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -99,6 +99,14 @@ public interface BackupRestoreConstants { String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on"; String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on"; + String OPTION_WAL_LOCATION_RESOLVER = "wal-location-resolver"; + String OPTION_WAL_LOCATION_RESOLVER_DESC = + "WAL file location resolver class for rack-aware incremental backup"; + + String OPTION_HFILE_LOCATION_RESOLVER = "hfile-location-resolver"; + String OPTION_HFILE_LOCATION_RESOLVER_DESC = + "HFile location resolver class for rack-aware bulk loading during incremental backup"; + String JOB_NAME_CONF_KEY = "mapreduce.job.name"; String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n" diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 66694f4384f4..2c78d0b50c11 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_HFILE_LOCATION_RESOLVER; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_IGNORECHECKSUM_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; @@ -37,6 +38,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WAL_LOCATION_RESOLVER; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; @@ -83,6 +85,14 @@ public final class BackupCommands { public final static String TOP_LEVEL_NOT_ALLOWED = "Top level (root) folder is not allowed to be a backup destination"; + // Configuration keys for location resolvers + // Must match WALPlayer.CONF_WAL_FILE_LOCATION_RESOLVER_CLASS + private static final String CONF_WAL_FILE_LOCATION_RESOLVER_CLASS = + "wal.backup.file.location.resolver.class"; + // Must match HFileInputFormat.CONF_HFILE_LOCATION_RESOLVER_CLASS + private static final String CONF_HFILE_LOCATION_RESOLVER_CLASS = + "hfile.backup.input.file.location.resolver.class"; + public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" + "where COMMAND is one of:\n" + " create create a new backup image\n" + " delete delete an existing backup image\n" @@ -148,6 +158,16 @@ public void execute() throws IOException { getConf().set("mapreduce.job.queuename", queueName); } + if (cmdline.hasOption(OPTION_WAL_LOCATION_RESOLVER)) { + String resolverClass = cmdline.getOptionValue(OPTION_WAL_LOCATION_RESOLVER); + getConf().set(CONF_WAL_FILE_LOCATION_RESOLVER_CLASS, resolverClass); + } + + if (cmdline.hasOption(OPTION_HFILE_LOCATION_RESOLVER)) { + String resolverClass = cmdline.getOptionValue(OPTION_HFILE_LOCATION_RESOLVER); + getConf().set(CONF_HFILE_LOCATION_RESOLVER_CLASS, resolverClass); + } + // Create connection conn = ConnectionFactory.createConnection(getConf()); if (requiresNoActiveSession()) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 1e07c026f0aa..14592806acec 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -412,6 +412,9 @@ protected void walToHFiles(List dirPaths, List tableList) throws conf.setBoolean(HFileOutputFormat2.TABLE_NAME_WITH_NAMESPACE_INCLUSIVE_KEY, true); conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); conf.set(JOB_NAME_CONF_KEY, jobname); + + // Rack-aware WAL processing configuration is set directly via command line to the same key + // WALPlayer uses String[] playerArgs = { dirs, StringUtils.join(tableList, ",") }; try { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java index 28db0c605f79..7d6ad00ddbf1 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java @@ -105,7 +105,11 @@ public Job createSubmittableJob(String[] args) throws IOException { job.getConfiguration().setBoolean(HFileOutputFormat2.EXTENDED_CELL_SERIALIZATION_ENABLED_KEY, true); job.setJarByClass(MapReduceHFileSplitterJob.class); + + // Use standard HFileInputFormat which now supports location resolver automatically + // HFileInputFormat will automatically detect and log rack-awareness configuration job.setInputFormatClass(HFileInputFormat.class); + job.setMapOutputKeyClass(ImmutableBytesWritable.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); if (hfileOutPath != null) { @@ -147,6 +151,11 @@ private void usage(final String errorMsg) { System.err.println("Other options:"); System.err.println(" -D " + JOB_NAME_CONF_KEY + "=jobName - use the specified mapreduce job name for the HFile splitter"); + + System.err.println("Rack-aware processing option:"); + System.err.println(" -D" + HFileInputFormat.CONF_HFILE_LOCATION_RESOLVER_CLASS + "= - " + + "HFile location resolver class for rack-aware processing"); + System.err.println("For performance also consider the following options:\n" + " -Dmapreduce.map.speculative=false\n" + " -Dmapreduce.reduce.speculative=false"); } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java index 1bbbe513f738..709e9a394533 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java @@ -19,8 +19,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.OptionalLong; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -39,6 +42,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +56,36 @@ public class HFileInputFormat extends FileInputFormat { private static final Logger LOG = LoggerFactory.getLogger(HFileInputFormat.class); + // Configuration key for pluggable location resolver class name + // Used to enable rack-aware processing by providing preferred data locality hints + public static final String CONF_HFILE_LOCATION_RESOLVER_CLASS = + "hfile.backup.input.file.location.resolver.class"; + + /** + * Interface for resolving file locations to influence InputSplit placement for HFile processing. + */ + @InterfaceAudience.Public + public interface HFileLocationResolver { + /** + * Get preferred locations for a group of HFiles to optimize rack-aware processing. + * @param hfiles Collection of HFile paths that will be processed together + * @return Set of preferred host names for processing these HFiles + */ + Set getLocationsForInputFiles(final Collection hfiles); + } + + /** + * Default no-op implementation of HFileLocationResolver. Provides backward compatibility by + * returning no location hints. + */ + public static class NoopHFileLocationResolver implements HFileLocationResolver { + @Override + public Set getLocationsForInputFiles(Collection hfiles) { + // No location hints - lets YARN scheduler decide + return Collections.emptySet(); + } + } + /** * File filter that removes all "hidden" files. This might be something worth removing from a more * general purpose utility; it accounts for the presence of metadata files created in the way @@ -183,4 +217,61 @@ protected boolean isSplitable(JobContext context, Path filename) { // This file isn't splittable. return false; } + + @Override + public List getSplits(JobContext context) throws IOException { + Configuration conf = context.getConfiguration(); + + // Check if location resolver is configured + String resolverClass = conf.get(CONF_HFILE_LOCATION_RESOLVER_CLASS); + if (resolverClass == null) { + LOG.debug("HFile rack-aware processing disabled - no location resolver configured"); + return super.getSplits(context); + } + + LOG.info("HFile rack-aware processing enabled with location resolver: {}", resolverClass); + try { + return createLocationAwareSplits(context, conf); + } catch (Exception e) { + LOG.error("Error creating location-aware splits, falling back to standard splits", e); + return super.getSplits(context); + } + } + + private List createLocationAwareSplits(JobContext context, Configuration conf) + throws IOException { + // Get all HFiles from input paths using existing listStatus logic + List files = listStatus(context); + + // Load configured location resolver + Class resolverClass = + conf.getClass(CONF_HFILE_LOCATION_RESOLVER_CLASS, NoopHFileLocationResolver.class, + HFileLocationResolver.class); + HFileLocationResolver fileLocationResolver = ReflectionUtils.newInstance(resolverClass, conf); + + // Create InputSplits with location hints + List splits = new ArrayList<>(); + + for (FileStatus file : files) { + Path path = file.getPath(); + long length = file.getLen(); + + if (length <= 0) { + LOG.warn("Skipping empty or invalid HFile: {} with length: {}", path, length); + continue; + } + + // Get location hints for this file + Set locations = + fileLocationResolver.getLocationsForInputFiles(Collections.singletonList(path.toString())); + String[] locationArray = locations.toArray(new String[0]); + + splits.add(new FileSplit(path, 0, length, locationArray)); + } + + LOG.info("Created {} location-aware InputSplits from {} HFiles", splits.size(), files.size()); + + return splits; + } + } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java index 03d3250f54a9..c4f3187b04dc 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALInputFormat.java @@ -71,6 +71,7 @@ static class WALSplit extends InputSplit implements Writable { private long fileSize; private long startTime; private long endTime; + private String[] locations; /** for serialization */ public WALSplit() { @@ -85,6 +86,19 @@ public WALSplit(String logFileName, long fileSize, long startTime, long endTime) this.fileSize = fileSize; this.startTime = startTime; this.endTime = endTime; + this.locations = new String[] {}; + } + + /** + * Represent an WALSplit with location hints for rack-aware processing. + */ + public WALSplit(String logFileName, long fileSize, long startTime, long endTime, + String[] locations) { + this.logFileName = logFileName; + this.fileSize = fileSize; + this.startTime = startTime; + this.endTime = endTime; + this.locations = locations != null ? locations : new String[] {}; } @Override @@ -94,8 +108,7 @@ public long getLength() throws IOException, InterruptedException { @Override public String[] getLocations() throws IOException, InterruptedException { - // TODO: Find the data node with the most blocks for this WAL? - return new String[] {}; + return locations; } public String getLogFileName() { @@ -116,6 +129,11 @@ public void readFields(DataInput in) throws IOException { fileSize = in.readLong(); startTime = in.readLong(); endTime = in.readLong(); + int locationsCount = in.readInt(); + locations = new String[locationsCount]; + for (int i = 0; i < locationsCount; i++) { + locations[i] = in.readUTF(); + } } @Override @@ -124,6 +142,10 @@ public void write(DataOutput out) throws IOException { out.writeLong(fileSize); out.writeLong(startTime); out.writeLong(endTime); + out.writeInt(locations.length); + for (String location : locations) { + out.writeUTF(location); + } } @Override @@ -328,10 +350,25 @@ List getSplits(final JobContext context, final String startKey, fina throw e; } } - List splits = new ArrayList(allFiles.size()); + // Create InputSplits with location hints for each WAL file + Class locationResolverClass = + conf.getClass(WALPlayer.CONF_WAL_FILE_LOCATION_RESOLVER_CLASS, + WALPlayer.NoopWALFileLocationResolver.class, WALPlayer.WALFileLocationResolver.class); + + WALPlayer.WALFileLocationResolver locationResolver = + org.apache.hadoop.util.ReflectionUtils.newInstance(locationResolverClass, conf); + + List splits = new ArrayList<>(); for (FileStatus file : allFiles) { - splits.add(new WALSplit(file.getPath().toString(), file.getLen(), startTime, endTime)); + // Get locations for this specific WAL file + String[] locations = locationResolver + .getLocationsForWALFiles(Collections.singletonList(file.getPath().toString())) + .toArray(new String[0]); + + splits + .add(new WALSplit(file.getPath().toString(), file.getLen(), startTime, endTime, locations)); } + return splits; } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index e06300848f68..189727f81b0f 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -21,6 +21,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -63,6 +64,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + /** * A tool to replay WAL files as a M/R job. The WAL can be replayed for a set of tables or all * tables, and a time range can be provided (in milliseconds). The WAL is filtered to the passed set @@ -85,6 +88,27 @@ public class WALPlayer extends Configured implements Tool { private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; + // Configuration key for pluggable WAL location resolver class name + // Used to enable rack-aware processing by providing preferred data locality hints for WAL files + public static final String CONF_WAL_FILE_LOCATION_RESOLVER_CLASS = + "wal.backup.file.location.resolver.class"; + + /** + * Interface for resolving file locations to influence InputSplit placement for rack-aware WAL + * processing. + */ + @InterfaceAudience.Public + public interface WALFileLocationResolver { + Set getLocationsForWALFiles(final Collection walFiles); + } + + public static class NoopWALFileLocationResolver implements WALFileLocationResolver { + @Override + public Set getLocationsForWALFiles(Collection walFiles) { + return ImmutableSet.of(); + } + } + public WALPlayer() { } From 84ac5b391f7d8ba6afae483f5e38dfe683354f5e Mon Sep 17 00:00:00 2001 From: ritika03494 <153202689+ritika03494@users.noreply.github.com> Date: Wed, 30 Jul 2025 16:01:47 +0100 Subject: [PATCH 22/78] HubSpot Edit: Use branched buildpack --- hubspot-client-bundles/.blazar.yaml | 1 + hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-client-bundle/.blazar.yaml | 1 + hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml | 1 + hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hubspot-client-bundles/.blazar.yaml b/hubspot-client-bundles/.blazar.yaml index a57d5eeb071b..8d5dac3de18f 100644 --- a/hubspot-client-bundles/.blazar.yaml +++ b/hubspot-client-bundles/.blazar.yaml @@ -1,5 +1,6 @@ buildpack: name: Blazar-Buildpack-Java + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml index 9399e5dc0aa4..509a0cc10fab 100644 --- a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -1,6 +1,6 @@ buildpack: name: Blazar-Buildpack-Java - + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml index 300be28892e8..aba96b1c7dd9 100644 --- a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -1,5 +1,6 @@ buildpack: name: Blazar-Buildpack-Java + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml index 5c020e374927..c79dbaaf6044 100644 --- a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -1,5 +1,6 @@ buildpack: name: Blazar-Buildpack-Java + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml index 26db8c8066b3..03fe644bf878 100644 --- a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -1,6 +1,6 @@ buildpack: name: Blazar-Buildpack-Java - + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may From 34e78ef35dce46b8f6a9c370211abba70dbfb2fe Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Thu, 7 Aug 2025 14:58:29 -0400 Subject: [PATCH 23/78] HubSpot Backport: HBASE-29502: Skip meta cache in RegionReplicaReplicationEndpoint when only one replica found (will be in 2.6.4) --- .../RegionReplicaReplicationEndpoint.java | 9 +- .../TestRegionReplicaReplicationEndpoint.java | 100 ++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java index 754811ce0e04..94b9daf836f9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java @@ -439,7 +439,7 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, // Replicas can take a while to come online. The cache may have only the primary. If we // keep going to the cache, we will not learn of the replicas and their locations after // they come online. - if (useCache && locations.size() == 1 && TableName.isMetaTableName(tableName)) { + if (useCache && locations.size() == 1) { if (tableDescriptors.get(tableName).getRegionReplication() > 1) { // Make an obnoxious log here. See how bad this issue is. Add a timer if happening // too much. @@ -488,6 +488,13 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, } if (locations.size() == 1) { + if (LOG.isTraceEnabled()) { + LOG.trace("Skipping {} entries in table {} because only one region location was found", + entries.size(), tableName); + for (Entry entry : entries) { + LOG.trace("Skipping: {}", entry); + } + } return; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java index 9a03536f7541..ffca0caabef6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java @@ -507,6 +507,106 @@ public void testRegionReplicaReplicationIgnores(boolean dropTable, boolean disab } } + @Test + public void testMetaCacheMissTriggersRefresh() throws Exception { + TableName tableName = TableName.valueOf(name.getMethodName()); + int regionReplication = 3; + HTableDescriptor htd = HTU.createTableDescriptor(tableName); + htd.setRegionReplication(regionReplication); + createOrEnableTableWithRetries(htd, true); + + Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); + Table table = connection.getTable(tableName); + + try { + HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 100); + + RegionLocator rl = connection.getRegionLocator(tableName); + HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); + rl.close(); + + AtomicLong skippedEdits = new AtomicLong(); + RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink = + mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class); + when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits); + + FSTableDescriptors fstd = + new FSTableDescriptors(FileSystem.get(HTU.getConfiguration()), HTU.getDefaultRootDirPath()); + + RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter = + new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink, + (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, + fstd); + + Cell cell = CellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("testRow")).setFamily(HBaseTestingUtility.fam1) + .setValue(Bytes.toBytes("testValue")).setType(Type.Put).build(); + + Entry entry = + new Entry(new WALKeyImpl(encodedRegionName, tableName, 1), new WALEdit().add(cell)); + + sinkWriter.append(tableName, encodedRegionName, Bytes.toBytes("testRow"), + Lists.newArrayList(entry)); + + assertEquals("No entries should be skipped for valid table", 0, skippedEdits.get()); + + } finally { + table.close(); + connection.close(); + } + } + + @Test + public void testMetaCacheSkippedForSingleReplicaTable() throws Exception { + TableName tableName = TableName.valueOf(name.getMethodName()); + int regionReplication = 1; + HTableDescriptor htd = HTU.createTableDescriptor(tableName); + htd.setRegionReplication(regionReplication); + createOrEnableTableWithRetries(htd, true); + + Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); + Table table = connection.getTable(tableName); + + try { + HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 100); + + RegionLocator rl = connection.getRegionLocator(tableName); + HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); + rl.close(); + + AtomicLong skippedEdits = new AtomicLong(); + RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink = + mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class); + when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits); + + FSTableDescriptors fstd = + new FSTableDescriptors(FileSystem.get(HTU.getConfiguration()), HTU.getDefaultRootDirPath()); + + RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter = + new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink, + (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, + fstd); + + Cell cell = CellBuilderFactory.create(CellBuilderType.DEEP_COPY) + .setRow(Bytes.toBytes("testRow")).setFamily(HBaseTestingUtility.fam1) + .setValue(Bytes.toBytes("testValue")).setType(Type.Put).build(); + + Entry entry = + new Entry(new WALKeyImpl(encodedRegionName, tableName, 1), new WALEdit().add(cell)); + + sinkWriter.append(tableName, encodedRegionName, Bytes.toBytes("testRow"), + Lists.newArrayList(entry)); + + assertEquals("No entries should be skipped for single replica table", 0, skippedEdits.get()); + + } finally { + table.close(); + connection.close(); + } + } + private void createOrEnableTableWithRetries(TableDescriptor htd, boolean createTableOperation) { // Helper function to run create/enable table operations with a retry feature boolean continueToRetry = true; From 3e424f70cba8acf7b2d67b9631c212d9fbb0e69a Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Wed, 20 Aug 2025 14:24:50 -0700 Subject: [PATCH 24/78] =?UTF-8?q?HubSpot=20Backport:=20HBASE-29469=20Add?= =?UTF-8?q?=20metrics=20with=20more=20detail=20for=20RpcThrottlingExceptio?= =?UTF-8?q?ns=20=E2=80=A6=20(will=20be=20in=202.6.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * HBASE-29469 Add metrics with more detail for RpcThrottlingExceptions (#7214) Co-authored-by: skhillon Signed-off by: cconnell Reviewed by: kgeisz * Removing unnecessary sanitization * Remove unnecessary tests --------- Co-authored-by: skhillon --- .../quotas/RegionServerRpcQuotaManager.java | 8 + .../regionserver/MetricsRegionServer.java | 16 ++ .../metrics/MetricsThrottleExceptions.java | 71 +++++ .../regionserver/TestMetricsRegionServer.java | 29 ++ .../TestMetricsThrottleExceptions.java | 251 ++++++++++++++++++ 5 files changed, 375 insertions(+) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 03fbfde47a13..958793dcdf00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -196,6 +196,10 @@ public OperationQuota checkScanQuota(final Region region, } catch (RpcThrottlingException e) { LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " scan=" + scanRequest.getScannerId() + ": " + e.getMessage()); + + rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + table.getNameAsString()); + throw e; } return quota; @@ -269,6 +273,10 @@ public OperationQuota checkBatchQuota(final Region region, final int numWrites, } catch (RpcThrottlingException e) { LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " numWrites=" + numWrites + " numReads=" + numReads + ": " + e.getMessage()); + + rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + table.getNameAsString()); + throw e; } return quota; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index a0bf25dc2eaa..580f77874992 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.metrics.MetricRegistries; import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.hadoop.hbase.regionserver.metrics.MetricsThrottleExceptions; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -47,6 +49,7 @@ public class MetricsRegionServer { private MetricsRegionServerQuotaSource quotaSource; private MetricRegistry metricRegistry; + private MetricsThrottleExceptions throttleMetrics; private Timer bulkLoadTimer; // Incremented once for each call to Scan#nextRaw private Meter serverReadQueryMeter; @@ -78,6 +81,8 @@ public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, Confi serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond"); serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond"); } + + throttleMetrics = new MetricsThrottleExceptions(metricRegistry); } MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, @@ -296,4 +301,15 @@ public void incrScannerLeaseExpired() { serverSource.incrScannerLeaseExpired(); } + /** + * Record a throttle exception with contextual information. + * @param throttleType the type of throttle exception from RpcThrottlingException.Type enum + * @param user the user who triggered the throttle + * @param table the table that was being accessed + */ + public void recordThrottleException(RpcThrottlingException.Type throttleType, String user, + String table) { + throttleMetrics.recordThrottleException(throttleType, user, table); + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java new file mode 100644 index 000000000000..90480c75cbcc --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/MetricsThrottleExceptions.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class MetricsThrottleExceptions { + + /** + * The name of the metrics + */ + private static final String METRICS_NAME = "ThrottleExceptions"; + + /** + * The name of the metrics context that metrics will be under. + */ + private static final String METRICS_CONTEXT = "regionserver"; + + /** + * Description + */ + private static final String METRICS_DESCRIPTION = "Metrics about RPC throttling exceptions"; + + /** + * The name of the metrics context that metrics will be under in jmx + */ + private static final String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME; + + private final MetricRegistry registry; + + public MetricsThrottleExceptions(MetricRegistry sharedRegistry) { + registry = sharedRegistry; + } + + /** + * Record a throttle exception with contextual information. + * @param throttleType the type of throttle exception + * @param user the user who triggered the throttle + * @param table the table that was being accessed + */ + public void recordThrottleException(RpcThrottlingException.Type throttleType, String user, + String table) { + String metricName = qualifyThrottleMetric(throttleType, user, table); + registry.counter(metricName).increment(); + } + + private static String qualifyThrottleMetric(RpcThrottlingException.Type throttleType, String user, + String table) { + return String.format("RpcThrottlingException_Type_%s_User_%s_Table_%s", throttleType.name(), + user, table); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index 2186aa7cc4dc..76378f784d49 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -26,11 +26,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.JvmPauseMonitor; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -68,6 +71,12 @@ public void setUp() { serverSource = rsm.getMetricsSource(); } + @After + public void tearDown() { + // Clean up global registries after each test to avoid interference + MetricRegistries.global().clear(); + } + @Test public void testWrapperSource() { HELPER.assertTag("serverName", "test", serverSource); @@ -314,4 +323,24 @@ public void testScannerMetrics() { HELPER.assertGauge("activeScanners", 0, serverSource); } + @Test + public void testThrottleExceptionMetricsIntegration() { + // Record different types of throttle exceptions + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + rsm.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", "logs"); + rsm.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, "charlie", + "metadata"); + + // Record the same exception multiple times to test increment + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + rsm.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", "users"); + + // Verify the specific counters were created and have correct values using HELPER + HELPER.assertCounter("RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", + 3L, serverSource); + HELPER.assertCounter("RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_logs", 1L, + serverSource); + HELPER.assertCounter("RpcThrottlingException_Type_ReadSizeExceeded_User_charlie_Table_metadata", + 1L, serverSource); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java new file mode 100644 index 000000000000..4f627dc507da --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestMetricsThrottleExceptions.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.quotas.RpcThrottlingException; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestMetricsThrottleExceptions { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetricsThrottleExceptions.class); + + private MetricRegistry testRegistry; + private MetricsThrottleExceptions throttleMetrics; + + @After + public void cleanup() { + // Clean up global registries after each test to avoid interference + MetricRegistries.global().clear(); + } + + @Test + public void testBasicThrottleMetricsRecording() { + setupTestMetrics(); + + // Record a throttle exception + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + + // Verify the counter exists and has correct value + Optional metric = + testRegistry.get("RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users"); + assertTrue("Counter metric should be present", metric.isPresent()); + assertTrue("Metric should be a counter", metric.get() instanceof Counter); + + Counter counter = (Counter) metric.get(); + assertEquals("Counter should have count of 1", 1, counter.getCount()); + } + + @Test + public void testMultipleThrottleTypes() { + setupTestMetrics(); + + // Record different types of throttle exceptions + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", + "logs"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, "charlie", + "metadata"); + + // Verify all three counters were created + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", 1); + verifyCounter(testRegistry, "RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_logs", + 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_ReadSizeExceeded_User_charlie_Table_metadata", 1); + } + + @Test + public void testCounterIncrement() { + setupTestMetrics(); + + // Record the same throttle exception multiple times + String metricName = "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users"; + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + + // Verify the counter incremented correctly + verifyCounter(testRegistry, metricName, 3); + } + + @Test + public void testConcurrentAccess() throws InterruptedException { + setupTestMetrics(); + + int numThreads = 10; + int incrementsPerThread = 100; + + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(numThreads); + AtomicInteger exceptions = new AtomicInteger(0); + + // Create multiple threads that increment the same counter concurrently + for (int i = 0; i < numThreads; i++) { + executor.submit(() -> { + try { + startLatch.await(); + for (int j = 0; j < incrementsPerThread; j++) { + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "alice", "users"); + } + } catch (Exception e) { + exceptions.incrementAndGet(); + } finally { + doneLatch.countDown(); + } + }); + } + + // Start all threads at once + startLatch.countDown(); + + // Wait for all threads to complete + boolean completed = doneLatch.await(30, TimeUnit.SECONDS); + assertTrue("All threads should complete within timeout", completed); + assertEquals("No exceptions should occur during concurrent access", 0, exceptions.get()); + + // Verify the final counter value + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_users", + numThreads * incrementsPerThread); + + executor.shutdown(); + } + + @Test + public void testCommonTableNamePatterns() { + setupTestMetrics(); + + // Test common HBase table name patterns that should be preserved + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, + "service-user", "my-app-logs"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, + "batch.process", "namespace:table-name"); + throttleMetrics.recordThrottleException(RpcThrottlingException.Type.ReadSizeExceeded, + "user_123", "test_table_v2"); + + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_service-user_Table_my-app-logs", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_WriteSizeExceeded_User_batch.process_Table_namespace:table-name", + 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_ReadSizeExceeded_User_user_123_Table_test_table_v2", 1); + } + + @Test + public void testAllThrottleExceptionTypes() { + setupTestMetrics(); + + // Test all 13 throttle exception types from RpcThrottlingException.Type enum + RpcThrottlingException.Type[] throttleTypes = RpcThrottlingException.Type.values(); + + // Record one exception for each type + for (RpcThrottlingException.Type throttleType : throttleTypes) { + throttleMetrics.recordThrottleException(throttleType, "testuser", "testtable"); + } + + // Verify all counters were created with correct values + for (RpcThrottlingException.Type throttleType : throttleTypes) { + String expectedMetricName = + "RpcThrottlingException_Type_" + throttleType.name() + "_User_testuser_Table_testtable"; + verifyCounter(testRegistry, expectedMetricName, 1); + } + } + + @Test + public void testMultipleInstances() { + setupTestMetrics(); + + // Test that multiple instances of MetricsThrottleExceptions work with the same registry + MetricsThrottleExceptions metrics1 = new MetricsThrottleExceptions(testRegistry); + MetricsThrottleExceptions metrics2 = new MetricsThrottleExceptions(testRegistry); + + // Record different exceptions on each instance + metrics1.recordThrottleException(RpcThrottlingException.Type.NumRequestsExceeded, "alice", + "table1"); + metrics2.recordThrottleException(RpcThrottlingException.Type.WriteSizeExceeded, "bob", + "table2"); + + // Verify both counters exist in the shared registry + verifyCounter(testRegistry, + "RpcThrottlingException_Type_NumRequestsExceeded_User_alice_Table_table1", 1); + verifyCounter(testRegistry, + "RpcThrottlingException_Type_WriteSizeExceeded_User_bob_Table_table2", 1); + } + + /** + * Helper method to set up test metrics registry and instance + */ + private void setupTestMetrics() { + MetricRegistryInfo registryInfo = getRegistryInfo(); + testRegistry = MetricRegistries.global().create(registryInfo); + throttleMetrics = new MetricsThrottleExceptions(testRegistry); + } + + /** + * Helper method to verify a counter exists and has the expected value + */ + private void verifyCounter(MetricRegistry registry, String metricName, long expectedCount) { + Optional metric = registry.get(metricName); + assertTrue("Counter metric '" + metricName + "' should be present", metric.isPresent()); + assertTrue("Metric should be a counter", metric.get() instanceof Counter); + + Counter counter = (Counter) metric.get(); + assertEquals("Counter '" + metricName + "' should have expected count", expectedCount, + counter.getCount()); + } + + /** + * Helper method to create the expected MetricRegistryInfo for ThrottleExceptions + */ + private MetricRegistryInfo getRegistryInfo() { + return new MetricRegistryInfo("ThrottleExceptions", "Metrics about RPC throttling exceptions", + "RegionServer,sub=ThrottleExceptions", "regionserver", false); + } +} From 61008534c6c93bed47863c2df5d3e3169c011a7d Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Wed, 3 Sep 2025 13:52:45 -0400 Subject: [PATCH 25/78] HubSpot Backport: HBASE-29479: QuotaCache should always return accurate information (will be in 2.6.4) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/QuotaCache.java | 209 ++++++++++-------- .../hbase/quotas/TestDefaultAtomicQuota.java | 9 - .../hadoop/hbase/quotas/TestQuotaCache.java | 118 +++++++++- 3 files changed, 227 insertions(+), 109 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index d77f6219ae59..2ec9d049f7da 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent; - import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -30,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; @@ -56,10 +55,7 @@ /** * Cache that keeps track of the quota settings for the users and tables that are interacting with - * it. To avoid blocking the operations if the requested quota is not in cache an "empty quota" will - * be returned and the request to fetch the quota information will be enqueued for the next refresh. - * TODO: At the moment the Cache has a Chore that will be triggered every 5min or on cache-miss - * events. Later the Quotas will be pushed using the notification system. + * it. */ @InterfaceAudience.Private @InterfaceStability.Evolving @@ -100,6 +96,62 @@ public class QuotaCache implements Stoppable { private QuotaRefresherChore refreshChore; private boolean stopped = true; + private final Fetcher userQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String user) { + final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); + final Set tables = QuotaCache.this.tableQuotaCache.keySet(); + return QuotaUtil.makeGetForUserQuotas(user, tables, namespaces); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, tableMachineQuotaFactors, + machineQuotaFactor); + } + }; + + private final Fetcher regionServerQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String regionServer) { + return QuotaUtil.makeGetForRegionServerQuotas(regionServer); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); + } + }; + + private final Fetcher tableQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final TableName table) { + return QuotaUtil.makeGetForTableQuotas(table); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, + tableMachineQuotaFactors); + } + }; + + private final Fetcher namespaceQuotaStateFetcher = + new Fetcher() { + @Override + public Get makeGet(final String namespace) { + return QuotaUtil.makeGetForNamespaceQuotas(namespace); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, machineQuotaFactor); + } + }; + public QuotaCache(final RegionServerServices rsServices) { this.rsServices = rsServices; this.userOverrideRequestAttributeKey = @@ -153,8 +205,13 @@ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableNa * @return the quota info associated to specified user */ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { - return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi), - () -> QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); + String user = getQuotaUserName(ugi); + if (!userQuotaCache.containsKey(user)) { + userQuotaCache.put(user, + QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); + fetch("user", userQuotaCache, userQuotaStateFetcher); + } + return userQuotaCache.get(user); } /** @@ -163,7 +220,11 @@ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { * @return the limiter associated to the specified table */ public QuotaLimiter getTableLimiter(final TableName table) { - return getQuotaState(this.tableQuotaCache, table).getGlobalLimiter(); + if (!tableQuotaCache.containsKey(table)) { + tableQuotaCache.put(table, new QuotaState()); + fetch("table", tableQuotaCache, tableQuotaStateFetcher); + } + return tableQuotaCache.get(table).getGlobalLimiter(); } /** @@ -172,7 +233,11 @@ public QuotaLimiter getTableLimiter(final TableName table) { * @return the limiter associated to the specified namespace */ public QuotaLimiter getNamespaceLimiter(final String namespace) { - return getQuotaState(this.namespaceQuotaCache, namespace).getGlobalLimiter(); + if (!namespaceQuotaCache.containsKey(namespace)) { + namespaceQuotaCache.put(namespace, new QuotaState()); + fetch("namespace", namespaceQuotaCache, namespaceQuotaStateFetcher); + } + return namespaceQuotaCache.get(namespace).getGlobalLimiter(); } /** @@ -181,13 +246,41 @@ public QuotaLimiter getNamespaceLimiter(final String namespace) { * @return the limiter associated to the specified region server */ public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) { - return getQuotaState(this.regionServerQuotaCache, regionServer).getGlobalLimiter(); + if (!regionServerQuotaCache.containsKey(regionServer)) { + regionServerQuotaCache.put(regionServer, new QuotaState()); + fetch("regionServer", regionServerQuotaCache, regionServerQuotaStateFetcher); + } + return regionServerQuotaCache.get(regionServer).getGlobalLimiter(); } protected boolean isExceedThrottleQuotaEnabled() { return exceedThrottleQuotaEnabled; } + private void fetch(final String type, final Map quotasMap, + final Fetcher fetcher) { + // Find the quota entries to update + List gets = quotasMap.keySet().stream().map(fetcher::makeGet).collect(Collectors.toList()); + + // fetch and update the quota entries + if (!gets.isEmpty()) { + try { + for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { + V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); + if (quotaInfo != null) { + quotaInfo.update(entry.getValue()); + } + + if (LOG.isTraceEnabled()) { + LOG.trace("Loading {} key={} quotas={}", type, entry.getKey(), quotaInfo); + } + } + } catch (IOException e) { + LOG.warn("Unable to read {} from quota table", type, e); + } + } + } + /** * Applies a request attribute user override if available, otherwise returns the UGI's short * username @@ -210,14 +303,6 @@ private String getQuotaUserName(final UserGroupInformation ugi) { return Bytes.toString(override); } - /** - * Returns the QuotaState requested. If the quota info is not in cache an empty one will be - * returned and the quota request will be enqueued for the next cache refresh. - */ - private QuotaState getQuotaState(final ConcurrentMap quotasMap, final K key) { - return computeIfAbsent(quotasMap, key, QuotaState::new); - } - void triggerCacheRefresh() { refreshChore.triggerNow(); } @@ -226,10 +311,6 @@ void forceSynchronousCacheRefresh() { refreshChore.chore(); } - long getLastUpdate() { - return refreshChore.lastUpdate; - } - Map getNamespaceQuotaCache() { return namespaceQuotaCache; } @@ -248,8 +329,6 @@ Map getUserQuotaCache() { // TODO: Remove this once we have the notification bus private class QuotaRefresherChore extends ScheduledChore { - private long lastUpdate = 0; - // Querying cluster metrics so often, per-RegionServer, limits horizontal scalability. // So we cache the results to reduce that load. private final RefreshableExpiringValueCache tableRegionStatesClusterMetrics; @@ -307,74 +386,12 @@ protected void chore() { .computeIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, key -> new QuotaState()); updateQuotaFactors(); - fetchNamespaceQuotaState(); - fetchTableQuotaState(); - fetchUserQuotaState(); - fetchRegionServerQuotaState(); + fetchAndEvict("namespace", QuotaCache.this.namespaceQuotaCache, namespaceQuotaStateFetcher); + fetchAndEvict("table", QuotaCache.this.tableQuotaCache, tableQuotaStateFetcher); + fetchAndEvict("user", QuotaCache.this.userQuotaCache, userQuotaStateFetcher); + fetchAndEvict("regionServer", QuotaCache.this.regionServerQuotaCache, + regionServerQuotaStateFetcher); fetchExceedThrottleQuota(); - lastUpdate = EnvironmentEdgeManager.currentTime(); - } - - private void fetchNamespaceQuotaState() { - fetch("namespace", QuotaCache.this.namespaceQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForNamespaceQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, - machineQuotaFactor); - } - }); - } - - private void fetchTableQuotaState() { - fetch("table", QuotaCache.this.tableQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForTableQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors); - } - }); - } - - private void fetchUserQuotaState() { - final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); - final Set tables = QuotaCache.this.tableQuotaCache.keySet(); - fetch("user", QuotaCache.this.userQuotaCache, new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForUserQuotas(entry.getKey(), tables, namespaces); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors, machineQuotaFactor); - } - }); - } - - private void fetchRegionServerQuotaState() { - fetch("regionServer", QuotaCache.this.regionServerQuotaCache, - new Fetcher() { - @Override - public Get makeGet(final Map.Entry entry) { - return QuotaUtil.makeGetForRegionServerQuotas(entry.getKey()); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); - } - }); } private void fetchExceedThrottleQuota() { @@ -386,7 +403,7 @@ private void fetchExceedThrottleQuota() { } } - private void fetch(final String type, + private void fetchAndEvict(final String type, final ConcurrentMap quotasMap, final Fetcher fetcher) { long now = EnvironmentEdgeManager.currentTime(); long evictPeriod = getPeriod() * EVICT_PERIOD_FACTOR; @@ -398,7 +415,7 @@ private void fetch(final String type, if (lastQuery > 0 && (now - lastQuery) >= evictPeriod) { toRemove.add(entry.getKey()); } else { - gets.add(fetcher.makeGet(entry)); + gets.add(fetcher.makeGet(entry.getKey())); } } @@ -543,8 +560,8 @@ static interface ThrowingSupplier { T get() throws Exception; } - static interface Fetcher { - Get makeGet(Map.Entry entry); + interface Fetcher { + Get makeGet(Key key); Map fetchEntries(List gets) throws IOException; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java index 966bce6bcdb9..31840cb8d2f2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultAtomicQuota.java @@ -81,10 +81,6 @@ public static void setUpBeforeClass() throws Exception { @Test public void testDefaultAtomicReadLimits() throws Exception { - // No write throttling - configureLenientThrottle(ThrottleType.ATOMIC_WRITE_SIZE); - refreshQuotas(); - // Should have a strict throttle by default TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100); @@ -102,11 +98,6 @@ public void testDefaultAtomicReadLimits() throws Exception { @Test public void testDefaultAtomicWriteLimits() throws Exception { - // No read throttling - configureLenientThrottle(ThrottleType.ATOMIC_REQUEST_NUMBER); - configureLenientThrottle(ThrottleType.ATOMIC_READ_SIZE); - refreshQuotas(); - // Should have a strict throttle by default TEST_UTIL.waitFor(60_000, () -> runIncTest(100) < 100); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java index 09e152369121..f4f876f104ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java @@ -20,13 +20,16 @@ import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota; import static org.junit.Assert.assertEquals; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; -import org.junit.After; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -42,8 +45,8 @@ public class TestQuotaCache { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final int REFRESH_TIME_MS = 1000; - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL); EnvironmentEdgeManager.reset(); TEST_UTIL.shutdownMiniCluster(); @@ -68,7 +71,6 @@ public void testDefaultUserRefreshFrequency() throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); - assertEquals(userQuotaState.getLastUpdate(), 0); QuotaCache.TEST_BLOCK_REFRESH = false; // new user should have refreshed immediately @@ -86,4 +88,112 @@ public void testDefaultUserRefreshFrequency() throws Exception { // should refresh after time has passed TEST_UTIL.waitFor(5_000, () -> lastUpdate != userQuotaState.getLastUpdate()); } + + @Test + public void testUserQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleUser("my_user", ThrottleType.READ_NUMBER, 3737, + TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("my_user"); + QuotaLimiter quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific user quota, fall back to default + ugi = UserGroupInformation.createRemoteUser("my_user2"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(1000, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + ugi = UserGroupInformation.createRemoteUser("my_user"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + ugi = UserGroupInformation.createRemoteUser("my_user2"); + quotaLimiter = quotaCache.getUserLimiter(ugi, TableName.valueOf("my_table")); + assertEquals(1000, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testTableQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleTable(TableName.valueOf("my_table"), + ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific table quota, fall back to default + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2")); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table")); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getTableLimiter(TableName.valueOf("my_table2")); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testNamespaceQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleNamespace("my_namespace", ThrottleType.READ_NUMBER, + 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific namespace quota, fall back to default + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getNamespaceLimiter("my_namespace2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } + + @Test + public void testRegionServerQuotaLookup() throws Exception { + QuotaCache quotaCache = + ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get(); + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleRegionServer("my_region_server", + ThrottleType.READ_NUMBER, 3737, TimeUnit.MINUTES)); + + // Setting a quota and then looking it up from the cache should work, even if the cache has not + // refreshed + QuotaLimiter quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + // if no specific server quota, fall back to default + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + + // still works after refresh + quotaCache.forceSynchronousCacheRefresh(); + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server"); + assertEquals(3737, quotaLimiter.getReadNumLimit()); + + quotaLimiter = quotaCache.getRegionServerQuotaLimiter("my_region_server2"); + assertEquals(Long.MAX_VALUE, quotaLimiter.getReadNumLimit()); + } } From 7075df1bcbf6adecef77149429138dcf526454e9 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 12 Sep 2025 10:52:49 -0400 Subject: [PATCH 26/78] HubSpot Backport: HBASE-29573: Fully load QuotaCache instead of reading individual rows on demand (will be in 2.6.4) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/QuotaTableUtil.java | 31 -- .../hadoop/hbase/quotas/QuotaCache.java | 302 +++++++----------- .../hadoop/hbase/quotas/QuotaState.java | 38 +-- .../apache/hadoop/hbase/quotas/QuotaUtil.java | 163 +++++----- .../hadoop/hbase/quotas/UserQuotaState.java | 22 +- .../hbase/quotas/TestAtomicReadQuota.java | 1 - .../quotas/TestBlockBytesScannedQuota.java | 1 - .../quotas/TestClusterScopeQuotaThrottle.java | 1 - .../hbase/quotas/TestDefaultAtomicQuota.java | 1 - .../quotas/TestDefaultHandlerUsageQuota.java | 1 - .../hadoop/hbase/quotas/TestDefaultQuota.java | 7 +- .../hadoop/hbase/quotas/TestQuotaCache.java | 40 +-- .../hadoop/hbase/quotas/TestQuotaCache2.java | 130 ++++++++ .../hadoop/hbase/quotas/TestQuotaState.java | 58 +--- .../hbase/quotas/TestQuotaThrottle.java | 1 - .../hbase/quotas/TestQuotaUserOverride.java | 1 - .../quotas/TestThreadHandlerUsageQuota.java | 8 +- 17 files changed, 362 insertions(+), 444 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 1afb15c0ac61..4bdf5e5af049 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -206,37 +206,6 @@ private static Quotas getQuotas(final Connection connection, final byte[] rowKey return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier)); } - public static Get makeGetForTableQuotas(final TableName table) { - Get get = new Get(getTableRowKey(table)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForNamespaceQuotas(final String namespace) { - Get get = new Get(getNamespaceRowKey(namespace)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForRegionServerQuotas(final String regionServer) { - Get get = new Get(getRegionServerRowKey(regionServer)); - get.addFamily(QUOTA_FAMILY_INFO); - return get; - } - - public static Get makeGetForUserQuotas(final String user, final Iterable tables, - final Iterable namespaces) { - Get get = new Get(getUserRowKey(user)); - get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); - for (final TableName table : tables) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserTable(table)); - } - for (final String ns : namespaces) { - get.addColumn(QUOTA_FAMILY_INFO, getSettingsQualifierForUserNamespace(ns)); - } - return get; - } - public static Scan makeScan(final QuotaFilter filter) { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 2ec9d049f7da..16681eb45f8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -19,30 +19,23 @@ import java.io.IOException; import java.time.Duration; -import java.util.ArrayList; import java.util.EnumSet; -import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -73,18 +66,15 @@ public class QuotaCache implements Stoppable { public static final String QUOTA_USER_REQUEST_ATTRIBUTE_OVERRIDE_KEY = "hbase.quota.user.override.key"; private static final int REFRESH_DEFAULT_PERIOD = 43_200_000; // 12 hours - private static final int EVICT_PERIOD_FACTOR = 5; - // for testing purpose only, enforce the cache to be always refreshed - static boolean TEST_FORCE_REFRESH = false; - // for testing purpose only, block cache refreshes to reliably verify state - static boolean TEST_BLOCK_REFRESH = false; + private final Object initializerLock = new Object(); + private volatile boolean initialized = false; + + private volatile Map namespaceQuotaCache = new HashMap<>(); + private volatile Map tableQuotaCache = new HashMap<>(); + private volatile Map userQuotaCache = new HashMap<>(); + private volatile Map regionServerQuotaCache = new HashMap<>(); - private final ConcurrentMap namespaceQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap tableQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap userQuotaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap regionServerQuotaCache = - new ConcurrentHashMap<>(); private volatile boolean exceedThrottleQuotaEnabled = false; // factors used to divide cluster scope quota into machine scope quota private volatile double machineQuotaFactor = 1; @@ -96,62 +86,6 @@ public class QuotaCache implements Stoppable { private QuotaRefresherChore refreshChore; private boolean stopped = true; - private final Fetcher userQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String user) { - final Set namespaces = QuotaCache.this.namespaceQuotaCache.keySet(); - final Set tables = QuotaCache.this.tableQuotaCache.keySet(); - return QuotaUtil.makeGetForUserQuotas(user, tables, namespaces); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), gets, tableMachineQuotaFactors, - machineQuotaFactor); - } - }; - - private final Fetcher regionServerQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String regionServer) { - return QuotaUtil.makeGetForRegionServerQuotas(regionServer); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); - } - }; - - private final Fetcher tableQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final TableName table) { - return QuotaUtil.makeGetForTableQuotas(table); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), gets, - tableMachineQuotaFactors); - } - }; - - private final Fetcher namespaceQuotaStateFetcher = - new Fetcher() { - @Override - public Get makeGet(final String namespace) { - return QuotaUtil.makeGetForNamespaceQuotas(namespace); - } - - @Override - public Map fetchEntries(final List gets) throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), gets, machineQuotaFactor); - } - }; - public QuotaCache(final RegionServerServices rsServices) { this.rsServices = rsServices; this.userOverrideRequestAttributeKey = @@ -163,10 +97,8 @@ public void start() throws IOException { Configuration conf = rsServices.getConfiguration(); // Refresh the cache every 12 hours, and every time a quota is changed, and every time a - // configuration - // reload is triggered. Periodic reloads are kept to a minimum to avoid flooding the - // RegionServer - // holding the hbase:quota table with requests. + // configuration reload is triggered. Periodic reloads are kept to a minimum to avoid + // flooding the RegionServer holding the hbase:quota table with requests. int period = conf.getInt(REFRESH_CONF_KEY, REFRESH_DEFAULT_PERIOD); refreshChore = new QuotaRefresherChore(conf, period, this); rsServices.getChoreService().scheduleChore(refreshChore); @@ -186,6 +118,34 @@ public boolean isStopped() { return stopped; } + private void ensureInitialized() { + if (!initialized) { + synchronized (initializerLock) { + if (!initialized) { + refreshChore.chore(); + initialized = true; + } + } + } + } + + private Map fetchUserQuotaStateEntries() throws IOException { + return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), tableMachineQuotaFactors, + machineQuotaFactor); + } + + private Map fetchRegionServerQuotaStateEntries() throws IOException { + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection()); + } + + private Map fetchTableQuotaStateEntries() throws IOException { + return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), tableMachineQuotaFactors); + } + + private Map fetchNamespaceQuotaStateEntries() throws IOException { + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), machineQuotaFactor); + } + /** * Returns the limiter associated to the specified user/table. * @param ugi the user to limit @@ -206,12 +166,13 @@ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableNa */ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { String user = getQuotaUserName(ugi); - if (!userQuotaCache.containsKey(user)) { - userQuotaCache.put(user, - QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L)); - fetch("user", userQuotaCache, userQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to userQuotaCache + Map cache = userQuotaCache; + if (!cache.containsKey(user)) { + cache.put(user, QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration())); } - return userQuotaCache.get(user); + return cache.get(user); } /** @@ -220,11 +181,13 @@ public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) { * @return the limiter associated to the specified table */ public QuotaLimiter getTableLimiter(final TableName table) { - if (!tableQuotaCache.containsKey(table)) { - tableQuotaCache.put(table, new QuotaState()); - fetch("table", tableQuotaCache, tableQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to tableQuotaCache + Map cache = tableQuotaCache; + if (!cache.containsKey(table)) { + cache.put(table, new QuotaState()); } - return tableQuotaCache.get(table).getGlobalLimiter(); + return cache.get(table).getGlobalLimiter(); } /** @@ -233,11 +196,13 @@ public QuotaLimiter getTableLimiter(final TableName table) { * @return the limiter associated to the specified namespace */ public QuotaLimiter getNamespaceLimiter(final String namespace) { - if (!namespaceQuotaCache.containsKey(namespace)) { - namespaceQuotaCache.put(namespace, new QuotaState()); - fetch("namespace", namespaceQuotaCache, namespaceQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to namespaceQuotaCache + Map cache = namespaceQuotaCache; + if (!cache.containsKey(namespace)) { + cache.put(namespace, new QuotaState()); } - return namespaceQuotaCache.get(namespace).getGlobalLimiter(); + return cache.get(namespace).getGlobalLimiter(); } /** @@ -246,41 +211,19 @@ public QuotaLimiter getNamespaceLimiter(final String namespace) { * @return the limiter associated to the specified region server */ public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) { - if (!regionServerQuotaCache.containsKey(regionServer)) { - regionServerQuotaCache.put(regionServer, new QuotaState()); - fetch("regionServer", regionServerQuotaCache, regionServerQuotaStateFetcher); + ensureInitialized(); + // local reference because the chore thread may assign to regionServerQuotaCache + Map cache = regionServerQuotaCache; + if (!cache.containsKey(regionServer)) { + cache.put(regionServer, new QuotaState()); } - return regionServerQuotaCache.get(regionServer).getGlobalLimiter(); + return cache.get(regionServer).getGlobalLimiter(); } protected boolean isExceedThrottleQuotaEnabled() { return exceedThrottleQuotaEnabled; } - private void fetch(final String type, final Map quotasMap, - final Fetcher fetcher) { - // Find the quota entries to update - List gets = quotasMap.keySet().stream().map(fetcher::makeGet).collect(Collectors.toList()); - - // fetch and update the quota entries - if (!gets.isEmpty()) { - try { - for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { - V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); - if (quotaInfo != null) { - quotaInfo.update(entry.getValue()); - } - - if (LOG.isTraceEnabled()) { - LOG.trace("Loading {} key={} quotas={}", type, entry.getKey(), quotaInfo); - } - } - } catch (IOException e) { - LOG.warn("Unable to read {} from quota table", type, e); - } - } - } - /** * Applies a request attribute user override if available, otherwise returns the UGI's short * username @@ -311,18 +254,22 @@ void forceSynchronousCacheRefresh() { refreshChore.chore(); } + /** visible for testing */ Map getNamespaceQuotaCache() { return namespaceQuotaCache; } + /** visible for testing */ Map getRegionServerQuotaCache() { return regionServerQuotaCache; } + /** visible for testing */ Map getTableQuotaCache() { return tableQuotaCache; } + /** visible for testing */ Map getUserQuotaCache() { return userQuotaCache; } @@ -359,38 +306,44 @@ public synchronized boolean triggerNow() { } @Override - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "GC_UNRELATED_TYPES", - justification = "I do not understand why the complaints, it looks good to me -- FIX") protected void chore() { - while (TEST_BLOCK_REFRESH) { - LOG.info("TEST_BLOCK_REFRESH=true, so blocking QuotaCache refresh until it is false"); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + updateQuotaFactors(); + + try { + Map newUserQuotaCache = new HashMap<>(fetchUserQuotaStateEntries()); + updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); + userQuotaCache = newUserQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching user quotas", e); } - // Prefetch online tables/namespaces - for (TableName table : ((HRegionServer) QuotaCache.this.rsServices).getOnlineTables()) { - if (table.isSystemTable()) { - continue; - } - QuotaCache.this.tableQuotaCache.computeIfAbsent(table, key -> new QuotaState()); - final String ns = table.getNamespaceAsString(); + try { + Map newRegionServerQuotaCache = + new HashMap<>(fetchRegionServerQuotaStateEntries()); + updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); + regionServerQuotaCache = newRegionServerQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching region server quotas", e); + } - QuotaCache.this.namespaceQuotaCache.computeIfAbsent(ns, key -> new QuotaState()); + try { + Map newTableQuotaCache = + new HashMap<>(fetchTableQuotaStateEntries()); + updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); + tableQuotaCache = newTableQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing table quotas", e); } - QuotaCache.this.regionServerQuotaCache - .computeIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, key -> new QuotaState()); + try { + Map newNamespaceQuotaCache = + new HashMap<>(fetchNamespaceQuotaStateEntries()); + updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); + namespaceQuotaCache = newNamespaceQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing namespace quotas", e); + } - updateQuotaFactors(); - fetchAndEvict("namespace", QuotaCache.this.namespaceQuotaCache, namespaceQuotaStateFetcher); - fetchAndEvict("table", QuotaCache.this.tableQuotaCache, tableQuotaStateFetcher); - fetchAndEvict("user", QuotaCache.this.userQuotaCache, userQuotaStateFetcher); - fetchAndEvict("regionServer", QuotaCache.this.regionServerQuotaCache, - regionServerQuotaStateFetcher); fetchExceedThrottleQuota(); } @@ -403,48 +356,6 @@ private void fetchExceedThrottleQuota() { } } - private void fetchAndEvict(final String type, - final ConcurrentMap quotasMap, final Fetcher fetcher) { - long now = EnvironmentEdgeManager.currentTime(); - long evictPeriod = getPeriod() * EVICT_PERIOD_FACTOR; - // Find the quota entries to update - List gets = new ArrayList<>(); - List toRemove = new ArrayList<>(); - for (Map.Entry entry : quotasMap.entrySet()) { - long lastQuery = entry.getValue().getLastQuery(); - if (lastQuery > 0 && (now - lastQuery) >= evictPeriod) { - toRemove.add(entry.getKey()); - } else { - gets.add(fetcher.makeGet(entry.getKey())); - } - } - - for (final K key : toRemove) { - if (LOG.isTraceEnabled()) { - LOG.trace("evict " + type + " key=" + key); - } - quotasMap.remove(key); - } - - // fetch and update the quota entries - if (!gets.isEmpty()) { - try { - for (Map.Entry entry : fetcher.fetchEntries(gets).entrySet()) { - V quotaInfo = quotasMap.putIfAbsent(entry.getKey(), entry.getValue()); - if (quotaInfo != null) { - quotaInfo.update(entry.getValue()); - } - - if (LOG.isTraceEnabled()) { - LOG.trace("refresh " + type + " key=" + entry.getKey() + " quotas=" + quotaInfo); - } - } - } catch (IOException e) { - LOG.warn("Unable to read " + type + " from quota table", e); - } - } - } - /** * Update quota factors which is used to divide cluster scope quota into machine scope quota For * user/namespace/user over namespace quota, use [1 / RSNum] as machine factor. For table/user @@ -520,6 +431,20 @@ private void updateMachineQuotaFactors(int rsSize) { } } + /** visible for testing */ + static void updateNewCacheFromOld(Map oldCache, + Map newCache) { + for (Map.Entry entry : oldCache.entrySet()) { + K key = entry.getKey(); + if (newCache.containsKey(key)) { + V newState = newCache.get(key); + V oldState = entry.getValue(); + oldState.update(newState); + newCache.put(key, oldState); + } + } + } + static class RefreshableExpiringValueCache { private final String name; private final LoadingCache> cache; @@ -560,9 +485,4 @@ static interface ThrowingSupplier { T get() throws Exception; } - interface Fetcher { - Get makeGet(Key key); - - Map fetchEntries(List gets) throws IOException; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java index 7c9445e15587..61aa9d7f068f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -32,33 +31,14 @@ justification = "FindBugs seems confused; says globalLimiter and lastUpdate " + "are mostly synchronized...but to me it looks like they are totally synchronized") public class QuotaState { - protected long lastUpdate = 0; - protected long lastQuery = 0; - protected QuotaLimiter globalLimiter = NoopQuotaLimiter.get(); - public QuotaState() { - this(0); - } - - public QuotaState(final long updateTs) { - lastUpdate = updateTs; - } - - public synchronized long getLastUpdate() { - return lastUpdate; - } - - public synchronized long getLastQuery() { - return lastQuery; - } - @Override public synchronized String toString() { StringBuilder builder = new StringBuilder(); - builder.append("QuotaState(ts=" + getLastUpdate()); + builder.append("QuotaState("); if (isBypass()) { - builder.append(" bypass"); + builder.append("bypass"); } else { if (globalLimiter != NoopQuotaLimiter.get()) { // builder.append(" global-limiter"); @@ -85,6 +65,11 @@ public synchronized void setQuotas(final Quotas quotas) { } } + /** visible for testing */ + void setGlobalLimiter(QuotaLimiter globalLimiter) { + this.globalLimiter = globalLimiter; + } + /** * Perform an update of the quota info based on the other quota info object. (This operation is * executed by the QuotaCache) @@ -97,7 +82,6 @@ public synchronized void update(final QuotaState other) { } else { globalLimiter = QuotaLimiterFactory.update(globalLimiter, other.globalLimiter); } - lastUpdate = other.lastUpdate; } /** @@ -105,15 +89,7 @@ public synchronized void update(final QuotaState other) { * @return the quota limiter */ public synchronized QuotaLimiter getGlobalLimiter() { - lastQuery = EnvironmentEdgeManager.currentTime(); return globalLimiter; } - /** - * Return the limiter associated with this quota without updating internal last query stats - * @return the quota limiter - */ - synchronized QuotaLimiter getGlobalLimiterWithoutUpdatingLastQuery() { - return globalLimiter; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 3ef704b666b3..6b38635eccc0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -39,10 +39,11 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -329,59 +330,56 @@ private static void deleteQuotas(final Connection connection, final byte[] rowKe } public static Map fetchUserQuotas(final Connection connection, - final List gets, Map tableMachineQuotaFactors, double factor) - throws IOException { - long nowTs = EnvironmentEdgeManager.currentTime(); - Result[] results = doGet(connection, gets); - - Map userQuotas = new HashMap<>(results.length); - for (int i = 0; i < results.length; ++i) { - byte[] key = gets.get(i).getRow(); - assert isUserRowKey(key); - String user = getUserFromRowKey(key); - - if (results[i].isEmpty()) { - userQuotas.put(user, buildDefaultUserQuotaState(connection.getConfiguration(), nowTs)); - continue; - } - - final UserQuotaState quotaInfo = new UserQuotaState(nowTs); - userQuotas.put(user, quotaInfo); - - assert Bytes.equals(key, results[i].getRow()); - - try { - parseUserResult(user, results[i], new UserQuotasVisitor() { - @Override - public void visitUserQuotas(String userName, String namespace, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(namespace, quotas); + Map tableMachineQuotaFactors, double factor) throws IOException { + Map userQuotas = new HashMap<>(); + try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_USER_ROW_KEY_PREFIX); + try (ResultScanner resultScanner = table.getScanner(scan)) { + for (Result result : resultScanner) { + byte[] key = result.getRow(); + assert isUserRowKey(key); + String user = getUserFromRowKey(key); + + final UserQuotaState quotaInfo = new UserQuotaState(); + userQuotas.put(user, quotaInfo); + + try { + parseUserResult(user, result, new UserQuotasVisitor() { + @Override + public void visitUserQuotas(String userName, String namespace, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, factor); + quotaInfo.setQuotas(namespace, quotas); + } + + @Override + public void visitUserQuotas(String userName, TableName table, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, + tableMachineQuotaFactors.containsKey(table) + ? tableMachineQuotaFactors.get(table) + : 1); + quotaInfo.setQuotas(table, quotas); + } + + @Override + public void visitUserQuotas(String userName, Quotas quotas) { + quotas = updateClusterQuotaToMachineQuota(quotas, factor); + quotaInfo.setQuotas(quotas); + } + }); + } catch (IOException e) { + LOG.error("Unable to parse user '" + user + "' quotas", e); + userQuotas.remove(user); } - - @Override - public void visitUserQuotas(String userName, TableName table, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, - tableMachineQuotaFactors.containsKey(table) - ? tableMachineQuotaFactors.get(table) - : 1); - quotaInfo.setQuotas(table, quotas); - } - - @Override - public void visitUserQuotas(String userName, Quotas quotas) { - quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(quotas); - } - }); - } catch (IOException e) { - LOG.error("Unable to parse user '" + user + "' quotas", e); - userQuotas.remove(user); + } } } + return userQuotas; } - protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf, long nowTs) { + protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) { QuotaProtos.Throttle.Builder throttleBuilder = QuotaProtos.Throttle.newBuilder(); buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_NUM) @@ -405,7 +403,7 @@ protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf, l buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_REQUEST_HANDLER_USAGE_MS) .ifPresent(throttleBuilder::setReqHandlerUsageMs); - UserQuotaState state = new UserQuotaState(nowTs); + UserQuotaState state = new UserQuotaState(); QuotaProtos.Quotas defaultQuotas = QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build(); state.setQuotas(defaultQuotas); @@ -422,8 +420,11 @@ private static Optional buildDefaultTimedQuota(Configuration conf, S } public static Map fetchTableQuotas(final Connection connection, - final List gets, Map tableMachineFactors) throws IOException { - return fetchGlobalQuotas("table", connection, gets, new KeyFromRow() { + Map tableMachineFactors) throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_TABLE_ROW_KEY_PREFIX); + return fetchGlobalQuotas("table", scan, connection, new KeyFromRow() { @Override public TableName getKeyFromRow(final byte[] row) { assert isTableRowKey(row); @@ -438,8 +439,11 @@ public double getFactor(TableName tableName) { } public static Map fetchNamespaceQuotas(final Connection connection, - final List gets, double factor) throws IOException { - return fetchGlobalQuotas("namespace", connection, gets, new KeyFromRow() { + double factor) throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_NAMESPACE_ROW_KEY_PREFIX); + return fetchGlobalQuotas("namespace", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isNamespaceRowKey(row); @@ -453,9 +457,12 @@ public double getFactor(String s) { }); } - public static Map fetchRegionServerQuotas(final Connection connection, - final List gets) throws IOException { - return fetchGlobalQuotas("regionServer", connection, gets, new KeyFromRow() { + public static Map fetchRegionServerQuotas(final Connection connection) + throws IOException { + Scan scan = new Scan(); + scan.addFamily(QUOTA_FAMILY_INFO); + scan.setStartStopRowForPrefixScan(QUOTA_REGION_SERVER_ROW_KEY_PREFIX); + return fetchGlobalQuotas("regionServer", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isRegionServerRowKey(row); @@ -469,32 +476,34 @@ public double getFactor(String s) { }); } - public static Map fetchGlobalQuotas(final String type, - final Connection connection, final List gets, final KeyFromRow kfr) throws IOException { - long nowTs = EnvironmentEdgeManager.currentTime(); - Result[] results = doGet(connection, gets); + public static Map fetchGlobalQuotas(final String type, final Scan scan, + final Connection connection, final KeyFromRow kfr) throws IOException { - Map globalQuotas = new HashMap<>(results.length); - for (int i = 0; i < results.length; ++i) { - byte[] row = gets.get(i).getRow(); - K key = kfr.getKeyFromRow(row); + Map globalQuotas = new HashMap<>(); + try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { + try (ResultScanner resultScanner = table.getScanner(scan)) { + for (Result result : resultScanner) { - QuotaState quotaInfo = new QuotaState(nowTs); - globalQuotas.put(key, quotaInfo); + byte[] row = result.getRow(); + K key = kfr.getKeyFromRow(row); - if (results[i].isEmpty()) continue; - assert Bytes.equals(row, results[i].getRow()); + QuotaState quotaInfo = new QuotaState(); + globalQuotas.put(key, quotaInfo); - byte[] data = results[i].getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); - if (data == null) continue; + byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); + if (data == null) { + continue; + } - try { - Quotas quotas = quotasFromData(data); - quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); - quotaInfo.setQuotas(quotas); - } catch (IOException e) { - LOG.error("Unable to parse " + type + " '" + key + "' quotas", e); - globalQuotas.remove(key); + try { + Quotas quotas = quotasFromData(data); + quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); + quotaInfo.setQuotas(quotas); + } catch (IOException e) { + LOG.error("Unable to parse {} '{}' quotas", type, key, e); + globalQuotas.remove(key); + } + } } } return globalQuotas; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java index a3ec97994363..877ad195c716 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -42,24 +41,18 @@ public class UserQuotaState extends QuotaState { private Map tableLimiters = null; private boolean bypassGlobals = false; - public UserQuotaState() { - super(); - } - - public UserQuotaState(final long updateTs) { - super(updateTs); - } - @Override public synchronized String toString() { StringBuilder builder = new StringBuilder(); - builder.append("UserQuotaState(ts=" + getLastUpdate()); - if (bypassGlobals) builder.append(" bypass-globals"); + builder.append("UserQuotaState("); + if (bypassGlobals) { + builder.append("bypass-globals"); + } if (isBypass()) { builder.append(" bypass"); } else { - if (getGlobalLimiterWithoutUpdatingLastQuery() != NoopQuotaLimiter.get()) { + if (getGlobalLimiter() != NoopQuotaLimiter.get()) { builder.append(" global-limiter"); } @@ -86,7 +79,7 @@ public synchronized String toString() { /** Returns true if there is no quota information associated to this object */ @Override public synchronized boolean isBypass() { - return !bypassGlobals && getGlobalLimiterWithoutUpdatingLastQuery() == NoopQuotaLimiter.get() + return !bypassGlobals && getGlobalLimiter() == NoopQuotaLimiter.get() && (tableLimiters == null || tableLimiters.isEmpty()) && (namespaceLimiters == null || namespaceLimiters.isEmpty()); } @@ -191,7 +184,6 @@ private static Map updateLimiters(final Map userQuotaState.getLastUpdate() != 0); - long lastUpdate = userQuotaState.getLastUpdate(); - - // refresh should not apply to recently refreshed quota - quotaCache.triggerCacheRefresh(); - Thread.sleep(250); - long newLastUpdate = userQuotaState.getLastUpdate(); - assertEquals(lastUpdate, newLastUpdate); - - quotaCache.triggerCacheRefresh(); - waitMinuteQuota(); - // should refresh after time has passed - TEST_UTIL.waitFor(5_000, () -> lastUpdate != userQuotaState.getLastUpdate()); - } - @Test public void testUserQuotaLookup() throws Exception { QuotaCache quotaCache = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java new file mode 100644 index 000000000000..2c33b265771a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; + +/** + * Tests of QuotaCache that don't require a minicluster, unlike in TestQuotaCache + */ +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestQuotaCache2 { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestQuotaCache2.class); + + @Test + public void testPreserveLimiterAvailability() throws Exception { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + oldState.setGlobalLimiter(limiter1); + + // consume one byte from the limiter, so 99 will be left + limiter1.consumeRead(1, 1, false); + + // establish new cache, also with a limiter for 100 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 99 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(99, updatedLimiter.getReadAvailable()); + } + + @Test + public void testClobberLimiterLimit() throws Exception { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + oldState.setGlobalLimiter(limiter1); + + // establish new cache, also with a limiter for 100 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(50).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 99 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(50, updatedLimiter.getReadLimit()); + } + + @Test + public void testForgetsDeletedQuota() { + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table1", oldState); + + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table2", newState); + + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + assertTrue(newCache.containsKey("my_table2")); + assertFalse(newCache.containsKey("my_table1")); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java index 59b26f3f0d91..ff4b6bc9949b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.quotas; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -81,67 +80,38 @@ public void testSimpleQuotaStateOperation() { assertThrottleException(quotaInfo.getTableLimiter(tableName), NUM_TABLE_THROTTLE); } - @Test - public void testQuotaStateUpdateBypassThrottle() { - final long LAST_UPDATE = 10; - - UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); - assertTrue(quotaInfo.isBypass()); - - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE); - assertEquals(LAST_UPDATE, otherQuotaState.getLastUpdate()); - assertTrue(otherQuotaState.isBypass()); - - quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE, quotaInfo.getLastUpdate()); - assertTrue(quotaInfo.isBypass()); - assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); - assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); - } - @Test public void testQuotaStateUpdateGlobalThrottle() { final int NUM_GLOBAL_THROTTLE_1 = 3; final int NUM_GLOBAL_THROTTLE_2 = 11; - final long LAST_UPDATE_1 = 10; - final long LAST_UPDATE_2 = 20; - final long LAST_UPDATE_3 = 30; QuotaState quotaInfo = new QuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add global throttle - QuotaState otherQuotaState = new QuotaState(LAST_UPDATE_1); + QuotaState otherQuotaState = new QuotaState(); otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_1); // Update global Throttle - otherQuotaState = new QuotaState(LAST_UPDATE_2); + otherQuotaState = new QuotaState(); otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); - assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getGlobalLimiter(), NUM_GLOBAL_THROTTLE_2 - NUM_GLOBAL_THROTTLE_1); // Remove global throttle - otherQuotaState = new QuotaState(LAST_UPDATE_3); - assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + otherQuotaState = new QuotaState(); assertTrue(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); assertNoopLimiter(quotaInfo.getGlobalLimiter()); } @@ -155,37 +125,29 @@ public void testQuotaStateUpdateTableThrottle() { final int TABLE_A_THROTTLE_2 = 11; final int TABLE_B_THROTTLE = 4; final int TABLE_C_THROTTLE = 5; - final long LAST_UPDATE_1 = 10; - final long LAST_UPDATE_2 = 20; - final long LAST_UPDATE_3 = 30; UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A B table limiters - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + UserQuotaState otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); otherQuotaState.setQuotas(tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getTableLimiter(tableNameA), TABLE_A_THROTTLE_1); assertThrottleException(quotaInfo.getTableLimiter(tableNameB), TABLE_B_THROTTLE); assertNoopLimiter(quotaInfo.getTableLimiter(tableNameC)); // Add C, Remove B, Update A table limiters - otherQuotaState = new UserQuotaState(LAST_UPDATE_2); + otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); otherQuotaState.setQuotas(tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); - assertEquals(LAST_UPDATE_2, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_2, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); assertThrottleException(quotaInfo.getTableLimiter(tableNameA), TABLE_A_THROTTLE_2 - TABLE_A_THROTTLE_1); @@ -193,12 +155,10 @@ public void testQuotaStateUpdateTableThrottle() { assertNoopLimiter(quotaInfo.getTableLimiter(tableNameB)); // Remove table limiters - otherQuotaState = new UserQuotaState(LAST_UPDATE_3); - assertEquals(LAST_UPDATE_3, otherQuotaState.getLastUpdate()); + otherQuotaState = new UserQuotaState(); assertTrue(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_3, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); assertNoopLimiter(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); } @@ -207,20 +167,16 @@ public void testQuotaStateUpdateTableThrottle() { public void testTableThrottleWithBatch() { final TableName TABLE_A = TableName.valueOf("TableA"); final int TABLE_A_THROTTLE_1 = 3; - final long LAST_UPDATE_1 = 10; UserQuotaState quotaInfo = new UserQuotaState(); - assertEquals(0, quotaInfo.getLastUpdate()); assertTrue(quotaInfo.isBypass()); // Add A table limiters - UserQuotaState otherQuotaState = new UserQuotaState(LAST_UPDATE_1); + UserQuotaState otherQuotaState = new UserQuotaState(); otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); - assertEquals(LAST_UPDATE_1, otherQuotaState.getLastUpdate()); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); - assertEquals(LAST_UPDATE_1, quotaInfo.getLastUpdate()); assertFalse(quotaInfo.isBypass()); QuotaLimiter limiter = quotaInfo.getTableLimiter(TABLE_A); try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java index 5ae9de1fbf16..66996f366610 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java @@ -88,7 +88,6 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); TEST_UTIL.startMiniCluster(1); TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; tables = new Table[TABLE_NAMES.length]; for (int i = 0; i < TABLE_NAMES.length; ++i) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java index 683d189b761b..7917f3c0847f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUserOverride.java @@ -65,7 +65,6 @@ public static void setUpBeforeClass() throws Exception { CUSTOM_OVERRIDE_KEY); TEST_UTIL.startMiniCluster(NUM_SERVERS); TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; TEST_UTIL.createTable(TABLE_NAME, FAMILY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java index 8a9863132e81..58b15ec24294 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestThreadHandlerUsageQuota.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.triggerUserCacheRefresh; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -74,7 +75,6 @@ public static void setUpBeforeClass() throws Exception { TEST_UTIL.createTable(TABLE_NAME, FAMILY); TEST_UTIL.waitTableAvailable(TABLE_NAME); - QuotaCache.TEST_FORCE_REFRESH = true; TEST_UTIL.flush(TABLE_NAME); } @@ -104,11 +104,12 @@ public void testHandlerUsageThrottleForWrites() throws Exception { } } - private void configureThrottle() throws IOException { + private void configureThrottle() throws Exception { try (Admin admin = TEST_UTIL.getAdmin()) { admin.setQuota(QuotaSettingsFactory.throttleUser(getUserName(), - ThrottleType.REQUEST_HANDLER_USAGE_MS, 10000, TimeUnit.SECONDS)); + ThrottleType.REQUEST_HANDLER_USAGE_MS, 1, TimeUnit.SECONDS)); } + triggerUserCacheRefresh(TEST_UTIL, false, TABLE_NAME); } private void unthrottleUser() throws Exception { @@ -116,6 +117,7 @@ private void unthrottleUser() throws Exception { admin.setQuota(QuotaSettingsFactory.unthrottleUserByThrottleType(getUserName(), ThrottleType.REQUEST_HANDLER_USAGE_MS)); } + triggerUserCacheRefresh(TEST_UTIL, true, TABLE_NAME); } private static String getUserName() throws IOException { From 62caea86329e47d36bf0f48b7270d05b1fc8e9fd Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 15 Sep 2025 08:31:55 -0400 Subject: [PATCH 27/78] HBASE-28440: Add support for using mapreduce sort in HFileOutputFormat2 (not yet merged upstream) (#197) --- .../impl/IncrementalTableBackupClient.java | 4 + .../mapreduce/MapReduceHFileSplitterJob.java | 37 +++- .../hbase/mapreduce/HFileOutputFormat2.java | 36 +++- .../apache/hadoop/hbase/mapreduce/Import.java | 4 + .../mapreduce/KeyOnlyCellComparable.java | 91 ++++++++++ .../mapreduce/PreSortedCellsReducer.java | 46 +++++ .../hadoop/hbase/mapreduce/WALPlayer.java | 38 +++- .../mapreduce/TestCellBasedWALPlayer2.java | 3 +- .../hadoop/hbase/mapreduce/TestWALPlayer.java | 166 +++++++++++++----- .../TestUnattainableBalancerCostGoal.java | 4 +- 10 files changed, 367 insertions(+), 62 deletions(-) create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 14592806acec..5f86f2a57b88 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -346,6 +346,7 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I LOG.debug("Setting incremental copy HFiles job name to : " + jobname); } conf.set(JOB_NAME_CONF_KEY, jobname); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); BackupCopyJob copyService = BackupRestoreFactory.getBackupCopyJob(conf); int res = copyService.copy(backupInfo, backupManager, conf, BackupType.INCREMENTAL, strArr); @@ -358,6 +359,7 @@ protected void incrementalCopyHFiles(String[] files, String backupDest) throws I + " finished."); } finally { deleteBulkLoadDirectory(); + conf.unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); } } @@ -411,6 +413,7 @@ protected void walToHFiles(List dirPaths, List tableList) throws conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); conf.setBoolean(HFileOutputFormat2.TABLE_NAME_WITH_NAMESPACE_INCLUSIVE_KEY, true); conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); + conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); conf.set(JOB_NAME_CONF_KEY, jobname); // Rack-aware WAL processing configuration is set directly via command line to the same key @@ -423,6 +426,7 @@ protected void walToHFiles(List dirPaths, List tableList) throws if (result != 0) { throw new IOException("WAL Player failed"); } + conf.unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY); conf.unset(JOB_NAME_CONF_KEY); } catch (IOException e) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java index 7d6ad00ddbf1..4938f9470540 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceHFileSplitterJob.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -33,11 +34,14 @@ import org.apache.hadoop.hbase.mapreduce.CellSortReducer; import org.apache.hadoop.hbase.mapreduce.HFileInputFormat; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; +import org.apache.hadoop.hbase.mapreduce.KeyOnlyCellComparable; +import org.apache.hadoop.hbase.mapreduce.PreSortedCellsReducer; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MapReduceExtendedCell; import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -71,18 +75,28 @@ protected MapReduceHFileSplitterJob(final Configuration c) { /** * A mapper that just writes out cells. This one can be used together with {@link CellSortReducer} */ - static class HFileCellMapper extends Mapper { + static class HFileCellMapper extends Mapper, Cell> { + + private boolean diskBasedSortingEnabled = false; @Override public void map(NullWritable key, Cell value, Context context) throws IOException, InterruptedException { - context.write(new ImmutableBytesWritable(CellUtil.cloneRow(value)), - new MapReduceExtendedCell(value)); + ExtendedCell extendedCell = (ExtendedCell) value; + context.write(wrap(extendedCell), new MapReduceExtendedCell(extendedCell)); } @Override public void setup(Context context) throws IOException { - // do nothing + diskBasedSortingEnabled = + HFileOutputFormat2.diskBasedSortingEnabled(context.getConfiguration()); + } + + private WritableComparable wrap(ExtendedCell cell) { + if (diskBasedSortingEnabled) { + return new KeyOnlyCellComparable(cell); + } + return new ImmutableBytesWritable(CellUtil.cloneRow(cell)); } } @@ -109,14 +123,23 @@ public Job createSubmittableJob(String[] args) throws IOException { // Use standard HFileInputFormat which now supports location resolver automatically // HFileInputFormat will automatically detect and log rack-awareness configuration job.setInputFormatClass(HFileInputFormat.class); - - job.setMapOutputKeyClass(ImmutableBytesWritable.class); String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); TableName tableName = TableName.valueOf(tabName); job.setMapperClass(HFileCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + } else { + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 6ab3bdd25048..7c297015cb03 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; @@ -50,6 +49,7 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -83,6 +83,7 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.OutputFormat; @@ -194,6 +195,11 @@ protected static byte[] combineTableNameSuffix(byte[] tableName, byte[] suffix) "hbase.mapreduce.hfileoutputformat.extendedcell.enabled"; static final boolean EXTENDED_CELL_SERIALIZATION_ENABLED_DEFULT = false; + @InterfaceAudience.Private + public static final String DISK_BASED_SORTING_ENABLED_KEY = + "hbase.mapreduce.hfileoutputformat.disk.based.sorting.enabled"; + private static final boolean DISK_BASED_SORTING_ENABLED_DEFAULT = false; + public static final String REMOTE_CLUSTER_CONF_PREFIX = "hbase.hfileoutputformat.remote.cluster."; public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY = REMOTE_CLUSTER_CONF_PREFIX + "zookeeper.quorum"; @@ -579,12 +585,19 @@ private static void writePartitions(Configuration conf, Path partitionsPath, // Write the actual file FileSystem fs = partitionsPath.getFileSystem(conf); - SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, partitionsPath, - ImmutableBytesWritable.class, NullWritable.class); + boolean diskBasedSortingEnabled = diskBasedSortingEnabled(conf); + Class keyClass = + diskBasedSortingEnabled ? KeyOnlyCellComparable.class : ImmutableBytesWritable.class; + SequenceFile.Writer writer = + SequenceFile.createWriter(fs, conf, partitionsPath, keyClass, NullWritable.class); try { for (ImmutableBytesWritable startKey : sorted) { - writer.append(startKey, NullWritable.get()); + Writable writable = diskBasedSortingEnabled + ? new KeyOnlyCellComparable(KeyValueUtil.createFirstOnRow(startKey.get())) + : startKey; + + writer.append(writable, NullWritable.get()); } } finally { writer.close(); @@ -631,6 +644,13 @@ public static void configureIncrementalLoad(Job job, TableDescriptor tableDescri configureIncrementalLoad(job, singleTableInfo, HFileOutputFormat2.class); } + public static boolean diskBasedSortingEnabled(Configuration conf) { + boolean res = + conf.getBoolean(DISK_BASED_SORTING_ENABLED_KEY, DISK_BASED_SORTING_ENABLED_DEFAULT); + LOG.info("Disk based sorting on: {}", res); + return res; + } + static void configureIncrementalLoad(Job job, List multiTableInfo, Class> cls) throws IOException { Configuration conf = job.getConfiguration(); @@ -652,7 +672,13 @@ static void configureIncrementalLoad(Job job, List multiTableInfo, // Based on the configured map output class, set the correct reducer to properly // sort the incoming values. // TODO it would be nice to pick one or the other of these formats. - if ( + boolean diskBasedSorting = diskBasedSortingEnabled(conf); + + if (diskBasedSorting) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + job.setReducerClass(PreSortedCellsReducer.class); + } else if ( KeyValue.class.equals(job.getMapOutputValueClass()) || MapReduceExtendedCell.class.equals(job.getMapOutputValueClass()) ) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index 4adcfbfcd3f6..03abcf159753 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -200,6 +200,10 @@ public CellWritableComparable(Cell kv) { this.kv = kv; } + public Cell getCell() { + return kv; + } + @Override public void write(DataOutput out) throws IOException { int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(kv); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java new file mode 100644 index 000000000000..a065abd63d08 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyOnlyCellComparable.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.ExtendedCell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableComparator; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class KeyOnlyCellComparable implements WritableComparable { + + static { + WritableComparator.define(KeyOnlyCellComparable.class, new KeyOnlyCellComparator()); + } + + private ExtendedCell cell = null; + + public KeyOnlyCellComparable() { + } + + public KeyOnlyCellComparable(ExtendedCell cell) { + this.cell = cell; + } + + public ExtendedCell getCell() { + return cell; + } + + @Override + public int compareTo(KeyOnlyCellComparable o) { + return CellComparator.getInstance().compare(cell, o.cell); + } + + @Override + public void write(DataOutput out) throws IOException { + int keyLen = PrivateCellUtil.estimatedSerializedSizeOfKey(cell); + int valueLen = 0; // We avoid writing value here. So just serialize as if an empty value. + out.writeInt(keyLen + valueLen + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + out.writeInt(keyLen); + out.writeInt(valueLen); + PrivateCellUtil.writeFlatKey(cell, out); + out.writeLong(cell.getSequenceId()); + } + + @Override + public void readFields(DataInput in) throws IOException { + cell = KeyValue.create(in); + long seqId = in.readLong(); + cell.setSequenceId(seqId); + } + + public static class KeyOnlyCellComparator extends WritableComparator { + + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { + try { + KeyOnlyCellComparable kv1 = new KeyOnlyCellComparable(); + kv1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1))); + KeyOnlyCellComparable kv2 = new KeyOnlyCellComparable(); + kv2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2))); + return compare(kv1, kv2); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java new file mode 100644 index 000000000000..81871ffb59c2 --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.IOException; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.MapReduceExtendedCell; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class PreSortedCellsReducer + extends Reducer { + + @Override + protected void reduce(KeyOnlyCellComparable key, Iterable values, Context context) + throws IOException, InterruptedException { + + int index = 0; + for (Cell cell : values) { + context.write(new ImmutableBytesWritable(CellUtil.cloneRow(key.getCell())), + new MapReduceExtendedCell(cell)); + + if (++index % 100 == 0) { + context.setStatus("Wrote " + index + " cells"); + } + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 189727f81b0f..d82531938faa 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; @@ -54,6 +55,7 @@ import org.apache.hadoop.hbase.util.MapReduceExtendedCell; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -63,6 +65,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; @@ -162,9 +165,10 @@ public void setup(Context context) throws IOException { /** * A mapper that just writes out Cells. This one can be used together with {@link CellSortReducer} */ - static class WALCellMapper extends Mapper { + static class WALCellMapper extends Mapper, Cell> { private Set tableSet = new HashSet<>(); private boolean multiTableSupport = false; + private boolean diskBasedSortingEnabled = false; @Override public void map(WALKey key, WALEdit value, Context context) throws IOException { @@ -185,7 +189,8 @@ public void map(WALKey key, WALEdit value, Context context) throws IOException { byte[] outKey = multiTableSupport ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell)) : CellUtil.cloneRow(cell); - context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell)); + ExtendedCell extendedCell = (ExtendedCell) cell; + context.write(wrapKey(outKey, extendedCell), new MapReduceExtendedCell(extendedCell)); } } } catch (InterruptedException e) { @@ -198,8 +203,23 @@ public void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); String[] tables = conf.getStrings(TABLES_KEY); this.multiTableSupport = conf.getBoolean(MULTI_TABLES_SUPPORT, false); + this.diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); Collections.addAll(tableSet, tables); } + + private WritableComparable wrapKey(byte[] key, ExtendedCell cell) { + if (this.diskBasedSortingEnabled) { + // Important to build a new cell with the updated key to maintain multi-table support + KeyValue kv = new KeyValue(key, 0, key.length, cell.getFamilyArray(), + cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), + cell.getQualifierOffset(), cell.getQualifierLength(), cell.getTimestamp(), + KeyValue.Type.codeToType(cell.getTypeByte()), null, 0, 0); + kv.setSequenceId(cell.getSequenceId()); + return new KeyOnlyCellComparable(kv); + } else { + return new ImmutableBytesWritable(key); + } + } } /** @@ -377,7 +397,13 @@ public Job createSubmittableJob(String[] args) throws IOException { job.setJarByClass(WALPlayer.class); job.setInputFormatClass(WALInputFormat.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); + boolean diskBasedSortingEnabled = HFileOutputFormat2.diskBasedSortingEnabled(conf); + if (diskBasedSortingEnabled) { + job.setMapOutputKeyClass(KeyOnlyCellComparable.class); + job.setSortComparatorClass(KeyOnlyCellComparable.KeyOnlyCellComparator.class); + } else { + job.setMapOutputKeyClass(ImmutableBytesWritable.class); + } String hfileOutPath = conf.get(BULK_OUTPUT_CONF_KEY); if (hfileOutPath != null) { @@ -396,7 +422,11 @@ public Job createSubmittableJob(String[] args) throws IOException { List tableNames = getTableNameList(tables); job.setMapperClass(WALCellMapper.class); - job.setReducerClass(CellSortReducer.class); + if (diskBasedSortingEnabled) { + job.setReducerClass(PreSortedCellsReducer.class); + } else { + job.setReducerClass(CellSortReducer.class); + } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); job.setMapOutputValueClass(MapReduceExtendedCell.class); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java index 283acbabf6e4..d6c4b623ad42 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellBasedWALPlayer2.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; +import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.util.ToolRunner; @@ -172,7 +173,7 @@ private void testWALKeyValueMapper(final String tableConfigKey) throws Exception WALKey key = mock(WALKey.class); when(key.getTableName()).thenReturn(TableName.valueOf("table")); @SuppressWarnings("unchecked") - Mapper.Context context = mock(Context.class); + Mapper, Cell>.Context context = mock(Context.class); when(context.getConfiguration()).thenReturn(configuration); WALEdit value = mock(WALEdit.class); diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index c6e51eee40ff..143043a039de 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -28,7 +28,6 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; @@ -114,6 +113,50 @@ public static void afterClass() throws Exception { logFs.delete(walRootDir, true); } + @Test + public void testDiskBasedSortingEnabled() throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + final byte[] COLUMN2 = Bytes.toBytes("c2"); + final byte[] ROW = Bytes.toBytes("row"); + Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + p.addColumn(FAMILY, COLUMN2, COLUMN2); + t1.put(p); + // delete one column + Delete d = new Delete(ROW); + d.addColumns(FAMILY, COLUMN1); + t1.delete(d); + + // replay the WAL, map table 1 to table 2 + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + + Configuration configuration = TEST_UTIL.getConfiguration(); + configuration.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + WALPlayer player = new WALPlayer(configuration); + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + } + /** * Test that WALPlayer can replay recovered.edits files. */ @@ -123,19 +166,22 @@ public void testPlayingRecoveredEdit() throws Exception { TEST_UTIL.createTable(tn, TestRecoveredEdits.RECOVEREDEDITS_COLUMNFAMILY); // Copy testing recovered.edits file that is over under hbase-server test resources // up into a dir in our little hdfs cluster here. - String hbaseServerTestResourcesEdits = - System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" - + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); - assertTrue(new File(hbaseServerTestResourcesEdits).exists()); - FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); - // Target dir. - Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); - assertTrue(dfs.mkdirs(targetDir)); - dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); - assertEquals(0, - ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); - // I don't know how many edits are in this file for this table... so just check more than 1. - assertTrue(TEST_UTIL.countRows(tn) > 0); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String hbaseServerTestResourcesEdits = + System.getProperty("test.build.classes") + "/../../../hbase-server/src/test/resources/" + + TestRecoveredEdits.RECOVEREDEDITS_PATH.getName(); + assertTrue(new File(hbaseServerTestResourcesEdits).exists()); + FileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); + // Target dir. + Path targetDir = new Path("edits").makeQualified(dfs.getUri(), dfs.getHomeDirectory()); + assertTrue(dfs.mkdirs(targetDir)); + dfs.copyFromLocalFile(new Path(hbaseServerTestResourcesEdits), targetDir); + assertEquals(0, + ToolRunner.run(new WALPlayer(this.conf), new String[] { targetDir.toString() })); + // I don't know how many edits are in this file for this table... so just check more than 1. + assertTrue(TEST_UTIL.countRows(tn) > 0); + dfs.delete(targetDir, true); + }); } /** @@ -150,7 +196,7 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { final byte[] column1 = Bytes.toBytes("c1"); final byte[] column2 = Bytes.toBytes("c2"); final byte[] row = Bytes.toBytes("row"); - Table table = TEST_UTIL.createTable(tableName, family); + final Table table = TEST_UTIL.createTable(tableName, family); long now = EnvironmentEdgeManager.currentTime(); // put a row into the first table @@ -188,28 +234,37 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { configuration.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); WALPlayer player = new WALPlayer(configuration); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName.getNameAsString() })); + final byte[] finalLastVal = lastVal; + + runWithDiskBasedSortingDisabledAndEnabled(() -> { + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName.getNameAsString() })); - Get g = new Get(row); - Result result = table.get(g); - byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + Get g = new Get(row); + Result result = table.get(g); + byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); - table = TEST_UTIL.truncateTable(tableName); - g = new Get(row); - result = table.get(g); - assertThat(result.listCells(), nullValue()); + TEST_UTIL.truncateTable(tableName); + g = new Get(row); + result = table.get(g); + assertThat(result.listCells(), nullValue()); - BulkLoadHFiles.create(configuration).bulkLoad(tableName, - new Path(outPath, tableName.getNameAsString())); + BulkLoadHFiles.create(configuration).bulkLoad(tableName, + new Path(outPath, tableName.getNamespaceAsString() + "/" + tableName.getNameAsString())); - g = new Get(row); - result = table.get(g); - value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); + g = new Get(row); + result = table.get(g); + value = CellUtil.cloneValue(result.getColumnLatestCell(family, column1)); - assertThat(result.listCells(), notNullValue()); - assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(lastVal))); + assertThat(result.listCells(), notNullValue()); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); + + // cleanup + Path out = new Path(outPath); + FileSystem fs = out.getFileSystem(configuration); + assertTrue(fs.delete(out, true)); + }); } /** @@ -244,18 +299,21 @@ public void testWALPlayer() throws Exception { Configuration configuration = TEST_UTIL.getConfiguration(); WALPlayer player = new WALPlayer(configuration); - String optionName = "_test_.name"; - configuration.set(optionName, "1000"); - player.setupTime(configuration, optionName); - assertEquals(1000, configuration.getLong(optionName, 0)); - assertEquals(0, ToolRunner.run(configuration, player, - new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); - // verify the WAL was player into table 2 - Get g = new Get(ROW); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + runWithDiskBasedSortingDisabledAndEnabled(() -> { + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + // verify the WAL was player into table 2 + Get g = new Get(ROW); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); + }); } /** @@ -335,7 +393,29 @@ public void testMainMethod() throws Exception { System.setErr(oldPrintStream); System.setSecurityManager(SECURITY_MANAGER); } + } + + private static void runWithDiskBasedSortingDisabledAndEnabled(TestMethod method) + throws Exception { + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + false); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + + TEST_UTIL.getConfiguration().setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, + true); + try { + method.run(); + } finally { + TEST_UTIL.getConfiguration().unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + } + private interface TestMethod { + void run() throws Exception; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java index 5e95564b6fee..cf3f241cab89 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestUnattainableBalancerCostGoal.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; @@ -33,7 +32,6 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -41,6 +39,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + /** * If your minCostNeedsBalance is set too low, then the balancer should still eventually stop making * moves as further cost improvements become impossible, and balancer plan calculation becomes From 1e4070c3d5d1536518dad681436a2dd76b08e924 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 15 Sep 2025 18:39:15 +0900 Subject: [PATCH 28/78] HubSpot Backport: HBASE-29577: Fix NPE from RegionServerRpcQuotaManager when reloading configuration (will be in 2.6.4) Signed-off-by: Wellington Chevreuil Signed-off-by: Charles Connell --- .../hadoop/hbase/quotas/RegionServerRpcQuotaManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 958793dcdf00..7a42d0f1aa31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -91,7 +91,9 @@ public void stop() { } public void reload() { - quotaCache.forceSynchronousCacheRefresh(); + if (isQuotaEnabled()) { + quotaCache.forceSynchronousCacheRefresh(); + } } @Override From 07c632f82f40c86edd6c0ee49d13d4d48cb81280 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Wed, 17 Sep 2025 16:00:12 -0400 Subject: [PATCH 29/78] HubSpot Backport: Thread safety in QuotaRefresherChore (not yet started upstream) --- .../hadoop/hbase/quotas/QuotaCache.java | 79 ++++++++++--------- .../hadoop/hbase/quotas/TestQuotaCache2.java | 46 +++++++++++ 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 16681eb45f8f..0238756251ab 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.time.Duration; import java.util.EnumSet; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -70,10 +69,10 @@ public class QuotaCache implements Stoppable { private final Object initializerLock = new Object(); private volatile boolean initialized = false; - private volatile Map namespaceQuotaCache = new HashMap<>(); - private volatile Map tableQuotaCache = new HashMap<>(); - private volatile Map userQuotaCache = new HashMap<>(); - private volatile Map regionServerQuotaCache = new HashMap<>(); + private volatile Map namespaceQuotaCache = new ConcurrentHashMap<>(); + private volatile Map tableQuotaCache = new ConcurrentHashMap<>(); + private volatile Map userQuotaCache = new ConcurrentHashMap<>(); + private volatile Map regionServerQuotaCache = new ConcurrentHashMap<>(); private volatile boolean exceedThrottleQuotaEnabled = false; // factors used to divide cluster scope quota into machine scope quota @@ -307,44 +306,48 @@ public synchronized boolean triggerNow() { @Override protected void chore() { - updateQuotaFactors(); + synchronized (this) { + LOG.info("Reloading quota cache from hbase:quota table"); + updateQuotaFactors(); + + try { + Map newUserQuotaCache = + new ConcurrentHashMap<>(fetchUserQuotaStateEntries()); + updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); + userQuotaCache = newUserQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching user quotas", e); + } - try { - Map newUserQuotaCache = new HashMap<>(fetchUserQuotaStateEntries()); - updateNewCacheFromOld(userQuotaCache, newUserQuotaCache); - userQuotaCache = newUserQuotaCache; - } catch (IOException e) { - LOG.error("Error while fetching user quotas", e); - } + try { + Map newRegionServerQuotaCache = + new ConcurrentHashMap<>(fetchRegionServerQuotaStateEntries()); + updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); + regionServerQuotaCache = newRegionServerQuotaCache; + } catch (IOException e) { + LOG.error("Error while fetching region server quotas", e); + } - try { - Map newRegionServerQuotaCache = - new HashMap<>(fetchRegionServerQuotaStateEntries()); - updateNewCacheFromOld(regionServerQuotaCache, newRegionServerQuotaCache); - regionServerQuotaCache = newRegionServerQuotaCache; - } catch (IOException e) { - LOG.error("Error while fetching region server quotas", e); - } + try { + Map newTableQuotaCache = + new ConcurrentHashMap<>(fetchTableQuotaStateEntries()); + updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); + tableQuotaCache = newTableQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing table quotas", e); + } - try { - Map newTableQuotaCache = - new HashMap<>(fetchTableQuotaStateEntries()); - updateNewCacheFromOld(tableQuotaCache, newTableQuotaCache); - tableQuotaCache = newTableQuotaCache; - } catch (IOException e) { - LOG.error("Error while refreshing table quotas", e); - } + try { + Map newNamespaceQuotaCache = + new ConcurrentHashMap<>(fetchNamespaceQuotaStateEntries()); + updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); + namespaceQuotaCache = newNamespaceQuotaCache; + } catch (IOException e) { + LOG.error("Error while refreshing namespace quotas", e); + } - try { - Map newNamespaceQuotaCache = - new HashMap<>(fetchNamespaceQuotaStateEntries()); - updateNewCacheFromOld(namespaceQuotaCache, newNamespaceQuotaCache); - namespaceQuotaCache = newNamespaceQuotaCache; - } catch (IOException e) { - LOG.error("Error while refreshing namespace quotas", e); + fetchExceedThrottleQuota(); } - - fetchExceedThrottleQuota(); } private void fetchExceedThrottleQuota() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java index 2c33b265771a..cd55ecd6fed8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -127,4 +127,50 @@ public void testForgetsDeletedQuota() { assertTrue(newCache.containsKey("my_table2")); assertFalse(newCache.containsKey("my_table1")); } + + @Test + public void testLearnsNewQuota() { + Map oldCache = new HashMap<>(); + + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table1", newState); + + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + assertTrue(newCache.containsKey("my_table1")); + } + + @Test + public void testUserSpecificOverridesDefaultNewQuota() { + // establish old cache with a limiter for 100 read bytes per second + QuotaState oldState = new QuotaState(); + Map oldCache = new HashMap<>(); + oldCache.put("my_table", oldState); + QuotaProtos.Throttle throttle1 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + oldState.setGlobalLimiter(limiter1); + + // establish new cache, with a limiter for 999 read bytes per second + QuotaState newState = new QuotaState(); + Map newCache = new HashMap<>(); + newCache.put("my_table", newState); + QuotaProtos.Throttle throttle2 = QuotaProtos.Throttle.newBuilder() + .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) + .setSoftLimit(999).setScope(QuotaProtos.QuotaScope.MACHINE).build()) + .build(); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + newState.setGlobalLimiter(limiter2); + + // update new cache from old cache + QuotaCache.updateNewCacheFromOld(oldCache, newCache); + + // verify that the 999 available bytes from the limiter was carried over + TimeBasedLimiter updatedLimiter = + (TimeBasedLimiter) newCache.get("my_table").getGlobalLimiter(); + assertEquals(999, updatedLimiter.getReadAvailable()); + } } From cd6a84bef374991ed2bd30a694a9d62849336c41 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 19 Sep 2025 11:16:42 -0400 Subject: [PATCH 30/78] HubSpot Edit: Exclude org.jspecify:jspecify (can remove when hbase-thirdparty is upgraded) --- hubspot-client-bundles/hbase-client-bundle/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hubspot-client-bundles/hbase-client-bundle/pom.xml b/hubspot-client-bundles/hbase-client-bundle/pom.xml index 24ce44daf93a..159949c1cd16 100644 --- a/hubspot-client-bundles/hbase-client-bundle/pom.xml +++ b/hubspot-client-bundles/hbase-client-bundle/pom.xml @@ -115,6 +115,8 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA + + org/jspecify/** From 2c50a4ad15f2ac158e264df1341196e157ac3dfa Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 23 Sep 2025 11:42:14 -0400 Subject: [PATCH 31/78] Incremental backups fail on archived bulkloaded HFiles (not yet upstream) (#201) --- .../impl/IncrementalTableBackupClient.java | 19 +++++++++++++++++-- .../hbase/mapreduce/HFileOutputFormat2.java | 1 + .../hadoop/hbase/mapreduce/WALPlayer.java | 1 - .../hadoop/hbase/mapreduce/TestWALPlayer.java | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 5f86f2a57b88..c5471a56cb47 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -249,9 +249,24 @@ private void updateFileLists(List activeFiles, List archiveFiles } } - if (newlyArchived.size() > 0) { + if (!newlyArchived.isEmpty()) { + String rootDir = CommonFSUtils.getRootDir(conf).toString(); + activeFiles.removeAll(newlyArchived); - archiveFiles.addAll(newlyArchived); + for (String file : newlyArchived) { + String archivedFile = file.substring(rootDir.length() + 1); + Path archivedFilePath = new Path(HFileArchiveUtil.getArchivePath(conf), archivedFile); + archivedFile = archivedFilePath.toString(); + + if (!fs.exists(archivedFilePath)) { + throw new IOException( + String.format("File %s not longer exists, and no archived file %s exists for it", file, + archivedFile)); + } + + LOG.debug("Archived file {} has been updated", archivedFile); + archiveFiles.add(archivedFile); + } } LOG.debug(newlyArchived.size() + " files have been archived."); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 7c297015cb03..9011e3b56b1e 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; + import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index d82531938faa..dc84debf49aa 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -65,7 +65,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index 143043a039de..57f4c9cf0095 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; From d738cf9c87ee9ca93731a81696919bd0691cd869 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 23 Sep 2025 14:21:23 -0400 Subject: [PATCH 32/78] Incremental backups fail on archived bulkloaded HFiles FIX (not yet upstream) (#202) --- .../hadoop/hbase/backup/impl/IncrementalTableBackupClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index c5471a56cb47..a47e62b4fa6f 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -197,6 +197,9 @@ private void mergeSplitAndCopyBulkloadedHFiles(List activeFiles, int numActiveFiles = activeFiles.size(); updateFileLists(activeFiles, archiveFiles); if (activeFiles.size() < numActiveFiles) { + // We've archived some files, delete bulkloads directory + // and re-try + deleteBulkLoadDirectory(); continue; } From 0bc74284d5b86bae20bc7f965678a64e246c72b2 Mon Sep 17 00:00:00 2001 From: Jared Stehler Date: Tue, 30 Sep 2025 13:07:15 -0400 Subject: [PATCH 33/78] use offstack buildpack for modules (#200) --- .blazar.yaml | 2 +- hubspot-client-bundles/.blazar.yaml | 7 +++-- .../hbase-backup-restore-bundle/.blazar.yaml | 26 ------------------- .../hbase-client-bundle/.blazar.yaml | 26 ------------------- .../hbase-mapreduce-bundle/.blazar.yaml | 26 ------------------- .../hbase-server-it-bundle/.blazar.yaml | 26 ------------------- pom.xml | 9 +++++++ 7 files changed, 15 insertions(+), 107 deletions(-) delete mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml delete mode 100644 hubspot-client-bundles/hbase-client-bundle/.blazar.yaml delete mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml delete mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml diff --git a/.blazar.yaml b/.blazar.yaml index e034ada7508d..4335a8f33d93 100644 --- a/.blazar.yaml +++ b/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java-single-module + name: Blazar-Buildpack-Java-oss-fork env: MAVEN_PHASE: "package assembly:single deploy" diff --git a/hubspot-client-bundles/.blazar.yaml b/hubspot-client-bundles/.blazar.yaml index 8d5dac3de18f..3e6be6dedcf8 100644 --- a/hubspot-client-bundles/.blazar.yaml +++ b/hubspot-client-bundles/.blazar.yaml @@ -1,6 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java - branch: rm-test-hbase + name: Blazar-Buildpack-Java-oss-fork env: # Below variables are generated in prepare_environment.sh. @@ -23,3 +22,7 @@ depends: - hbase provides: - hubspot-client-bundles + - hbase-backup-restore-bundle + - hbase-client-bundle + - hbase-mapreduce-bundle + - hbase-server-it-bundle diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml deleted file mode 100644 index 509a0cc10fab..000000000000 --- a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml +++ /dev/null @@ -1,26 +0,0 @@ -buildpack: - name: Blazar-Buildpack-Java - branch: rm-test-hbase -env: - # Below variables are generated in prepare_environment.sh. - # The build environment requires environment variables to be explicitly defined before they may - # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable - # throughout a build - REPO_NAME: "" - SET_VERSION: "" - HBASE_VERSION: "" - PKG_RELEASE: "" - FULL_BUILD_VERSION: "" - MAVEN_BUILD_ARGS: "" - -before: - - description: "Prepare build environment" - commands: - - $WORKSPACE/build-scripts/prepare_environment.sh - -depends: - - hubspot-client-bundles - - hbase-client-bundle - - hbase-mapreduce-bundle -provides: - - hbase-backup-restore-bundle diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml deleted file mode 100644 index aba96b1c7dd9..000000000000 --- a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml +++ /dev/null @@ -1,26 +0,0 @@ -buildpack: - name: Blazar-Buildpack-Java - branch: rm-test-hbase - -env: - # Below variables are generated in prepare_environment.sh. - # The build environment requires environment variables to be explicitly defined before they may - # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable - # throughout a build - REPO_NAME: "" - SET_VERSION: "" - HBASE_VERSION: "" - PKG_RELEASE: "" - FULL_BUILD_VERSION: "" - MAVEN_BUILD_ARGS: "" - -before: - - description: "Prepare build environment" - commands: - - $WORKSPACE/build-scripts/prepare_environment.sh - -depends: - - hubspot-client-bundles -provides: - - hbase-client-bundle - diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml deleted file mode 100644 index c79dbaaf6044..000000000000 --- a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml +++ /dev/null @@ -1,26 +0,0 @@ -buildpack: - name: Blazar-Buildpack-Java - branch: rm-test-hbase - -env: - # Below variables are generated in prepare_environment.sh. - # The build environment requires environment variables to be explicitly defined before they may - # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable - # throughout a build - REPO_NAME: "" - SET_VERSION: "" - HBASE_VERSION: "" - PKG_RELEASE: "" - FULL_BUILD_VERSION: "" - MAVEN_BUILD_ARGS: "" - -before: - - description: "Prepare build environment" - commands: - - $WORKSPACE/build-scripts/prepare_environment.sh - -depends: - - hubspot-client-bundles - - hbase-client-bundle -provides: - - hbase-mapreduce-bundle diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml deleted file mode 100644 index 03fe644bf878..000000000000 --- a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml +++ /dev/null @@ -1,26 +0,0 @@ -buildpack: - name: Blazar-Buildpack-Java - branch: rm-test-hbase -env: - # Below variables are generated in prepare_environment.sh. - # The build environment requires environment variables to be explicitly defined before they may - # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable - # throughout a build - YUM_REPO_UPLOAD_OVERRIDE_CENTOS_8: "" - SET_VERSION: "" - HBASE_VERSION: "" - PKG_RELEASE: "" - FULL_BUILD_VERSION: "" - MAVEN_BUILD_ARGS: "" - REPO_NAME: "" - -before: - - description: "Prepare build environment" - commands: - - $WORKSPACE/build-scripts/prepare_environment.sh - -depends: - - hbase -provides: - - hbase-server-it-bundle - diff --git a/pom.xml b/pom.xml index d6ee14697144..28f9facfab35 100644 --- a/pom.xml +++ b/pom.xml @@ -1536,6 +1536,15 @@ org.apache.hbase.thirdparty hbase-shaded-miscellaneous ${hbase-thirdparty.version} + + + + org.jspecify + jspecify + + org.apache.hbase.thirdparty From 6c171c3bb05735819504dbf8d48f382716f3d068 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 30 Sep 2025 15:06:47 -0400 Subject: [PATCH 34/78] SnapshotRegionLocator should filter out offline regions and split regions (not yet upstream) (#204) --- .../apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java index 5c90dd900f77..c3a42e45de42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java @@ -75,6 +75,10 @@ public static SnapshotRegionLocator create(Configuration conf, TableName table) HBaseProtos.RegionInfo ri = region.getRegionInfo(); byte[] key = ri.getStartKey().toByteArray(); + if (ri.getOffline() || ri.getSplit()) { + continue; + } + SnapshotHRegionLocation location = toLocation(ri, tableName); rawLocations.add(location); HRegionReplicas hrr = replicas.get(key); From 4feb44325447a6c7e0475f53c9be26c1c663b999 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 30 Sep 2025 16:00:15 -0400 Subject: [PATCH 35/78] Revert "use offstack buildpack for modules (#200)" (#205) This reverts commit 70f6120227f9050c8b3cb7c6bb33a768264cf5c4. --- .blazar.yaml | 2 +- hubspot-client-bundles/.blazar.yaml | 7 ++--- .../hbase-backup-restore-bundle/.blazar.yaml | 26 +++++++++++++++++++ .../hbase-client-bundle/.blazar.yaml | 26 +++++++++++++++++++ .../hbase-mapreduce-bundle/.blazar.yaml | 26 +++++++++++++++++++ .../hbase-server-it-bundle/.blazar.yaml | 26 +++++++++++++++++++ pom.xml | 9 ------- 7 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-client-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml diff --git a/.blazar.yaml b/.blazar.yaml index 4335a8f33d93..e034ada7508d 100644 --- a/.blazar.yaml +++ b/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java-oss-fork + name: Blazar-Buildpack-Java-single-module env: MAVEN_PHASE: "package assembly:single deploy" diff --git a/hubspot-client-bundles/.blazar.yaml b/hubspot-client-bundles/.blazar.yaml index 3e6be6dedcf8..8d5dac3de18f 100644 --- a/hubspot-client-bundles/.blazar.yaml +++ b/hubspot-client-bundles/.blazar.yaml @@ -1,5 +1,6 @@ buildpack: - name: Blazar-Buildpack-Java-oss-fork + name: Blazar-Buildpack-Java + branch: rm-test-hbase env: # Below variables are generated in prepare_environment.sh. @@ -22,7 +23,3 @@ depends: - hbase provides: - hubspot-client-bundles - - hbase-backup-restore-bundle - - hbase-client-bundle - - hbase-mapreduce-bundle - - hbase-server-it-bundle diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml new file mode 100644 index 000000000000..509a0cc10fab --- /dev/null +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + branch: rm-test-hbase +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle + - hbase-mapreduce-bundle +provides: + - hbase-backup-restore-bundle diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml new file mode 100644 index 000000000000..aba96b1c7dd9 --- /dev/null +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + branch: rm-test-hbase + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles +provides: + - hbase-client-bundle + diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml new file mode 100644 index 000000000000..c79dbaaf6044 --- /dev/null +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + branch: rm-test-hbase + +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles + - hbase-client-bundle +provides: + - hbase-mapreduce-bundle diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml new file mode 100644 index 000000000000..03fe644bf878 --- /dev/null +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java + branch: rm-test-hbase +env: + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + YUM_REPO_UPLOAD_OVERRIDE_CENTOS_8: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + REPO_NAME: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hbase +provides: + - hbase-server-it-bundle + diff --git a/pom.xml b/pom.xml index 28f9facfab35..d6ee14697144 100644 --- a/pom.xml +++ b/pom.xml @@ -1536,15 +1536,6 @@ org.apache.hbase.thirdparty hbase-shaded-miscellaneous ${hbase-thirdparty.version} - - - - org.jspecify - jspecify - - org.apache.hbase.thirdparty From f0c5cb99e0ca0aeee9f2e0a6bc9b3f1edfb47f64 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Tue, 7 Oct 2025 13:13:11 -0400 Subject: [PATCH 36/78] HubSpot Backport: HBASE-29448: Modern backup failures can cause backup system to lock up (#206) Co-authored-by: Ray Mattingly --- .../hbase/backup/impl/BackupSystemTable.java | 4 +- .../master/TestRestoreBackupSystemTable.java | 84 +++++++++ .../org/apache/hadoop/hbase/client/Admin.java | 11 ++ .../hadoop/hbase/client/AsyncAdmin.java | 3 + .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 + .../client/ConnectionImplementation.java | 8 + .../hadoop/hbase/client/HBaseAdmin.java | 16 ++ .../hbase/client/RawAsyncHBaseAdmin.java | 25 +++ .../client/ShortCircuitMasterConnection.java | 7 + .../src/main/protobuf/Master.proto | 10 + .../src/main/protobuf/MasterProcedure.proto | 7 + .../hbase/master/MasterRpcServices.java | 19 ++ .../RestoreBackupSystemTableProcedure.java | 171 ++++++++++++++++++ .../procedure/TableProcedureInterface.java | 3 +- .../hbase/master/procedure/TableQueue.java | 1 + .../hbase/thrift2/client/ThriftAdmin.java | 5 + 16 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreBackupSystemTableProcedure.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index c2253a46d04b..0ece064d2355 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -1403,9 +1403,7 @@ public static void restoreFromSnapshot(Connection conn) throws IOException { try (Admin admin = conn.getAdmin()) { String snapshotName = BackupSystemTable.getSnapshotName(conf); if (snapshotExists(admin, snapshotName)) { - admin.disableTable(BackupSystemTable.getTableName(conf)); - admin.restoreSnapshot(snapshotName); - admin.enableTable(BackupSystemTable.getTableName(conf)); + admin.restoreBackupSystemTable(snapshotName); LOG.debug("Done restoring backup system table"); } else { // Snapshot does not exists, i.e completeBackup failed after diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java new file mode 100644 index 000000000000..6ea984f72bf2 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestRestoreBackupSystemTable.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.master; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestRestoreBackupSystemTable { + private static final String BACKUP_ROOT = "root"; + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.startMiniCluster(); + } + + @Test + public void itRestoresFromSnapshot() throws Exception { + BackupSystemTable table = new BackupSystemTable(UTIL.getConnection()); + Set tables = new HashSet<>(); + + tables.add(TableName.valueOf("test1")); + tables.add(TableName.valueOf("test2")); + tables.add(TableName.valueOf("test3")); + + Map rsTimestampMap = new HashMap<>(); + rsTimestampMap.put("rs1:100", 100L); + rsTimestampMap.put("rs2:100", 101L); + rsTimestampMap.put("rs3:100", 103L); + + table.writeRegionServerLogTimestamp(tables, rsTimestampMap, BACKUP_ROOT); + BackupSystemTable.snapshot(UTIL.getConnection()); + + Admin admin = UTIL.getAdmin(); + TableName backupSystemTn = BackupSystemTable.getTableName(UTIL.getConfiguration()); + admin.disableTable(backupSystemTn); + admin.truncateTable(backupSystemTn, true); + + BackupSystemTable.restoreFromSnapshot(UTIL.getConnection()); + Map> results = table.readLogTimestampMap(BACKUP_ROOT); + + assertEquals(results.size(), tables.size()); + + for (TableName tableName : tables) { + Map resultMap = results.get(tableName); + assertEquals(resultMap, rsTimestampMap); + } + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index c341cf4a9895..cefb84d5a52a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -3356,4 +3356,15 @@ List getLogEntries(Set serverNames, String logType, Server * Get the list of cached files */ List getCachedFilesList(ServerName serverName) throws IOException; + + @InterfaceAudience.Private + default void restoreBackupSystemTable(String snapshotName) throws IOException { + SnapshotDescription snapshot = + listSnapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst() + .orElseThrow(() -> new IOException("Snapshot " + snapshotName + " not found")); + TableName tn = snapshot.getTableName(); + disableTable(tn); + restoreSnapshot(snapshotName); + enableTable(tn); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index ae651c3e0074..ea0bcb7a6d77 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1721,4 +1721,7 @@ CompletableFuture> getLogEntries(Set serverNames, Str * Get the list of cached files */ CompletableFuture> getCachedFilesList(ServerName serverName); + + @InterfaceAudience.Private + CompletableFuture restoreBackupSystemTable(String snapshotName); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 448cfc9c36ee..650f80470ea0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -921,4 +921,9 @@ public CompletableFuture flushMasterStore() { public CompletableFuture> getCachedFilesList(ServerName serverName) { return wrap(rawAdmin.getCachedFilesList(serverName)); } + + @Override + public CompletableFuture restoreBackupSystemTable(String snapshotName) { + return wrap(rawAdmin.restoreBackupSystemTable(snapshotName)); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 28b26ba648b2..71e7dbf06c66 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -2082,6 +2082,14 @@ public FlushMasterStoreResponse flushMasterStore(RpcController controller, return stub.flushMasterStore(controller, request); } + @Override + public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable( + RpcController rpcController, + MasterProtos.RestoreBackupSystemTableRequest restoreBackupSystemTableRequest) + throws ServiceException { + return stub.restoreBackupSystemTable(rpcController, restoreBackupSystemTableRequest); + } + @Override public ReplicationPeerModificationSwitchResponse replicationPeerModificationSwitch( RpcController controller, ReplicationPeerModificationSwitchRequest request) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index d8b9c9dd9a68..7963f1cf6684 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -2095,6 +2095,22 @@ public List getCachedFilesList(ServerName serverName) throws IOException this.connection.getAdmin(serverName)); } + @Override + public void restoreBackupSystemTable(String snapshotName) throws IOException { + long pid = + executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { + @Override + protected Long rpcCall() throws Exception { + return master.restoreBackupSystemTable(getRpcController(), + MasterProtos.RestoreBackupSystemTableRequest.newBuilder().setSnapshotName(snapshotName) + .build()) + .getProcId(); + } + }); + ProcedureFuture future = new ProcedureFuture<>(this, pid); + get(future, getProcedureTimeout, TimeUnit.MILLISECONDS); + } + private MasterCallable getTruncateRegionCallable(TableName tableName, RegionInfo hri) { return new MasterCallable(getConnection(), diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index bd01b2d247b1..8000e3ad396a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -2716,6 +2716,19 @@ void onError(Throwable error) { } } + private static class RestoreBackupSystemTableProcedureBiConsumer extends ProcedureBiConsumer { + + @Override + void onFinished() { + LOG.info("RestoreBackupSystemTableProcedure completed"); + } + + @Override + void onError(Throwable error) { + LOG.info("RestoreBackupSystemTableProcedure failed with {}", error.getMessage()); + } + } + private static class CreateTableProcedureBiConsumer extends TableProcedureBiConsumer { CreateTableProcedureBiConsumer(TableName tableName) { @@ -4299,4 +4312,16 @@ List> adminCall(controller, stub, request.build(), resp -> resp.getCachedFilesList())) .serverName(serverName).call(); } + + @Override + public CompletableFuture restoreBackupSystemTable(String snapshotName) { + MasterProtos.RestoreBackupSystemTableRequest request = + MasterProtos.RestoreBackupSystemTableRequest.newBuilder().setSnapshotName(snapshotName) + .build(); + return this. procedureCall(request, + MasterService.Interface::restoreBackupSystemTable, + MasterProtos.RestoreBackupSystemTableResponse::getProcId, + new RestoreBackupSystemTableProcedureBiConsumer()); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java index 57f3cba86053..072572c57ebb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java @@ -796,6 +796,13 @@ public FlushMasterStoreResponse flushMasterStore(RpcController controller, return stub.flushMasterStore(controller, request); } + @Override + public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable( + RpcController controller, MasterProtos.RestoreBackupSystemTableRequest request) + throws ServiceException { + return stub.restoreBackupSystemTable(controller, request); + } + @Override public ReplicationPeerModificationSwitchResponse replicationPeerModificationSwitch( RpcController controller, ReplicationPeerModificationSwitchRequest request) diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index a70587d34c39..e0182506dd12 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -1228,6 +1228,9 @@ service MasterService { rpc FlushMasterStore(FlushMasterStoreRequest) returns(FlushMasterStoreResponse); + + rpc RestoreBackupSystemTable(RestoreBackupSystemTableRequest) + returns(RestoreBackupSystemTableResponse); } // HBCK Service definitions. @@ -1313,6 +1316,13 @@ message FixMetaRequest {} message FixMetaResponse {} +message RestoreBackupSystemTableRequest { + required string snapshot_name = 1; +} +message RestoreBackupSystemTableResponse { + optional uint64 proc_id = 1; +} + service HbckService { /** Update state of the table in meta only*/ rpc SetTableStateInMeta(SetTableStateInMetaRequest) diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 024cb7b8b002..8e5d302142ad 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -730,3 +730,10 @@ message ReloadQuotasProcedureStateData { required ServerName target_server = 1; optional ForeignExceptionMessage error = 2; } + +enum RestoreBackupSystemTableState { + RESTORE_BACKUP_SYSTEM_TABLE_PREPARE = 1; + RESTORE_BACKUP_SYSTEM_TABLE_DISABLE = 2; + RESTORE_BACKUP_SYSTEM_TABLE_RESTORE = 3; + RESTORE_BACKUP_SYSTEM_TABLE_ENABLE = 4; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 194cdcca65b2..9ae3f4550e95 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -84,6 +84,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil; import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.NonceProcedureRunnable; +import org.apache.hadoop.hbase.master.procedure.RestoreBackupSystemTableProcedure; import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; import org.apache.hadoop.hbase.master.replication.AbstractPeerProcedure; import org.apache.hadoop.hbase.mob.MobUtils; @@ -3168,4 +3169,22 @@ public FlushTableResponse flushTable(RpcController controller, FlushTableRequest throw new ServiceException(ioe); } } + + @Override + public MasterProtos.RestoreBackupSystemTableResponse restoreBackupSystemTable( + RpcController rpcController, + MasterProtos.RestoreBackupSystemTableRequest restoreBackupSystemTableRequest) + throws ServiceException { + try { + String snapshotName = restoreBackupSystemTableRequest.getSnapshotName(); + SnapshotDescription snapshot = master.snapshotManager.getCompletedSnapshots().stream() + .filter(s -> s.getName().equals(snapshotName)).findFirst().orElseThrow( + () -> new ServiceException(String.format("Snapshot %s not found", snapshotName))); + long pid = master.getMasterProcedureExecutor() + .submitProcedure(new RestoreBackupSystemTableProcedure(snapshot)); + return MasterProtos.RestoreBackupSystemTableResponse.newBuilder().setProcId(pid).build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreBackupSystemTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreBackupSystemTableProcedure.java new file mode 100644 index 000000000000..3a204d42a2c8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreBackupSystemTableProcedure.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException; +import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; +import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreBackupSystemTableState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; + +@InterfaceAudience.Private +public class RestoreBackupSystemTableProcedure + extends AbstractStateMachineTableProcedure { + private static final Logger LOG = + LoggerFactory.getLogger(RestoreBackupSystemTableProcedure.class); + + private final SnapshotDescription snapshot; + private boolean enableOnRollback = false; + + // Necessary for the procedure framework. Do not remove. + public RestoreBackupSystemTableProcedure() { + this(null); + } + + public RestoreBackupSystemTableProcedure(SnapshotDescription snapshot) { + this.snapshot = snapshot; + } + + @Override + public TableName getTableName() { + return TableName.valueOf(snapshot.getTable()); + } + + @Override + public TableOperationType getTableOperationType() { + return TableOperationType.RESTORE_BACKUP_SYSTEM_TABLE; + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, RestoreBackupSystemTableState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + LOG.info("{} execute state={}", this, state); + + try { + switch (state) { + case RESTORE_BACKUP_SYSTEM_TABLE_PREPARE: + prepare(env); + return moreState(RestoreBackupSystemTableState.RESTORE_BACKUP_SYSTEM_TABLE_DISABLE); + case RESTORE_BACKUP_SYSTEM_TABLE_DISABLE: + TableState tableState = + env.getMasterServices().getTableStateManager().getTableState(getTableName()); + if (tableState.isEnabled()) { + addChildProcedure(createDisableTableProcedure(env)); + } + return moreState(RestoreBackupSystemTableState.RESTORE_BACKUP_SYSTEM_TABLE_RESTORE); + case RESTORE_BACKUP_SYSTEM_TABLE_RESTORE: + addChildProcedure(createRestoreSnapshotProcedure(env)); + return moreState(RestoreBackupSystemTableState.RESTORE_BACKUP_SYSTEM_TABLE_ENABLE); + case RESTORE_BACKUP_SYSTEM_TABLE_ENABLE: + addChildProcedure(createEnableTableProcedure(env)); + return Flow.NO_MORE_STATE; + default: + throw new UnsupportedOperationException("unhandled state=" + state); + } + } catch (Exception e) { + setFailure("restore-backup-system-table", e); + LOG.warn("unexpected exception while execute {}. Mark procedure Failed.", this, e); + return Flow.NO_MORE_STATE; + } + } + + @Override + protected void rollbackState(MasterProcedureEnv env, RestoreBackupSystemTableState state) + throws IOException, InterruptedException { + switch (state) { + case RESTORE_BACKUP_SYSTEM_TABLE_DISABLE: + case RESTORE_BACKUP_SYSTEM_TABLE_PREPARE: + return; + case RESTORE_BACKUP_SYSTEM_TABLE_RESTORE: + case RESTORE_BACKUP_SYSTEM_TABLE_ENABLE: + if (enableOnRollback) { + addChildProcedure(createEnableTableProcedure(env)); + } + return; + default: + throw new UnsupportedOperationException("unhandled state=" + state); + } + } + + @Override + protected RestoreBackupSystemTableState getState(int stateId) { + return RestoreBackupSystemTableState.forNumber(stateId); + } + + @Override + protected int getStateId(RestoreBackupSystemTableState state) { + return state.getNumber(); + } + + @Override + protected RestoreBackupSystemTableState getInitialState() { + return RestoreBackupSystemTableState.RESTORE_BACKUP_SYSTEM_TABLE_PREPARE; + } + + private Flow moreState(RestoreBackupSystemTableState next) { + setNextState(next); + return Flow.HAS_MORE_STATE; + } + + private Procedure[] createDisableTableProcedure(MasterProcedureEnv env) + throws HBaseIOException { + DisableTableProcedure disableTableProcedure = + new DisableTableProcedure(env, getTableName(), true); + return new DisableTableProcedure[] { disableTableProcedure }; + } + + private Procedure[] createEnableTableProcedure(MasterProcedureEnv env) { + EnableTableProcedure enableTableProcedure = new EnableTableProcedure(env, getTableName()); + return new EnableTableProcedure[] { enableTableProcedure }; + } + + private Procedure[] createRestoreSnapshotProcedure(MasterProcedureEnv env) + throws IOException { + TableDescriptor desc = env.getMasterServices().getTableDescriptors().get(getTableName()); + RestoreSnapshotProcedure restoreSnapshotProcedure = + new RestoreSnapshotProcedure(env, desc, snapshot); + return new RestoreSnapshotProcedure[] { restoreSnapshotProcedure }; + } + + private void prepare(MasterProcedureEnv env) throws IOException { + List snapshots = + env.getMasterServices().getSnapshotManager().getCompletedSnapshots(); + boolean exists = snapshots.stream().anyMatch(s -> s.getName().equals(snapshot.getName())); + if (!exists) { + throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot)); + } + + TableState tableState = + env.getMasterServices().getTableStateManager().getTableState(getTableName()); + if (tableState.isEnabled()) { + enableOnRollback = true; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java index 1018535a1734..c12e57a2a639 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java @@ -43,7 +43,8 @@ public enum TableOperationType { REGION_UNASSIGN, REGION_GC, MERGED_REGIONS_GC/* region operations */, - REGION_TRUNCATE + REGION_TRUNCATE, + RESTORE_BACKUP_SYSTEM_TABLE } /** Returns the name of the table the procedure is operating on */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java index 8fd44079e11a..5be78a513a66 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java @@ -54,6 +54,7 @@ static boolean requireTableExclusiveLock(TableProcedureInterface proc) { case DISABLE: case SNAPSHOT: case ENABLE: + case RESTORE_BACKUP_SYSTEM_TABLE: return true; case EDIT: // we allow concurrent edit on the NS table diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 94da27933cf8..83e3c5402b3e 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -1505,6 +1505,11 @@ public Future modifyTableStoreFileTrackerAsync(TableName tableName, String "modifyTableStoreFileTrackerAsync not supported in ThriftAdmin"); } + @Override + public void restoreBackupSystemTable(String snapshotName) throws IOException { + throw new NotImplementedException("restoreBackupSystemTable not supported in ThriftAdmin"); + } + @Override public boolean replicationPeerModificationSwitch(boolean on, boolean drainProcedures) throws IOException { From 08cf765141fa36013680277497b53744422f5814 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Wed, 8 Oct 2025 10:56:16 -0700 Subject: [PATCH 37/78] HubSpot Backport HBASE-29629 Record the quota user name value on metrics for RpcThrottlingExceptions (#7345) (#208) Signed-off-by: Wellington Chevreuil --- .../main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java | 2 +- .../hadoop/hbase/quotas/RegionServerRpcQuotaManager.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 0238756251ab..910aefd9142d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -228,7 +228,7 @@ protected boolean isExceedThrottleQuotaEnabled() { * username * @param ugi The request's UserGroupInformation */ - private String getQuotaUserName(final UserGroupInformation ugi) { + String getQuotaUserName(final UserGroupInformation ugi) { if (userOverrideRequestAttributeKey == null) { return ugi.getShortUserName(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 7a42d0f1aa31..34fc57cb0814 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -199,7 +199,7 @@ public OperationQuota checkScanQuota(final Region region, LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " scan=" + scanRequest.getScannerId() + ": " + e.getMessage()); - rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + rsServices.getMetrics().recordThrottleException(e.getType(), quotaCache.getQuotaUserName(ugi), table.getNameAsString()); throw e; @@ -276,7 +276,7 @@ public OperationQuota checkBatchQuota(final Region region, final int numWrites, LOG.debug("Throttling exception for user=" + ugi.getUserName() + " table=" + table + " numWrites=" + numWrites + " numReads=" + numReads + ": " + e.getMessage()); - rsServices.getMetrics().recordThrottleException(e.getType(), ugi.getShortUserName(), + rsServices.getMetrics().recordThrottleException(e.getType(), quotaCache.getQuotaUserName(ugi), table.getNameAsString()); throw e; From 7ec74b538f010254154247ff9434f255dccf7eb9 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 14 Oct 2025 15:58:08 +0200 Subject: [PATCH 38/78] HubSpot Backport: HBASE-29604 BackupHFileCleaner uses flawed time based check (#7360) (will be in 2.6.4) (#209) Adds javadoc mentioning the concurrent usage and thread-safety need of FileCleanerDelegate#getDeletableFiles. Fixes a potential thread-safety issue in BackupHFileCleaner: this class tracks timestamps to block the deletion of recently loaded HFiles that might be needed for backup purposes. The timestamps were being registered from inside the concurrent method, which could result in recently added files getting deleted. Moved the timestamp registration to the postClean method, which is called only a single time per cleaner run, so recently loaded HFiles are in fact protected from deletion. Signed-off-by: Nick Dimiduk Co-authored-by: DieterDP <90392398+DieterDP-ng@users.noreply.github.com> --- .../hadoop/hbase/backup/BackupHFileCleaner.java | 17 ++++++++++------- .../hbase/backup/TestBackupHFileCleaner.java | 13 ++++++++++--- .../master/cleaner/BaseFileCleanerDelegate.java | 4 ++++ .../master/cleaner/FileCleanerDelegate.java | 4 ++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java index c9a76bef2891..bbbae2d631fe 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java @@ -52,10 +52,13 @@ public class BackupHFileCleaner extends BaseHFileCleanerDelegate implements Abor private boolean stopped = false; private boolean aborted = false; private Connection connection; - // timestamp of most recent read from backup system table - private long prevReadFromBackupTbl = 0; - // timestamp of 2nd most recent read from backup system table - private long secondPrevReadFromBackupTbl = 0; + // timestamp of most recent completed cleaning run + private volatile long previousCleaningCompletionTimestamp = 0; + + @Override + public void postClean() { + previousCleaningCompletionTimestamp = EnvironmentEdgeManager.currentTime(); + } @Override public Iterable getDeletableFiles(Iterable files) { @@ -79,12 +82,12 @@ public Iterable getDeletableFiles(Iterable files) { return Collections.emptyList(); } - secondPrevReadFromBackupTbl = prevReadFromBackupTbl; - prevReadFromBackupTbl = EnvironmentEdgeManager.currentTime(); + // Pin the threshold, we don't want the result to change depending on evaluation time. + final long recentFileThreshold = previousCleaningCompletionTimestamp; return Iterables.filter(files, file -> { // If the file is recent, be conservative and wait for one more scan of the bulk loads - if (file.getModificationTime() > secondPrevReadFromBackupTbl) { + if (file.getModificationTime() > recentFileThreshold) { LOG.debug("Preventing deletion due to timestamp: {}", file.getPath().toString()); return false; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java index 7fba9c02e94e..3ffb265cfce0 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupHFileCleaner.java @@ -108,11 +108,11 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { Iterable deletable; // The first call will not allow any deletions because of the timestamp mechanism. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Collections.emptySet(), Sets.newHashSet(deletable)); // No bulk loads registered, so all files can be deleted. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file1, file1Archived, file2, file3), Sets.newHashSet(deletable)); // Register some bulk loads. @@ -125,10 +125,17 @@ protected Set fetchFullyBackedUpTables(BackupSystemTable tbl) { } // File 1 can no longer be deleted, because it is registered as a bulk load. - deletable = cleaner.getDeletableFiles(Arrays.asList(file1, file1Archived, file2, file3)); + deletable = callCleaner(cleaner, Arrays.asList(file1, file1Archived, file2, file3)); assertEquals(Sets.newHashSet(file2, file3), Sets.newHashSet(deletable)); } + private Iterable callCleaner(BackupHFileCleaner cleaner, Iterable files) { + cleaner.preClean(); + Iterable deletable = cleaner.getDeletableFiles(files); + cleaner.postClean(); + return deletable; + } + private FileStatus createFile(String fileName) throws IOException { Path file = new Path(root, fileName); fs.createNewFile(file); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java index 4c24ba1f81c5..700914f07b90 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseFileCleanerDelegate.java @@ -44,6 +44,10 @@ public void init(Map params) { /** * Should the master delete the file or keep it? + *

+ * This method can be called concurrently by multiple threads. Implementations must be thread + * safe. + *

* @param fStat file status of the file to check * @return true if the file is deletable, false if not */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java index d37bb6202730..438f34a891ce 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java @@ -33,6 +33,10 @@ public interface FileCleanerDelegate extends Configurable, Stoppable { /** * Determines which of the given files are safe to delete + *

+ * This method can be called concurrently by multiple threads. Implementations must be thread + * safe. + *

* @param files files to check for deletion * @return files that are ok to delete according to this cleaner */ From e6cb40059816fdf26467f22c0b142ad55873e15e Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Tue, 14 Oct 2025 15:47:43 -0400 Subject: [PATCH 39/78] Operation interceptor (not yet merged upstream) (#207) simplify the interceptor Co-authored-by: Ray Mattingly --- .../client/ConnectionImplementation.java | 21 +- .../client/NoOpOperationInterceptor.java | 48 ++++ .../hbase/client/OperationInterceptor.java | 88 +++++++ .../client/OperationInterceptorFactory.java | 60 +++++ .../client/RpcRetryingCallerFactory.java | 27 +- .../hbase/client/RpcRetryingCallerImpl.java | 32 ++- .../client/TestOperationInterceptor.java | 231 ++++++++++++++++++ 7 files changed, 499 insertions(+), 8 deletions(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java create mode 100644 hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 71e7dbf06c66..03636a2d38d9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -257,6 +257,8 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { private final RetryingCallerInterceptor interceptor; + private final OperationInterceptorFactory operationInterceptorFactory; + /** * Cluster registry of basic info such as clusterid and meta region location. */ @@ -337,6 +339,7 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { this.stats = ServerStatisticTracker.create(conf); this.interceptor = new RetryingCallerInterceptorFactory(conf).build(); + this.operationInterceptorFactory = createOperationInterceptorFactory(conf); this.backoffPolicy = ClientBackoffPolicyFactory.create(conf); @@ -370,7 +373,7 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { connectionAttributes); this.rpcControllerFactory = RpcControllerFactory.instantiate(conf); this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf, connectionConfig, - interceptor, this.stats, this.metrics); + interceptor, this.stats, this.metrics, operationInterceptorFactory); this.asyncProcess = new AsyncProcess(this, conf, rpcCallerFactory, rpcControllerFactory); // Do we publish the status? @@ -2341,7 +2344,7 @@ public TableState getTableState(TableName tableName) throws IOException { @Override public RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf) { return RpcRetryingCallerFactory.instantiate(conf, connectionConfig, this.interceptor, - this.getStatisticsTracker(), metrics); + this.stats, metrics, createOperationInterceptorFactory(conf)); } @Override @@ -2419,4 +2422,18 @@ public String getClusterId() { } return null; } + + private static OperationInterceptorFactory createOperationInterceptorFactory(Configuration conf) { + String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); + if (clazz == null || clazz.isEmpty()) { + return OperationInterceptorFactory.NO_OP; + } + try { + Class factoryClass = + conf.getClassByName(clazz).asSubclass(OperationInterceptorFactory.class); + return ReflectionUtils.newInstance(factoryClass, conf); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to load OperationInterceptorFactory class: " + clazz, e); + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java new file mode 100644 index 000000000000..c8a3d7d6d9f4 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpOperationInterceptor.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * No-op implementation of OperationInterceptor that does nothing. Used as the default when no + * custom interceptor is configured. + */ +@InterfaceAudience.Private +class NoOpOperationInterceptor extends OperationInterceptor { + + NoOpOperationInterceptor() { + super(); + } + + @Override + public void beforeAttempt(RetryingCallable callable) { + } + + @Override + public void afterAttemptSuccess(RetryingCallable callable, Object result) { + } + + @Override + public void afterAttemptFailure(RetryingCallable callable, Throwable cause) { + } + + @Override + public void afterOperation() { + } +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java new file mode 100644 index 000000000000..ddb78554df54 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptor.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import java.io.IOException; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Modern per-operation interceptor for HBase client operations. Each operation gets a fresh + * interceptor instance, eliminating thread-safety concerns and making implementation simple. + *

+ * This interceptor tracks both single operations (get, put, delete) and batch operations (multi-row + * operations). Batch operations are treated as single operations for simplicity. + *

+ * All fields are automatically populated by the HBase client. + *

+ * Usage example: + * + * 

+ * public class MyInterceptor extends OperationInterceptor {
+ *   public MyInterceptor(long operationStartTime) {
+ *     super(operationStartTime);
+ *   }
+ *
+ *   public void afterAttemptFailure(RetryingCallable callable, Throwable cause) {
+ *     long attemptDuration = System.currentTimeMillis() - getCurrentAttemptStartTime();
+ *     long operationDuration = System.currentTimeMillis() - getOperationStartTime();
+ *
+ *     // Fast-fail after 5 attempts or 30 seconds
+ *     if (getAttemptNumber() >= 4 || operationDuration > 30000) {
+ *       throw new FastFailException("Operation taking too long");
+ *     }
+ *
+ *     recordMetric("attempt.failure.duration", attemptDuration);
+ *     recordMetric("attempt.failure.type", cause.getClass().getSimpleName());
+ *   }
+ * }
+ *
+ */ +@InterfaceAudience.Public +public abstract class OperationInterceptor { + + /** + * Called before each attempt. + * @param callable the callable about to be executed + * @throws IOException the implementer may throw if they find issue with the request + */ + public abstract void beforeAttempt(RetryingCallable callable) throws IOException; + + /** + * Called after successful attempt completion. + * @param callable the callable that was executed + * @param result the result returned by the attempt (may be null) + * @throws IOException the implementer may throw if they find issue with the result + */ + public abstract void afterAttemptSuccess(RetryingCallable callable, Object result) + throws IOException; + + /** + * Called after attempt failure, before retry logic. + * @param callable the callable that failed + * @param cause the exception that caused the failure + * @throws IOException the implementer may throw an alternative exception + */ + public abstract void afterAttemptFailure(RetryingCallable callable, Throwable cause) + throws IOException; + + /** + * Called at the end of the operation, regardless of success or failure. This is guaranteed to be + * called exactly once per operation. + */ + public abstract void afterOperation(); +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java new file mode 100644 index 000000000000..88e15a87b17d --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationInterceptorFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Factory for creating per-operation interceptors. Each HBase client operation (get, put, batch, + * etc.) will call createInterceptor() to get a fresh interceptor instance, ensuring thread safety + * and simplicity. + *

+ * Implementations should be lightweight and thread-safe, as the factory itself may be shared across + * multiple threads. However, the interceptors created by the factory are used by only one operation + * at a time. + *

+ * Configuration is injected once during factory construction and can be used to customize + * interceptor behavior. + */ +@InterfaceAudience.Public +public interface OperationInterceptorFactory { + + /** + * Configuration key for specifying the OperationInterceptorFactory implementation. The specified + * class must implement OperationInterceptorFactory and have a no-argument constructor. + */ + String HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL = "hbase.client.operation.interceptor.impl"; + + /** + * Create a new interceptor instance for a single operation. This method may be called + * concurrently from multiple threads. + * @return a new OperationInterceptor instance, never null + */ + OperationInterceptor createInterceptor(); + + /** + * A no-op factory that creates interceptors that do nothing. Used as the default when no custom + * interceptor is configured. + */ + OperationInterceptorFactory NO_OP = new OperationInterceptorFactory() { + @Override + public OperationInterceptor createInterceptor() { + return new NoOpOperationInterceptor(); + } + }; +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java index c062ad43e253..8f737d3c9ce6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java @@ -34,18 +34,27 @@ public class RpcRetryingCallerFactory { private final RetryingCallerInterceptor interceptor; private final int startLogErrorsCnt; private final MetricsConnection metrics; + private final OperationInterceptorFactory operationInterceptorFactory; public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf) { - this(conf, connectionConf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null); + this(conf, connectionConf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, + OperationInterceptorFactory.NO_OP); } public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, MetricsConnection metrics) { + this(conf, connectionConf, interceptor, metrics, OperationInterceptorFactory.NO_OP); + } + + public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, + RetryingCallerInterceptor interceptor, MetricsConnection metrics, + OperationInterceptorFactory operationInterceptorFactory) { this.connectionConf = connectionConf; startLogErrorsCnt = conf.getInt(AsyncProcess.START_LOG_ERRORS_AFTER_COUNT_KEY, AsyncProcess.DEFAULT_START_LOG_ERRORS_AFTER_COUNT); this.interceptor = interceptor; this.metrics = metrics; + this.operationInterceptorFactory = operationInterceptorFactory; } /** @@ -56,7 +65,7 @@ public RpcRetryingCaller newCaller(int rpcTimeout) { // is cheap as it does not require parsing a complex structure. return new RpcRetryingCallerImpl<>(connectionConf.getPauseMillis(), connectionConf.getPauseMillisForServerOverloaded(), connectionConf.getRetriesNumber(), - interceptor, startLogErrorsCnt, rpcTimeout, metrics); + interceptor, startLogErrorsCnt, rpcTimeout, metrics, operationInterceptorFactory); } /** @@ -67,7 +76,8 @@ public RpcRetryingCaller newCaller() { // is cheap as it does not require parsing a complex structure. return new RpcRetryingCallerImpl<>(connectionConf.getPauseMillis(), connectionConf.getPauseMillisForServerOverloaded(), connectionConf.getRetriesNumber(), - interceptor, startLogErrorsCnt, connectionConf.getRpcTimeout(), metrics); + interceptor, startLogErrorsCnt, connectionConf.getRpcTimeout(), metrics, + operationInterceptorFactory); } @RestrictedApi(explanation = "Should only be called on process initialization", link = "", @@ -93,12 +103,21 @@ public static RpcRetryingCallerFactory instantiate(Configuration configuration, public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, ServerStatisticTracker stats, MetricsConnection metrics) { + return instantiate(configuration, connectionConf, interceptor, stats, metrics, + OperationInterceptorFactory.NO_OP); + } + + public static RpcRetryingCallerFactory instantiate(Configuration configuration, + ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, + ServerStatisticTracker stats, MetricsConnection metrics, + OperationInterceptorFactory operationInterceptorFactory) { String clazzName = RpcRetryingCallerFactory.class.getName(); String rpcCallerFactoryClazz = configuration.get(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, clazzName); RpcRetryingCallerFactory factory; if (rpcCallerFactoryClazz.equals(clazzName)) { - factory = new RpcRetryingCallerFactory(configuration, connectionConf, interceptor, metrics); + factory = new RpcRetryingCallerFactory(configuration, connectionConf, interceptor, metrics, + operationInterceptorFactory); } else { factory = ReflectionUtils.instantiateWithCustomCtor(rpcCallerFactoryClazz, new Class[] { Configuration.class, ConnectionConfiguration.class }, diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java index c692a2757d6f..6c73a391ced3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerImpl.java @@ -66,10 +66,18 @@ public class RpcRetryingCallerImpl implements RpcRetryingCaller { private final RetryingCallerInterceptorContext context; private final RetryingTimeTracker tracker; private final MetricsConnection metrics; + private final OperationInterceptorFactory operationInterceptorFactory; public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retries, RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout, MetricsConnection metricsConnection) { + this(pause, pauseForServerOverloaded, retries, interceptor, startLogErrorsCnt, rpcTimeout, + metricsConnection, OperationInterceptorFactory.NO_OP); + } + + public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retries, + RetryingCallerInterceptor interceptor, int startLogErrorsCnt, int rpcTimeout, + MetricsConnection metricsConnection, OperationInterceptorFactory operationInterceptorFactory) { this.pause = pause; this.pauseForServerOverloaded = pauseForServerOverloaded; this.maxAttempts = retries2Attempts(retries); @@ -79,6 +87,7 @@ public RpcRetryingCallerImpl(long pause, long pauseForServerOverloaded, int retr this.tracker = new RetryingTimeTracker(); this.rpcTimeout = rpcTimeout; this.metrics = metricsConnection; + this.operationInterceptorFactory = operationInterceptorFactory; } @Override @@ -95,16 +104,25 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) List exceptions = new ArrayList<>(); tracker.start(); context.clear(); + + OperationInterceptor operationInterceptor = operationInterceptorFactory.createInterceptor(); + for (int tries = 0;; tries++) { long expectedSleep; + try { // bad cache entries are cleared in the call to RetryingCallable#throwable() in catch block callable.prepare(tries != 0); interceptor.intercept(context.prepare(callable, tries)); - return callable.call(getTimeout(callTimeout)); + operationInterceptor.beforeAttempt(callable); + T result = callable.call(getTimeout(callTimeout)); + operationInterceptor.afterAttemptSuccess(callable, result); + operationInterceptor.afterOperation(); + return result; } catch (PreemptiveFastFailException e) { throw e; } catch (Throwable t) { + operationInterceptor.afterAttemptFailure(callable, t); ExceptionUtil.rethrowIfInterrupt(t); Throwable cause = t.getCause(); if (cause instanceof DoNotRetryIOException) { @@ -139,6 +157,7 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) EnvironmentEdgeManager.currentTime(), toString()); exceptions.add(qt); if (tries >= maxAttempts - 1) { + operationInterceptor.afterOperation(); throw new RetriesExhaustedException(tries, exceptions); } @@ -201,10 +220,19 @@ private long singleCallDuration(final long expectedSleep) { public T callWithoutRetries(RetryingCallable callable, int callTimeout) throws IOException, RuntimeException { // The code of this method should be shared with withRetries. + long startTime = EnvironmentEdgeManager.currentTime(); + OperationInterceptor operationInterceptor = operationInterceptorFactory.createInterceptor(); + try { callable.prepare(false); - return callable.call(callTimeout); + operationInterceptor.beforeAttempt(callable); + T result = callable.call(callTimeout); + operationInterceptor.afterAttemptSuccess(callable, result); + operationInterceptor.afterOperation(); + return result; } catch (Throwable t) { + operationInterceptor.afterAttemptFailure(callable, t); + operationInterceptor.afterOperation(); Throwable t2 = translateException(t); ExceptionUtil.rethrowIfInterrupt(t2); // It would be nice to clear the location cache here. diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java new file mode 100644 index 000000000000..16c8cb2cb2e5 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperationInterceptor.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ClientTests.class, SmallTests.class }) +public class TestOperationInterceptor { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestOperationInterceptor.class); + + private static class MockOperationInterceptor extends OperationInterceptor { + final List events = new ArrayList<>(); + volatile Throwable lastException; + volatile Object lastResult; + + public MockOperationInterceptor() { + super(); + } + + @Override + public void beforeAttempt(RetryingCallable callable) { + events.add("beforeAttempt"); + } + + @Override + public void afterAttemptSuccess(RetryingCallable callable, Object result) { + events.add("afterAttemptSuccess"); + this.lastResult = result; + } + + @Override + public void afterAttemptFailure(RetryingCallable callable, Throwable cause) { + events.add("afterAttemptFailure"); + this.lastException = cause; + } + + @Override + public void afterOperation() { + events.add("afterOperation"); + } + } + + private static class TestOperationInterceptorFactory implements OperationInterceptorFactory { + final AtomicInteger createCount = new AtomicInteger(0); + + @Override + public OperationInterceptor createInterceptor() { + createCount.incrementAndGet(); + return new MockOperationInterceptor(); + } + } + + // Helper method to create RpcRetryingCallerImpl with test factory + private RpcRetryingCallerImpl createCaller(TestOperationInterceptorFactory factory) { + return new RpcRetryingCallerImpl<>(100, 500, 3, + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, 0, 0, null, factory); + } + + // Helper method to create RpcRetryingCallerImpl with fast retry settings + private RpcRetryingCallerImpl + createFastRetryCaller(TestOperationInterceptorFactory factory) { + return new RpcRetryingCallerImpl<>(10, 50, 2, + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, 0, 0, null, factory); + } + + // Base RetryingCallable implementation with common no-op methods + private static abstract class BaseRetryingCallable implements RetryingCallable { + @Override + public void prepare(boolean reload) { + } + + @Override + public void throwable(Throwable t, boolean retrying) { + } + + @Override + public String getExceptionMessageAdditionalDetail() { + return null; + } + + @Override + public long sleep(long pause, int tries) { + return tries == 0 ? 1 : 0; // Short sleep for tests + } + } + + // Simple success callable + private static class SuccessCallable extends BaseRetryingCallable { + private final String result; + + SuccessCallable(String result) { + this.result = result; + } + + @Override + public String call(int callTimeout) { + return result; + } + } + + // Simple failure callable + private static class FailureCallable extends BaseRetryingCallable { + private final Exception exception; + + FailureCallable(Exception exception) { + this.exception = exception; + } + + @Override + public String call(int callTimeout) throws Exception { + throw exception; + } + } + + // Callable that fails on first attempt, succeeds on second + private static class RetrySuccessCallable extends BaseRetryingCallable { + private final AtomicInteger callCount = new AtomicInteger(0); + private final String successResult; + + RetrySuccessCallable(String successResult) { + this.successResult = successResult; + } + + @Override + public String call(int callTimeout) throws Exception { + int attempt = callCount.incrementAndGet(); + if (attempt == 1) { + throw new IOException("first failure"); + } + return successResult + " " + attempt; + } + + int getCallCount() { + return callCount.get(); + } + } + + @Test + public void testSuccessfulSingleOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createCaller(factory); + + String result = caller.callWithoutRetries(new SuccessCallable("success"), 1000); + assertEquals("success", result); + assertEquals(1, factory.createCount.get()); + } + + @Test + public void testFailedSingleOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createCaller(factory); + + try { + caller.callWithoutRetries(new FailureCallable(new IOException("test failure")), 1000); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("test failure", e.getMessage()); + } + assertEquals(1, factory.createCount.get()); + } + + @Test + public void testRetryingOperation() throws Exception { + TestOperationInterceptorFactory factory = new TestOperationInterceptorFactory(); + RpcRetryingCallerImpl caller = createFastRetryCaller(factory); + + RetrySuccessCallable callable = new RetrySuccessCallable("success on attempt"); + String result = caller.callWithRetries(callable, 5000); + assertEquals("success on attempt 2", result); + assertEquals(1, factory.createCount.get()); + assertEquals(2, callable.getCallCount()); + } + + @Test + public void testNoOpFactory() { + OperationInterceptorFactory factory = OperationInterceptorFactory.NO_OP; + OperationInterceptor interceptor = factory.createInterceptor(); + assertNotNull(interceptor); + + // Verify it creates new instances each time (no longer singleton) + OperationInterceptor interceptor2 = factory.createInterceptor(); + assertNotNull(interceptor2); + // They should be different instances + assertNotSame(interceptor, interceptor2); + } + + @Test + public void testConfigurationBasedFactory() { + Configuration conf = new Configuration(); + conf.set(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL, + TestOperationInterceptorFactory.class.getName()); + + // Test factory creation + String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); + assertNotNull(clazz); + assertEquals(TestOperationInterceptorFactory.class.getName(), clazz); + } +} From 4e9241b7be1b5da62c3504c48feae6d488b9b933 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Thu, 16 Oct 2025 11:02:11 -0400 Subject: [PATCH 40/78] Improved operation interceptor instantiation (not yet merged upstream) (#210) Co-authored-by: Ray Mattingly --- .../client/ConnectionImplementation.java | 21 ++---------- .../client/RpcRetryingCallerFactory.java | 32 ++++++++++++++++--- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 03636a2d38d9..25e111c31916 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -257,8 +257,6 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { private final RetryingCallerInterceptor interceptor; - private final OperationInterceptorFactory operationInterceptorFactory; - /** * Cluster registry of basic info such as clusterid and meta region location. */ @@ -339,7 +337,6 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { this.stats = ServerStatisticTracker.create(conf); this.interceptor = new RetryingCallerInterceptorFactory(conf).build(); - this.operationInterceptorFactory = createOperationInterceptorFactory(conf); this.backoffPolicy = ClientBackoffPolicyFactory.create(conf); @@ -373,7 +370,7 @@ public class ConnectionImplementation implements ClusterConnection, Closeable { connectionAttributes); this.rpcControllerFactory = RpcControllerFactory.instantiate(conf); this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf, connectionConfig, - interceptor, this.stats, this.metrics, operationInterceptorFactory); + interceptor, this.stats, this.metrics); this.asyncProcess = new AsyncProcess(this, conf, rpcCallerFactory, rpcControllerFactory); // Do we publish the status? @@ -2344,7 +2341,7 @@ public TableState getTableState(TableName tableName) throws IOException { @Override public RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf) { return RpcRetryingCallerFactory.instantiate(conf, connectionConfig, this.interceptor, - this.stats, metrics, createOperationInterceptorFactory(conf)); + this.stats, metrics); } @Override @@ -2422,18 +2419,4 @@ public String getClusterId() { } return null; } - - private static OperationInterceptorFactory createOperationInterceptorFactory(Configuration conf) { - String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); - if (clazz == null || clazz.isEmpty()) { - return OperationInterceptorFactory.NO_OP; - } - try { - Class factoryClass = - conf.getClassByName(clazz).asSubclass(OperationInterceptorFactory.class); - return ReflectionUtils.newInstance(factoryClass, conf); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Failed to load OperationInterceptorFactory class: " + clazz, e); - } - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java index 8f737d3c9ce6..d913166b12c8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java @@ -21,6 +21,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory to create an {@link RpcRetryingCaller} @@ -28,6 +30,8 @@ @InterfaceAudience.Private public class RpcRetryingCallerFactory { + private static final Logger LOG = LoggerFactory.getLogger(RpcRetryingCallerFactory.class); + /** Configuration key for a custom {@link RpcRetryingCaller} */ public static final String CUSTOM_CALLER_CONF_KEY = "hbase.rpc.callerfactory.class"; private final ConnectionConfiguration connectionConf; @@ -38,12 +42,12 @@ public class RpcRetryingCallerFactory { public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf) { this(conf, connectionConf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, - OperationInterceptorFactory.NO_OP); + createOperationInterceptorFactory(conf)); } public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, MetricsConnection metrics) { - this(conf, connectionConf, interceptor, metrics, OperationInterceptorFactory.NO_OP); + this(conf, connectionConf, interceptor, metrics, createOperationInterceptorFactory(conf)); } public RpcRetryingCallerFactory(Configuration conf, ConnectionConfiguration connectionConf, @@ -90,21 +94,23 @@ public static RpcRetryingCallerFactory instantiate(Configuration configuration, public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, MetricsConnection metrics) { return instantiate(configuration, connectionConf, - RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, metrics); + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, null, metrics, + createOperationInterceptorFactory(configuration)); } public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, ServerStatisticTracker stats, MetricsConnection metrics) { return instantiate(configuration, connectionConf, - RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, stats, metrics); + RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR, stats, metrics, + createOperationInterceptorFactory(configuration)); } public static RpcRetryingCallerFactory instantiate(Configuration configuration, ConnectionConfiguration connectionConf, RetryingCallerInterceptor interceptor, ServerStatisticTracker stats, MetricsConnection metrics) { return instantiate(configuration, connectionConf, interceptor, stats, metrics, - OperationInterceptorFactory.NO_OP); + createOperationInterceptorFactory(configuration)); } public static RpcRetryingCallerFactory instantiate(Configuration configuration, @@ -125,4 +131,20 @@ public static RpcRetryingCallerFactory instantiate(Configuration configuration, } return factory; } + + private static OperationInterceptorFactory createOperationInterceptorFactory(Configuration conf) { + String clazz = conf.get(OperationInterceptorFactory.HBASE_CLIENT_OPERATION_INTERCEPTOR_IMPL); + if (clazz == null || clazz.isEmpty()) { + return OperationInterceptorFactory.NO_OP; + } + try { + Class factoryClass = + conf.getClassByName(clazz).asSubclass(OperationInterceptorFactory.class); + return ReflectionUtils.newInstance(factoryClass, conf); + } catch (ClassNotFoundException e) { + LOG.warn("Failed to load OperationInterceptorFactory class: {}, using NO_OP instead", clazz, + e); + return OperationInterceptorFactory.NO_OP; + } + } } From 8618a3c7cf7f192578533b0d664054f137900cd1 Mon Sep 17 00:00:00 2001 From: ritika03494 <153202689+ritika03494@users.noreply.github.com> Date: Fri, 17 Oct 2025 13:08:32 +0100 Subject: [PATCH 41/78] Sets PRE_RUN_BEFORE_STEPS: false (#211) --- hubspot-client-bundles/.blazar.yaml | 2 +- .../hbase-backup-restore-bundle/.blazar.yaml | 3 ++- hubspot-client-bundles/hbase-client-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hubspot-client-bundles/.blazar.yaml b/hubspot-client-bundles/.blazar.yaml index 8d5dac3de18f..c925e07bed17 100644 --- a/hubspot-client-bundles/.blazar.yaml +++ b/hubspot-client-bundles/.blazar.yaml @@ -1,8 +1,8 @@ buildpack: name: Blazar-Buildpack-Java - branch: rm-test-hbase env: + PRE_RUN_BEFORE_STEPS: false # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml index 509a0cc10fab..fa25d831cb59 100644 --- a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -1,7 +1,8 @@ buildpack: name: Blazar-Buildpack-Java - branch: rm-test-hbase + env: + PRE_RUN_BEFORE_STEPS: false # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml index aba96b1c7dd9..a1f316b94c11 100644 --- a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -1,8 +1,8 @@ buildpack: name: Blazar-Buildpack-Java - branch: rm-test-hbase env: + PRE_RUN_BEFORE_STEPS: false # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml index c79dbaaf6044..9258a31d30b3 100644 --- a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -1,8 +1,8 @@ buildpack: name: Blazar-Buildpack-Java - branch: rm-test-hbase env: + PRE_RUN_BEFORE_STEPS: false # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml index 03fe644bf878..6f01acc4db6b 100644 --- a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -1,7 +1,8 @@ buildpack: name: Blazar-Buildpack-Java - branch: rm-test-hbase + env: + PRE_RUN_BEFORE_STEPS: false # Below variables are generated in prepare_environment.sh. # The build environment requires environment variables to be explicitly defined before they may # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable From 7fdaa7d4372ee119b058d66430591e3df8c5ed07 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 28 Oct 2025 09:10:44 -0400 Subject: [PATCH 42/78] HBASE-29679: Suppress stack trace in RpcThrottlingException (will be in 2.6.4) Signed-off by: Ray Mattingly --- .../hadoop/hbase/quotas/RpcThrottlingException.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java index d4ab38f5bf73..b08179a27a58 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/RpcThrottlingException.java @@ -205,4 +205,15 @@ protected static long timeFromString(String timeDiff) { } return -1; } + + /** + * There is little value in an RpcThrottlingException having a stack trace, since its cause is + * well understood without one. When a RegionServer is under heavy load and needs to serve many + * RpcThrottlingExceptions, skipping fillInStackTrace() will save CPU time and allocations, both + * here and later when the exception must be serialized over the wire. + */ + @Override + public synchronized Throwable fillInStackTrace() { + return this; + } } From bbd5f7174798bc330147b8a1110adec9b2ea7633 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Tue, 28 Oct 2025 11:44:51 -0400 Subject: [PATCH 43/78] Try Blazar-Buildpack-Java-oss-fork again --- .blazar.yaml | 2 +- hubspot-client-bundles/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-client-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml | 2 +- hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.blazar.yaml b/.blazar.yaml index e034ada7508d..4335a8f33d93 100644 --- a/.blazar.yaml +++ b/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java-single-module + name: Blazar-Buildpack-Java-oss-fork env: MAVEN_PHASE: "package assembly:single deploy" diff --git a/hubspot-client-bundles/.blazar.yaml b/hubspot-client-bundles/.blazar.yaml index c925e07bed17..5c4e00582b1c 100644 --- a/hubspot-client-bundles/.blazar.yaml +++ b/hubspot-client-bundles/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java + name: Blazar-Buildpack-Java-oss-fork env: PRE_RUN_BEFORE_STEPS: false diff --git a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml index fa25d831cb59..7fbd207a1895 100644 --- a/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-backup-restore-bundle/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java + name: Blazar-Buildpack-Java-oss-fork env: PRE_RUN_BEFORE_STEPS: false diff --git a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml index a1f316b94c11..c31e15b9a5db 100644 --- a/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-client-bundle/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java + name: Blazar-Buildpack-Java-oss-fork env: PRE_RUN_BEFORE_STEPS: false diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml index 9258a31d30b3..51517fcdd8eb 100644 --- a/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java + name: Blazar-Buildpack-Java-oss-fork env: PRE_RUN_BEFORE_STEPS: false diff --git a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml index 6f01acc4db6b..66c4e7eea66a 100644 --- a/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml +++ b/hubspot-client-bundles/hbase-server-it-bundle/.blazar.yaml @@ -1,5 +1,5 @@ buildpack: - name: Blazar-Buildpack-Java + name: Blazar-Buildpack-Java-oss-fork env: PRE_RUN_BEFORE_STEPS: false From f7a27d79f2ca6db0ef9414b227acb3b8b7c55002 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Tue, 28 Oct 2025 19:37:59 -0400 Subject: [PATCH 44/78] (DO NOT UPSTREAM) Update incremental backups to support skipping log roll (#213) --- .../hadoop/hbase/backup/BackupInfo.java | 11 ++++++++++- .../hadoop/hbase/backup/BackupRequest.java | 14 ++++++++++++++ .../hbase/backup/impl/BackupAdminImpl.java | 3 ++- .../hbase/backup/impl/BackupManager.java | 19 +++++++++++++++++++ .../backup/impl/IncrementalBackupManager.java | 16 ++++++++++------ .../hbase/backup/impl/TableBackupClient.java | 2 +- 6 files changed, 56 insertions(+), 9 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index f0dc10b83619..a4dec6fc83ff 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -34,7 +34,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; @@ -170,6 +169,8 @@ public enum BackupPhase { */ private boolean noChecksumVerify; + private boolean usePreviousLogRoll = false; + public BackupInfo() { backupTableInfoMap = new HashMap<>(); } @@ -211,6 +212,14 @@ public boolean getNoChecksumVerify() { return noChecksumVerify; } + public void setUsePreviousLogRoll(boolean usePreviousLogRoll) { + this.usePreviousLogRoll = usePreviousLogRoll; + } + + public boolean getUsePreviousLogRoll() { + return usePreviousLogRoll; + } + public void setBackupTableInfoMap(Map backupTableInfoMap) { this.backupTableInfoMap = backupTableInfoMap; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java index aa2d5b44259f..c13bc1f95754 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java @@ -75,6 +75,11 @@ public Builder withYarnPoolName(String name) { return this; } + public Builder withUsePreviousLogRoll(boolean usePreviousLogRoll) { + request.setUsePreviousLogRoll(usePreviousLogRoll); + return this; + } + public BackupRequest build() { return request; } @@ -89,6 +94,7 @@ public BackupRequest build() { private boolean noChecksumVerify = false; private String backupSetName; private String yarnPoolName; + private boolean usePreviousLogRoll = false; private BackupRequest() { } @@ -163,4 +169,12 @@ public String getYarnPoolName() { public void setYarnPoolName(String yarnPoolName) { this.yarnPoolName = yarnPoolName; } + + private void setUsePreviousLogRoll(boolean usePreviousLogRoll) { + this.usePreviousLogRoll = usePreviousLogRoll; + } + + public boolean getUsePreviousLogRoll() { + return this.usePreviousLogRoll; + } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index c36b398e5e86..8c019a0615f9 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -581,7 +581,8 @@ public String backupTables(BackupRequest request) throws IOException { request = builder.withBackupType(request.getBackupType()).withTableList(tableList) .withTargetRootDir(request.getTargetRootDir()).withBackupSetName(request.getBackupSetName()) .withTotalTasks(request.getTotalTasks()).withBandwidthPerTasks((int) request.getBandwidth()) - .withNoChecksumVerify(request.getNoChecksumVerify()).build(); + .withNoChecksumVerify(request.getNoChecksumVerify()) + .withUsePreviousLogRoll(request.getUsePreviousLogRoll()).build(); TableBackupClient client; try { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 5e5098a7b66a..349ad4dee345 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -200,6 +200,24 @@ public void close() { public BackupInfo createBackupInfo(String backupId, BackupType type, List tableList, String targetRootDir, int workers, long bandwidth, boolean noChecksumVerify) throws BackupException { + return createBackupInfo(backupId, type, tableList, targetRootDir, workers, bandwidth, + noChecksumVerify, false); + } + + /** + * Creates a backup info based on input backup request with optional provided timestamps. + * @param backupId backup id + * @param type type + * @param tableList table list + * @param targetRootDir root dir + * @param workers number of parallel workers + * @param bandwidth bandwidth per worker in MB per sec + * @param noChecksumVerify whether to skip checksum verification + * @throws BackupException exception + */ + public BackupInfo createBackupInfo(String backupId, BackupType type, List tableList, + String targetRootDir, int workers, long bandwidth, boolean noChecksumVerify, + boolean usePreviousLogRoll) throws BackupException { if (targetRootDir == null) { throw new BackupException("Wrong backup request parameter: target backup root directory"); } @@ -237,6 +255,7 @@ public BackupInfo createBackupInfo(String backupId, BackupType type, List getIncrBackupLogFileMap() throws IOException { + "In order to create an incremental backup, at least one full backup is needed."); } - LOG.info("Execute roll log procedure for incremental backup ..."); - HashMap props = new HashMap<>(); - props.put("backupRoot", backupInfo.getBackupRootDir()); + if (backupInfo.getUsePreviousLogRoll()) { + LOG.info("Using previous WAL roll for backup, skipping WAL roll procedure"); + } else { + LOG.info("Execute roll log procedure for incremental backup ..."); + HashMap props = new HashMap<>(); + props.put("backupRoot", backupInfo.getBackupRootDir()); - try (Admin admin = conn.getAdmin()) { - admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, - LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); + try (Admin admin = conn.getAdmin()) { + admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, + LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); + } } newTimestamps = readRegionServerLastLogRollResult(); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index 30c27f01faaf..a228f75055a6 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -92,7 +92,7 @@ public void init(final Connection conn, final String backupId, BackupRequest req this.fs = CommonFSUtils.getCurrentFileSystem(conf); backupInfo = backupManager.createBackupInfo(backupId, request.getBackupType(), tableList, request.getTargetRootDir(), request.getTotalTasks(), request.getBandwidth(), - request.getNoChecksumVerify()); + request.getNoChecksumVerify(), request.getUsePreviousLogRoll()); if (tableList == null || tableList.isEmpty()) { this.tableList = new ArrayList<>(backupInfo.getTables()); } From b4a0fe658835e4dc674ebcb8fdaabffdcdea9434 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Mon, 3 Nov 2025 13:01:44 -0500 Subject: [PATCH 45/78] HubSpot Edit: Warn when RegionCoprocessorRpcChannelImpl detects region name changed, instead of failing RPC --- .../hbase/client/RegionCoprocessorRpcChannelImpl.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java index 8acadafbb068..16d2bbb726c0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorRpcChannelImpl.java @@ -29,13 +29,14 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; @@ -46,6 +47,8 @@ @InterfaceAudience.Private class RegionCoprocessorRpcChannelImpl implements RpcChannel { + private static final Logger LOG = LoggerFactory.getLogger(RegionCoprocessorRpcChannelImpl.class); + private final AsyncConnectionImpl conn; private final TableName tableName; @@ -76,10 +79,8 @@ private CompletableFuture rpcCall(MethodDescriptor method, Message requ if ( region != null && !Bytes.equals(loc.getRegionInfo().getRegionName(), region.getRegionName()) ) { - future.completeExceptionally(new DoNotRetryIOException( - "Region name is changed, expected " + region.getRegionNameAsString() + ", actual " - + loc.getRegionInfo().getRegionNameAsString())); - return future; + LOG.warn("Region name is changed, expected {}, actual {}", region.getRegionNameAsString(), + loc.getRegionInfo().getRegionNameAsString()); } CoprocessorServiceRequest csr = CoprocessorRpcUtils.getCoprocessorServiceRequest(method, request, row, loc.getRegionInfo().getRegionName()); From 3ac3d23612446f9686ba997bf924b6cc4fdb5aee Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Fri, 7 Nov 2025 09:35:40 -0500 Subject: [PATCH 46/78] HubSpot Edit (not yet upstreamed): Preserve WAL edit order (#215) --- .../hadoop/hbase/backup/BackupInfo.java | 1 + .../hbase/backup/impl/BackupManager.java | 14 +-- .../hbase/mapreduce/HFileOutputFormat2.java | 5 + ...derPreservedExtendedCellSerialization.java | 102 ++++++++++++++++++ .../mapreduce/PreSortedCellsReducer.java | 25 +++-- .../hadoop/hbase/mapreduce/WALPlayer.java | 15 ++- .../OrderPreservedMapReduceExtendedCell.java | 36 +++++++ .../hadoop/hbase/mapreduce/TestWALPlayer.java | 49 +++++++++ 8 files changed, 231 insertions(+), 16 deletions(-) create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java create mode 100644 hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index a4dec6fc83ff..28ff70eaf0df 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -34,6 +34,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 349ad4dee345..54328817f512 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -206,13 +206,13 @@ public BackupInfo createBackupInfo(String backupId, BackupType type, List tableList, diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 9011e3b56b1e..66b309313951 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -756,6 +756,11 @@ private static void mergeSerializations(Configuration conf) { // SerializationFactory runs through serializations in the order they are registered. // We want to register ExtendedCellSerialization before CellSerialization because both // work for ExtendedCells but only ExtendedCellSerialization handles them properly. + + if (diskBasedSortingEnabled(conf)) { + serializations.add(OrderPreservedExtendedCellSerialization.class.getName()); + } + if ( conf.getBoolean(EXTENDED_CELL_SERIALIZATION_ENABLED_KEY, EXTENDED_CELL_SERIALIZATION_ENABLED_DEFULT) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java new file mode 100644 index 000000000000..1a7deeff5f9d --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/OrderPreservedExtendedCellSerialization.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.mapreduce; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; +import org.apache.hadoop.io.serializer.Deserializer; +import org.apache.hadoop.io.serializer.Serialization; +import org.apache.hadoop.io.serializer.Serializer; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class OrderPreservedExtendedCellSerialization + implements Serialization { + + @Override + public boolean accept(Class c) { + return OrderPreservedMapReduceExtendedCell.class.isAssignableFrom(c); + } + + @Override + public Serializer + getSerializer(Class c) { + return new OrderPreservedExtendedCellSerializer(); + } + + @Override + public Deserializer + getDeserializer(Class c) { + return new OrderPreservedExtendedCellDeserializer(); + } + + public static class OrderPreservedExtendedCellSerializer + implements Serializer { + private DataOutputStream dos; + + @Override + public void open(OutputStream os) throws IOException { + this.dos = new DataOutputStream(os); + } + + @Override + public void serialize(OrderPreservedMapReduceExtendedCell kv) throws IOException { + dos.writeInt(PrivateCellUtil.estimatedSerializedSizeOf(kv) - Bytes.SIZEOF_INT); + PrivateCellUtil.writeCell(kv, dos, true); + dos.writeLong(kv.getSequenceId()); + dos.writeInt(kv.getOrder()); + } + + @Override + public void close() throws IOException { + dos.close(); + } + } + + public static class OrderPreservedExtendedCellDeserializer + implements Deserializer { + private DataInputStream dis; + + @Override + public void open(InputStream is) throws IOException { + this.dis = new DataInputStream(is); + } + + @Override + public OrderPreservedMapReduceExtendedCell + deserialize(OrderPreservedMapReduceExtendedCell ignore) throws IOException { + KeyValue kv = KeyValueUtil.create(this.dis); + PrivateCellUtil.setSequenceId(kv, this.dis.readLong()); + int order = dis.readInt(); + return new OrderPreservedMapReduceExtendedCell(kv, order); + } + + @Override + public void close() throws IOException { + dis.close(); + } + } +} diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java index 81871ffb59c2..113235bfeb42 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/PreSortedCellsReducer.java @@ -18,25 +18,36 @@ package org.apache.hadoop.hbase.mapreduce; import java.io.IOException; +import java.util.Comparator; +import java.util.PriorityQueue; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.MapReduceExtendedCell; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; import org.apache.hadoop.mapreduce.Reducer; import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private -public class PreSortedCellsReducer - extends Reducer { +public class PreSortedCellsReducer extends Reducer { @Override - protected void reduce(KeyOnlyCellComparable key, Iterable values, Context context) + protected void reduce(KeyOnlyCellComparable key, + Iterable values, Context context) throws IOException, InterruptedException { + PriorityQueue cells = new PriorityQueue<>( + Comparator.comparingInt(OrderPreservedMapReduceExtendedCell::getOrder).reversed()); + + for (OrderPreservedMapReduceExtendedCell cell : values) { + OrderPreservedMapReduceExtendedCell copy = + new OrderPreservedMapReduceExtendedCell(cell.deepClone(), cell.getOrder()); + cells.add(copy); + } + int index = 0; - for (Cell cell : values) { - context.write(new ImmutableBytesWritable(CellUtil.cloneRow(key.getCell())), - new MapReduceExtendedCell(cell)); + for (OrderPreservedMapReduceExtendedCell cell : cells) { + context.write(new ImmutableBytesWritable(CellUtil.cloneRow(key.getCell())), cell); if (++index % 100 == 0) { context.setStatus("Wrote " + index + " cells"); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index dc84debf49aa..5e38ebe3f7fc 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.MapReduceExtendedCell; +import org.apache.hadoop.hbase.util.OrderPreservedMapReduceExtendedCell; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.io.WritableComparable; @@ -174,6 +175,7 @@ public void map(WALKey key, WALEdit value, Context context) throws IOException { try { TableName table = key.getTableName(); if (tableSet.contains(table.getNameAsString())) { + int order = 0; for (Cell cell : value.getCells()) { if (WALEdit.isMetaEditFamily(cell)) { continue; @@ -189,7 +191,8 @@ public void map(WALKey key, WALEdit value, Context context) throws IOException { ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell)) : CellUtil.cloneRow(cell); ExtendedCell extendedCell = (ExtendedCell) cell; - context.write(wrapKey(outKey, extendedCell), new MapReduceExtendedCell(extendedCell)); + context.write(wrapKey(outKey, extendedCell), wrapCell(extendedCell, order)); + ++order; } } } catch (InterruptedException e) { @@ -206,6 +209,13 @@ public void setup(Context context) throws IOException { Collections.addAll(tableSet, tables); } + private MapReduceExtendedCell wrapCell(ExtendedCell cell, int order) { + if (this.diskBasedSortingEnabled) { + return new OrderPreservedMapReduceExtendedCell(cell, order); + } + return new MapReduceExtendedCell(cell); + } + private WritableComparable wrapKey(byte[] key, ExtendedCell cell) { if (this.diskBasedSortingEnabled) { // Important to build a new cell with the updated key to maintain multi-table support @@ -423,12 +433,13 @@ public Job createSubmittableJob(String[] args) throws IOException { job.setMapperClass(WALCellMapper.class); if (diskBasedSortingEnabled) { job.setReducerClass(PreSortedCellsReducer.class); + job.setMapOutputValueClass(OrderPreservedMapReduceExtendedCell.class); } else { + job.setMapOutputValueClass(MapReduceExtendedCell.class); job.setReducerClass(CellSortReducer.class); } Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); - job.setMapOutputValueClass(MapReduceExtendedCell.class); try (Connection conn = ConnectionFactory.createConnection(conf);) { List tableInfoList = new ArrayList<>(); for (TableName tableName : tableNames) { diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java new file mode 100644 index 000000000000..7154af70245a --- /dev/null +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/OrderPreservedMapReduceExtendedCell.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.Cell; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class OrderPreservedMapReduceExtendedCell extends MapReduceExtendedCell { + + private final int order; + + public OrderPreservedMapReduceExtendedCell(Cell cell, int order) { + super(cell); + this.order = order; + } + + public int getOrder() { + return order; + } +} diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index 57f4c9cf0095..96330329f0c4 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -330,6 +330,55 @@ public void testWALKeyValueMapperWithDeprecatedConfig() throws Exception { testWALKeyValueMapper("hlog.input.tables"); } + @Test + public void testInsertionOrderPreserved() throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN = Bytes.toBytes("c1"); + final byte[] ROW = Bytes.toBytes("row"); + Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + + // put a row into the first table + Put p = new Put(ROW); + p.addColumn(FAMILY, COLUMN, Bytes.toBytes("aaa")); + p.addColumn(FAMILY, COLUMN, Bytes.toBytes("zzz")); + t1.put(p); + + // replay the WAL, map table 1 to table 2 + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + + Configuration configuration = TEST_UTIL.getConfiguration(); + WALPlayer player = new WALPlayer(configuration); + + configuration.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); + + try { + String optionName = "_test_.name"; + configuration.set(optionName, "1000"); + player.setupTime(configuration, optionName); + String outPath = "/tmp/" + name.getMethodName(); + configuration.set(WALPlayer.BULK_OUTPUT_CONF_KEY, outPath); + assertEquals(1000, configuration.getLong(optionName, 0)); + assertEquals(0, ToolRunner.run(configuration, player, + new String[] { walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); + + BulkLoadHFiles.create(configuration).bulkLoad(tableName2, new Path(outPath)); + + Get g = new Get(ROW); + Result r = t2.get(g); + Cell cell = r.getColumnLatestCell(FAMILY, COLUMN); + String value = Bytes.toString(CellUtil.cloneValue(cell)); + assertEquals("zzz", value); + } finally { + configuration.unset(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY); + } + } + private void testWALKeyValueMapper(final String tableConfigKey) throws Exception { Configuration configuration = new Configuration(); configuration.set(tableConfigKey, "table"); From 031bc4f3f4d6d8e8437ef56ac071343bacb929e4 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Mon, 24 Nov 2025 14:32:57 -0500 Subject: [PATCH 47/78] HubSpot Backport: HBASE-29663 TimeBasedLimiters should support dynamic configuration refresh (will be in 2.6.4) (#220) Signed-off-by: Charles Connell Signed-off-by: Nick Dimiduk Co-authored-by: Ray Mattingly --- .../quotas/FixedIntervalRateLimiter.java | 18 ++++++--- .../hadoop/hbase/quotas/QuotaCache.java | 13 ++++--- .../hbase/quotas/QuotaLimiterFactory.java | 5 ++- .../hadoop/hbase/quotas/QuotaState.java | 5 ++- .../apache/hadoop/hbase/quotas/QuotaUtil.java | 38 ++++++++++--------- .../hadoop/hbase/quotas/TimeBasedLimiter.java | 8 ++-- .../hadoop/hbase/quotas/UserQuotaState.java | 19 +++++----- .../TestRegionCoprocessorQuotaUsage.java | 12 +++++- .../quotas/TestDefaultOperationQuota.java | 16 ++++---- .../hadoop/hbase/quotas/TestQuotaCache2.java | 16 +++++--- .../hadoop/hbase/quotas/TestQuotaState.java | 22 ++++++----- 11 files changed, 101 insertions(+), 71 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java index c5b2fc7f5d83..a71b5d4b2fba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FixedIntervalRateLimiter.java @@ -20,8 +20,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; - -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * With this limiter resources will be refilled only after a fixed interval of time. @@ -43,6 +43,8 @@ public class FixedIntervalRateLimiter extends RateLimiter { public static final String RATE_LIMITER_REFILL_INTERVAL_MS = "hbase.quota.rate.limiter.refill.interval.ms"; + private static final Logger LOG = LoggerFactory.getLogger(FixedIntervalRateLimiter.class); + private long nextRefillTime = -1L; private final long refillInterval; @@ -52,10 +54,14 @@ public FixedIntervalRateLimiter() { public FixedIntervalRateLimiter(long refillInterval) { super(); - Preconditions.checkArgument(getTimeUnitInMillis() >= refillInterval, - String.format("Refill interval %s must be less than or equal to TimeUnit millis %s", - refillInterval, getTimeUnitInMillis())); - this.refillInterval = refillInterval; + long timeUnit = getTimeUnitInMillis(); + if (refillInterval > timeUnit) { + LOG.warn( + "Refill interval {} is larger than time unit {}. This is invalid. " + + "Instead, we will use the time unit {} as the refill interval", + refillInterval, timeUnit, timeUnit); + } + this.refillInterval = Math.min(timeUnit, refillInterval); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 910aefd9142d..34104752e81d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -129,20 +129,23 @@ private void ensureInitialized() { } private Map fetchUserQuotaStateEntries() throws IOException { - return QuotaUtil.fetchUserQuotas(rsServices.getConnection(), tableMachineQuotaFactors, - machineQuotaFactor); + return QuotaUtil.fetchUserQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + tableMachineQuotaFactors, machineQuotaFactor); } private Map fetchRegionServerQuotaStateEntries() throws IOException { - return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection()); + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConfiguration(), + rsServices.getConnection()); } private Map fetchTableQuotaStateEntries() throws IOException { - return QuotaUtil.fetchTableQuotas(rsServices.getConnection(), tableMachineQuotaFactors); + return QuotaUtil.fetchTableQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + tableMachineQuotaFactors); } private Map fetchNamespaceQuotaStateEntries() throws IOException { - return QuotaUtil.fetchNamespaceQuotas(rsServices.getConnection(), machineQuotaFactor); + return QuotaUtil.fetchNamespaceQuotas(rsServices.getConfiguration(), rsServices.getConnection(), + machineQuotaFactor); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java index 762896773fc7..63d8df65d25d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaLimiterFactory.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -25,8 +26,8 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class QuotaLimiterFactory { - public static QuotaLimiter fromThrottle(final Throttle throttle) { - return TimeBasedLimiter.fromThrottle(throttle); + public static QuotaLimiter fromThrottle(Configuration conf, final Throttle throttle) { + return TimeBasedLimiter.fromThrottle(conf, throttle); } public static QuotaLimiter update(final QuotaLimiter a, final QuotaLimiter b) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java index 61aa9d7f068f..4a0b634abec5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaState.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.quotas; +import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -57,9 +58,9 @@ public synchronized boolean isBypass() { /** * Setup the global quota information. (This operation is part of the QuotaState setup) */ - public synchronized void setQuotas(final Quotas quotas) { + public synchronized void setQuotas(Configuration conf, final Quotas quotas) { if (quotas.hasThrottle()) { - globalLimiter = QuotaLimiterFactory.fromThrottle(quotas.getThrottle()); + globalLimiter = QuotaLimiterFactory.fromThrottle(conf, quotas.getThrottle()); } else { globalLimiter = NoopQuotaLimiter.get(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 6b38635eccc0..f7df09801e0f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -329,8 +329,9 @@ private static void deleteQuotas(final Connection connection, final byte[] rowKe doDelete(connection, delete); } - public static Map fetchUserQuotas(final Connection connection, - Map tableMachineQuotaFactors, double factor) throws IOException { + public static Map fetchUserQuotas(final Configuration conf, + final Connection connection, Map tableMachineQuotaFactors, double factor) + throws IOException { Map userQuotas = new HashMap<>(); try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { Scan scan = new Scan(); @@ -350,7 +351,7 @@ public static Map fetchUserQuotas(final Connection conne @Override public void visitUserQuotas(String userName, String namespace, Quotas quotas) { quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(namespace, quotas); + quotaInfo.setQuotas(conf, namespace, quotas); } @Override @@ -359,13 +360,13 @@ public void visitUserQuotas(String userName, TableName table, Quotas quotas) { tableMachineQuotaFactors.containsKey(table) ? tableMachineQuotaFactors.get(table) : 1); - quotaInfo.setQuotas(table, quotas); + quotaInfo.setQuotas(conf, table, quotas); } @Override public void visitUserQuotas(String userName, Quotas quotas) { quotas = updateClusterQuotaToMachineQuota(quotas, factor); - quotaInfo.setQuotas(quotas); + quotaInfo.setQuotas(conf, quotas); } }); } catch (IOException e) { @@ -406,7 +407,7 @@ protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) { UserQuotaState state = new UserQuotaState(); QuotaProtos.Quotas defaultQuotas = QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build(); - state.setQuotas(defaultQuotas); + state.setQuotas(conf, defaultQuotas); return state; } @@ -419,12 +420,12 @@ private static Optional buildDefaultTimedQuota(Configuration conf, S java.util.concurrent.TimeUnit.SECONDS, org.apache.hadoop.hbase.quotas.QuotaScope.MACHINE)); } - public static Map fetchTableQuotas(final Connection connection, - Map tableMachineFactors) throws IOException { + public static Map fetchTableQuotas(final Configuration conf, + final Connection connection, Map tableMachineFactors) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_TABLE_ROW_KEY_PREFIX); - return fetchGlobalQuotas("table", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "table", scan, connection, new KeyFromRow() { @Override public TableName getKeyFromRow(final byte[] row) { assert isTableRowKey(row); @@ -438,12 +439,12 @@ public double getFactor(TableName tableName) { }); } - public static Map fetchNamespaceQuotas(final Connection connection, - double factor) throws IOException { + public static Map fetchNamespaceQuotas(final Configuration conf, + final Connection connection, double factor) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_NAMESPACE_ROW_KEY_PREFIX); - return fetchGlobalQuotas("namespace", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "namespace", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isNamespaceRowKey(row); @@ -457,12 +458,12 @@ public double getFactor(String s) { }); } - public static Map fetchRegionServerQuotas(final Connection connection) - throws IOException { + public static Map fetchRegionServerQuotas(final Configuration conf, + final Connection connection) throws IOException { Scan scan = new Scan(); scan.addFamily(QUOTA_FAMILY_INFO); scan.setStartStopRowForPrefixScan(QUOTA_REGION_SERVER_ROW_KEY_PREFIX); - return fetchGlobalQuotas("regionServer", scan, connection, new KeyFromRow() { + return fetchGlobalQuotas(conf, "regionServer", scan, connection, new KeyFromRow() { @Override public String getKeyFromRow(final byte[] row) { assert isRegionServerRowKey(row); @@ -476,8 +477,9 @@ public double getFactor(String s) { }); } - public static Map fetchGlobalQuotas(final String type, final Scan scan, - final Connection connection, final KeyFromRow kfr) throws IOException { + public static Map fetchGlobalQuotas(final Configuration conf, + final String type, final Scan scan, final Connection connection, final KeyFromRow kfr) + throws IOException { Map globalQuotas = new HashMap<>(); try (Table table = connection.getTable(QUOTA_TABLE_NAME)) { @@ -498,7 +500,7 @@ public static Map fetchGlobalQuotas(final String type, final try { Quotas quotas = quotasFromData(data); quotas = updateClusterQuotaToMachineQuota(quotas, kfr.getFactor(key)); - quotaInfo.setQuotas(quotas); + quotaInfo.setQuotas(conf, quotas); } catch (IOException e) { LOG.error("Unable to parse {} '{}' quotas", type, key, e); globalQuotas.remove(key); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java index 232ceb894ef6..38d171f1bf9a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TimeBasedLimiter.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.quotas; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -32,7 +31,6 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class TimeBasedLimiter implements QuotaLimiter { - private static final Configuration conf = HBaseConfiguration.create(); private RateLimiter reqsLimiter = null; private RateLimiter reqSizeLimiter = null; private RateLimiter writeReqsLimiter = null; @@ -47,7 +45,7 @@ public class TimeBasedLimiter implements QuotaLimiter { private RateLimiter atomicWriteSizeLimiter = null; private RateLimiter reqHandlerUsageTimeLimiter = null; - private TimeBasedLimiter() { + private TimeBasedLimiter(Configuration conf) { String limiterClassName = conf.getClass(RateLimiter.QUOTA_RATE_LIMITER_CONF_KEY, AverageIntervalRateLimiter.class) .getName(); @@ -100,8 +98,8 @@ private TimeBasedLimiter() { } } - static QuotaLimiter fromThrottle(final Throttle throttle) { - TimeBasedLimiter limiter = new TimeBasedLimiter(); + static QuotaLimiter fromThrottle(Configuration conf, final Throttle throttle) { + TimeBasedLimiter limiter = new TimeBasedLimiter(conf); boolean isBypass = true; if (throttle.hasReqNum()) { setFromTimedQuota(limiter.reqsLimiter, throttle.getReqNum()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java index 877ad195c716..0704e869239b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -89,8 +90,8 @@ public synchronized boolean hasBypassGlobals() { } @Override - public synchronized void setQuotas(final Quotas quotas) { - super.setQuotas(quotas); + public synchronized void setQuotas(Configuration conf, final Quotas quotas) { + super.setQuotas(conf, quotas); bypassGlobals = quotas.getBypassGlobals(); } @@ -98,30 +99,30 @@ public synchronized void setQuotas(final Quotas quotas) { * Add the quota information of the specified table. (This operation is part of the QuotaState * setup) */ - public synchronized void setQuotas(final TableName table, Quotas quotas) { - tableLimiters = setLimiter(tableLimiters, table, quotas); + public synchronized void setQuotas(Configuration conf, final TableName table, Quotas quotas) { + tableLimiters = setLimiter(conf, tableLimiters, table, quotas); } /** * Add the quota information of the specified namespace. (This operation is part of the QuotaState * setup) */ - public void setQuotas(final String namespace, Quotas quotas) { - namespaceLimiters = setLimiter(namespaceLimiters, namespace, quotas); + public void setQuotas(Configuration conf, final String namespace, Quotas quotas) { + namespaceLimiters = setLimiter(conf, namespaceLimiters, namespace, quotas); } public boolean hasTableLimiters() { return tableLimiters != null && !tableLimiters.isEmpty(); } - private Map setLimiter(Map limiters, final K key, - final Quotas quotas) { + private Map setLimiter(Configuration conf, Map limiters, + final K key, final Quotas quotas) { if (limiters == null) { limiters = new HashMap<>(); } QuotaLimiter limiter = - quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(quotas.getThrottle()) : null; + quotas.hasThrottle() ? QuotaLimiterFactory.fromThrottle(conf, quotas.getThrottle()) : null; if (limiter != null && !limiter.isBypass()) { limiters.put(key, limiter); } else { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java index 4a638d965b38..e614e71b3350 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionCoprocessorQuotaUsage.java @@ -43,6 +43,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category({ MediumTests.class, CoprocessorTests.class }) public class TestRegionCoprocessorQuotaUsage { @@ -52,6 +54,7 @@ public class TestRegionCoprocessorQuotaUsage { HBaseClassTestRule.forClass(TestRegionCoprocessorQuotaUsage.class); private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Logger LOG = LoggerFactory.getLogger(TestRegionCoprocessorQuotaUsage.class); private static TableName TABLE_NAME = TableName.valueOf("TestRegionCoprocessorQuotaUsage"); private static byte[] CF = Bytes.toBytes("CF"); private static byte[] CQ = Bytes.toBytes("CQ"); @@ -66,11 +69,14 @@ public void preGetOp(ObserverContext c, Get get, // For the purposes of this test, we only need to catch a throttle happening once, then // let future requests pass through so we don't make this test take any longer than necessary + LOG.info("Intercepting GetOp"); if (!THROTTLING_OCCURRED.get()) { try { c.getEnvironment().checkBatchQuota(c.getEnvironment().getRegion(), OperationQuota.OperationType.GET); + LOG.info("Request was not throttled"); } catch (RpcThrottlingException e) { + LOG.info("Intercepting was throttled"); THROTTLING_OCCURRED.set(true); throw e; } @@ -91,9 +97,8 @@ public Optional getRegionObserver() { public static void setUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setBoolean("hbase.quota.enabled", true); - conf.setInt("hbase.quota.default.user.machine.read.num", 2); + conf.setInt("hbase.quota.default.user.machine.read.num", 1); conf.set("hbase.quota.rate.limiter", "org.apache.hadoop.hbase.quotas.FixedIntervalRateLimiter"); - conf.set("hbase.quota.rate.limiter.refill.interval.ms", "300000"); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MyCoprocessor.class.getName()); UTIL.startMiniCluster(3); byte[][] splitKeys = new byte[8][]; @@ -116,6 +121,9 @@ public void testGet() throws InterruptedException, ExecutionException, IOExcepti // Hit the table 5 times which ought to be enough to make a throttle happen for (int i = 0; i < 5; i++) { TABLE.get(new Get(Bytes.toBytes("000"))); + if (THROTTLING_OCCURRED.get()) { + break; + } } assertTrue("Throttling did not happen as expected", THROTTLING_OCCURRED.get()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java index c22a03f8db00..2b9200ab6465 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestDefaultOperationQuota.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -41,6 +42,7 @@ public class TestDefaultOperationQuota { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestDefaultOperationQuota.class); + private static final Configuration conf = HBaseConfiguration.create(); private static final int DEFAULT_REQUESTS_PER_SECOND = 1000; private static ManualEnvironmentEdge envEdge = new ManualEnvironmentEdge(); static { @@ -150,7 +152,7 @@ public void testLargeBatchSaturatesReadNumLimit() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -172,7 +174,7 @@ public void testLargeBatchSaturatesReadWriteLimit() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -194,7 +196,7 @@ public void testTooLargeReadBatchIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -216,7 +218,7 @@ public void testTooLargeWriteBatchIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteNum(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -238,7 +240,7 @@ public void testTooLargeWriteSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setWriteSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), 65536, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -261,7 +263,7 @@ public void testTooLargeReadSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReadSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), (int) blockSize, DEFAULT_REQUESTS_PER_SECOND, limiter); @@ -284,7 +286,7 @@ public void testTooLargeRequestSizeIsNotBlocked() QuotaProtos.Throttle throttle = QuotaProtos.Throttle.newBuilder().setReqSize(QuotaProtos.TimedQuota.newBuilder() .setSoftLimit(limit).setTimeUnit(HBaseProtos.TimeUnit.SECONDS).build()).build(); - QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(throttle); + QuotaLimiter limiter = TimeBasedLimiter.fromThrottle(conf, throttle); DefaultOperationQuota quota = new DefaultOperationQuota(new Configuration(), (int) blockSize, DEFAULT_REQUESTS_PER_SECOND, limiter); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java index cd55ecd6fed8..3e829b5c08af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache2.java @@ -23,7 +23,9 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.ClassRule; @@ -43,6 +45,8 @@ public class TestQuotaCache2 { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestQuotaCache2.class); + private static final Configuration conf = HBaseConfiguration.create(); + @Test public void testPreserveLimiterAvailability() throws Exception { // establish old cache with a limiter for 100 read bytes per second @@ -53,7 +57,7 @@ public void testPreserveLimiterAvailability() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); oldState.setGlobalLimiter(limiter1); // consume one byte from the limiter, so 99 will be left @@ -67,7 +71,7 @@ public void testPreserveLimiterAvailability() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); newState.setGlobalLimiter(limiter2); // update new cache from old cache @@ -89,7 +93,7 @@ public void testClobberLimiterLimit() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); oldState.setGlobalLimiter(limiter1); // establish new cache, also with a limiter for 100 read bytes per second @@ -100,7 +104,7 @@ public void testClobberLimiterLimit() throws Exception { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(50).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); newState.setGlobalLimiter(limiter2); // update new cache from old cache @@ -151,7 +155,7 @@ public void testUserSpecificOverridesDefaultNewQuota() { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(100).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(throttle1); + QuotaLimiter limiter1 = TimeBasedLimiter.fromThrottle(conf, throttle1); oldState.setGlobalLimiter(limiter1); // establish new cache, with a limiter for 999 read bytes per second @@ -162,7 +166,7 @@ public void testUserSpecificOverridesDefaultNewQuota() { .setReadSize(QuotaProtos.TimedQuota.newBuilder().setTimeUnit(HBaseProtos.TimeUnit.SECONDS) .setSoftLimit(999).setScope(QuotaProtos.QuotaScope.MACHINE).build()) .build(); - QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(throttle2); + QuotaLimiter limiter2 = TimeBasedLimiter.fromThrottle(conf, throttle2); newState.setGlobalLimiter(limiter2); // update new cache from old cache diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java index ff4b6bc9949b..b45f78b07653 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaState.java @@ -22,7 +22,9 @@ import static org.junit.Assert.fail; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -48,6 +50,8 @@ public class TestQuotaState { @Rule public TestName name = new TestName(); + private static final Configuration conf = HBaseConfiguration.create(); + @Test public void testQuotaStateBypass() { QuotaState quotaInfo = new QuotaState(); @@ -69,11 +73,11 @@ public void testSimpleQuotaStateOperation() { assertTrue(quotaInfo.isBypass()); // Set global quota - quotaInfo.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE)); + quotaInfo.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE)); assertFalse(quotaInfo.isBypass()); // Set table quota - quotaInfo.setQuotas(tableName, buildReqNumThrottle(NUM_TABLE_THROTTLE)); + quotaInfo.setQuotas(conf, tableName, buildReqNumThrottle(NUM_TABLE_THROTTLE)); assertFalse(quotaInfo.isBypass()); assertTrue(quotaInfo.getGlobalLimiter() == quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME)); assertThrottleException(quotaInfo.getTableLimiter(UNKNOWN_TABLE_NAME), NUM_GLOBAL_THROTTLE); @@ -90,7 +94,7 @@ public void testQuotaStateUpdateGlobalThrottle() { // Add global throttle QuotaState otherQuotaState = new QuotaState(); - otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); + otherQuotaState.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE_1)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -99,7 +103,7 @@ public void testQuotaStateUpdateGlobalThrottle() { // Update global Throttle otherQuotaState = new QuotaState(); - otherQuotaState.setQuotas(buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); + otherQuotaState.setQuotas(conf, buildReqNumThrottle(NUM_GLOBAL_THROTTLE_2)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -131,8 +135,8 @@ public void testQuotaStateUpdateTableThrottle() { // Add A B table limiters UserQuotaState otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); - otherQuotaState.setQuotas(tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); + otherQuotaState.setQuotas(conf, tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + otherQuotaState.setQuotas(conf, tableNameB, buildReqNumThrottle(TABLE_B_THROTTLE)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -143,8 +147,8 @@ public void testQuotaStateUpdateTableThrottle() { // Add C, Remove B, Update A table limiters otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); - otherQuotaState.setQuotas(tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); + otherQuotaState.setQuotas(conf, tableNameA, buildReqNumThrottle(TABLE_A_THROTTLE_2)); + otherQuotaState.setQuotas(conf, tableNameC, buildReqNumThrottle(TABLE_C_THROTTLE)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); @@ -173,7 +177,7 @@ public void testTableThrottleWithBatch() { // Add A table limiters UserQuotaState otherQuotaState = new UserQuotaState(); - otherQuotaState.setQuotas(TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); + otherQuotaState.setQuotas(conf, TABLE_A, buildReqNumThrottle(TABLE_A_THROTTLE_1)); assertFalse(otherQuotaState.isBypass()); quotaInfo.update(otherQuotaState); From 64a249fb7b435d2d1ce3f38ab3774df5b683333a Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 24 Nov 2025 20:39:00 +0000 Subject: [PATCH 48/78] Fix walplayer mapping (not yet upstream) (#218) --- .../java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 5e38ebe3f7fc..ce490fb420e0 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -418,7 +418,7 @@ public Job createSubmittableJob(String[] args) throws IOException { if (hfileOutPath != null) { LOG.debug("add incremental job :" + hfileOutPath + " from " + inputDirs); - if (!multiTableSupport && tables.length != 1) { + if (!multiTableSupport && tableMap.length != 1) { throw new IOException("Exactly one table must be specified for the bulk export option"); } @@ -428,7 +428,7 @@ public Job createSubmittableJob(String[] args) throws IOException { true); // the bulk HFile case - List tableNames = getTableNameList(tables); + List tableNames = getTableNameList(tableMap); job.setMapperClass(WALCellMapper.class); if (diskBasedSortingEnabled) { From ad62e7f31743e74b932cff72c55ef80e6c9f76e4 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Tue, 25 Nov 2025 19:01:04 +0000 Subject: [PATCH 49/78] HBASE-29716 Include sequence ID on incremental backup HFiles (#221) --- .../impl/IncrementalTableBackupClient.java | 1 + .../hbase/mapreduce/HFileOutputFormat2.java | 76 +++++++----- .../hadoop/hbase/mapreduce/TestWALPlayer.java | 114 ++++++++++++++++++ 3 files changed, 160 insertions(+), 31 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index a47e62b4fa6f..d800f07f5fed 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -431,6 +431,7 @@ protected void walToHFiles(List dirPaths, List tableList) throws conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";"); conf.setBoolean(HFileOutputFormat2.TABLE_NAME_WITH_NAMESPACE_INCLUSIVE_KEY, true); conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); + conf.setBoolean(HFileOutputFormat2.SET_MAX_SEQ_ID_KEY, true); conf.setBoolean(HFileOutputFormat2.DISK_BASED_SORTING_ENABLED_KEY, true); conf.set(JOB_NAME_CONF_KEY, jobname); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 66b309313951..8e6ae72121bc 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; +import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -209,6 +210,13 @@ protected static byte[] combineTableNameSuffix(byte[] tableName, byte[] suffix) public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY = REMOTE_CLUSTER_CONF_PREFIX + HConstants.ZOOKEEPER_ZNODE_PARENT; + /** + * Set the MAX_SEQ_ID metadata on the resulting HFile. This will ensure the HFiles will be sorted + * properly when read by tools such as the ClientSideRegionScanner. Will have no effect if the + * HFile is bulkloaded, as the sequence ID generated when bulkloading will override this metadata. + */ + public static final String SET_MAX_SEQ_ID_KEY = "hbase.hfileoutputformat.set.max.seq.id"; + public static final String STORAGE_POLICY_PROPERTY = HStore.BLOCK_STORAGE_POLICY_KEY; public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; @@ -270,7 +278,7 @@ static RecordWriter createRecordWrit return new RecordWriter() { // Map of families to writers and how much has been output on the writer. - private final Map writers = new TreeMap<>(Bytes.BYTES_COMPARATOR); + private final Map writers = new TreeMap<>(Bytes.BYTES_COMPARATOR); private final Map previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR); private final long now = EnvironmentEdgeManager.currentTime(); private byte[] tableNameBytes = writeMultipleTables ? null : Bytes.toBytes(writeTableNames); @@ -300,10 +308,10 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { } byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableNameBytes, family); - WriterLength wl = this.writers.get(tableAndFamily); + WriterInfo wi = this.writers.get(tableAndFamily); // If this is a new column family, verify that the directory exists - if (wl == null) { + if (wi == null) { Path writerPath = null; if (writeMultipleTables) { Path tableRelPath = getTableRelativePath(tableNameBytes); @@ -317,14 +325,14 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { // This can only happen once a row is finished though if ( - wl != null && wl.written + length >= maxsize + wi != null && wi.written + length >= maxsize && Bytes.compareTo(this.previousRows.get(family), rowKey) != 0 ) { - rollWriters(wl); + rollWriters(wi); } // create a new WAL writer, if necessary - if (wl == null || wl.writer == null) { + if (wi == null || wi.writer == null) { InetSocketAddress[] favoredNodes = null; if (conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) { HRegionLocation loc = null; @@ -355,14 +363,15 @@ public void write(ImmutableBytesWritable row, V cell) throws IOException { } } } - wl = getNewWriter(tableNameBytes, family, conf, favoredNodes); + wi = getNewWriter(tableNameBytes, family, conf, favoredNodes); } // we now have the proper WAL writer. full steam ahead PrivateCellUtil.updateLatestStamp(cell, this.now); - wl.writer.append(kv); - wl.written += length; + wi.writer.append(kv); + wi.written += length; + wi.maxSequenceId = Math.max(kv.getSequenceId(), wi.maxSequenceId); // Copy the row so we know when a row transition. this.previousRows.put(family, rowKey); @@ -378,24 +387,25 @@ private Path getTableRelativePath(byte[] tableNameBytes) { return tableRelPath; } - private void rollWriters(WriterLength writerLength) throws IOException { - if (writerLength != null) { - closeWriter(writerLength); + private void rollWriters(WriterInfo writerInfo) throws IOException { + if (writerInfo != null) { + closeWriter(writerInfo); } else { - for (WriterLength wl : this.writers.values()) { - closeWriter(wl); + for (WriterInfo wi : this.writers.values()) { + closeWriter(wi); } } } - private void closeWriter(WriterLength wl) throws IOException { - if (wl.writer != null) { + private void closeWriter(WriterInfo wi) throws IOException { + if (wi.writer != null) { LOG.info( - "Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); - close(wl.writer); - wl.writer = null; + "Writer=" + wi.writer.getPath() + ((wi.written == 0) ? "" : ", wrote=" + wi.written)); + close(wi.writer, wi); + wi.writer = null; } - wl.written = 0; + wi.written = 0; + wi.maxSequenceId = -1; } private Configuration createRemoteClusterConf(Configuration conf) { @@ -435,11 +445,11 @@ private Configuration createRemoteClusterConf(Configuration conf) { /* * Create a new StoreFile.Writer. - * @return A WriterLength, containing a new StoreFile.Writer. + * @return A WriterInfo, containing a new StoreFile.Writer. */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "BX_UNBOXING_IMMEDIATELY_REBOXED", justification = "Not important") - private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf, + private WriterInfo getNewWriter(byte[] tableName, byte[] family, Configuration conf, InetSocketAddress[] favoredNodes) throws IOException { byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family); Path familydir = new Path(outputDir, Bytes.toString(family)); @@ -447,7 +457,7 @@ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration familydir = new Path(outputDir, new Path(getTableRelativePath(tableName), Bytes.toString(family))); } - WriterLength wl = new WriterLength(); + WriterInfo wi = new WriterInfo(); Algorithm compression = overriddenCompression; compression = compression == null ? compressionMap.get(tableAndFamily) : compression; compression = compression == null ? defaultCompression : compression; @@ -474,23 +484,26 @@ private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration HFileContext hFileContext = contextBuilder.build(); if (null == favoredNodes) { - wl.writer = + wi.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs).withOutputDir(familydir) .withBloomType(bloomType).withFileContext(hFileContext).build(); } else { - wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) + wi.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) .withOutputDir(familydir).withBloomType(bloomType).withFileContext(hFileContext) .withFavoredNodes(favoredNodes).build(); } - this.writers.put(tableAndFamily, wl); - return wl; + this.writers.put(tableAndFamily, wi); + return wi; } - private void close(final StoreFileWriter w) throws IOException { + private void close(final StoreFileWriter w, final WriterInfo wl) throws IOException { if (w != null) { w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(EnvironmentEdgeManager.currentTime())); w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); + if (conf.getBoolean(SET_MAX_SEQ_ID_KEY, false) && wl.maxSequenceId >= 0) { + w.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(wl.maxSequenceId)); + } w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); @@ -500,8 +513,8 @@ private void close(final StoreFileWriter w) throws IOException { @Override public void close(TaskAttemptContext c) throws IOException, InterruptedException { - for (WriterLength wl : this.writers.values()) { - close(wl.writer); + for (WriterInfo wi : this.writers.values()) { + close(wi.writer, wi); } } }; @@ -524,8 +537,9 @@ static void configureStoragePolicy(final Configuration conf, final FileSystem fs /* * Data structure to hold a Writer and amount of data written on it. */ - static class WriterLength { + static class WriterInfo { long written = 0; + long maxSequenceId = -1; StoreFileWriter writer = null; } diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index 96330329f0c4..60db315ebcbb 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -36,7 +36,10 @@ import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -45,11 +48,15 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClientSideRegionScanner; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.WALPlayer.WALKeyValueMapper; import org.apache.hadoop.hbase.regionserver.TestRecoveredEdits; @@ -268,6 +275,113 @@ public void testWALPlayerBulkLoadWithOverriddenTimestamps() throws Exception { }); } + /** + * Tests that the sequence IDs of cells are retained in the resulting HFile and usable by a + * RegionScanner. It does this by running the WALPlayer multiple times and using a RegionScanner + * to read the files; without sequence IDs, the files will be sorted by size or name and will not + * always return the correct result. + */ + @Test + public void testMaxSeqIdHFileMetadata() throws Exception { + final int numEdits = 20; + final int flushInterval = 10; + + // Phase 1: Setup test data and configuration + final TableName tableName = TableName.valueOf(name.getMethodName()); + final byte[] family = Bytes.toBytes("family"); + final byte[] column = Bytes.toBytes("c1"); + final byte[] row = Bytes.toBytes("row"); + final Table table = TEST_UTIL.createTable(tableName, family); + + long now = EnvironmentEdgeManager.currentTime(); + { + Put put = new Put(row); + put.addColumn(family, column, now, column); + table.put(put); + } + + String walInputDir = new Path(cluster.getMaster().getMasterFileSystem().getWALRootDir(), + HConstants.HREGION_LOGDIR_NAME).toString(); + String walPlayerOutputRoot = "/tmp/" + name.getMethodName(); + + Configuration walPlayerConfig = new Configuration(TEST_UTIL.getConfiguration()); + walPlayerConfig.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true); + walPlayerConfig.setBoolean(HFileOutputFormat2.SET_MAX_SEQ_ID_KEY, true); + + // Phase 2: Write edits with periodic WAL rolling and WALPlayer execution + int walPlayerRunCount = 0; + byte[] lastVal = null; + + for (int i = 0; i < numEdits; i++) { + lastVal = new byte[12]; + ThreadLocalRandom.current().nextBytes(lastVal); + + Put put = new Put(row); + put.addColumn(family, column, now, lastVal); + table.put(put); + + // Roll WALs and run WALPlayer every flushInterval iterations + if (i > 0 && (i % flushInterval == 0) || i + 1 == numEdits) { + WAL log = cluster.getRegionServer(0).getWAL(null); + log.rollWriter(); + + walPlayerRunCount++; + String walPlayerRunDir = walPlayerOutputRoot + "/run_" + walPlayerRunCount; + Configuration runConfig = new Configuration(walPlayerConfig); + runConfig.set(WALPlayer.BULK_OUTPUT_CONF_KEY, walPlayerRunDir); + + WALPlayer player = new WALPlayer(runConfig); + assertEquals(0, ToolRunner.run(runConfig, player, + new String[] { walInputDir, tableName.getNameAsString() })); + } + } + + table.close(); + + final byte[] finalLastVal = lastVal; + + // Phase 3: Collect all generated HFiles into proper structure for region scanner + TableDescriptor htd = TEST_UTIL.getAdmin().getDescriptor(tableName); + RegionInfo regionInfo = cluster.getRegions(tableName).get(0).getRegionInfo(); + FileSystem fs = cluster.getRegionServer(0).getFileSystem(); + + Path regionOutPath = CommonFSUtils.getRegionDir(new Path(walPlayerOutputRoot), + htd.getTableName(), regionInfo.getEncodedName()); + Path familyOutPath = new Path(regionOutPath, new String(family)); + fs.mkdirs(familyOutPath); + + // Copy all HFiles from each WALPlayer run + for (int i = 1; i <= walPlayerRunCount; i++) { + Path walPlayerRunPath = new Path(walPlayerOutputRoot, "run_" + i); + RemoteIterator files = + fs.listFiles(new Path(walPlayerRunPath, tableName.getNamespaceAsString()), true); + + while (files.hasNext()) { + LocatedFileStatus fileStatus = files.next(); + // Skip hidden/metadata files (starting with '.') + if (fileStatus.isFile() && !fileStatus.getPath().getName().startsWith(".")) { + FileUtil.copy(fs, fileStatus.getPath(), fs, + new Path(familyOutPath, fileStatus.getPath().getName()), false, walPlayerConfig); + } + } + } + + // Phase 4: Verify sequence IDs are preserved correctly + Scan scan = new Scan(); + try (ClientSideRegionScanner scanner = new ClientSideRegionScanner(walPlayerConfig, fs, + new Path(walPlayerOutputRoot), htd, regionInfo, scan, null)) { + + // Verify exactly one row returned + Result result = scanner.next(); + assertThat(result, notNullValue()); + assertThat(result.listCells(), notNullValue()); + + // Verify the value with highest sequence ID (from last iteration) wins + byte[] value = CellUtil.cloneValue(result.getColumnLatestCell(family, column)); + assertThat(Bytes.toStringBinary(value), equalTo(Bytes.toStringBinary(finalLastVal))); + } + } + /** * Simple end-to-end test */ From 5c35dad9c74f426c4ecece4fd8d13e630089f231 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Wed, 3 Dec 2025 14:56:41 +0000 Subject: [PATCH 50/78] Dont fail on snapshot timeout (#222) Co-authored-by: Hernan Gelaf-Romer --- .../hbase/backup/impl/FullTableBackupClient.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index 7fb7a5768805..969d0945b8fa 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupCopyJob; @@ -217,8 +218,20 @@ protected void snapshotTable(Admin admin, TableName tableName, String snapshotNa int pause = conf.getInt(BACKUP_ATTEMPTS_PAUSE_MS_KEY, DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS); int attempts = 0; + Pattern regex = Pattern.compile(snapshotName); + while (attempts++ < maxAttempts) { try { + if ( + admin.listSnapshots(regex).stream() + .anyMatch(snapshot -> snapshot.getName().equals(snapshotName)) + ) { + // If a snapshot takes a long time to run, we may get a TimeoutException from the + // admin, in that case re-attempts to take a snapshot will always fail b/c we'll + // eventually + // attempt to take a snapshot with a name that already exists + return; + } admin.snapshot(snapshotName, tableName); return; } catch (IOException ee) { From 6d4f77fe54ad2218be6efad298db018464d85c3a Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Thu, 11 Dec 2025 08:00:48 -0500 Subject: [PATCH 51/78] Expose SnapshotManifest for HubSpot applications (#225) Co-authored-by: Hernan Gelaf-Romer --- .../hadoop/hbase/snapshot/SnapshotRegionLocator.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java index c3a42e45de42..c7e543d89a25 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotRegionLocator.java @@ -56,6 +56,7 @@ public final class SnapshotRegionLocator implements RegionLocator { private final TableName tableName; private final TreeMap regions; + private final SnapshotManifest manifest; private final List rawLocations; @@ -92,16 +93,21 @@ public static SnapshotRegionLocator create(Configuration conf, TableName table) replicas.put(key, hrr); } - return new SnapshotRegionLocator(tableName, replicas, rawLocations); + return new SnapshotRegionLocator(tableName, replicas, manifest, rawLocations); } private SnapshotRegionLocator(TableName tableName, TreeMap regions, - List rawLocations) { + SnapshotManifest manifest, List rawLocations) { this.tableName = tableName; this.regions = regions; + this.manifest = manifest; this.rawLocations = rawLocations; } + public SnapshotManifest getManifest() { + return manifest; + } + @Override public HRegionLocation getRegionLocation(byte[] row, int replicaId, boolean reload) throws IOException { From e3fc7cd4dcfb6cb730cc256c531ae9a600dcebbe Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Fri, 12 Dec 2025 13:36:03 -0500 Subject: [PATCH 52/78] Log filtering in IncrementalBackupManager can lead to data loss (#226) Co-authored-by: Hernan Gelaf-Romer --- .../hadoop/hbase/backup/impl/IncrementalBackupManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index a9ebc519809d..ff77e6782eb8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -246,7 +246,7 @@ private List getLogFilesForNewBackup(Map olderTimestamps, // Even if these logs belong to a obsolete region server, we still need // to include they to avoid loss of edits for backup. Long newTimestamp = newestTimestamps.get(host); - if (newTimestamp == null || currentLogTS > newTimestamp) { + if (newTimestamp != null && currentLogTS > newTimestamp) { newestLogs.add(currentLogFile); } } From 10740103ae08579cb47c405359e342129d601c93 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Fri, 12 Dec 2025 14:37:33 -0500 Subject: [PATCH 53/78] (Not upstreamed yet) HBASE-29776: Log filtering in IncrementalBackupManager can lead to data loss (#227) --- .../hbase/backup/impl/IncrementalBackupManager.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index ff77e6782eb8..cfa222936293 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -240,15 +240,6 @@ private List getLogFilesForNewBackup(Map olderTimestamps, } else if (currentLogTS > oldTimeStamp) { resultLogFiles.add(currentLogFile); } - - // It is possible that a host in .oldlogs is an obsolete region server - // so newestTimestamps.get(host) here can be null. - // Even if these logs belong to a obsolete region server, we still need - // to include they to avoid loss of edits for backup. - Long newTimestamp = newestTimestamps.get(host); - if (newTimestamp != null && currentLogTS > newTimestamp) { - newestLogs.add(currentLogFile); - } } // remove newest log per host because they are still in use resultLogFiles.removeAll(newestLogs); From 2711ed69aac6d2f2bb6fec46248e7d322c75bb88 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 15 Dec 2025 10:25:16 -0500 Subject: [PATCH 54/78] HBASE-29744: Data loss scenario for WAL files belonging to RS added between backups (#224) Co-authored-by: Hernan Gelaf-Romer --- .../hbase/backup/master/BackupLogCleaner.java | 82 +++------- .../hbase/backup/util/BackupBoundaries.java | 149 ++++++++++++++++++ .../backup/master/TestBackupLogCleaner.java | 149 ++++++++++++++++-- 3 files changed, 308 insertions(+), 72 deletions(-) create mode 100644 hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 6bf1edc7bf03..263191df8049 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.backup.master; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,7 +33,8 @@ import org.apache.hadoop.hbase.backup.BackupInfo; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.impl.BackupManager; -import org.apache.hadoop.hbase.backup.util.BackupUtils; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupBoundaries; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.master.HMaster; @@ -41,7 +43,6 @@ import org.apache.hadoop.hbase.master.region.MasterRegionFactory; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +57,8 @@ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) public class BackupLogCleaner extends BaseLogCleanerDelegate { private static final Logger LOG = LoggerFactory.getLogger(BackupLogCleaner.class); + private static final long TS_BUFFER_DEFAULT = Duration.ofHours(1).toMillis(); + static final String TS_BUFFER_KEY = "hbase.backup.log.cleaner.timestamp.buffer.ms"; private boolean stopped = false; private Connection conn; @@ -86,8 +89,9 @@ public void init(Map params) { * I.e. WALs with a lower (= older) or equal timestamp are no longer needed for future incremental * backups. */ - private Map serverToPreservationBoundaryTs(List backups) + private BackupBoundaries serverToPreservationBoundaryTs(BackupSystemTable sysTable) throws IOException { + List backups = sysTable.getBackupHistory(true); if (LOG.isDebugEnabled()) { LOG.debug( "Cleaning WALs if they are older than the WAL cleanup time-boundary. " @@ -112,27 +116,25 @@ private Map serverToPreservationBoundaryTs(List backu .collect(Collectors.joining(", "))); } - // This map tracks, for every RegionServer, the least recent (= oldest / lowest timestamp) - // inclusion in any backup. In other words, it is the timestamp boundary up to which all backup - // roots have included the WAL in their backup. - Map boundaries = new HashMap<>(); + BackupBoundaries.BackupBoundariesBuilder builder = + BackupBoundaries.builder(getConf().getLong(TS_BUFFER_KEY, TS_BUFFER_DEFAULT)); for (BackupInfo backupInfo : newestBackupPerRootDir.values()) { + long startCode = Long.parseLong(sysTable.readBackupStartCode(backupInfo.getBackupRootDir())); // Iterate over all tables in the timestamp map, which contains all tables covered in the // backup root, not just the tables included in that specific backup (which could be a subset) for (TableName table : backupInfo.getTableSetTimestampMap().keySet()) { for (Map.Entry entry : backupInfo.getTableSetTimestampMap().get(table) .entrySet()) { - Address address = Address.fromString(entry.getKey()); - Long storedTs = boundaries.get(address); - if (storedTs == null || entry.getValue() < storedTs) { - boundaries.put(address, entry.getValue()); - } + builder.addBackupTimestamps(entry.getKey(), entry.getValue(), startCode); } } } + BackupBoundaries boundaries = builder.build(); + if (LOG.isDebugEnabled()) { - for (Map.Entry entry : boundaries.entrySet()) { + LOG.debug("Boundaries oldestStartCode: {}", boundaries.getOldestStartCode()); + for (Map.Entry entry : boundaries.getBoundaries().entrySet()) { LOG.debug("Server: {}, WAL cleanup boundary: {}", entry.getKey().getHostName(), entry.getValue()); } @@ -153,11 +155,10 @@ public Iterable getDeletableFiles(Iterable files) { return files; } - Map serverToPreservationBoundaryTs; + BackupBoundaries boundaries; try { - try (BackupManager backupManager = new BackupManager(conn, getConf())) { - serverToPreservationBoundaryTs = - serverToPreservationBoundaryTs(backupManager.getBackupHistory(true)); + try (BackupSystemTable sysTable = new BackupSystemTable(conn)) { + boundaries = serverToPreservationBoundaryTs(sysTable); } } catch (IOException ex) { LOG.error("Failed to analyse backup history with exception: {}. Retaining all logs", @@ -165,7 +166,7 @@ public Iterable getDeletableFiles(Iterable files) { return Collections.emptyList(); } for (FileStatus file : files) { - if (canDeleteFile(serverToPreservationBoundaryTs, file.getPath())) { + if (canDeleteFile(boundaries, file.getPath())) { filteredFiles.add(file); } } @@ -200,54 +201,17 @@ public boolean isStopped() { return this.stopped; } - protected static boolean canDeleteFile(Map addressToBoundaryTs, Path path) { + protected static boolean canDeleteFile(BackupBoundaries boundaries, Path path) { if (isHMasterWAL(path)) { return true; } - - try { - String hostname = BackupUtils.parseHostNameFromLogFile(path); - if (hostname == null) { - LOG.warn( - "Cannot parse hostname from RegionServer WAL file: {}. Ignoring cleanup of this log", - path); - return false; - } - Address walServerAddress = Address.fromString(hostname); - long walTimestamp = WAL.getTimestamp(path.getName()); - - if (!addressToBoundaryTs.containsKey(walServerAddress)) { - if (LOG.isDebugEnabled()) { - LOG.debug("No cleanup WAL time-boundary found for server: {}. Ok to delete file: {}", - walServerAddress.getHostName(), path); - } - return true; - } - - Long backupBoundary = addressToBoundaryTs.get(walServerAddress); - if (backupBoundary >= walTimestamp) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "WAL cleanup time-boundary found for server {}: {}. Ok to delete older file: {}", - walServerAddress.getHostName(), backupBoundary, path); - } - return true; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("WAL cleanup time-boundary found for server {}: {}. Keeping younger file: {}", - walServerAddress.getHostName(), backupBoundary, path); - } - } catch (Exception ex) { - LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", path, ex); - return false; - } - return false; + return boundaries.isDeletable(path); } private static boolean isHMasterWAL(Path path) { String fn = path.getName(); return fn.startsWith(WALProcedureStore.LOG_PREFIX) - || fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX); + || fn.endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX) + || path.toString().contains("/" + MasterRegionFactory.MASTER_STORE_DIR + "/"); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java new file mode 100644 index 000000000000..b38c1bdb68d7 --- /dev/null +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup.util; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.wal.WAL; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks time boundaries for WAL file cleanup during backup operations. Maintains the oldest + * timestamp per RegionServer included in any backup, enabling safe determination of which WAL files + * can be deleted without compromising backup integrity. + */ +@InterfaceAudience.Private +public class BackupBoundaries { + private static final Logger LOG = LoggerFactory.getLogger(BackupBoundaries.class); + private static final BackupBoundaries EMPTY_BOUNDARIES = + new BackupBoundaries(Collections.emptyMap(), Long.MAX_VALUE); + + // This map tracks, for every RegionServer, the least recent (= oldest / lowest timestamp) + // inclusion in any backup. In other words, it is the timestamp boundary up to which all backup + // roots have included the WAL in their backup. + private final Map boundaries; + + // The minimum WAL roll timestamp from the most recent backup of each backup root, used as a + // fallback cleanup boundary for RegionServers without explicit backup boundaries (e.g., servers + // that joined after backups began) + private final long oldestStartCode; + + private BackupBoundaries(Map boundaries, long oldestStartCode) { + this.boundaries = boundaries; + this.oldestStartCode = oldestStartCode; + } + + public boolean isDeletable(Path walLogPath) { + try { + String hostname = BackupUtils.parseHostNameFromLogFile(walLogPath); + + if (hostname == null) { + LOG.warn( + "Cannot parse hostname from RegionServer WAL file: {}. Ignoring cleanup of this log", + walLogPath); + return false; + } + + Address address = Address.fromString(hostname); + long pathTs = WAL.getTimestamp(walLogPath.getName()); + + if (!boundaries.containsKey(address)) { + boolean isDeletable = pathTs <= oldestStartCode; + if (LOG.isDebugEnabled()) { + LOG.debug( + "Boundary for {} not found. isDeletable = {} based on oldestStartCode = {} and WAL ts of {}", + walLogPath, isDeletable, oldestStartCode, pathTs); + } + return isDeletable; + } + + long backupTs = boundaries.get(address); + if (pathTs <= backupTs) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "WAL cleanup time-boundary found for server {}: {}. Ok to delete older file: {}", + address.getHostName(), pathTs, walLogPath); + } + return true; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("WAL cleanup time-boundary found for server {}: {}. Keeping younger file: {}", + address.getHostName(), backupTs, walLogPath); + } + + return false; + } catch (Exception e) { + LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", walLogPath, + e); + return false; + } + } + + public Map getBoundaries() { + return boundaries; + } + + public long getOldestStartCode() { + return oldestStartCode; + } + + public static BackupBoundariesBuilder builder(long tsCleanupBuffer) { + return new BackupBoundariesBuilder(tsCleanupBuffer); + } + + public static class BackupBoundariesBuilder { + private final Map boundaries = new HashMap<>(); + private final long tsCleanupBuffer; + + private long oldestStartCode = Long.MAX_VALUE; + + private BackupBoundariesBuilder(long tsCleanupBuffer) { + this.tsCleanupBuffer = tsCleanupBuffer; + } + + public BackupBoundariesBuilder addBackupTimestamps(String host, long hostLogRollTs, + long backupStartCode) { + Address address = Address.fromString(host); + Long storedTs = boundaries.get(address); + if (storedTs == null || hostLogRollTs < storedTs) { + boundaries.put(address, hostLogRollTs); + } + + if (oldestStartCode > backupStartCode) { + oldestStartCode = backupStartCode; + } + + return this; + } + + public BackupBoundaries build() { + if (boundaries.isEmpty()) { + return EMPTY_BOUNDARIES; + } + + oldestStartCode -= tsCleanupBuffer; + return new BackupBoundaries(boundaries, oldestStartCode); + } + } +} diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java index 0602327f9bd3..57e067148f30 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java @@ -21,9 +21,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -32,16 +33,23 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.backup.TestBackupBase; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.backup.util.BackupBoundaries; +import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -60,6 +68,11 @@ public class TestBackupLogCleaner extends TestBackupBase { // implements all test cases in 1 test since incremental full backup/ // incremental backup has dependencies + @BeforeClass + public static void before() { + TEST_UTIL.getConfiguration().setLong(BackupLogCleaner.TS_BUFFER_KEY, 0); + } + @Test public void testBackupLogCleaner() throws Exception { Path backupRoot1 = new Path(BACKUP_ROOT_DIR, "root1"); @@ -74,9 +87,9 @@ public void testBackupLogCleaner() throws Exception { assertFalse(systemTable.hasBackupSessions()); BackupLogCleaner cleaner = new BackupLogCleaner(); - cleaner.setConf(TEST_UTIL.getConfiguration()); Map params = new HashMap<>(1); params.put(HMaster.MASTER, TEST_UTIL.getHBaseCluster().getMaster()); + cleaner.setConf(TEST_UTIL.getConfiguration()); cleaner.init(params); // All WAL files can be deleted because we do not have backups @@ -197,35 +210,145 @@ public void testBackupLogCleaner() throws Exception { // Taking the minimum timestamp (= 2), this means all WALs preceding B3 can be deleted. deletable = cleaner.getDeletableFiles(walFilesAfterB5); assertEquals(toSet(walFilesAfterB2), toSet(deletable)); + } finally { + TEST_UTIL.truncateTable(BackupSystemTable.getTableName(TEST_UTIL.getConfiguration())).close(); } } - private Set mergeAsSet(Collection toCopy, Collection toAdd) { - Set result = new LinkedHashSet<>(toCopy); - result.addAll(toAdd); - return result; + @Test + public void testDoesNotDeleteWALsFromNewServers() throws Exception { + Path backupRoot1 = new Path(BACKUP_ROOT_DIR, "backup1"); + List tableSetFull = Arrays.asList(table1, table2, table3, table4); + + JVMClusterUtil.RegionServerThread rsThread = null; + try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + LOG.info("Creating initial backup B1"); + String backupIdB1 = backupTables(BackupType.FULL, tableSetFull, backupRoot1.toString()); + assertTrue(checkSucceeded(backupIdB1)); + + List walsAfterB1 = getListOfWALFiles(TEST_UTIL.getConfiguration()); + LOG.info("WALs after B1: {}", walsAfterB1.size()); + + String startCodeStr = systemTable.readBackupStartCode(backupRoot1.toString()); + long b1StartCode = Long.parseLong(startCodeStr); + LOG.info("B1 startCode: {}", b1StartCode); + + // Add a new RegionServer to the cluster + LOG.info("Adding new RegionServer to cluster"); + rsThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer(); + ServerName newServerName = rsThread.getRegionServer().getServerName(); + LOG.info("New RegionServer started: {}", newServerName); + + // Move a region to the new server to ensure it creates a WAL + List regions = TEST_UTIL.getAdmin().getRegions(table1); + RegionInfo regionToMove = regions.get(0); + + LOG.info("Moving region {} to new server {}", regionToMove.getEncodedName(), newServerName); + TEST_UTIL.getAdmin().move(regionToMove.getEncodedNameAsBytes(), newServerName); + + TEST_UTIL.waitFor(30000, () -> { + try { + HRegionLocation location = TEST_UTIL.getConnection().getRegionLocator(table1) + .getRegionLocation(regionToMove.getStartKey()); + return location.getServerName().equals(newServerName); + } catch (IOException e) { + return false; + } + }); + + // Write some data to trigger WAL creation on the new server + try (Table t1 = TEST_UTIL.getConnection().getTable(table1)) { + for (int i = 0; i < 100; i++) { + Put p = new Put(Bytes.toBytes("newserver-row-" + i)); + p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p); + } + } + TEST_UTIL.getAdmin().flushRegion(regionToMove.getEncodedNameAsBytes()); + + List walsAfterNewServer = getListOfWALFiles(TEST_UTIL.getConfiguration()); + LOG.info("WALs after adding new server: {}", walsAfterNewServer.size()); + assertTrue("Should have more WALs after new server", + walsAfterNewServer.size() > walsAfterB1.size()); + + List newServerWALs = new ArrayList<>(walsAfterNewServer); + newServerWALs.removeAll(walsAfterB1); + assertFalse("Should have WALs from new server", newServerWALs.isEmpty()); + + BackupLogCleaner cleaner = new BackupLogCleaner(); + cleaner.setConf(TEST_UTIL.getConfiguration()); + Map params = new HashMap<>(1); + params.put(HMaster.MASTER, TEST_UTIL.getHBaseCluster().getMaster()); + cleaner.init(params); + + Set deletable = toSet(cleaner.getDeletableFiles(walsAfterNewServer)); + for (FileStatus newWAL : newServerWALs) { + assertFalse("WAL from new server should NOT be deletable: " + newWAL.getPath(), + deletable.contains(newWAL)); + } + } finally { + TEST_UTIL.truncateTable(BackupSystemTable.getTableName(TEST_UTIL.getConfiguration())).close(); + // Clean up the RegionServer we added + if (rsThread != null) { + LOG.info("Stopping the RegionServer added for test"); + TEST_UTIL.getMiniHBaseCluster() + .stopRegionServer(rsThread.getRegionServer().getServerName()); + TEST_UTIL.getMiniHBaseCluster() + .waitForRegionServerToStop(rsThread.getRegionServer().getServerName(), 30000); + } + } } - private Set toSet(Iterable iterable) { - Set result = new LinkedHashSet<>(); - iterable.forEach(result::add); - return result; + @Test + public void testCanDeleteFileWithNewServerWALs() { + long backupStartCode = 1000000L; + // Old WAL from before the backup + Path oldWAL = new Path("/hbase/oldWALs/server1%2C60020%2C12345.500000"); + String host = BackupUtils.parseHostNameFromLogFile(oldWAL); + BackupBoundaries boundaries = BackupBoundaries.builder(0L) + .addBackupTimestamps(host, backupStartCode, backupStartCode).build(); + + assertTrue("WAL older than backup should be deletable", + BackupLogCleaner.canDeleteFile(boundaries, oldWAL)); + + // WAL from exactly at the backup boundary + Path boundaryWAL = new Path("/hbase/oldWALs/server1%2C60020%2C12345.1000000"); + assertTrue("WAL at boundary should be deletable", + BackupLogCleaner.canDeleteFile(boundaries, boundaryWAL)); + + // WAL from a server that joined AFTER the backup + Path newServerWAL = new Path("/hbase/oldWALs/newserver%2C60020%2C99999.1500000"); + assertFalse("WAL from new server (after backup) should NOT be deletable", + BackupLogCleaner.canDeleteFile(boundaries, newServerWAL)); } @Test public void testCleansUpHMasterWal() { Path path = new Path("/hbase/MasterData/WALs/hmaster,60000,1718808578163"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), path)); + assertTrue(BackupLogCleaner.canDeleteFile(BackupBoundaries.builder(0L).build(), path)); } @Test public void testCleansUpArchivedHMasterWal() { + BackupBoundaries empty = BackupBoundaries.builder(0L).build(); Path normalPath = new Path("/hbase/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), normalPath)); + assertTrue(BackupLogCleaner.canDeleteFile(empty, normalPath)); Path masterPath = new Path( "/hbase/MasterData/oldWALs/hmaster%2C60000%2C1716224062663.1716247552189$masterlocalwal$"); - assertTrue(BackupLogCleaner.canDeleteFile(Collections.emptyMap(), masterPath)); + assertTrue(BackupLogCleaner.canDeleteFile(empty, masterPath)); + } + + private Set mergeAsSet(Collection toCopy, Collection toAdd) { + Set result = new LinkedHashSet<>(toCopy); + result.addAll(toAdd); + return result; + } + + private Set toSet(Iterable iterable) { + Set result = new LinkedHashSet<>(); + iterable.forEach(result::add); + return result; } } From 82ee29e09a7752cf98b9ae1a446fe78e2f589de6 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Tue, 16 Dec 2025 11:31:38 -0500 Subject: [PATCH 55/78] Fix an issue with duplicated backups of decommissioned host WAL files (#228) * Fix an issue with duplicated backups of inactive host WAL files * Skip meta region WAL files --- .../backup/impl/FullTableBackupClient.java | 27 +++ .../backup/impl/IncrementalBackupManager.java | 22 ++ .../hbase/backup/TestBackupOfflineRS.java | 221 ++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index 969d0945b8fa..8854e5843f53 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -30,6 +30,10 @@ import java.util.Map; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupCopyJob; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -42,7 +46,9 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -190,6 +196,27 @@ public void execute() throws IOException { backupInfo.setState(BackupState.COMPLETE); // The table list in backupInfo is good for both full backup and incremental backup. // For incremental backup, it contains the incremental backup table set. + + // Scan oldlogs for dead/decommissioned hosts and add their max WAL timestamps + // to newTimestamps. This ensures subsequent incremental backups won't try to back up + // WALs that are already covered by this full backup's snapshot. + Path walRootDir = CommonFSUtils.getWALRootDir(conf); + Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + FileSystem fs = walRootDir.getFileSystem(conf); + if (fs.exists(oldLogDir)) { + for (FileStatus oldlog : fs.listStatus(oldLogDir)) { + if (AbstractFSWALProvider.isMetaFile(oldlog.getPath())) { + continue; + } + String host = BackupUtils.parseHostFromOldLog(oldlog.getPath()); + if (host != null && !newTimestamps.containsKey(host)) { + long ts = BackupUtils.getCreationTime(oldlog.getPath()); + newTimestamps.put(host, ts); + LOG.info("Updating backup boundary for inactive host {}: timestamp={}", host, ts); + } + } + } + backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); Map> newTableSetTimestampMap = diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index cfa222936293..d4934ccde27e 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -243,6 +243,28 @@ private List getLogFilesForNewBackup(Map olderTimestamps, } // remove newest log per host because they are still in use resultLogFiles.removeAll(newestLogs); + + // Update newestTimestamps with max timestamp of files we're actually backing up. + // This ensures dead/decommissioned hosts get their boundaries recorded in trslm, + // preventing re-backup of the same WAL files on subsequent incremental backups. + for (String logFile : resultLogFiles) { + Path logPath = new Path(logFile); + String logHost = BackupUtils.parseHostFromOldLog(logPath); + if (logHost == null) { + logHost = BackupUtils.parseHostNameFromLogFile(logPath.getParent()); + } + if (logHost != null) { + long logTs = BackupUtils.getCreationTime(logPath); + Long existingTs = newestTimestamps.get(logHost); + if (existingTs == null || logTs > existingTs) { + newestTimestamps.put(logHost, logTs); + if (existingTs == null) { + LOG.info("Updating backup boundary for inactive host {}: timestamp={}", logHost, logTs); + } + } + } + } + return resultLogFiles; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java new file mode 100644 index 000000000000..831cb128e105 --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Tests that WAL files from offline/inactive RegionServers are handled correctly during backup. + * Specifically verifies that WALs from an offline RS are: + *

    + *
  1. Backed up once in the first backup after the RS goes offline
    2. + *
  2. NOT re-backed up in subsequent backups
    3. + *
+ */ +@Category(LargeTests.class) +public class TestBackupOfflineRS extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBackupOfflineRS.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestBackupOfflineRS.class); + + @BeforeClass + public static void setUp() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + conf1 = TEST_UTIL.getConfiguration(); + conf1.setInt("hbase.regionserver.info.port", -1); + autoRestoreOnFailure = true; + useSecondCluster = false; + setUpHelper(); + // Start an additional RS so we have at least 2 + TEST_UTIL.getMiniHBaseCluster().startRegionServer(); + TEST_UTIL.waitTableAvailable(table1); + } + + /** + * Tests that when a RegionServer goes offline, its WAL files are backed up once in the first + * incremental backup and NOT re-backed up in subsequent incremental backups. + */ + @Test + public void testIncrementalBackupWithOfflineRS() throws Exception { + LOG.info("Starting testIncrementalBackupWithOfflineRS"); + + MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + List tables = Lists.newArrayList(table1); + + // 1. Run full backup to establish baseline + LOG.info("Taking full backup"); + String fullBackupId = fullTableBackup(tables); + assertTrue("Full backup should succeed", checkSucceeded(fullBackupId)); + + // 2. Insert some data to generate WAL entries + LOG.info("Inserting data to generate WAL entries"); + try (Connection conn = ConnectionFactory.createConnection(conf1)) { + insertIntoTable(conn, table1, famName, 1, 100); + } + + // 3. Stop one RS to simulate it going offline + int rsToStop = 0; + HRegionServer rsBeforeStop = cluster.getRegionServer(rsToStop); + String offlineHost = rsBeforeStop.getServerName().getHostAndPort(); + String offlineHostPrefix = offlineHost.split(",")[0]; + LOG.info("Stopping RS: {} (prefix: {})", offlineHost, offlineHostPrefix); + + cluster.stopRegionServer(rsToStop); + // Wait for WALs to be moved to oldlogs + Thread.sleep(5000); + + // 4. Run first incremental backup - should include offline host's WALs + LOG.info("Taking first incremental backup (should include offline RS WALs)"); + String incr1 = incrementalTableBackup(tables); + assertTrue("First incremental backup should succeed", checkSucceeded(incr1)); + + // 5. Verify offline host is recorded in trslm + try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + Map> timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); + Map rsTimestamps = timestamps.get(table1); + LOG.info("RS timestamps after first incremental: {}", rsTimestamps); + + boolean offlineHostRecorded = + rsTimestamps.keySet().stream().anyMatch(k -> k.contains(offlineHostPrefix)); + assertTrue("Offline host should have timestamp recorded in trslm", offlineHostRecorded); + + // 6. Get WAL file list for first incremental + BackupInfo backupInfo1 = sysTable.readBackupInfo(incr1); + List walFiles1 = backupInfo1.getIncrBackupFileList(); + LOG.info("WAL files in first incremental: {}", walFiles1); + + long offlineHostWalCount1 = + walFiles1.stream().filter(f -> f.contains(offlineHostPrefix)).count(); + LOG.info("Offline host WAL count in first incremental: {}", offlineHostWalCount1); + assertTrue("First incremental should include offline host WALs", offlineHostWalCount1 > 0); + + // 7. Run second incremental backup - should NOT include offline host's WALs + LOG.info("Taking second incremental backup (should NOT include offline RS WALs)"); + String incr2 = incrementalTableBackup(tables); + assertTrue("Second incremental backup should succeed", checkSucceeded(incr2)); + + // 8. Verify second incremental does not include offline host's WALs + BackupInfo backupInfo2 = sysTable.readBackupInfo(incr2); + List walFiles2 = backupInfo2.getIncrBackupFileList(); + LOG.info("WAL files in second incremental: {}", walFiles2); + + long offlineHostWalCount2 = + walFiles2.stream().filter(f -> f.contains(offlineHostPrefix)).count(); + LOG.info("Offline host WAL count in second incremental: {}", offlineHostWalCount2); + assertEquals("Second incremental should NOT include offline host WALs", 0, + offlineHostWalCount2); + } + + LOG.info("testIncrementalBackupWithOfflineRS completed successfully"); + } + + /** + * Tests that when a full backup is taken while an RS is offline (with WALs in oldlogs), the + * offline host's timestamps are recorded so subsequent incremental backups don't re-include those + * WALs. + */ + @Test + public void testFullBackupWithOfflineRS() throws Exception { + LOG.info("Starting testFullBackupWithOfflineRS"); + + MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + List tables = Lists.newArrayList(table1); + + // Ensure we have at least 2 RSes + if (cluster.getNumLiveRegionServers() < 2) { + cluster.startRegionServer(); + Thread.sleep(2000); + } + + // 1. Insert some data to generate WAL entries + LOG.info("Inserting data to generate WAL entries"); + try (Connection conn = ConnectionFactory.createConnection(conf1)) { + insertIntoTable(conn, table1, famName, 2, 100); + } + + // 2. Stop one RS to simulate it going offline + int rsToStop = 0; + HRegionServer rsBeforeStop = cluster.getRegionServer(rsToStop); + String offlineHost = rsBeforeStop.getServerName().getHostAndPort(); + String offlineHostPrefix = offlineHost.split(",")[0]; + LOG.info("Stopping RS: {} (prefix: {})", offlineHost, offlineHostPrefix); + + cluster.stopRegionServer(rsToStop); + // Wait for WALs to be moved to oldlogs + Thread.sleep(5000); + + // 3. Run full backup - should record offline host timestamps + LOG.info("Taking full backup (with offline RS WALs in oldlogs)"); + String fullBackupId = fullTableBackup(tables); + assertTrue("Full backup should succeed", checkSucceeded(fullBackupId)); + + // 4. Verify offline host is recorded in trslm + try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + Map> timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); + Map rsTimestamps = timestamps.get(table1); + LOG.info("RS timestamps after full backup: {}", rsTimestamps); + + boolean offlineHostRecorded = + rsTimestamps.keySet().stream().anyMatch(k -> k.contains(offlineHostPrefix)); + assertTrue("Offline host should have timestamp recorded in trslm after full backup", + offlineHostRecorded); + + // 5. Run incremental backup - should NOT include offline host's WALs from before full backup + LOG.info("Taking incremental backup (should NOT include offline RS WALs)"); + String incrBackupId = incrementalTableBackup(tables); + assertTrue("Incremental backup should succeed", checkSucceeded(incrBackupId)); + + // 6. Verify incremental does not include offline host's WALs + BackupInfo backupInfo = sysTable.readBackupInfo(incrBackupId); + List walFiles = backupInfo.getIncrBackupFileList(); + LOG.info("WAL files in incremental: {}", walFiles); + + long offlineHostWalCount = + walFiles.stream().filter(f -> f.contains(offlineHostPrefix)).count(); + LOG.info("Offline host WAL count in incremental: {}", offlineHostWalCount); + assertEquals("Incremental after full should NOT include offline host WALs", 0, + offlineHostWalCount); + } + + LOG.info("testFullBackupWithOfflineRS completed successfully"); + } +} From f851ba4c67e10cb5b8ff76e7c754a51d14e6ed78 Mon Sep 17 00:00:00 2001 From: Alex Hughes Date: Fri, 19 Dec 2025 08:11:00 +0000 Subject: [PATCH 56/78] HubSpot Backport: HBASE-29729 Add per-region table descriptor hash to regionServer JMX Metric (#7481) (#229) * HBASE-29729 Add table descriptor hash --- .../hadoop/hbase/client/TableDescriptor.java | 26 +++ .../hbase/client/TableDescriptorBuilder.java | 13 ++ .../regionserver/MetricsRegionSource.java | 2 + .../regionserver/MetricsRegionWrapper.java | 8 + .../regionserver/MetricsRegionSourceImpl.java | 7 + .../TestMetricsRegionSourceImpl.java | 5 + .../MetricsRegionWrapperImpl.java | 15 ++ .../MetricsRegionWrapperStub.java | 5 + ...tricsRegionWrapperTableDescriptorHash.java | 155 ++++++++++++++++++ .../regionserver/TestRegionServerMetrics.java | 18 ++ .../TestTableDescriptorHashComputation.java | 127 ++++++++++++++ 11 files changed, 381 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionWrapperTableDescriptorHash.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTableDescriptorHashComputation.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java index c017387d67fc..dfeadb972296 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.client; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; @@ -26,10 +27,16 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.CRC32; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; /** * TableDescriptor contains the details about an HBase table such as the descriptors of all the @@ -337,4 +344,23 @@ default boolean matchReplicationScope(boolean enabled) { * {@link org.apache.hadoop.hbase.rsgroup.RSGroupInfo#DEFAULT_GROUP}. */ Optional getRegionServerGroup(); + + /** + * Computes a CRC32 hash of the table descriptor's protobuf representation. This hash can be used + * to detect changes in the table descriptor configuration. + * @return A hex string representation of the CRC32 hash, or "UNKNOWN" if computation fails + */ + default String getDescriptorHash() { + try { + HBaseProtos.TableSchema tableSchema = ProtobufUtil.toTableSchema(this); + ByteBuffer byteBuffer = ByteBuffer.wrap(tableSchema.toByteArray()); + CRC32 crc32 = new CRC32(); + crc32.update(byteBuffer); + return Long.toHexString(crc32.getValue()); + } catch (Exception e) { + Logger log = LoggerFactory.getLogger(TableDescriptor.class); + log.error("Failed to compute table descriptor hash for table {}", getTableName(), e); + return "UNKNOWN"; + } + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java index 3c8b7ad34b71..2ec5202f7fb5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java @@ -633,6 +633,11 @@ public static class ModifyableTableDescriptor private final Map families = new TreeMap<>(Bytes.BYTES_RAWCOMPARATOR); + /** + * Cached hash of the table descriptor. Computed lazily on first access. + */ + private volatile String descriptorHash; + /** * Construct a table descriptor specifying a TableName object * @param name Table name. TODO: make this private after removing the HTableDescriptor @@ -1619,6 +1624,14 @@ public Optional getRegionServerGroup() { return Optional.empty(); } } + + @Override + public String getDescriptorHash() { + if (descriptorHash == null) { + descriptorHash = TableDescriptor.super.getDescriptorHash(); + } + return descriptorHash; + } } private static Optional toCoprocessorDescriptor(String spec) { diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java index c3d955592d6a..41267de1981a 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java @@ -56,6 +56,8 @@ public interface MetricsRegionSource extends Comparable { String ROW_READS_ONLY_ON_MEMSTORE_DESC = "Row reads happening completely out of memstore"; String MIXED_ROW_READS = "mixedRowReadsCount"; String MIXED_ROW_READS_ON_STORE_DESC = "Row reads happening out of files and memstore on store"; + String TABLE_DESCRIPTOR_HASH = "tableDescriptorHash"; + String TABLE_DESCRIPTOR_HASH_DESC = "The hash of the current table descriptor"; /** * Close the region's metrics as this region is closing. diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 4d8a028d89b1..c3e37586e283 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -161,4 +161,12 @@ public interface MetricsRegionWrapper { /** Returns the number of row reads on memstore and file per store */ Map getMixedRowReadsCount(); + /** + * Returns a hash of the table descriptor that this region was opened with. This hash uniquely + * identifies the table configuration (column families, compression, TTL, block size, etc.) and + * can be used to determine if a region needs to be reopened to pick up descriptor changes. + * @return hex-encoded hash of the serialized TableDescriptor + */ + String getTableDescriptorHash(); + } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 92ecaa580885..5e8f01a7579d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -284,6 +284,13 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { MetricsRegionSource.ROW_READS_ONLY_ON_MEMSTORE_DESC); addCounter(mrb, this.regionWrapper.getMixedRowReadsCount(), MetricsRegionSource.MIXED_ROW_READS, MetricsRegionSource.MIXED_ROW_READS_ON_STORE_DESC); + mrb.add( + Interns.tag( + regionNamePrefix + MetricsRegionSource.TABLE_DESCRIPTOR_HASH, + MetricsRegionSource.TABLE_DESCRIPTOR_HASH_DESC, + this.regionWrapper.getTableDescriptorHash() + ) + ); } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 2c8205085d1e..994ce6dcad93 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -232,5 +232,10 @@ public Map getMixedRowReadsCount() { map.put("info", 0L); return map; } + + @Override + public String getTableDescriptorHash() { + return "testhash"; + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index bce961e8f279..eef58ed9f3ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -64,9 +64,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable private ScheduledFuture regionMetricsUpdateTask; private float currentRegionCacheRatio; + private final String tableDescriptorHash; public MetricsRegionWrapperImpl(HRegion region) { this.region = region; + this.tableDescriptorHash = determineTableDescriptorHash(); this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); this.runnable = new HRegionMetricsWrapperRunnable(); this.regionMetricsUpdateTask = @@ -352,6 +354,19 @@ public void run() { } } + @Override + public String getTableDescriptorHash() { + return tableDescriptorHash; + } + + private String determineTableDescriptorHash() { + TableDescriptor tableDesc = this.region.getTableDescriptor(); + if (tableDesc == null) { + return UNKNOWN; + } + return tableDesc.getDescriptorHash(); + } + @Override public void close() throws IOException { regionMetricsUpdateTask.cancel(true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 0995b0faee05..ac77279882af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -198,4 +198,9 @@ public Map getMixedRowReadsCount() { map.put("info", 0L); return map; } + + @Override + public String getTableDescriptorHash() { + return "testhash123abc"; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionWrapperTableDescriptorHash.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionWrapperTableDescriptorHash.java new file mode 100644 index 000000000000..7c70b56ebddf --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionWrapperTableDescriptorHash.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestMetricsRegionWrapperTableDescriptorHash { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetricsRegionWrapperTableDescriptorHash.class); + + private HBaseTestingUtility testUtil; + private Configuration conf; + + @Before + public void setUp() throws Exception { + conf = HBaseConfiguration.create(); + testUtil = new HBaseTestingUtility(conf); + } + + @After + public void tearDown() throws Exception { + if (testUtil != null) { + testUtil.cleanupTestDir(); + } + } + + @Test + public void testTableDescriptorHashGeneration() throws Exception { + TableName tableName = TableName.valueOf("testTable"); + TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("a")) + .setEndKey(Bytes.toBytes("z")).build(); + + Path testDir = testUtil.getDataTestDir("testTableDescriptorHashGeneration"); + HRegion region = + HBaseTestingUtility.createRegionAndWAL(regionInfo, testDir, conf, tableDescriptor); + + try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) { + String hash = wrapper.getTableDescriptorHash(); + assertNotNull(hash); + assertNotEquals("unknown", hash); + assertEquals(8, hash.length()); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + } + + @Test + public void testHashConsistency() throws Exception { + TableName tableName = TableName.valueOf("testTable2"); + TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + RegionInfo regionInfo1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("a")) + .setEndKey(Bytes.toBytes("m")).build(); + RegionInfo regionInfo2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("m")) + .setEndKey(Bytes.toBytes("z")).build(); + + Path testDir1 = testUtil.getDataTestDir("testHashConsistency1"); + HRegion region1 = + HBaseTestingUtility.createRegionAndWAL(regionInfo1, testDir1, conf, tableDescriptor); + + Path testDir2 = testUtil.getDataTestDir("testHashConsistency2"); + HRegion region2 = + HBaseTestingUtility.createRegionAndWAL(regionInfo2, testDir2, conf, tableDescriptor); + try (MetricsRegionWrapperImpl wrapper1 = new MetricsRegionWrapperImpl(region1); + MetricsRegionWrapperImpl wrapper2 = new MetricsRegionWrapperImpl(region2)) { + + String hash1 = wrapper1.getTableDescriptorHash(); + String hash2 = wrapper2.getTableDescriptorHash(); + + assertEquals(hash1, hash2); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region1); + HBaseTestingUtility.closeRegionAndWAL(region2); + } + } + + @Test + public void testHashChangeOnDescriptorChange() throws Exception { + TableName tableName = TableName.valueOf("testTable3"); + TableDescriptor tableDescriptor1 = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + TableDescriptor tableDescriptor2 = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder("cf".getBytes()).setTimeToLive(86400).build()) + .build(); + + RegionInfo regionInfo1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("a")) + .setEndKey(Bytes.toBytes("m")).build(); + RegionInfo regionInfo2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("m")) + .setEndKey(Bytes.toBytes("z")).build(); + + Path testDir1 = testUtil.getDataTestDir("testHashChangeOnDescriptorChange1"); + HRegion region1 = + HBaseTestingUtility.createRegionAndWAL(regionInfo1, testDir1, conf, tableDescriptor1); + + Path testDir2 = testUtil.getDataTestDir("testHashChangeOnDescriptorChange2"); + HRegion region2 = + HBaseTestingUtility.createRegionAndWAL(regionInfo2, testDir2, conf, tableDescriptor2); + + try (MetricsRegionWrapperImpl wrapper1 = new MetricsRegionWrapperImpl(region1); + MetricsRegionWrapperImpl wrapper2 = new MetricsRegionWrapperImpl(region2)) { + String hash1 = wrapper1.getTableDescriptorHash(); + String hash2 = wrapper2.getTableDescriptorHash(); + + assertNotEquals(hash1, hash2); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region1); + HBaseTestingUtility.closeRegionAndWAL(region2); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index dba0c141952c..5f75391f0e2b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -649,4 +650,21 @@ public void testReadBytes() throws Exception { assertEquals("Total zero-byte read bytes should be equal to 0", 0, metricsRegionServer.getRegionServerWrapper().getZeroCopyBytesRead()); } + + @Test + public void testTableDescriptorHashMetric() throws Exception { + doNPuts(1, false); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + + HRegion region = rs.getRegions(tableName).get(0); + assertNotNull("Region should exist", region); + + try (MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region)) { + String hash = wrapper.getTableDescriptorHash(); + + assertNotNull("TableDescriptorHash should not be null", hash); + assertNotEquals("TableDescriptorHash should not be 'UNKNOWN'", "UNKNOWN", hash); + assertEquals("Hash should be 8 characters (CRC32 hex)", 8, hash.length()); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTableDescriptorHashComputation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTableDescriptorHashComputation.java new file mode 100644 index 000000000000..9c793ef7fb4b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTableDescriptorHashComputation.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestTableDescriptorHashComputation { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTableDescriptorHashComputation.class); + + @Test + public void testHashLength() { + TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + String hash = td.getDescriptorHash(); + assertNotNull(hash); + assertEquals(8, hash.length()); + } + + @Test + public void testIdenticalDescriptorsProduceSameHash() { + TableDescriptor td1 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + TableDescriptor td2 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + String hash1 = td1.getDescriptorHash(); + String hash2 = td2.getDescriptorHash(); + + assertEquals(hash1, hash2); + } + + @Test + public void testDifferentDescriptorsProduceDifferentHashes() { + TableDescriptor td1 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + TableDescriptor td2 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder("cf".getBytes()).setTimeToLive(86400).build()) + .build(); + + String hash1 = td1.getDescriptorHash(); + String hash2 = td2.getDescriptorHash(); + + assertNotEquals(hash1, hash2); + } + + @Test + public void testDifferentCompressionProducesDifferentHash() { + TableDescriptor td1 = TableDescriptorBuilder + .newBuilder(TableName.valueOf("testTable")).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder("cf".getBytes()).setCompressionType(Compression.Algorithm.NONE).build()) + .build(); + + TableDescriptor td2 = TableDescriptorBuilder + .newBuilder(TableName.valueOf("testTable")).setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder("cf".getBytes()).setCompressionType(Compression.Algorithm.SNAPPY).build()) + .build(); + + String hash1 = td1.getDescriptorHash(); + String hash2 = td2.getDescriptorHash(); + + assertNotEquals(hash1, hash2); + } + + @Test + public void testMultipleColumnFamilies() { + TableDescriptor td1 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2")).build(); + + TableDescriptor td2 = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1")).build(); + + String hash1 = td1.getDescriptorHash(); + String hash2 = td2.getDescriptorHash(); + + assertNotEquals(hash1, hash2); + } + + @Test + public void testHashCaching() { + TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("testTable")) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build(); + + String hash1 = td.getDescriptorHash(); + String hash2 = td.getDescriptorHash(); + + assertNotNull(hash1); + assertEquals(hash1, hash2); + } +} From 3cc8905f701fa7e4cfb7af30823ebaafc6e6d9a9 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 29 Dec 2025 10:03:10 -0500 Subject: [PATCH 57/78] Incremental Backups Superfluously Write Region Info (#230) Co-authored-by: Hernan Gelaf-Romer --- .../backup/impl/IncrementalTableBackupClient.java | 3 +-- .../hadoop/hbase/backup/util/BackupUtils.java | 15 ++------------- .../hadoop/hbase/backup/TestBackupBase.java | 3 +-- .../regionserver/MetricsRegionSourceImpl.java | 10 +++------- 4 files changed, 7 insertions(+), 24 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index d800f07f5fed..5cc68fd43b86 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -302,8 +302,7 @@ public void execute() throws IOException, ColumnFamilyMismatchException { // case INCREMENTAL_COPY: try { - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + BackupUtils.copyTableDescriptor(conn, backupInfo, conf); setupRegionLocator(); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index bb3d80a5c1bd..b8b82ef31cbc 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -40,7 +40,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -116,14 +115,14 @@ private BackupUtils() { } /** - * copy out Table RegionInfo into incremental backup image need to consider move this logic into + * copy out Table descriptor into incremental backup image need to consider move this logic into * HBackupFileSystem * @param conn connection * @param backupInfo backup info * @param conf configuration * @throws IOException exception */ - public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf) + public static void copyTableDescriptor(Connection conn, BackupInfo backupInfo, Configuration conf) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); @@ -147,16 +146,6 @@ public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, C LOG.debug("Attempting to copy table info for:" + table + " target: " + target + " descriptor: " + orig); LOG.debug("Finished copying tableinfo."); - List regions = MetaTableAccessor.getTableRegions(conn, table); - // For each region, write the region info to disk - LOG.debug("Starting to write region info for table " + table); - for (RegionInfo regionInfo : regions) { - Path regionDir = FSUtils - .getRegionDirFromTableDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo); - regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName()); - writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); - } - LOG.debug("Finished writing region info for table " + table); } } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 2a0be003cab9..32598af94e90 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -163,8 +163,7 @@ public void execute() throws IOException { LOG.debug("For incremental backup, current table set is " + backupManager.getIncrementalBackupTableSet()); newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap(); - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + BackupUtils.copyTableDescriptor(conn, backupInfo, conf); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(); incrementalCopyHFiles(new String[] { getBulkOutputDir().toString() }, diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 5e8f01a7579d..d26653e9e2d6 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -284,13 +284,9 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { MetricsRegionSource.ROW_READS_ONLY_ON_MEMSTORE_DESC); addCounter(mrb, this.regionWrapper.getMixedRowReadsCount(), MetricsRegionSource.MIXED_ROW_READS, MetricsRegionSource.MIXED_ROW_READS_ON_STORE_DESC); - mrb.add( - Interns.tag( - regionNamePrefix + MetricsRegionSource.TABLE_DESCRIPTOR_HASH, - MetricsRegionSource.TABLE_DESCRIPTOR_HASH_DESC, - this.regionWrapper.getTableDescriptorHash() - ) - ); + mrb.add(Interns.tag(regionNamePrefix + MetricsRegionSource.TABLE_DESCRIPTOR_HASH, + MetricsRegionSource.TABLE_DESCRIPTOR_HASH_DESC, + this.regionWrapper.getTableDescriptorHash())); } } From b0c61be4a8e70049cb0b713a3ff9a559c23eee89 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 5 Jan 2026 08:55:00 -0500 Subject: [PATCH 58/78] HBASE-29776: Log filtering in IncrementalBackupManager can lead to data loss (#231) Co-authored-by: Hernan Gelaf-Romer --- .../backup/impl/FullTableBackupClient.java | 51 ++++++-- .../backup/impl/IncrementalBackupManager.java | 61 +++++---- .../hbase/backup/TestBackupOfflineRS.java | 118 ++---------------- 3 files changed, 88 insertions(+), 142 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java index 8854e5843f53..a9381b1409a3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -165,13 +166,14 @@ public void execute() throws IOException { // will be part of the snapshot being taken). We gather this list before taking the actual // snapshots for the same reason as the log rolls. List bulkLoadsToDelete = backupManager.readBulkloadRows(tableList); + Map previousLogRollsByHost = backupManager.readRegionServerLastLogRollResult(); Map props = new HashMap<>(); props.put("backupRoot", backupInfo.getBackupRootDir()); admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); - newTimestamps = backupManager.readRegionServerLastLogRollResult(); + Map latestLogRollsByHost = backupManager.readRegionServerLastLogRollResult(); // SNAPSHOT_TABLES: backupInfo.setPhase(BackupPhase.SNAPSHOT); @@ -194,29 +196,52 @@ public void execute() throws IOException { // set overall backup status: complete. Here we make sure to complete the backup. // After this checkpoint, even if entering cancel process, will let the backup finished backupInfo.setState(BackupState.COMPLETE); - // The table list in backupInfo is good for both full backup and incremental backup. - // For incremental backup, it contains the incremental backup table set. // Scan oldlogs for dead/decommissioned hosts and add their max WAL timestamps // to newTimestamps. This ensures subsequent incremental backups won't try to back up // WALs that are already covered by this full backup's snapshot. Path walRootDir = CommonFSUtils.getWALRootDir(conf); + Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME); Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); FileSystem fs = walRootDir.getFileSystem(conf); - if (fs.exists(oldLogDir)) { - for (FileStatus oldlog : fs.listStatus(oldLogDir)) { - if (AbstractFSWALProvider.isMetaFile(oldlog.getPath())) { - continue; - } - String host = BackupUtils.parseHostFromOldLog(oldlog.getPath()); - if (host != null && !newTimestamps.containsKey(host)) { - long ts = BackupUtils.getCreationTime(oldlog.getPath()); - newTimestamps.put(host, ts); - LOG.info("Updating backup boundary for inactive host {}: timestamp={}", host, ts); + + List allLogs = new ArrayList<>(); + for (FileStatus hostLogDir : fs.listStatus(logDir)) { + String host = BackupUtils.parseHostNameFromLogFile(hostLogDir.getPath()); + if (host == null) { + continue; + } + allLogs.addAll(Arrays.asList(fs.listStatus(hostLogDir.getPath()))); + } + allLogs.addAll(Arrays.asList(fs.listStatus(oldLogDir))); + + newTimestamps = new HashMap<>(); + + for (FileStatus log : allLogs) { + if (AbstractFSWALProvider.isMetaFile(log.getPath())) { + continue; + } + String host = BackupUtils.parseHostNameFromLogFile(log.getPath()); + if (host == null) { + continue; + } + long timestamp = BackupUtils.getCreationTime(log.getPath()); + Long previousLogRoll = previousLogRollsByHost.get(host); + Long latestLogRoll = latestLogRollsByHost.get(host); + boolean isInactive = latestLogRoll == null || latestLogRoll.equals(previousLogRoll); + + if (isInactive) { + long currentTs = newTimestamps.getOrDefault(host, 0L); + if (timestamp > currentTs) { + newTimestamps.put(host, timestamp); } + } else { + newTimestamps.put(host, latestLogRoll); } } + // The table list in backupInfo is good for both full backup and incremental backup. + // For incremental backup, it contains the incremental backup table set. backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); Map> newTableSetTimestampMap = diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index d4934ccde27e..ff6855e5c166 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -61,7 +61,6 @@ public IncrementalBackupManager(Connection conn, Configuration conf) throws IOEx */ public Map getIncrBackupLogFileMap() throws IOException { List logList; - Map newTimestamps; Map previousTimestampMins; String savedStartCode = readBackupStartCode(); @@ -95,12 +94,48 @@ public Map getIncrBackupLogFileMap() throws IOException { LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); } } - newTimestamps = readRegionServerLastLogRollResult(); + Map newTimestamps = readRegionServerLastLogRollResult(); + + Map latestLogRollByHost = readRegionServerLastLogRollResult(); + for (Map.Entry entry : latestLogRollByHost.entrySet()) { + String host = entry.getKey(); + long latestLogRoll = entry.getValue(); + Long earliestTimestampToIncludeInBackup = previousTimestampMins.get(host); + + boolean isInactive = earliestTimestampToIncludeInBackup != null + && earliestTimestampToIncludeInBackup > latestLogRoll; + + long latestTimestampToIncludeInBackup; + if (isInactive) { + LOG.debug("Avoided resetting latest timestamp boundary for {} from {} to {}", host, + earliestTimestampToIncludeInBackup, latestLogRoll); + latestTimestampToIncludeInBackup = earliestTimestampToIncludeInBackup; + } else { + latestTimestampToIncludeInBackup = latestLogRoll; + } + newTimestamps.put(host, latestTimestampToIncludeInBackup); + } logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode); logList = excludeProcV2WALs(logList); backupInfo.setIncrBackupFileList(logList); + // Update boundaries based on WALs that will be backed up + for (String logFile : logList) { + Path logPath = new Path(logFile); + String logHost = BackupUtils.parseHostFromOldLog(logPath); + if (logHost == null) { + logHost = BackupUtils.parseHostNameFromLogFile(logPath.getParent()); + } + if (logHost != null) { + long logTs = BackupUtils.getCreationTime(logPath); + Long latestTimestampToIncludeInBackup = newTimestamps.get(logHost); + if (latestTimestampToIncludeInBackup == null || logTs > latestTimestampToIncludeInBackup) { + LOG.info("Updating backup boundary for inactive host {}: timestamp={}", logHost, logTs); + newTimestamps.put(logHost, logTs); + } + } + } return newTimestamps; } @@ -243,28 +278,6 @@ private List getLogFilesForNewBackup(Map olderTimestamps, } // remove newest log per host because they are still in use resultLogFiles.removeAll(newestLogs); - - // Update newestTimestamps with max timestamp of files we're actually backing up. - // This ensures dead/decommissioned hosts get their boundaries recorded in trslm, - // preventing re-backup of the same WAL files on subsequent incremental backups. - for (String logFile : resultLogFiles) { - Path logPath = new Path(logFile); - String logHost = BackupUtils.parseHostFromOldLog(logPath); - if (logHost == null) { - logHost = BackupUtils.parseHostNameFromLogFile(logPath.getParent()); - } - if (logHost != null) { - long logTs = BackupUtils.getCreationTime(logPath); - Long existingTs = newestTimestamps.get(logHost); - if (existingTs == null || logTs > existingTs) { - newestTimestamps.put(logHost, logTs); - if (existingTs == null) { - LOG.info("Updating backup boundary for inactive host {}: timestamp={}", logHost, logTs); - } - } - } - } - return resultLogFiles; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java index 831cb128e105..10e33380f111 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupOfflineRS.java @@ -17,7 +17,8 @@ */ package org.apache.hadoop.hbase.backup; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.List; @@ -70,152 +71,59 @@ public static void setUp() throws Exception { TEST_UTIL.waitTableAvailable(table1); } - /** - * Tests that when a RegionServer goes offline, its WAL files are backed up once in the first - * incremental backup and NOT re-backed up in subsequent incremental backups. - */ - @Test - public void testIncrementalBackupWithOfflineRS() throws Exception { - LOG.info("Starting testIncrementalBackupWithOfflineRS"); - - MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); - List tables = Lists.newArrayList(table1); - - // 1. Run full backup to establish baseline - LOG.info("Taking full backup"); - String fullBackupId = fullTableBackup(tables); - assertTrue("Full backup should succeed", checkSucceeded(fullBackupId)); - - // 2. Insert some data to generate WAL entries - LOG.info("Inserting data to generate WAL entries"); - try (Connection conn = ConnectionFactory.createConnection(conf1)) { - insertIntoTable(conn, table1, famName, 1, 100); - } - - // 3. Stop one RS to simulate it going offline - int rsToStop = 0; - HRegionServer rsBeforeStop = cluster.getRegionServer(rsToStop); - String offlineHost = rsBeforeStop.getServerName().getHostAndPort(); - String offlineHostPrefix = offlineHost.split(",")[0]; - LOG.info("Stopping RS: {} (prefix: {})", offlineHost, offlineHostPrefix); - - cluster.stopRegionServer(rsToStop); - // Wait for WALs to be moved to oldlogs - Thread.sleep(5000); - - // 4. Run first incremental backup - should include offline host's WALs - LOG.info("Taking first incremental backup (should include offline RS WALs)"); - String incr1 = incrementalTableBackup(tables); - assertTrue("First incremental backup should succeed", checkSucceeded(incr1)); - - // 5. Verify offline host is recorded in trslm - try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { - Map> timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); - Map rsTimestamps = timestamps.get(table1); - LOG.info("RS timestamps after first incremental: {}", rsTimestamps); - - boolean offlineHostRecorded = - rsTimestamps.keySet().stream().anyMatch(k -> k.contains(offlineHostPrefix)); - assertTrue("Offline host should have timestamp recorded in trslm", offlineHostRecorded); - - // 6. Get WAL file list for first incremental - BackupInfo backupInfo1 = sysTable.readBackupInfo(incr1); - List walFiles1 = backupInfo1.getIncrBackupFileList(); - LOG.info("WAL files in first incremental: {}", walFiles1); - - long offlineHostWalCount1 = - walFiles1.stream().filter(f -> f.contains(offlineHostPrefix)).count(); - LOG.info("Offline host WAL count in first incremental: {}", offlineHostWalCount1); - assertTrue("First incremental should include offline host WALs", offlineHostWalCount1 > 0); - - // 7. Run second incremental backup - should NOT include offline host's WALs - LOG.info("Taking second incremental backup (should NOT include offline RS WALs)"); - String incr2 = incrementalTableBackup(tables); - assertTrue("Second incremental backup should succeed", checkSucceeded(incr2)); - - // 8. Verify second incremental does not include offline host's WALs - BackupInfo backupInfo2 = sysTable.readBackupInfo(incr2); - List walFiles2 = backupInfo2.getIncrBackupFileList(); - LOG.info("WAL files in second incremental: {}", walFiles2); - - long offlineHostWalCount2 = - walFiles2.stream().filter(f -> f.contains(offlineHostPrefix)).count(); - LOG.info("Offline host WAL count in second incremental: {}", offlineHostWalCount2); - assertEquals("Second incremental should NOT include offline host WALs", 0, - offlineHostWalCount2); - } - - LOG.info("testIncrementalBackupWithOfflineRS completed successfully"); - } - /** * Tests that when a full backup is taken while an RS is offline (with WALs in oldlogs), the * offline host's timestamps are recorded so subsequent incremental backups don't re-include those * WALs. */ @Test - public void testFullBackupWithOfflineRS() throws Exception { + public void testBackupWithOfflineRS() throws Exception { LOG.info("Starting testFullBackupWithOfflineRS"); MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); List tables = Lists.newArrayList(table1); - // Ensure we have at least 2 RSes if (cluster.getNumLiveRegionServers() < 2) { cluster.startRegionServer(); Thread.sleep(2000); } - // 1. Insert some data to generate WAL entries LOG.info("Inserting data to generate WAL entries"); try (Connection conn = ConnectionFactory.createConnection(conf1)) { insertIntoTable(conn, table1, famName, 2, 100); } - // 2. Stop one RS to simulate it going offline int rsToStop = 0; HRegionServer rsBeforeStop = cluster.getRegionServer(rsToStop); - String offlineHost = rsBeforeStop.getServerName().getHostAndPort(); - String offlineHostPrefix = offlineHost.split(",")[0]; - LOG.info("Stopping RS: {} (prefix: {})", offlineHost, offlineHostPrefix); + String offlineHost = + rsBeforeStop.getServerName().getHostname() + ":" + rsBeforeStop.getServerName().getPort(); + LOG.info("Stopping RS: {}", offlineHost); cluster.stopRegionServer(rsToStop); // Wait for WALs to be moved to oldlogs Thread.sleep(5000); - // 3. Run full backup - should record offline host timestamps LOG.info("Taking full backup (with offline RS WALs in oldlogs)"); String fullBackupId = fullTableBackup(tables); assertTrue("Full backup should succeed", checkSucceeded(fullBackupId)); - // 4. Verify offline host is recorded in trslm try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { Map> timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); Map rsTimestamps = timestamps.get(table1); LOG.info("RS timestamps after full backup: {}", rsTimestamps); - boolean offlineHostRecorded = - rsTimestamps.keySet().stream().anyMatch(k -> k.contains(offlineHostPrefix)); - assertTrue("Offline host should have timestamp recorded in trslm after full backup", - offlineHostRecorded); + Long tsAfterFullBackup = rsTimestamps.get(offlineHost); + assertNotNull("Offline host should have timestamp recorded in trslm after full backup", + tsAfterFullBackup); - // 5. Run incremental backup - should NOT include offline host's WALs from before full backup LOG.info("Taking incremental backup (should NOT include offline RS WALs)"); String incrBackupId = incrementalTableBackup(tables); assertTrue("Incremental backup should succeed", checkSucceeded(incrBackupId)); - // 6. Verify incremental does not include offline host's WALs - BackupInfo backupInfo = sysTable.readBackupInfo(incrBackupId); - List walFiles = backupInfo.getIncrBackupFileList(); - LOG.info("WAL files in incremental: {}", walFiles); - - long offlineHostWalCount = - walFiles.stream().filter(f -> f.contains(offlineHostPrefix)).count(); - LOG.info("Offline host WAL count in incremental: {}", offlineHostWalCount); - assertEquals("Incremental after full should NOT include offline host WALs", 0, - offlineHostWalCount); + timestamps = sysTable.readLogTimestampMap(BACKUP_ROOT_DIR); + rsTimestamps = timestamps.get(table1); + assertFalse("Offline host should not have a boundary ", + rsTimestamps.containsKey(offlineHost)); } - - LOG.info("testFullBackupWithOfflineRS completed successfully"); } } From 39274a873a9ab1a701b92e9c03c5d071c32007e2 Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Sun, 11 Jan 2026 15:03:45 -0500 Subject: [PATCH 59/78] HubSpot Backport: HBASE-29782 Expose public Admin API to reopen table regions without moving (#7563) (#7611) (#232) (will be in 2.7) Signed-off-by: Ray Mattingly Co-authored-by: Alex Hughes Co-authored-by: Alex Hughes --- .../org/apache/hadoop/hbase/client/Admin.java | 41 ++ .../hadoop/hbase/client/AsyncAdmin.java | 18 + .../hadoop/hbase/client/AsyncHBaseAdmin.java | 10 + .../client/ConnectionImplementation.java | 6 + .../hadoop/hbase/client/HBaseAdmin.java | 43 ++ .../hbase/client/RawAsyncHBaseAdmin.java | 30 ++ .../client/ShortCircuitMasterConnection.java | 6 + .../shaded/protobuf/RequestConverter.java | 26 ++ .../src/main/protobuf/Master.proto | 18 + .../apache/hadoop/hbase/master/HMaster.java | 49 ++ .../hbase/master/MasterRpcServices.java | 25 + .../ReopenTableRegionsProcedure.java | 161 ++++--- .../TestReopenTableRegionsIntegration.java | 318 +++++++++++++ ...nTableRegionsProcedureSpecificRegions.java | 442 ++++++++++++++++++ .../hbase/thrift2/client/ThriftAdmin.java | 10 + 15 files changed, 1145 insertions(+), 58 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsIntegration.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureSpecificRegions.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index cefb84d5a52a..ccbbf9a08e48 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1618,6 +1618,47 @@ default Future modifyTableAsync(TableDescriptor td) throws IOException { */ Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) throws IOException; + /** + * Reopen all regions of a table. This is useful after calling + * {@link #modifyTableAsync(TableDescriptor, boolean)} with reopenRegions=false to gradually roll + * out table descriptor changes to regions. Regions are reopened in-place (no move). + * @param tableName table whose regions to reopen + * @throws IOException if a remote or network exception occurs + */ + default void reopenTableRegions(TableName tableName) throws IOException { + get(reopenTableRegionsAsync(tableName), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); + } + + /** + * Reopen specific regions of a table. Useful for canary testing table descriptor changes on a + * subset of regions before rolling out to the entire table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @throws IOException if a remote or network exception occurs + */ + default void reopenTableRegions(TableName tableName, List regions) + throws IOException { + get(reopenTableRegionsAsync(tableName, regions), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); + } + + /** + * Asynchronously reopen all regions of a table. + * @param tableName table whose regions to reopen + * @return Future for tracking completion + * @throws IOException if a remote or network exception occurs + */ + Future reopenTableRegionsAsync(TableName tableName) throws IOException; + + /** + * Asynchronously reopen specific regions of a table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @return Future for tracking completion + * @throws IOException if a remote or network exception occurs + */ + Future reopenTableRegionsAsync(TableName tableName, List regions) + throws IOException; + /** * Change the store file tracker of the given table. * @param tableName the table you want to change diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index ea0bcb7a6d77..6b38c13547df 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -208,6 +208,24 @@ default CompletableFuture modifyTable(TableDescriptor desc) { */ CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions); + /** + * Reopen all regions of a table. This is useful after calling + * {@link #modifyTable(TableDescriptor, boolean)} with reopenRegions=false to gradually roll out + * table descriptor changes to regions. Regions are reopened in-place (no move). + * @param tableName table whose regions to reopen + * @return CompletableFuture that completes when all regions have been reopened + */ + CompletableFuture reopenTableRegions(TableName tableName); + + /** + * Reopen specific regions of a table. Useful for canary testing table descriptor changes on a + * subset of regions before rolling out to the entire table. + * @param tableName table whose regions to reopen + * @param regions specific regions to reopen + * @return CompletableFuture that completes when specified regions have been reopened + */ + CompletableFuture reopenTableRegions(TableName tableName, List regions); + /** * Change the store file tracker of the given table. * @param tableName the table you want to change diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 650f80470ea0..fc7d5758e14e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -156,6 +156,16 @@ public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenR return wrap(rawAdmin.modifyTable(desc, reopenRegions)); } + @Override + public CompletableFuture reopenTableRegions(TableName tableName) { + return wrap(rawAdmin.reopenTableRegions(tableName)); + } + + @Override + public CompletableFuture reopenTableRegions(TableName tableName, List regions) { + return wrap(rawAdmin.reopenTableRegions(tableName, regions)); + } + @Override public CompletableFuture modifyTableStoreFileTracker(TableName tableName, String dstSFT) { return wrap(rawAdmin.modifyTableStoreFileTracker(tableName, dstSFT)); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 25e111c31916..ab6b25b340d4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -2062,6 +2062,12 @@ public HBaseProtos.LogEntry getLogEntries(RpcController controller, return stub.getLogEntries(controller, request); } + @Override + public MasterProtos.ReopenTableRegionsResponse reopenTableRegions(RpcController controller, + MasterProtos.ReopenTableRegionsRequest request) throws ServiceException { + return stub.reopenTableRegions(controller, request); + } + @Override public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker( RpcController controller, ModifyTableStoreFileTrackerRequest request) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 7963f1cf6684..be13415a6b59 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -213,6 +213,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest; @@ -409,6 +411,34 @@ protected ModifyTableResponse rpcCall() throws Exception { return new ModifyTableFuture(this, td.getTableName(), response); } + @Override + public Future reopenTableRegionsAsync(TableName tableName) throws IOException { + return reopenTableRegionsAsync(tableName, Collections.emptyList()); + } + + @Override + public Future reopenTableRegionsAsync(TableName tableName, List regions) + throws IOException { + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsResponse response = executeCallable( + new MasterCallable(getConnection(), getRpcControllerFactory()) { + long nonceGroup = ng.getNonceGroup(); + long nonce = ng.newNonce(); + + @Override + protected ReopenTableRegionsResponse rpcCall() throws Exception { + setPriority(tableName); + ReopenTableRegionsRequest request = RequestConverter + .buildReopenTableRegionsRequest(tableName, regionNames, nonceGroup, nonce); + return master.reopenTableRegions(getRpcController(), request); + + } + }); + return new ReopenTableRegionsFuture(this, tableName, response); + } + @Override public Future modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT) throws IOException { @@ -2141,6 +2171,19 @@ public String getOperationType() { } } + private static class ReopenTableRegionsFuture extends TableFuture { + public ReopenTableRegionsFuture(HBaseAdmin admin, TableName tableName, + ReopenTableRegionsResponse response) { + super(admin, tableName, + (response != null && response.hasProcId()) ? response.getProcId() : null); + } + + @Override + public String getOperationType() { + return "REOPEN_TABLE_REGIONS"; + } + } + private static class ModifyTableFuture extends TableFuture { public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName, final ModifyTableResponse response) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 8000e3ad396a..35d54412075d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -256,6 +256,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCatalogScanRequest; @@ -674,6 +676,22 @@ public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenR new ModifyTableProcedureBiConsumer(this, desc.getTableName())); } + @Override + public CompletableFuture reopenTableRegions(TableName tableName) { + return reopenTableRegions(tableName, Collections.emptyList()); + } + + @Override + public CompletableFuture reopenTableRegions(TableName tableName, List regions) { + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + return this. procedureCall(tableName, + RequestConverter.buildReopenTableRegionsRequest(tableName, regionNames, ng.getNonceGroup(), + ng.newNonce()), + (s, c, req, done) -> s.reopenTableRegions(c, req, done), (resp) -> resp.getProcId(), + new ReopenTableRegionsProcedureBiConsumer(this, tableName)); + } + @Override public CompletableFuture modifyTableStoreFileTracker(TableName tableName, String dstSFT) { return this. regionNames, final long nonceGroup, final long nonce) { + ReopenTableRegionsRequest.Builder builder = ReopenTableRegionsRequest.newBuilder(); + builder.setTableName(ProtobufUtil.toProtoTableName(tableName)); + + if (regionNames != null && !regionNames.isEmpty()) { + for (byte[] regionName : regionNames) { + builder.addRegionNames(UnsafeByteOperations.unsafeWrap(regionName)); + } + } + + builder.setNonceGroup(nonceGroup); + builder.setNonce(nonce); + + return builder.build(); + } + public static ModifyTableStoreFileTrackerRequest buildModifyTableStoreFileTrackerRequest( final TableName tableName, final String dstSFT, final long nonceGroup, final long nonce) { ModifyTableStoreFileTrackerRequest.Builder builder = diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index e0182506dd12..06404e15b771 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -210,6 +210,17 @@ message ModifyTableResponse { optional uint64 proc_id = 1; } +message ReopenTableRegionsRequest { + required TableName table_name = 1; + repeated bytes region_names = 2; // empty = all regions + optional uint64 nonce_group = 3 [default = 0]; + optional uint64 nonce = 4 [default = 0]; +} + +message ReopenTableRegionsResponse { + optional uint64 proc_id = 1; +} + message FlushTableRequest { required TableName table_name = 1; repeated bytes column_family = 2; @@ -900,6 +911,13 @@ service MasterService { rpc ModifyTable(ModifyTableRequest) returns(ModifyTableResponse); + /** + * Reopen regions of a table. Regions are reopened in-place without moving. + * Useful for rolling out table descriptor changes after modifyTable(reopenRegions=false). + */ + rpc ReopenTableRegions(ReopenTableRegionsRequest) + returns(ReopenTableRegionsResponse); + /** Flush a table's memstore */ rpc FlushTable(FlushTableRequest) returns(FlushTableResponse); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 84c782aaab72..b93f95ca8f2f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -4169,6 +4169,7 @@ long reopenRegions(final TableName tableName, final List regionNames, @Override protected void run() throws IOException { + submitProcedure(new ReopenTableRegionsProcedure(tableName, regionNames)); } @@ -4181,6 +4182,54 @@ protected String getDescription() { } + /** + * Reopen regions provided in the argument. Applies throttling to the procedure to avoid + * overwhelming the system. This is used by the reopenTableRegions methods in the Admin API via + * HMaster. + * @param tableName The current table name + * @param regionNames The region names of the regions to reopen + * @param nonceGroup Identifier for the source of the request, a client or process + * @param nonce A unique identifier for this operation from the client or process identified + * by nonceGroup (the source must ensure each operation gets a + * unique id). + * @return procedure Id + * @throws IOException if reopening region fails while running procedure + */ + long reopenRegionsThrottled(final TableName tableName, final List regionNames, + final long nonceGroup, final long nonce) throws IOException { + + checkInitialized(); + + if (!tableStateManager.isTablePresent(tableName)) { + throw new TableNotFoundException(tableName); + } + + return MasterProcedureUtil + .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) { + @Override + protected void run() throws IOException { + ReopenTableRegionsProcedure proc; + if (regionNames.isEmpty()) { + proc = ReopenTableRegionsProcedure.throttled(getConfiguration(), + getTableDescriptors().get(tableName)); + } else { + proc = ReopenTableRegionsProcedure.throttled(getConfiguration(), + getTableDescriptors().get(tableName), regionNames); + } + + LOG.info("{} throttled reopening {} regions for table {}", getClientIdAuditPrefix(), + regionNames.isEmpty() ? "all" : regionNames.size(), tableName); + + submitProcedure(proc); + } + + @Override + protected String getDescription() { + return "Throttled ReopenTableRegionsProcedure for " + tableName; + } + }); + } + @Override public ReplicationPeerManager getReplicationPeerManager() { return replicationPeerManager; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 9ae3f4550e95..668ccaa2c236 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -290,6 +290,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RegionSpecifierAndState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ReopenTableRegionsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RunCatalogScanRequest; @@ -1492,6 +1494,29 @@ public ModifyTableResponse modifyTable(RpcController controller, ModifyTableRequ } } + @Override + public ReopenTableRegionsResponse reopenTableRegions(RpcController controller, + ReopenTableRegionsRequest request) throws ServiceException { + try { + master.checkInitialized(); + + final TableName tableName = ProtobufUtil.toTableName(request.getTableName()); + final List regionNames = request.getRegionNamesList().stream() + .map(ByteString::toByteArray).collect(Collectors.toList()); + + LOG.info("Reopening regions for table={}, regionCount={}", tableName, + regionNames.isEmpty() ? "all" : regionNames.size()); + + long procId = master.reopenRegionsThrottled(tableName, regionNames, request.getNonceGroup(), + request.getNonce()); + + return ReopenTableRegionsResponse.newBuilder().setProcId(procId).build(); + + } catch (IOException ioe) { + throw new ServiceException(ioe); + } + } + @Override public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker(RpcController controller, ModifyTableStoreFileTrackerRequest req) throws ServiceException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java index 03f04792af55..bf1c94116c8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ReopenTableRegionsProcedure.java @@ -25,8 +25,10 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.conf.ConfigKey; import org.apache.hadoop.hbase.master.assignment.RegionStateNode; @@ -89,7 +91,7 @@ public class ReopenTableRegionsProcedure /** * Create a new ReopenTableRegionsProcedure respecting the throttling configuration for the table. * First check the table descriptor, then fall back to the global configuration. Only used in - * ModifyTableProcedure. + * ModifyTableProcedure and in HMaster#reopenRegionsThrottled. */ public static ReopenTableRegionsProcedure throttled(final Configuration conf, final TableDescriptor desc) { @@ -103,6 +105,24 @@ public static ReopenTableRegionsProcedure throttled(final Configuration conf, return new ReopenTableRegionsProcedure(desc.getTableName(), backoffMillis, batchSizeMax); } + /** + * Create a new ReopenTableRegionsProcedure for specific regions, respecting the throttling + * configuration for the table. First check the table descriptor, then fall back to the global + * configuration. Only used in HMaster#reopenRegionsThrottled. + */ + public static ReopenTableRegionsProcedure throttled(final Configuration conf, + final TableDescriptor desc, final List regionNames) { + long backoffMillis = Optional.ofNullable(desc.getValue(PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY)) + .map(Long::parseLong).orElseGet(() -> conf.getLong(PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, + PROGRESSIVE_BATCH_BACKOFF_MILLIS_DEFAULT)); + int batchSizeMax = Optional.ofNullable(desc.getValue(PROGRESSIVE_BATCH_SIZE_MAX_KEY)) + .map(Integer::parseInt).orElseGet( + () -> conf.getInt(PROGRESSIVE_BATCH_SIZE_MAX_KEY, PROGRESSIVE_BATCH_SIZE_MAX_DISABLED)); + + return new ReopenTableRegionsProcedure(desc.getTableName(), regionNames, backoffMillis, + batchSizeMax); + } + public ReopenTableRegionsProcedure() { this(null); } @@ -116,12 +136,12 @@ public ReopenTableRegionsProcedure(final TableName tableName, final List PROGRESSIVE_BATCH_SIZE_MAX_DISABLED); } - ReopenTableRegionsProcedure(final TableName tableName, long reopenBatchBackoffMillis, + public ReopenTableRegionsProcedure(final TableName tableName, long reopenBatchBackoffMillis, int reopenBatchSizeMax) { this(tableName, Collections.emptyList(), reopenBatchBackoffMillis, reopenBatchSizeMax); } - private ReopenTableRegionsProcedure(final TableName tableName, final List regionNames, + public ReopenTableRegionsProcedure(final TableName tableName, final List regionNames, long reopenBatchBackoffMillis, int reopenBatchSizeMax) { this.tableName = tableName; this.regionNames = regionNames; @@ -190,67 +210,78 @@ private boolean canSchedule(MasterProcedureEnv env, HRegionLocation loc) { @Override protected Flow executeFromState(MasterProcedureEnv env, ReopenTableRegionsState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { - switch (state) { - case REOPEN_TABLE_REGIONS_GET_REGIONS: - if (!isTableEnabled(env)) { - LOG.info("Table {} is disabled, give up reopening its regions", tableName); - return Flow.NO_MORE_STATE; - } - List tableRegions = - env.getAssignmentManager().getRegionStates().getRegionsOfTableForReopen(tableName); - regions = getRegionLocationsForReopen(tableRegions); - setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); - return Flow.HAS_MORE_STATE; - case REOPEN_TABLE_REGIONS_REOPEN_REGIONS: - // if we didn't finish reopening the last batch yet, let's keep trying until we do. - // at that point, the batch will be empty and we can generate a new batch - if (!regions.isEmpty() && currentRegionBatch.isEmpty()) { - currentRegionBatch = regions.stream().limit(reopenBatchSize).collect(Collectors.toList()); - batchesProcessed++; - } - for (HRegionLocation loc : currentRegionBatch) { - RegionStateNode regionNode = - env.getAssignmentManager().getRegionStates().getRegionStateNode(loc.getRegion()); - // this possible, maybe the region has already been merged or split, see HBASE-20921 - if (regionNode == null) { - continue; + try { + switch (state) { + case REOPEN_TABLE_REGIONS_GET_REGIONS: + if (!isTableEnabled(env)) { + LOG.info("Table {} is disabled, give up reopening its regions", tableName); + return Flow.NO_MORE_STATE; } - TransitRegionStateProcedure proc; - regionNode.lock(); - try { - if (regionNode.getProcedure() != null) { + List tableRegions = + env.getAssignmentManager().getRegionStates().getRegionsOfTableForReopen(tableName); + regions = getRegionLocationsForReopen(tableRegions); + setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_REOPEN_REGIONS); + return Flow.HAS_MORE_STATE; + case REOPEN_TABLE_REGIONS_REOPEN_REGIONS: + // if we didn't finish reopening the last batch yet, let's keep trying until we do. + // at that point, the batch will be empty and we can generate a new batch + if (!regions.isEmpty() && currentRegionBatch.isEmpty()) { + currentRegionBatch = + regions.stream().limit(reopenBatchSize).collect(Collectors.toList()); + batchesProcessed++; + } + for (HRegionLocation loc : currentRegionBatch) { + RegionStateNode regionNode = + env.getAssignmentManager().getRegionStates().getRegionStateNode(loc.getRegion()); + // this possible, maybe the region has already been merged or split, see HBASE-20921 + if (regionNode == null) { continue; } - proc = TransitRegionStateProcedure.reopen(env, regionNode.getRegionInfo()); - regionNode.setProcedure(proc); - } finally { - regionNode.unlock(); + TransitRegionStateProcedure proc; + regionNode.lock(); + try { + if (regionNode.getProcedure() != null) { + continue; + } + proc = TransitRegionStateProcedure.reopen(env, regionNode.getRegionInfo()); + regionNode.setProcedure(proc); + } finally { + regionNode.unlock(); + } + addChildProcedure(proc); + regionsReopened++; + } + setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_CONFIRM_REOPENED); + return Flow.HAS_MORE_STATE; + case REOPEN_TABLE_REGIONS_CONFIRM_REOPENED: + // update region lists based on what's been reopened + regions = filterReopened(env, regions); + currentRegionBatch = filterReopened(env, currentRegionBatch); + + // existing batch didn't fully reopen, so try to resolve that first. + // since this is a retry, don't do the batch backoff + if (!currentRegionBatch.isEmpty()) { + return reopenIfSchedulable(env, currentRegionBatch, false); } - addChildProcedure(proc); - regionsReopened++; - } - setNextState(ReopenTableRegionsState.REOPEN_TABLE_REGIONS_CONFIRM_REOPENED); - return Flow.HAS_MORE_STATE; - case REOPEN_TABLE_REGIONS_CONFIRM_REOPENED: - // update region lists based on what's been reopened - regions = filterReopened(env, regions); - currentRegionBatch = filterReopened(env, currentRegionBatch); - - // existing batch didn't fully reopen, so try to resolve that first. - // since this is a retry, don't do the batch backoff - if (!currentRegionBatch.isEmpty()) { - return reopenIfSchedulable(env, currentRegionBatch, false); - } - if (regions.isEmpty()) { - return Flow.NO_MORE_STATE; - } + if (regions.isEmpty()) { + return Flow.NO_MORE_STATE; + } - // current batch is finished, schedule more regions - return reopenIfSchedulable(env, regions, true); - default: - throw new UnsupportedOperationException("unhandled state=" + state); + // current batch is finished, schedule more regions + return reopenIfSchedulable(env, regions, true); + default: + throw new UnsupportedOperationException("unhandled state=" + state); + } + } catch (IOException e) { + if (isRollbackSupported(state) || e instanceof DoNotRetryIOException) { + setFailure("master-reopen-table-regions", e); + } else { + LOG.warn("Retriable error trying to reopen regions for table={} (in state={})", tableName, + state, e); + } } + return Flow.HAS_MORE_STATE; } private List filterReopened(MasterProcedureEnv env, @@ -296,19 +327,33 @@ private void setBackoffState(long millis) { } private List - getRegionLocationsForReopen(List tableRegionsForReopen) { + getRegionLocationsForReopen(List tableRegionsForReopen) throws IOException { List regionsToReopen = new ArrayList<>(); if ( CollectionUtils.isNotEmpty(regionNames) && CollectionUtils.isNotEmpty(tableRegionsForReopen) ) { + List notFoundRegions = new ArrayList<>(); + for (byte[] regionName : regionNames) { + boolean found = false; for (HRegionLocation hRegionLocation : tableRegionsForReopen) { if (Bytes.equals(regionName, hRegionLocation.getRegion().getRegionName())) { regionsToReopen.add(hRegionLocation); + found = true; break; } } + if (!found) { + notFoundRegions.add(regionName); + } + } + + if (!notFoundRegions.isEmpty()) { + String regionNamesStr = + notFoundRegions.stream().map(Bytes::toStringBinary).collect(Collectors.joining(", ")); + throw new UnknownRegionException( + "The following regions do not belong to table " + tableName + ": " + regionNamesStr); } } else { regionsToReopen = tableRegionsForReopen; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsIntegration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsIntegration.java new file mode 100644 index 000000000000..52cac259ae95 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsIntegration.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.MetricsRegionWrapperImpl; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestReopenTableRegionsIntegration { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReopenTableRegionsIntegration.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final TableName TABLE_NAME = TableName.valueOf("testLazyUpdateReopen"); + private static final byte[] CF = Bytes.toBytes("cf"); + + @BeforeClass + public static void setupCluster() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testLazyUpdateThenReopenUpdatesTableDescriptorHash() throws Exception { + // Step 1: Create table with column family and 3 regions + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(CF).setMaxVersions(1).build(); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(cfd) + .setMaxFileSize(100 * 1024 * 1024L).build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 3); + UTIL.waitTableAvailable(TABLE_NAME); + + try { + // Step 2: Capture initial tableDescriptorHash from all regions + List regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + assertEquals("Expected 3 regions", 3, regions.size()); + + Map initialHashes = new HashMap<>(); + + for (HRegion region : regions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String hash = wrapper.getTableDescriptorHash(); + initialHashes.put(region.getRegionInfo().getRegionName(), hash); + } + + // Verify all regions have same hash + Set uniqueHashes = new HashSet<>(initialHashes.values()); + assertEquals("All regions should have same hash", 1, uniqueHashes.size()); + String initialHash = uniqueHashes.iterator().next(); + + // Step 3: Perform lazy table descriptor update + ColumnFamilyDescriptor newCfd = + ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(5).build(); + + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).modifyColumnFamily(newCfd) + .setMaxFileSize(200 * 1024 * 1024L).build(); + + // Perform lazy update (reopenRegions = false) + UTIL.getAdmin().modifyTable(newTd, false); + + // Wait for modification to complete + UTIL.waitFor(30000, () -> { + try { + TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(TABLE_NAME); + return currentTd.getMaxFileSize() == 200 * 1024 * 1024L; + } catch (Exception e) { + return false; + } + }); + + // Step 4: Verify tableDescriptorHash has NOT changed in region metrics + List regionsAfterLazyUpdate = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + for (HRegion region : regionsAfterLazyUpdate) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String currentHash = wrapper.getTableDescriptorHash(); + + assertEquals("Hash should NOT change without region reopen", + initialHashes.get(region.getRegionInfo().getRegionName()), currentHash); + } + + // Verify the table descriptor itself has changed + TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(TABLE_NAME); + String newDescriptorHash = currentTd.getDescriptorHash(); + assertNotEquals("Table descriptor should have new hash", initialHash, newDescriptorHash); + + // Step 5: Use new Admin API to reopen all regions + UTIL.getAdmin().reopenTableRegions(TABLE_NAME); + + // Wait for all regions to be reopened + UTIL.waitFor(60000, () -> { + try { + List currentRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + if (currentRegions.size() != 3) { + return false; + } + + // Check if all regions now have the new hash + for (HRegion region : currentRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String hash = wrapper.getTableDescriptorHash(); + if (hash.equals(initialHash)) { + return false; + } + } + return true; + } catch (Exception e) { + return false; + } + }); + + // Step 6: Verify tableDescriptorHash HAS changed in all region metrics + List reopenedRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + assertEquals("Should still have 3 regions", 3, reopenedRegions.size()); + + for (HRegion region : reopenedRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String currentHash = wrapper.getTableDescriptorHash(); + + assertNotEquals("Hash SHOULD change after region reopen", initialHash, currentHash); + assertEquals("Hash should match current table descriptor", newDescriptorHash, currentHash); + } + + // Verify all regions show the same new hash + Set newHashes = new HashSet<>(); + for (HRegion region : reopenedRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + newHashes.add(wrapper.getTableDescriptorHash()); + } + assertEquals("All regions should have same new hash", 1, newHashes.size()); + + } finally { + UTIL.deleteTable(TABLE_NAME); + } + } + + @Test + public void testLazyUpdateThenReopenSpecificRegions() throws Exception { + TableName tableName = TableName.valueOf("testSpecificRegionsReopen"); + + // Step 1: Create table with 5 regions + ColumnFamilyDescriptor cfd = + ColumnFamilyDescriptorBuilder.newBuilder(CF).setMaxVersions(1).build(); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(cfd) + .setMaxFileSize(100 * 1024 * 1024L).build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5); + UTIL.waitTableAvailable(tableName); + + try { + // Step 2: Capture initial hashes + List regions = UTIL.getHBaseCluster().getRegions(tableName); + assertEquals("Expected 5 regions", 5, regions.size()); + + Map initialHashes = new HashMap<>(); + + for (HRegion region : regions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String hash = wrapper.getTableDescriptorHash(); + initialHashes.put(region.getRegionInfo().getRegionName(), hash); + } + + String initialHash = initialHashes.values().iterator().next(); + + // Step 3: Perform lazy update + ColumnFamilyDescriptor newCfd = + ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(10).build(); + + TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).modifyColumnFamily(newCfd) + .setMaxFileSize(300 * 1024 * 1024L).build(); + + UTIL.getAdmin().modifyTable(newTd, false); + + UTIL.waitFor(30000, () -> { + try { + TableDescriptor currentTd = UTIL.getAdmin().getDescriptor(tableName); + return currentTd.getMaxFileSize() == 300 * 1024 * 1024L; + } catch (Exception e) { + return false; + } + }); + + String newDescriptorHash = UTIL.getAdmin().getDescriptor(tableName).getDescriptorHash(); + + // Step 4: Reopen only first 2 regions + List regionsToReopen = new ArrayList<>(); + regionsToReopen.add(regions.get(0).getRegionInfo()); + regionsToReopen.add(regions.get(1).getRegionInfo()); + + UTIL.getAdmin().reopenTableRegions(tableName, regionsToReopen); + + // Wait for those regions to reopen + UTIL.waitFor(60000, () -> { + try { + List currentRegions = UTIL.getHBaseCluster().getRegions(tableName); + int newHashCount = 0; + for (HRegion region : currentRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String hash = wrapper.getTableDescriptorHash(); + if (!hash.equals(initialHash)) { + newHashCount++; + } + } + return newHashCount >= 2; + } catch (Exception e) { + return false; + } + }); + + // Step 5: Verify only reopened regions have new hash + List regionsAfterFirstReopen = UTIL.getHBaseCluster().getRegions(tableName); + int newHashCount = 0; + int oldHashCount = 0; + + for (HRegion region : regionsAfterFirstReopen) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String currentHash = wrapper.getTableDescriptorHash(); + + if (currentHash.equals(newDescriptorHash)) { + newHashCount++; + } else if (currentHash.equals(initialHash)) { + oldHashCount++; + } + } + + assertEquals("Should have 2 regions with new hash", 2, newHashCount); + assertEquals("Should have 3 regions with old hash", 3, oldHashCount); + + // Step 6: Reopen remaining regions + List remainingRegions = new ArrayList<>(); + for (int i = 2; i < regions.size(); i++) { + remainingRegions.add(regions.get(i).getRegionInfo()); + } + + UTIL.getAdmin().reopenTableRegions(tableName, remainingRegions); + + // Wait for all regions to have new hash + UTIL.waitFor(60000, () -> { + try { + List currentRegions = UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : currentRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String hash = wrapper.getTableDescriptorHash(); + if (!hash.equals(newDescriptorHash)) { + return false; + } + } + return true; + } catch (Exception e) { + return false; + } + }); + + // Step 7: Verify all regions now have new hash + List finalRegions = UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : finalRegions) { + MetricsRegionWrapperImpl wrapper = new MetricsRegionWrapperImpl(region); + String currentHash = wrapper.getTableDescriptorHash(); + + assertEquals("All regions should now have new hash", newDescriptorHash, currentHash); + } + + } finally { + UTIL.deleteTable(tableName); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureSpecificRegions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureSpecificRegions.java new file mode 100644 index 000000000000..ff4e7a6e0c1b --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestReopenTableRegionsProcedureSpecificRegions.java @@ -0,0 +1,442 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestReopenTableRegionsProcedureSpecificRegions { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReopenTableRegionsProcedureSpecificRegions.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final byte[] CF = Bytes.toBytes("cf"); + + private static MiniHBaseCluster singleProcessHBaseCluster; + + @BeforeClass + public static void setupCluster() throws Exception { + Configuration conf = UTIL.getConfiguration(); + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + singleProcessHBaseCluster = UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + if (Objects.nonNull(singleProcessHBaseCluster)) { + singleProcessHBaseCluster.close(); + } + } + + private ProcedureExecutor getProcExec() { + return UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); + } + + @Test + public void testInvalidRegionNamesThrowsException() throws Exception { + TableName tableName = TableName.valueOf("TestInvalidRegions"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List regions = UTIL.getAdmin().getRegions(tableName); + assertFalse("Table should have at least one region", regions.isEmpty()); + + List invalidRegionNames = + Collections.singletonList(Bytes.toBytes("non-existent-region-name")); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, invalidRegionNames, 0L, Integer.MAX_VALUE); + + long procId = getProcExec().submitProcedure(proc); + UTIL.waitFor(60000, proc::isFailed); + + Throwable cause = ProcedureTestingUtility.getExceptionCause(proc); + assertTrue("Expected UnknownRegionException, got: " + cause.getClass().getName(), + cause instanceof UnknownRegionException); + assertTrue("Error message should contain region name", + cause.getMessage().contains("non-existent-region-name")); + assertTrue("Error message should contain table name", + cause.getMessage().contains(tableName.getNameAsString())); + } + } + + @Test + public void testMixedValidInvalidRegions() throws Exception { + TableName tableName = TableName.valueOf("TestMixedRegions"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List actualRegions = UTIL.getAdmin().getRegions(tableName); + assertFalse("Table should have at least one region", actualRegions.isEmpty()); + + List mixedRegionNames = new ArrayList<>(); + mixedRegionNames.add(actualRegions.get(0).getRegionName()); + mixedRegionNames.add(Bytes.toBytes("invalid-region-1")); + mixedRegionNames.add(Bytes.toBytes("invalid-region-2")); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, mixedRegionNames, 0L, Integer.MAX_VALUE); + + long procId = getProcExec().submitProcedure(proc); + UTIL.waitFor(60000, proc::isFailed); + + Throwable cause = ProcedureTestingUtility.getExceptionCause(proc); + assertTrue("Expected UnknownRegionException", cause instanceof UnknownRegionException); + assertTrue("Error message should contain first invalid region", + cause.getMessage().contains("invalid-region-1")); + assertTrue("Error message should contain second invalid region", + cause.getMessage().contains("invalid-region-2")); + } + } + + @Test + public void testSpecificRegionsReopenWithThrottling() throws Exception { + TableName tableName = TableName.valueOf("TestSpecificThrottled"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "100") + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, "2").build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5); + + List allRegions = UTIL.getAdmin().getRegions(tableName); + assertEquals(5, allRegions.size()); + + List specificRegionNames = + allRegions.subList(0, 3).stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure.throttled( + UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName), specificRegionNames); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should succeed", proc.isFailed()); + assertEquals("Should reopen exactly 3 regions", 3, proc.getRegionsReopened()); + assertTrue("Should process multiple batches with batch size 2", + proc.getBatchesProcessed() >= 2); + } + + @Test + public void testEmptyRegionListReopensAll() throws Exception { + TableName tableName = TableName.valueOf("TestEmptyList"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5); + + List allRegions = UTIL.getAdmin().getRegions(tableName); + assertEquals(5, allRegions.size()); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName)); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should succeed", proc.isFailed()); + assertEquals("Should reopen all 5 regions", 5, proc.getRegionsReopened()); + } + + @Test + public void testDisabledTableSkipsReopen() throws Exception { + TableName tableName = TableName.valueOf("TestDisabledTable"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + UTIL.getAdmin().disableTable(tableName); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName)); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should succeed", proc.isFailed()); + assertEquals("Should not reopen any regions for disabled table", 0, + proc.getRegionsReopened()); + } + } + + @Test + public void testReopenRegionsThrottledWithLargeTable() throws Exception { + TableName tableName = TableName.valueOf("TestLargeTable"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "50") + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, "3").build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 10); + + List regions = UTIL.getAdmin().getRegions(tableName); + assertEquals(10, regions.size()); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName)); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should succeed", proc.isFailed()); + assertEquals("Should reopen all 10 regions", 10, proc.getRegionsReopened()); + assertTrue("Should process multiple batches", proc.getBatchesProcessed() >= 4); + } + + @Test + public void testConfigurationPrecedence() throws Exception { + TableName tableName = TableName.valueOf("TestConfigPrecedence"); + + Configuration conf = UTIL.getConfiguration(); + conf.setLong(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, 1000); + conf.setInt(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, 5); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "2000") + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, "2").build(); + + UTIL.getAdmin().createTable(td); + + ReopenTableRegionsProcedure proc = + ReopenTableRegionsProcedure.throttled(conf, UTIL.getAdmin().getDescriptor(tableName)); + + assertEquals("Table descriptor config should override global config", 2000, + proc.getReopenBatchBackoffMillis()); + } + + @Test + public void testThrottledVsUnthrottled() throws Exception { + TableName tableName = TableName.valueOf("TestThrottledVsUnthrottled"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "1000") + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, "2").build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5); + + List regions = UTIL.getAdmin().getRegions(tableName); + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsProcedure unthrottledProc = + new ReopenTableRegionsProcedure(tableName, regionNames); + assertEquals("Unthrottled should use default (0ms)", 0, + unthrottledProc.getReopenBatchBackoffMillis()); + + ReopenTableRegionsProcedure throttledProc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName), regionNames); + assertEquals("Throttled should use table config (1000ms)", 1000, + throttledProc.getReopenBatchBackoffMillis()); + } + + @Test + public void testExceptionInProcedureExecution() throws Exception { + TableName tableName = TableName.valueOf("TestExceptionInExecution"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List invalidRegionNames = + Collections.singletonList(Bytes.toBytes("nonexistent-region")); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, invalidRegionNames, 0L, Integer.MAX_VALUE); + + long procId = getProcExec().submitProcedure(proc); + UTIL.waitFor(60000, () -> getProcExec().isFinished(procId)); + + Procedure result = getProcExec().getResult(procId); + assertTrue("Procedure should have failed", result.isFailed()); + + Throwable cause = ProcedureTestingUtility.getExceptionCause(result); + assertTrue("Should be UnknownRegionException", cause instanceof UnknownRegionException); + } + } + + @Test + public void testSerializationWithRegionNames() throws Exception { + TableName tableName = TableName.valueOf("TestSerialization"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List regions = UTIL.getAdmin().getRegions(tableName); + List regionNames = + regions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, regionNames, 500L, 3); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertEquals("TableName should be preserved", tableName, proc.getTableName()); + assertEquals("Backoff should be preserved", 500L, proc.getReopenBatchBackoffMillis()); + } + } + + @Test + public void testAllRegionsWithValidNames() throws Exception { + TableName tableName = TableName.valueOf("TestAllValidRegions"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List actualRegions = UTIL.getAdmin().getRegions(tableName); + assertFalse("Table should have regions", actualRegions.isEmpty()); + + List validRegionNames = + actualRegions.stream().map(RegionInfo::getRegionName).collect(Collectors.toList()); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, validRegionNames, 0L, Integer.MAX_VALUE); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should succeed with all valid regions", proc.isFailed()); + assertEquals("Should reopen all specified regions", actualRegions.size(), + proc.getRegionsReopened()); + } + } + + @Test + public void testSingleInvalidRegion() throws Exception { + TableName tableName = TableName.valueOf("TestSingleInvalid"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List invalidRegionNames = + Collections.singletonList(Bytes.toBytes("totally-fake-region")); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, invalidRegionNames, 0L, Integer.MAX_VALUE); + + long procId = getProcExec().submitProcedure(proc); + UTIL.waitFor(60000, proc::isFailed); + + Throwable cause = ProcedureTestingUtility.getExceptionCause(proc); + assertTrue("Expected UnknownRegionException", cause instanceof UnknownRegionException); + assertTrue("Error message should list the invalid region", + cause.getMessage().contains("totally-fake-region")); + } + } + + @Test + public void testRecoveryAfterValidationFailure() throws Exception { + TableName tableName = TableName.valueOf("TestRecoveryValidation"); + try (Table ignored = UTIL.createTable(tableName, CF)) { + + List invalidRegionNames = + Collections.singletonList(Bytes.toBytes("invalid-for-recovery")); + + ReopenTableRegionsProcedure proc = + new ReopenTableRegionsProcedure(tableName, invalidRegionNames, 0L, Integer.MAX_VALUE); + + ProcedureExecutor procExec = getProcExec(); + long procId = procExec.submitProcedure(proc); + + UTIL.waitFor(60000, () -> procExec.isFinished(procId)); + + Procedure result = procExec.getResult(procId); + assertTrue("Procedure should fail validation", result.isFailed()); + + Throwable cause = ProcedureTestingUtility.getExceptionCause(result); + assertTrue("Should be UnknownRegionException", cause instanceof UnknownRegionException); + assertTrue("Error should mention the invalid region", + cause.getMessage().contains("invalid-for-recovery")); + } + } + + @Test + public void testEmptyTableWithNoRegions() throws Exception { + TableName tableName = TableName.valueOf("TestEmptyTable"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build(); + + UTIL.getAdmin().createTable(td); + + List regions = UTIL.getAdmin().getRegions(tableName); + int regionCount = regions.size(); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName)); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should complete successfully even with no regions", proc.isFailed()); + assertEquals("Should handle empty table gracefully", regionCount, proc.getRegionsReopened()); + } + + @Test + public void testConfigChangeDoesNotAffectRunningProcedure() throws Exception { + TableName tableName = TableName.valueOf("TestConfigChange"); + + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "1000") + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_SIZE_MAX_KEY, "2").build(); + + UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 5); + + ReopenTableRegionsProcedure proc = ReopenTableRegionsProcedure + .throttled(UTIL.getConfiguration(), UTIL.getAdmin().getDescriptor(tableName)); + + assertEquals("Initial config should be 1000ms", 1000L, proc.getReopenBatchBackoffMillis()); + + TableDescriptor modifiedTd = TableDescriptorBuilder.newBuilder(td) + .setValue(ReopenTableRegionsProcedure.PROGRESSIVE_BATCH_BACKOFF_MILLIS_KEY, "5000").build(); + UTIL.getAdmin().modifyTable(modifiedTd); + + assertEquals("Running procedure should keep original config", 1000L, + proc.getReopenBatchBackoffMillis()); + + long procId = getProcExec().submitProcedure(proc); + ProcedureTestingUtility.waitProcedure(getProcExec(), procId); + + assertFalse("Procedure should complete successfully", proc.isFailed()); + } +} diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 83e3c5402b3e..ed0e94085da2 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -926,6 +926,16 @@ public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) throw new NotImplementedException("modifyTableAsync not supported in ThriftAdmin"); } + @Override + public Future reopenTableRegionsAsync(TableName tableName) { + throw new NotImplementedException("reopenTableRegionsAsync not supported in ThriftAdmin"); + } + + @Override + public Future reopenTableRegionsAsync(TableName tableName, List regions) { + throw new NotImplementedException("reopenTableRegionsAsync not supported in ThriftAdmin"); + } + @Override public void shutdown() { throw new NotImplementedException("shutdown not supported in ThriftAdmin"); From 9b7d72713537c2cdd11c8b75ee44306ba6d3f8b6 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Wed, 14 Jan 2026 09:17:23 -0500 Subject: [PATCH 60/78] HBASE-29827: BackupTables should return BackupInfo (#233) Co-authored-by: Hernan Gelaf-Romer --- .../apache/hadoop/hbase/backup/BackupAdmin.java | 4 ++-- .../hbase/backup/impl/BackupAdminImpl.java | 4 ++-- .../hbase/backup/impl/BackupCommands.java | 4 +--- .../hbase/backup/impl/TableBackupClient.java | 4 ++++ .../hadoop/hbase/backup/TestBackupBase.java | 2 +- .../hadoop/hbase/backup/TestBackupMerge.java | 12 ++++++------ .../hbase/backup/TestBackupMultipleDeletes.java | 14 +++++++------- .../TestBackupRestoreOnEmptyEnvironment.java | 2 +- .../TestBackupRestoreWithModifications.java | 2 +- .../hbase/backup/TestIncrementalBackup.java | 16 ++++++++-------- .../TestIncrementalBackupDeleteTable.java | 4 ++-- .../TestIncrementalBackupMergeWithBulkLoad.java | 2 +- .../TestIncrementalBackupMergeWithFailures.java | 6 +++--- .../TestIncrementalBackupWithDataLoss.java | 17 ++++++++++------- .../TestIncrementalBackupWithFailures.java | 2 +- .../hbase/IntegrationTestBackupRestore.java | 2 +- 16 files changed, 51 insertions(+), 46 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java index 25055fd5e8e6..269afd666239 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java @@ -39,10 +39,10 @@ public interface BackupAdmin extends Closeable { * Backup given list of tables fully. This is a synchronous operation. It returns backup id on * success or throw exception on failure. * @param userRequest BackupRequest instance - * @return the backup Id + * @return backup info */ - String backupTables(final BackupRequest userRequest) throws IOException; + BackupInfo backupTables(final BackupRequest userRequest) throws IOException; /** * Restore backup diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index 8c019a0615f9..31bd75ba4b2e 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -510,7 +510,7 @@ public void restore(RestoreRequest request) throws IOException { } @Override - public String backupTables(BackupRequest request) throws IOException { + public BackupInfo backupTables(BackupRequest request) throws IOException { BackupType type = request.getBackupType(); String targetRootDir = request.getTargetRootDir(); List tableList = request.getTableList(); @@ -594,7 +594,7 @@ public String backupTables(BackupRequest request) throws IOException { client.execute(); - return backupId; + return client.getBackupInfo(); } private List excludeNonExistingTables(List tableList, diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 2c78d0b50c11..c8322fdba76d 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -43,7 +43,6 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; - import java.io.IOException; import java.net.URI; import java.util.List; @@ -68,7 +67,6 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; - import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; @@ -367,7 +365,7 @@ public void execute() throws IOException { .withTargetRootDir(targetBackupDir).withTotalTasks(workers) .withBandwidthPerTasks(bandwidth).withNoChecksumVerify(ignoreChecksum) .withBackupSetName(setName).build(); - String backupId = admin.backupTables(request); + String backupId = admin.backupTables(request).getBackupId(); System.out.println("Backup session " + backupId + " finished. Status: SUCCESS"); } catch (IOException e) { System.out.println("Backup session finished. Status: FAILURE"); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index a228f75055a6..8bf91ef86fa3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -100,6 +100,10 @@ public void init(final Connection conn, final String backupId, BackupRequest req backupManager.startBackupSession(); } + public BackupInfo getBackupInfo() { + return backupInfo; + } + /** * Begin the overall backup. * @param backupInfo backup info diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 32598af94e90..d99221bdd9db 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -414,7 +414,7 @@ protected String backupTables(BackupType type, List tables, String pa conn = ConnectionFactory.createConnection(conf1); badmin = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(type, new ArrayList<>(tables), path); - backupId = badmin.backupTables(request); + backupId = badmin.backupTables(request).getBackupId(); } finally { if (badmin != null) { badmin.close(); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java index 38204f68e31a..f72fc5dc3f71 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java @@ -64,7 +64,7 @@ public void TestIncBackupMergeRestore() throws Exception { BackupAdminImpl client = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); @@ -85,7 +85,7 @@ public void TestIncBackupMergeRestore() throws Exception { // #3 - incremental backup for multiple tables tables = Lists.newArrayList(table1, table2); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); + String backupIdIncMultiple = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple)); @@ -97,7 +97,7 @@ public void TestIncBackupMergeRestore() throws Exception { // #3 - incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); + String backupIdIncMultiple2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple2)); try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) { @@ -139,15 +139,15 @@ public void testIncBackupMergeRestoreSeparateFs() throws Exception { List tables = Lists.newArrayList(table1, table2); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, true); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String backupIdIncMultiple = client.backupTables(request); + String backupIdIncMultiple = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple)); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String backupIdIncMultiple2 = client.backupTables(request); + String backupIdIncMultiple2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple2)); try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) { diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java index 36cecd3faf58..5149880820dd 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMultipleDeletes.java @@ -63,7 +63,7 @@ public void testBackupMultipleDeletes() throws Exception { Admin admin = conn.getAdmin(); BackupAdmin client = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); // #2 - insert some data to table table1 Table t1 = conn.getTable(table1); @@ -78,7 +78,7 @@ public void testBackupMultipleDeletes() throws Exception { // #3 - incremental backup for table1 tables = Lists.newArrayList(table1); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc1 = client.backupTables(request); + String backupIdInc1 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdInc1)); // #4 - insert some data to table table2 Table t2 = conn.getTable(table2); @@ -91,7 +91,7 @@ public void testBackupMultipleDeletes() throws Exception { // #5 - incremental backup for table1, table2 tables = Lists.newArrayList(table1, table2); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc2 = client.backupTables(request); + String backupIdInc2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdInc2)); // #6 - insert some data to table table1 t1 = conn.getTable(table1); @@ -103,7 +103,7 @@ public void testBackupMultipleDeletes() throws Exception { // #7 - incremental backup for table1 tables = Lists.newArrayList(table1); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc3 = client.backupTables(request); + String backupIdInc3 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdInc3)); // #8 - insert some data to table table2 t2 = conn.getTable(table2); @@ -115,17 +115,17 @@ public void testBackupMultipleDeletes() throws Exception { // #9 - incremental backup for table1, table2 tables = Lists.newArrayList(table1, table2); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc4 = client.backupTables(request); + String backupIdInc4 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdInc4)); // #10 full backup for table3 tables = Lists.newArrayList(table3); request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull2 = client.backupTables(request); + String backupIdFull2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull2)); // #11 - incremental backup for table3 tables = Lists.newArrayList(table3); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdInc5 = client.backupTables(request); + String backupIdInc5 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdInc5)); LOG.error("Delete backupIdInc2"); client.deleteBackups(new String[] { backupIdInc2 }); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java index 6fb538fe629a..faf43f38ce8d 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreOnEmptyEnvironment.java @@ -220,7 +220,7 @@ private String backup(BackupType backupType, List tables) throws IOEx BackupRequest backupRequest = new BackupRequest.Builder().withTargetRootDir(BACKUP_ROOT_DIR.toString()) .withTableList(new ArrayList<>(tables)).withBackupType(backupType).build(); - return backupAdmin.backupTables(backupRequest); + return backupAdmin.backupTables(backupRequest).getBackupId(); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java index 72c4a464e94d..c806dded9040 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupRestoreWithModifications.java @@ -217,7 +217,7 @@ private String backup(BackupType backupType, List tables) throws IOEx BackupRequest backupRequest = new BackupRequest.Builder().withTargetRootDir(BACKUP_ROOT_DIR.toString()) .withTableList(new ArrayList<>(tables)).withBackupType(backupType).build(); - return backupAdmin.backupTables(backupRequest); + return backupAdmin.backupTables(backupRequest).getBackupId(); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java index df187f752959..a5dad55ece9f 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java @@ -200,7 +200,7 @@ public void TestIncBackupRestore() throws Exception { // #3 - incremental backup for multiple tables tables = Lists.newArrayList(table1, table2); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); + String backupIdIncMultiple = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple)); BackupManifest manifest = HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple); @@ -231,7 +231,7 @@ public void TestIncBackupRestore() throws Exception { // #4 - additional incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); + String backupIdIncMultiple2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple2)); validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple2); @@ -299,7 +299,7 @@ public void TestIncBackupRestoreWithOriginalSplits() throws Exception { Connection conn = TEST_UTIL.getConnection(); BackupAdminImpl backupAdmin = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String fullBackupId = backupAdmin.backupTables(request); + String fullBackupId = backupAdmin.backupTables(request).getBackupId(); assertTrue(checkSucceeded(fullBackupId)); TableName[] fromTables = new TableName[] { table1 }; @@ -332,7 +332,7 @@ public void TestIncBackupRestoreWithOriginalSplits() throws Exception { assertNotEquals(currentRegions, TEST_UTIL.getHBaseCluster().getRegions(table1)); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String incrementalBackupId = backupAdmin.backupTables(request); + String incrementalBackupId = backupAdmin.backupTables(request).getBackupId(); assertTrue(checkSucceeded(incrementalBackupId)); preRestoreBackupFiles = getBackupFiles(); backupAdmin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, @@ -364,7 +364,7 @@ public void TestIncBackupRestoreWithOriginalSplits() throws Exception { } request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - incrementalBackupId = backupAdmin.backupTables(request); + incrementalBackupId = backupAdmin.backupTables(request).getBackupId(); assertTrue(checkSucceeded(incrementalBackupId)); preRestoreBackupFiles = getBackupFiles(); @@ -404,7 +404,7 @@ public void TestIncBackupRestoreWithOriginalSplitsSeperateFs() throws Exception BackupRequest request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String incrementalBackupId = admin.backupTables(request); + String incrementalBackupId = admin.backupTables(request).getBackupId(); assertTrue(checkSucceeded(incrementalBackupId)); TableName[] fromTable = new TableName[] { table1 }; @@ -483,7 +483,7 @@ public void TestIncBackupRestoreHandlesArchivedFiles() throws Exception { BackupRequest request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true); - String incrementalBackupId = admin.backupTables(request); + String incrementalBackupId = admin.backupTables(request).getBackupId(); assertTrue(checkSucceeded(incrementalBackupId)); TableName[] fromTable = new TableName[] { table1 }; @@ -514,7 +514,7 @@ private String takeFullBackup(List tables, BackupAdminImpl backupAdmi boolean noChecksumVerify) throws IOException { BackupRequest req = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR, noChecksumVerify); - String backupId = backupAdmin.backupTables(req); + String backupId = backupAdmin.backupTables(req).getBackupId(); checkSucceeded(backupId); return backupId; } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java index a5eec87fb06b..0d7d5528558d 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java @@ -65,7 +65,7 @@ public void testIncBackupDeleteTable() throws Exception { BackupAdminImpl client = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); @@ -88,7 +88,7 @@ public void testIncBackupDeleteTable() throws Exception { // #3 - incremental backup for table1 tables = Lists.newArrayList(table1); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); + String backupIdIncMultiple = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple)); // #4 - restore full backup for all tables, without overwrite diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java index c383a8545a8b..e83f943f0728 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithBulkLoad.java @@ -216,7 +216,7 @@ private String backup(BackupType backupType, List tables) throws IOEx BackupRequest backupRequest = new BackupRequest.Builder().withTargetRootDir(BACKUP_ROOT_DIR.toString()) .withTableList(new ArrayList<>(tables)).withBackupType(backupType).build(); - return backupAdmin.backupTables(backupRequest); + return backupAdmin.backupTables(backupRequest).getBackupId(); } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java index 1ece1770489b..0e4b3f32cbf7 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupMergeWithFailures.java @@ -240,7 +240,7 @@ public void TestIncBackupMergeRestore() throws Exception { BackupAdminImpl client = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); @@ -261,7 +261,7 @@ public void TestIncBackupMergeRestore() throws Exception { // #3 - incremental backup for multiple tables tables = Lists.newArrayList(table1, table2); request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple = client.backupTables(request); + String backupIdIncMultiple = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple)); @@ -273,7 +273,7 @@ public void TestIncBackupMergeRestore() throws Exception { // #3 - incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); - String backupIdIncMultiple2 = client.backupTables(request); + String backupIdIncMultiple2 = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdIncMultiple2)); // #4 Merge backup images with failures diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java index cf442f5f0dd7..26a585fde779 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithDataLoss.java @@ -54,11 +54,12 @@ public void testFullBackupBreaksDependencyOnOlderBackups() throws Exception { List tables = Lists.newArrayList(table1); insertIntoTable(conn, table1, famName, 1, 1).close(); - String backup1 = - client.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); + String backup1 = client + .backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)).getBackupId(); insertIntoTable(conn, table1, famName, 2, 1).close(); String backup2 = - client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)); + client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)) + .getBackupId(); assertTrue(checkSucceeded(backup1)); assertTrue(checkSucceeded(backup2)); @@ -67,14 +68,16 @@ public void testFullBackupBreaksDependencyOnOlderBackups() throws Exception { TEST_UTIL.getTestFileSystem().delete(new Path(BACKUP_ROOT_DIR, backup2), true); insertIntoTable(conn, table1, famName, 4, 1).close(); - String backup4 = - client.backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)); + String backup4 = client + .backupTables(createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR)).getBackupId(); insertIntoTable(conn, table1, famName, 5, 1).close(); String backup5 = - client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)); + client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)) + .getBackupId(); insertIntoTable(conn, table1, famName, 6, 1).close(); String backup6 = - client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)); + client.backupTables(createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR)) + .getBackupId(); assertTrue(checkSucceeded(backup4)); assertTrue(checkSucceeded(backup5)); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java index c8d536564188..f2c122632a64 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithFailures.java @@ -95,7 +95,7 @@ public void testIncBackupRestore() throws Exception { BackupAdminImpl client = new BackupAdminImpl(conn); BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR); - String backupIdFull = client.backupTables(request); + String backupIdFull = client.backupTables(request).getBackupId(); assertTrue(checkSucceeded(backupIdFull)); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java index 4326c9852e35..beb0c11da139 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java @@ -229,7 +229,7 @@ private void loadData(TableName table, int numRows) throws IOException { } private String backup(BackupRequest request, BackupAdmin client) throws IOException { - String backupId = client.backupTables(request); + String backupId = client.backupTables(request).getBackupId(); return backupId; } From 626b554b4c13628ed0c8a18ec4c412f2538a00c4 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Fri, 16 Jan 2026 10:42:56 -0500 Subject: [PATCH 61/78] Fix for NPE in region replication --- .../regionserver/RegionReplicaReplicationEndpoint.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java index 94b9daf836f9..bf8316626dd3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java @@ -440,7 +440,8 @@ public void append(TableName tableName, byte[] encodedRegionName, byte[] row, // keep going to the cache, we will not learn of the replicas and their locations after // they come online. if (useCache && locations.size() == 1) { - if (tableDescriptors.get(tableName).getRegionReplication() > 1) { + TableDescriptor td = tableDescriptors.get(tableName); + if (td != null && td.getRegionReplication() > 1) { // Make an obnoxious log here. See how bad this issue is. Add a timer if happening // too much. LOG.info("Skipping location cache; only one location found for {}", tableName); From 440e576c94f75994b69a529a0c562cc25af06d9f Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Tue, 6 Jan 2026 13:27:15 -0800 Subject: [PATCH 62/78] HBASE-29796 [branch-2] Allow sleepForRetry replication config to be overridden by replication peers (#7578) Co-authored-by: skhillon Signed-off by: --- .../PeerProcedureHandlerImpl.java | 29 ++++ .../regionserver/ReplicationSource.java | 4 + .../ReplicationSourceManager.java | 14 +- .../ReplicationSourceShipper.java | 4 + .../ReplicationSourceWALReader.java | 4 + .../TestPeerProcedureHandlerImpl.java | 130 ++++++++++++++++++ .../TestReplicationSourceManager.java | 120 ++++++++++++++++ 7 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java index 429276806f1e..b2d5d3e6deaf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.replication.regionserver; import java.io.IOException; +import java.util.Map; import java.util.concurrent.locks.Lock; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.replication.ReplicationException; @@ -98,6 +99,33 @@ public void disablePeer(String peerId) throws ReplicationException, IOException refreshPeerState(peerId); } + private boolean hasReplicationConfigChange(ReplicationPeerConfig oldConfig, + ReplicationPeerConfig newConfig) { + Map oldReplicationConfigs = oldConfig.getConfiguration(); + Map newReplicationConfigs = newConfig.getConfiguration(); + + // Check if any replication.source.* keys have changed values + for (Map.Entry entry : newReplicationConfigs.entrySet()) { + String key = entry.getKey(); + if (key.startsWith("replication.source.")) { + String oldValue = oldReplicationConfigs.get(key); + String newValue = entry.getValue(); + if (!newValue.equals(oldValue)) { + return true; + } + } + } + + // Check if any replication.source.* keys were removed + for (String key : oldReplicationConfigs.keySet()) { + if (key.startsWith("replication.source.") && !newReplicationConfigs.containsKey(key)) { + return true; + } + } + + return false; + } + @Override public void updatePeerConfig(String peerId) throws ReplicationException, IOException { Lock peerLock = peersLock.acquireLock(peerId); @@ -121,6 +149,7 @@ public void updatePeerConfig(String peerId) throws ReplicationException, IOExcep if ( !ReplicationUtils.isNamespacesAndTableCFsEqual(oldConfig, newConfig) || oldConfig.isSerial() != newConfig.isSerial() + || hasReplicationConfigChange(oldConfig, newConfig) || (oldState.equals(PeerState.ENABLED) && newState.equals(PeerState.DISABLED)) ) { replicationSourceManager.refreshSources(peerId); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index aebdccdc92dc..182aa7bbab88 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -834,4 +834,8 @@ public String logPeerId() { public long getTotalReplicatedEdits() { return totalReplicatedEdits.get(); } + + long getSleepForRetries() { + return sleepForRetries; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index 35c217940eee..ba799831c287 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CompoundConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; @@ -316,7 +317,18 @@ private ReplicationSourceInterface createSource(String queueId, ReplicationPeer WALFileLengthProvider walFileLengthProvider = this.walFactory.getWALProvider() != null ? this.walFactory.getWALProvider().getWALFileLengthProvider() : p -> OptionalLong.empty(); - src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId, + + // Create merged configuration with peer overrides as higher priority and + // global config as lower priority + Configuration mergedConf = conf; + if (!replicationPeer.getPeerConfig().getConfiguration().isEmpty()) { + CompoundConfiguration compound = new CompoundConfiguration(); + compound.add(conf); + compound.addStringMap(replicationPeer.getPeerConfig().getConfiguration()); + mergedConf = compound; + } + + src.init(mergedConf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId, walFileLengthProvider, new MetricsSource(queueId)); return src; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java index 50dbaca7ff6b..d7328d939d23 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java @@ -371,4 +371,8 @@ void clearWALEntryBatch() { totalReleasedBytes); } } + + long getSleepForRetries() { + return sleepForRetries; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java index 26360cbe3ea1..e617fe6d0162 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java @@ -440,4 +440,8 @@ public void setReaderRunning(boolean readerRunning) { private ReplicationSourceManager getSourceManager() { return this.source.getSourceManager(); } + + long getSleepForRetries() { + return sleepForRetries; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java new file mode 100644 index 000000000000..193d1123c0eb --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication.regionserver; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState; +import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerImpl; +import org.apache.hadoop.hbase.replication.ReplicationPeers; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(ReplicationTests.TAG) +@Tag(SmallTests.TAG) +public class TestPeerProcedureHandlerImpl { + + private ReplicationSourceManager mockSourceManager; + private ReplicationPeers mockReplicationPeers; + private ReplicationPeerImpl mockPeer; + private PeerProcedureHandlerImpl handler; + private static final String PEER_ID = "testPeer"; + + @BeforeEach + public void setup() throws Exception { + mockSourceManager = mock(ReplicationSourceManager.class); + mockReplicationPeers = mock(ReplicationPeers.class); + mockPeer = mock(ReplicationPeerImpl.class); + + when(mockSourceManager.getReplicationPeers()).thenReturn(mockReplicationPeers); + when(mockReplicationPeers.getPeer(PEER_ID)).thenReturn(mockPeer); + + handler = new PeerProcedureHandlerImpl(mockSourceManager); + } + + @Test + public void testReplicationSourceConfigChangeTriggers() throws Exception { + ReplicationPeerConfig oldConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("replication.source.sleepforretries", "1000").build(); + + ReplicationPeerConfig newConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("replication.source.sleepforretries", "5000").build(); + + when(mockPeer.getPeerConfig()).thenReturn(oldConfig); + when(mockPeer.getPeerState()).thenReturn(PeerState.ENABLED); + when(mockReplicationPeers.refreshPeerConfig(PEER_ID)).thenReturn(newConfig); + when(mockReplicationPeers.refreshPeerState(PEER_ID)).thenReturn(PeerState.ENABLED); + + handler.updatePeerConfig(PEER_ID); + + verify(mockSourceManager, times(1)).refreshSources(PEER_ID); + } + + @Test + public void testNonReplicationSourceConfigDoesNotTrigger() throws Exception { + ReplicationPeerConfig oldConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("some.other.config", "value1").build(); + + ReplicationPeerConfig newConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("some.other.config", "value2").build(); + + when(mockPeer.getPeerConfig()).thenReturn(oldConfig); + when(mockPeer.getPeerState()).thenReturn(PeerState.ENABLED); + when(mockReplicationPeers.refreshPeerConfig(PEER_ID)).thenReturn(newConfig); + when(mockReplicationPeers.refreshPeerState(PEER_ID)).thenReturn(PeerState.ENABLED); + + handler.updatePeerConfig(PEER_ID); + + verify(mockSourceManager, never()).refreshSources(anyString()); + } + + @Test + public void testNewReplicationSourceConfigTriggers() throws Exception { + ReplicationPeerConfig oldConfig = + ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster").build(); + + ReplicationPeerConfig newConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("replication.source.sleepforretries", "5000").build(); + + when(mockPeer.getPeerConfig()).thenReturn(oldConfig); + when(mockPeer.getPeerState()).thenReturn(PeerState.ENABLED); + when(mockReplicationPeers.refreshPeerConfig(PEER_ID)).thenReturn(newConfig); + when(mockReplicationPeers.refreshPeerState(PEER_ID)).thenReturn(PeerState.ENABLED); + + handler.updatePeerConfig(PEER_ID); + + verify(mockSourceManager, times(1)).refreshSources(PEER_ID); + } + + @Test + public void testRemovedReplicationSourceConfigTriggers() throws Exception { + ReplicationPeerConfig oldConfig = ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster") + .putConfiguration("replication.source.sleepforretries", "2000").build(); + + ReplicationPeerConfig newConfig = + ReplicationPeerConfig.newBuilder().setClusterKey("oldCluster").build(); + + when(mockPeer.getPeerConfig()).thenReturn(oldConfig); + when(mockPeer.getPeerState()).thenReturn(PeerState.ENABLED); + when(mockReplicationPeers.refreshPeerConfig(PEER_ID)).thenReturn(newConfig); + when(mockReplicationPeers.refreshPeerState(PEER_ID)).thenReturn(PeerState.ENABLED); + + handler.updatePeerConfig(PEER_ID); + + verify(mockSourceManager, times(1)).refreshSources(PEER_ID); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index fc5d14c7091a..2b5c22ced148 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.replication.DummyReplicationEndpoint; import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationPeer; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; @@ -667,6 +668,125 @@ public void testSameWALPrefix() throws IOException { assertTrue(latestWals.contains(walName2)); } + @Test + public void testPeerConfigurationOverridesPropagate() throws Exception { + String replicationSourceImplName = conf.get("replication.replicationsource.implementation"); + String peerId = "testConfigOverridePeer"; + try { + conf.set("replication.replicationsource.implementation", ReplicationSource.class.getName()); + + Configuration globalConf = utility.getConfiguration(); + long globalSleepValue = 1000L; + globalConf.setLong("replication.source.sleepforretries", globalSleepValue); + + long peerSleepOverride = 5000L; + String clusterKey = "testPeerConfigOverride"; + + ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder() + .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/" + clusterKey) + .setReplicationEndpointImpl(DummyReplicationEndpoint.class.getName()) + .putConfiguration("replication.source.sleepforretries", String.valueOf(peerSleepOverride)) + .build(); + + manager.getReplicationPeers().getPeerStorage().addPeer(peerId, peerConfig, true); + manager.addPeer(peerId); + utility.waitFor(20000, () -> { + ReplicationSourceInterface rs = manager.getSource(peerId); + return rs != null && rs.isSourceActive(); + }); + + ReplicationSource source = (ReplicationSource) manager.getSources().stream() + .filter(s -> s.getPeerId().equals(peerId)).findFirst().orElse(null); + assertNotNull("Source should be created for peer", source); + + assertEquals("ReplicationSource should use peer config override for sleepForRetries", + peerSleepOverride, source.getSleepForRetries()); + + Map workers = source.workerThreads; + if (!workers.isEmpty()) { + ReplicationSourceShipper shipper = workers.values().iterator().next(); + assertEquals("ReplicationSourceShipper should use peer config override for sleepForRetries", + peerSleepOverride, shipper.getSleepForRetries()); + + ReplicationSourceWALReader reader = shipper.entryReader; + if (reader != null) { + assertEquals( + "ReplicationSourceWALReader should use peer config override for sleepForRetries", + peerSleepOverride, reader.getSleepForRetries()); + } + } + } finally { + conf.set("replication.replicationsource.implementation", replicationSourceImplName); + removePeerAndWait(peerId); + } + } + + @Test + public void testPeerConfigurationIsolation() throws Exception { + String replicationSourceImplName = conf.get("replication.replicationsource.implementation"); + String peerIdWithOverride = "peerWithOverride"; + String peerIdWithoutOverride = "peerWithoutOverride"; + try { + conf.set("replication.replicationsource.implementation", ReplicationSource.class.getName()); + + Configuration globalConf = utility.getConfiguration(); + long globalSleepValue = 1000L; + globalConf.setLong("replication.source.sleepforretries", globalSleepValue); + + // Create first peer WITH config override + long peerSleepOverride = 5000L; + String clusterKeyWithOverride = "testPeerWithOverride"; + + ReplicationPeerConfig configWithOverride = ReplicationPeerConfig.newBuilder() + .setClusterKey( + utility.getZkCluster().getAddress().toString() + ":/" + clusterKeyWithOverride) + .setReplicationEndpointImpl(DummyReplicationEndpoint.class.getName()) + .putConfiguration("replication.source.sleepforretries", String.valueOf(peerSleepOverride)) + .build(); + + manager.getReplicationPeers().getPeerStorage().addPeer(peerIdWithOverride, configWithOverride, + true); + manager.addPeer(peerIdWithOverride); + + // Create second peer WITHOUT config override + String clusterKeyWithoutOverride = "testPeerWithoutOverride"; + + ReplicationPeerConfig configWithoutOverride = ReplicationPeerConfig.newBuilder() + .setClusterKey( + utility.getZkCluster().getAddress().toString() + ":/" + clusterKeyWithoutOverride) + .setReplicationEndpointImpl(DummyReplicationEndpoint.class.getName()).build(); + + manager.getReplicationPeers().getPeerStorage().addPeer(peerIdWithoutOverride, + configWithoutOverride, true); + manager.addPeer(peerIdWithoutOverride); + + // Wait for both peers to be active + utility.waitFor(20000, () -> { + ReplicationSourceInterface rs1 = manager.getSource(peerIdWithOverride); + ReplicationSourceInterface rs2 = manager.getSource(peerIdWithoutOverride); + return rs1 != null && rs1.isSourceActive() && rs2 != null && rs2.isSourceActive(); + }); + + // Verify peer with override uses the override value + ReplicationSource sourceWithOverride = (ReplicationSource) manager.getSources().stream() + .filter(s -> s.getPeerId().equals(peerIdWithOverride)).findFirst().orElse(null); + assertNotNull("Source with override should be created", sourceWithOverride); + assertEquals("Peer with override should use override value", peerSleepOverride, + sourceWithOverride.getSleepForRetries()); + + // Verify peer without override uses global config + ReplicationSource sourceWithoutOverride = (ReplicationSource) manager.getSources().stream() + .filter(s -> s.getPeerId().equals(peerIdWithoutOverride)).findFirst().orElse(null); + assertNotNull("Source without override should be created", sourceWithoutOverride); + assertEquals("Peer without override should use global config", globalSleepValue, + sourceWithoutOverride.getSleepForRetries()); + } finally { + conf.set("replication.replicationsource.implementation", replicationSourceImplName); + removePeerAndWait(peerIdWithOverride); + removePeerAndWait(peerIdWithoutOverride); + } + } + private WALEdit getBulkLoadWALEdit(NavigableMap scope) { // 1. Create store files for the families Map> storeFiles = new HashMap<>(1); From 6c46410a75a00423c66027c108d08028d74a8500 Mon Sep 17 00:00:00 2001 From: skhillon Date: Thu, 8 Jan 2026 06:44:14 -0800 Subject: [PATCH 63/78] Stop using new version of junit --- .../regionserver/TestPeerProcedureHandlerImpl.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java index 193d1123c0eb..99259f225dfb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestPeerProcedureHandlerImpl.java @@ -30,12 +30,11 @@ import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.testclassification.ReplicationTests; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; -@Tag(ReplicationTests.TAG) -@Tag(SmallTests.TAG) +@Category({ ReplicationTests.class, SmallTests.class }) public class TestPeerProcedureHandlerImpl { private ReplicationSourceManager mockSourceManager; @@ -44,7 +43,7 @@ public class TestPeerProcedureHandlerImpl { private PeerProcedureHandlerImpl handler; private static final String PEER_ID = "testPeer"; - @BeforeEach + @Before public void setup() throws Exception { mockSourceManager = mock(ReplicationSourceManager.class); mockReplicationPeers = mock(ReplicationPeers.class); From 39d766a5883441a93fd8571df4981176a0dd2c64 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 17 Feb 2026 09:07:25 -0500 Subject: [PATCH 64/78] HBASE-29808 Simplify backup history retrieval (#7595) (#238) This commit condenses several redundant APIs used to access the backup history. No functional changes. Signed-off-by: Nick Dimiduk Co-authored-by: DieterDP <90392398+DieterDP-ng@users.noreply.github.com> --- .../hadoop/hbase/backup/BackupAdmin.java | 8 - .../hadoop/hbase/backup/BackupInfo.java | 27 ++- .../hbase/backup/impl/BackupAdminImpl.java | 59 ++----- .../hbase/backup/impl/BackupCommands.java | 53 +++--- .../hbase/backup/impl/BackupManager.java | 16 +- .../hbase/backup/impl/BackupSystemTable.java | 162 +++++------------- .../hbase/backup/impl/TableBackupClient.java | 5 +- .../hbase/backup/master/BackupLogCleaner.java | 5 +- .../hadoop/hbase/backup/util/BackupUtils.java | 45 +---- .../hbase/backup/TestBackupShowHistory.java | 3 +- 10 files changed, 118 insertions(+), 265 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java index 269afd666239..86cd8d987401 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java @@ -76,14 +76,6 @@ public interface BackupAdmin extends Closeable { */ void mergeBackups(String[] backupIds) throws IOException; - /** - * Show backup history command - * @param n last n backup sessions - * @return list of backup info objects - * @throws IOException exception - */ - List getHistory(int n) throws IOException; - /** * Show backup history command with filters * @param n last n backup sessions diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java index 28ff70eaf0df..801d14e5264e 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; @@ -46,13 +47,22 @@ public class BackupInfo implements Comparable { private static final Logger LOG = LoggerFactory.getLogger(BackupInfo.class); private static final int MAX_FAILED_MESSAGE_LENGTH = 1024; - public interface Filter { - /** - * Filter interface - * @param info backup info - * @return true if info passes filter, false otherwise - */ - boolean apply(BackupInfo info); + public interface Filter extends Predicate { + /** Returns true if the BackupInfo passes the filter, false otherwise */ + @Override + boolean test(BackupInfo backupInfo); + } + + public static Filter withRoot(String backupRoot) { + return info -> info.getBackupRootDir().equals(backupRoot); + } + + public static Filter withType(BackupType type) { + return info -> info.getType() == type; + } + + public static Filter withState(BackupState state) { + return info -> info.getState() == state; } /** @@ -61,8 +71,7 @@ public interface Filter { public enum BackupState { RUNNING, COMPLETE, - FAILED, - ANY + FAILED } /** diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index 31bd75ba4b2e..84082e7db107 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.BackupInfo.withType; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -74,7 +78,7 @@ public BackupInfo getBackupInfo(String backupId) throws IOException { BackupInfo backupInfo; try (final BackupSystemTable table = new BackupSystemTable(conn)) { if (backupId == null) { - ArrayList recentSessions = table.getBackupInfos(BackupState.RUNNING); + List recentSessions = table.getBackupInfos(withState(BackupState.RUNNING)); if (recentSessions.isEmpty()) { LOG.warn("No ongoing sessions found."); return null; @@ -111,7 +115,7 @@ public int deleteBackups(String[] backupIds) throws IOException { } // Step 2: Make sure there is no failed session - List list = sysTable.getBackupInfos(BackupState.RUNNING); + List list = sysTable.getBackupInfos(withState(BackupState.RUNNING)); if (list.size() != 0) { // ailed sessions found LOG.warn("Failed backup session found. Run backup repair tool first."); @@ -301,7 +305,7 @@ private List getAffectedBackupSessions(BackupInfo backupInfo, TableN LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); long ts = backupInfo.getStartTs(); List list = new ArrayList<>(); - List history = table.getBackupHistory(backupInfo.getBackupRootDir()); + List history = table.getBackupHistory(withRoot(backupInfo.getBackupRootDir())); // Scan from most recent to backupInfo // break when backupInfo reached for (BackupInfo info : history) { @@ -367,49 +371,10 @@ private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long return false; } - @Override - public List getHistory(int n) throws IOException { - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - - if (history.size() <= n) { - return history; - } - - List list = new ArrayList<>(); - for (int i = 0; i < n; i++) { - list.add(history.get(i)); - } - return list; - } - } - @Override public List getHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) { - return getHistory(n); - } - try (final BackupSystemTable table = new BackupSystemTable(conn)) { - List history = table.getBackupHistory(); - List result = new ArrayList<>(); - for (BackupInfo bi : history) { - if (result.size() == n) { - break; - } - - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; + return table.getBackupInfos(n, filters); } } @@ -673,7 +638,7 @@ private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) // Filter 1 : backupRoot // Filter 2 : time range filter // Filter 3 : table filter - BackupInfo.Filter destinationFilter = info -> info.getBackupRootDir().equals(backupDest); + BackupInfo.Filter destinationFilter = withRoot(backupDest); BackupInfo.Filter timeRangeFilter = info -> { long time = info.getStartTs(); @@ -685,10 +650,10 @@ private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) return !Collections.disjoint(allTables, tables); }; - BackupInfo.Filter typeFilter = info -> info.getType() == BackupType.INCREMENTAL; - BackupInfo.Filter stateFilter = info -> info.getState() == BackupState.COMPLETE; + BackupInfo.Filter typeFilter = withType(BackupType.INCREMENTAL); + BackupInfo.Filter stateFilter = withState(BackupState.COMPLETE); - List allInfos = table.getBackupHistory(-1, destinationFilter, timeRangeFilter, + List allInfos = table.getBackupHistory(destinationFilter, timeRangeFilter, tableFilter, typeFilter, stateFilter); if (allInfos.size() != allBackups.size()) { // Yes we have at least one hole in backup image sequence diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index c8322fdba76d..7e11f8a3cee1 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; @@ -171,7 +172,7 @@ public void execute() throws IOException { if (requiresNoActiveSession()) { // Check active session try (BackupSystemTable table = new BackupSystemTable(conn)) { - List sessions = table.getBackupInfos(BackupState.RUNNING); + List sessions = table.getBackupInfos(withState(BackupState.RUNNING)); if (sessions.size() > 0) { System.err.println("Found backup session in a RUNNING state: "); @@ -546,7 +547,7 @@ public void execute() throws IOException { if (backupId != null) { info = sysTable.readBackupInfo(backupId); } else { - List infos = sysTable.getBackupInfos(BackupState.RUNNING); + List infos = sysTable.getBackupInfos(withState(BackupState.RUNNING)); if (infos != null && infos.size() > 0) { info = infos.get(0); backupId = info.getBackupId(); @@ -612,18 +613,15 @@ private void executeDeleteOlderThan(CommandLine cmdline) throws IOException { throw new IOException(value + " is not an integer number"); } final long fdays = days; - BackupInfo.Filter dateFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - long currentTime = EnvironmentEdgeManager.currentTime(); - long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; - return info.getCompleteTs() <= maxTsToDelete; - } + BackupInfo.Filter dateFilter = info -> { + long currentTime = EnvironmentEdgeManager.currentTime(); + long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; + return info.getCompleteTs() <= maxTsToDelete; }; List history = null; try (final BackupSystemTable sysTable = new BackupSystemTable(conn); BackupAdminImpl admin = new BackupAdminImpl(conn)) { - history = sysTable.getBackupHistory(-1, dateFilter); + history = sysTable.getBackupHistory(dateFilter); String[] backupIds = convertToBackupIds(history); int deleted = admin.deleteBackups(backupIds); System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " @@ -697,7 +695,7 @@ public void execute() throws IOException { final BackupSystemTable sysTable = new BackupSystemTable(conn)) { // Failed backup BackupInfo backupInfo; - List list = sysTable.getBackupInfos(BackupState.RUNNING); + List list = sysTable.getBackupInfos(withState(BackupState.RUNNING)); if (list.size() == 0) { // No failed sessions found System.out.println("REPAIR status: no failed sessions found." @@ -878,27 +876,21 @@ public void execute() throws IOException { int n = parseHistoryLength(); final TableName tableName = getTableName(); final String setName = getTableSetName(); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (tableName == null) { - return true; - } - - List names = info.getTableNames(); - return names.contains(tableName); + BackupInfo.Filter tableNameFilter = info -> { + if (tableName == null) { + return true; } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (setName == null) { - return true; - } - String backupId = info.getBackupId(); - return backupId.startsWith(setName); + List names = info.getTableNames(); + return names.contains(tableName); + }; + BackupInfo.Filter tableSetFilter = info -> { + if (setName == null) { + return true; } + + String backupId = info.getBackupId(); + return backupId.startsWith(setName); }; Path backupRootPath = getBackupRootPath(); List history; @@ -906,7 +898,8 @@ public boolean apply(BackupInfo info) { // Load from backup system table super.execute(); try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) { - history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); + history = sysTable.getBackupHistory(tableNameFilter, tableSetFilter); + history = history.subList(0, Math.min(n, history.size())); } } else { // load from backup FS diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 54328817f512..66e3463f5b58 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; + import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -267,7 +269,7 @@ public BackupInfo createBackupInfo(String backupId, BackupType type, List sessions = systemTable.getBackupInfos(BackupState.RUNNING); + List sessions = systemTable.getBackupInfos(withState(BackupState.RUNNING)); if (sessions.size() == 0) { return null; } @@ -389,16 +391,10 @@ public void deleteBulkLoadedRows(List rows) throws IOException { } /** - * Get all completed backup information (in desc order by time) - * @return history info of BackupCompleteData - * @throws IOException exception + * Get all backup information, ordered by descending start time. I.e. from newest to oldest. */ - public List getBackupHistory() throws IOException { - return systemTable.getBackupHistory(); - } - - public ArrayList getBackupHistory(boolean completed) throws IOException { - return systemTable.getBackupHistory(completed); + public List getBackupHistory(BackupInfo.Filter... filters) throws IOException { + return systemTable.getBackupHistory(filters); } /** diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index 0ece064d2355..a9847f5c77ea 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.BackupInfo.withType; + import edu.umd.cs.findbugs.annotations.Nullable; import java.io.Closeable; import java.io.IOException; @@ -26,6 +30,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -36,7 +41,9 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -54,7 +61,6 @@ import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; @@ -592,123 +598,26 @@ public void writeRegionServerLastLogRollResult(String server, Long ts, String ba } /** - * Get all completed backup information (in desc order by time) - * @param onlyCompleted true, if only successfully completed sessions - * @return history info of BackupCompleteData - * @throws IOException exception + * Get all backup information passing the given filters, ordered by descending start time. I.e. + * from newest to oldest. */ - public ArrayList getBackupHistory(boolean onlyCompleted) throws IOException { + public List getBackupHistory(BackupInfo.Filter... toInclude) throws IOException { LOG.trace("get backup history from backup system table"); - BackupState state = onlyCompleted ? BackupState.COMPLETE : BackupState.ANY; - ArrayList list = getBackupInfos(state); - return BackupUtils.sortHistoryListDesc(list); - } - - /** - * Get all backups history - * @return list of backup info - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory() throws IOException { - return getBackupHistory(false); - } - - /** - * Get first n backup history records - * @param n number of records, if n== -1 - max number is ignored - * @return list of records - * @throws IOException if getting the backup history fails - */ - public List getHistory(int n) throws IOException { - List history = getBackupHistory(); - if (n == -1 || history.size() <= n) { - return history; - } - return Collections.unmodifiableList(history.subList(0, n)); - } - - /** - * Get backup history records filtered by list of filters. - * @param n max number of records, if n == -1 , then max number is ignored - * @param filters list of filters - * @return backup records - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory(int n, BackupInfo.Filter... filters) throws IOException { - if (filters.length == 0) { - return getHistory(n); - } - - List history = getBackupHistory(); - List result = new ArrayList<>(); - for (BackupInfo bi : history) { - if (n >= 0 && result.size() == n) { - break; - } - - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(bi)) { - passed = false; - break; - } - } - if (passed) { - result.add(bi); - } - } - return result; + List list = getBackupInfos(toInclude); + list.sort(Comparator.comparing(BackupInfo::getStartTs).reversed()); + return list; } /** - * Retrieve all table names that are part of any known backup + * Retrieve all table names that are part of any known completed backup */ public Set getTablesIncludedInBackups() throws IOException { - Set names = new HashSet<>(); - List infos = getBackupHistory(true); - for (BackupInfo info : infos) { - // Incremental backups have the same tables as the preceding full backups - if (info.getType() == BackupType.FULL) { - names.addAll(info.getTableNames()); - } - } - return names; - } - - /** - * Get history for backup destination - * @param backupRoot backup destination path - * @return List of backup info - * @throws IOException if getting the backup history fails - */ - public List getBackupHistory(String backupRoot) throws IOException { - ArrayList history = getBackupHistory(false); - for (Iterator iterator = history.iterator(); iterator.hasNext();) { - BackupInfo info = iterator.next(); - if (!backupRoot.equals(info.getBackupRootDir())) { - iterator.remove(); - } - } - return history; - } - - /** - * Get history for a table - * @param name table name - * @return history for a table - * @throws IOException if getting the backup history fails - */ - public List getBackupHistoryForTable(TableName name) throws IOException { - List history = getBackupHistory(); - List tableHistory = new ArrayList<>(); - for (BackupInfo info : history) { - List tables = info.getTableNames(); - if (tables.contains(name)) { - tableHistory.add(info); - } - } - return tableHistory; + // Incremental backups have the same tables as the preceding full backups + List infos = + getBackupInfos(withState(BackupState.COMPLETE), withType(BackupType.FULL)); + return infos.stream().flatMap(info -> info.getTableNames().stream()) + .collect(Collectors.toSet()); } /** @@ -722,7 +631,7 @@ public List getBackupHistoryForTable(TableName name) throws IOExcept */ public Map> getBackupHistoryForTableSet(Set set, String backupRoot) throws IOException { - List history = getBackupHistory(backupRoot); + List history = getBackupHistory(withRoot(backupRoot)); Map> tableHistoryMap = new HashMap<>(); for (BackupInfo info : history) { List tables = info.getTableNames(); @@ -738,16 +647,27 @@ public Map> getBackupHistoryForTableSet(Set getBackupInfos(BackupState state) throws IOException { + public List getBackupInfos(BackupInfo.Filter... toInclude) throws IOException { + return getBackupInfos(Integer.MAX_VALUE, toInclude); + } + + /** + * Get the first n backup infos passing the given filters (ordered by ascending backup id) + */ + public List getBackupInfos(int n, BackupInfo.Filter... toInclude) throws IOException { LOG.trace("get backup infos from backup system table"); + if (n <= 0) { + return Collections.emptyList(); + } + + Predicate combinedPredicate = Stream.of(toInclude) + .map(filter -> (Predicate) filter).reduce(Predicate::and).orElse(x -> true); + Scan scan = createScanForBackupHistory(); - ArrayList list = new ArrayList<>(); + List list = new ArrayList<>(); try (Table table = connection.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) { @@ -755,10 +675,12 @@ public ArrayList getBackupInfos(BackupState state) throws IOExceptio while ((res = scanner.next()) != null) { res.advance(); BackupInfo context = cellToBackupInfo(res.current()); - if (state != BackupState.ANY && context.getState() != state) { - continue; + if (combinedPredicate.test(context)) { + list.add(context); + if (list.size() == n) { + break; + } } - list.add(context); } return list; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java index 8bf91ef86fa3..e0d56f7bda6f 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/TableBackupClient.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -314,7 +316,8 @@ protected List getAncestors(BackupInfo backupInfo) throws IOExcepti Set tablesToCover = new HashSet<>(backupInfo.getTables()); // Go over the backup history list from newest to oldest - List allHistoryList = backupManager.getBackupHistory(true); + List allHistoryList = + backupManager.getBackupHistory(withState(BackupState.COMPLETE)); for (BackupInfo backup : allHistoryList) { // If the image has a different rootDir, it cannot be an ancestor. if (!Objects.equals(backup.getBackupRootDir(), backupInfo.getBackupRootDir())) { diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 263191df8049..03f006948721 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.backup.master; +import static org.apache.hadoop.hbase.backup.BackupInfo.withState; + import java.io.IOException; import java.time.Duration; import java.util.ArrayList; @@ -31,6 +33,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; import org.apache.hadoop.hbase.backup.BackupRestoreConstants; import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; @@ -91,7 +94,7 @@ public void init(Map params) { */ private BackupBoundaries serverToPreservationBoundaryTs(BackupSystemTable sysTable) throws IOException { - List backups = sysTable.getBackupHistory(true); + List backups = sysTable.getBackupHistory(withState(BackupState.COMPLETE)); if (LOG.isDebugEnabled()) { LOG.debug( "Cleaning WALs if they are older than the WAL cleanup time-boundary. " diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index b8b82ef31cbc..a301a3e3734f 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -24,12 +24,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -479,24 +480,6 @@ public static String getTableBackupDir(String backupRootDir, String backupId, + Path.SEPARATOR; } - /** - * Sort history list by start time in descending order. - * @param historyList history list - * @return sorted list of BackupCompleteData - */ - public static ArrayList sortHistoryListDesc(ArrayList historyList) { - ArrayList list = new ArrayList<>(); - TreeMap map = new TreeMap<>(); - for (BackupInfo h : historyList) { - map.put(Long.toString(h.getStartTs()), h); - } - Iterator i = map.descendingKeySet().iterator(); - while (i.hasNext()) { - list.add(map.get(i.next())); - } - return list; - } - /** * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and @@ -597,23 +580,11 @@ private long getTimestamp(String backupId) { public static List getHistory(Configuration conf, int n, Path backupRootPath, BackupInfo.Filter... filters) throws IOException { List infos = getHistory(conf, backupRootPath); - List ret = new ArrayList<>(); - for (BackupInfo info : infos) { - if (ret.size() == n) { - break; - } - boolean passed = true; - for (int i = 0; i < filters.length; i++) { - if (!filters[i].apply(info)) { - passed = false; - break; - } - } - if (passed) { - ret.add(info); - } - } - return ret; + + Predicate combinedPredicate = Stream.of(filters) + .map(filter -> (Predicate) filter).reduce(Predicate::and).orElse(x -> true); + + return infos.stream().filter(combinedPredicate).limit(n).collect(Collectors.toList()); } public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs) diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index fa624250929d..165003fdb5cb 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -74,8 +74,7 @@ public void testBackupHistory() throws Exception { List history = getBackupAdmin().getHistory(10); assertTrue(findBackup(history, backupId)); - BackupInfo.Filter nullFilter = info -> true; - history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), nullFilter); + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); assertTrue(findBackup(history, backupId)); ByteArrayOutputStream baos = new ByteArrayOutputStream(); From 5d865a4d36bff67c84f68d02289f53192109f9b7 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Wed, 18 Feb 2026 12:37:45 -0500 Subject: [PATCH 65/78] HBASE-29846 Fix backup history ordering (#239) Restores the ordering of BackupAdmin#getHistory that was accidentally reversed in HBASE-29808. Extends & refactors TestBackupShowHistory to verify correct behavior. Fixes a possible FileNotFoundException in BackupUtils#getHistory. Merges BackupSystemTable#getBackupHistory with BackupSystemTable#getBackupInfos, to further simplify backup info retrieval. Optimized various usages of history retrieval. Clarified some javadoc regarding backup history retrieval. (cherry picked from commit ed04e2e1fe5c61d744fc08585a2e2cfa6becc6d1) Co-authored-by: Dieter De Paepe --- .../hadoop/hbase/backup/BackupAdmin.java | 8 +- .../hbase/backup/impl/BackupAdminImpl.java | 9 +- .../hbase/backup/impl/BackupCommands.java | 10 +- .../hbase/backup/impl/BackupManager.java | 4 +- .../hbase/backup/impl/BackupSystemTable.java | 74 +++++---- .../hadoop/hbase/backup/util/BackupUtils.java | 34 ++-- .../hbase/backup/TestBackupShowHistory.java | 155 +++++++++--------- 7 files changed, 156 insertions(+), 138 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java index 86cd8d987401..9e022a8716e3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupAdmin.java @@ -77,10 +77,10 @@ public interface BackupAdmin extends Closeable { void mergeBackups(String[] backupIds) throws IOException; /** - * Show backup history command with filters - * @param n last n backup sessions - * @param f list of filters - * @return list of backup info objects + * Retrieve info about the most recent backups. + * @param n number of backup infos desired + * @param f optional filters, only entries passing the filters will be returned + * @return a list of at most n entries, ordered from newest (most recent) to oldest (least recent) * @throws IOException exception */ List getHistory(int n, BackupInfo.Filter... f) throws IOException; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index 84082e7db107..66b0479ec687 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hbase.backup.BackupInfo.withRoot; import static org.apache.hadoop.hbase.backup.BackupInfo.withState; import static org.apache.hadoop.hbase.backup.BackupInfo.withType; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; import java.io.IOException; import java.util.ArrayList; @@ -78,7 +79,8 @@ public BackupInfo getBackupInfo(String backupId) throws IOException { BackupInfo backupInfo; try (final BackupSystemTable table = new BackupSystemTable(conn)) { if (backupId == null) { - List recentSessions = table.getBackupInfos(withState(BackupState.RUNNING)); + List recentSessions = + table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (recentSessions.isEmpty()) { LOG.warn("No ongoing sessions found."); return null; @@ -115,7 +117,8 @@ public int deleteBackups(String[] backupIds) throws IOException { } // Step 2: Make sure there is no failed session - List list = sysTable.getBackupInfos(withState(BackupState.RUNNING)); + List list = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (list.size() != 0) { // ailed sessions found LOG.warn("Failed backup session found. Run backup repair tool first."); @@ -374,7 +377,7 @@ private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long @Override public List getHistory(int n, BackupInfo.Filter... filters) throws IOException { try (final BackupSystemTable table = new BackupSystemTable(conn)) { - return table.getBackupInfos(n, filters); + return table.getBackupHistory(NEW_TO_OLD, n, filters); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 7e11f8a3cee1..ef3a3bcaebb3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -44,6 +44,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; import java.io.IOException; import java.net.URI; import java.util.List; @@ -172,7 +173,8 @@ public void execute() throws IOException { if (requiresNoActiveSession()) { // Check active session try (BackupSystemTable table = new BackupSystemTable(conn)) { - List sessions = table.getBackupInfos(withState(BackupState.RUNNING)); + List sessions = + table.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (sessions.size() > 0) { System.err.println("Found backup session in a RUNNING state: "); @@ -547,7 +549,8 @@ public void execute() throws IOException { if (backupId != null) { info = sysTable.readBackupInfo(backupId); } else { - List infos = sysTable.getBackupInfos(withState(BackupState.RUNNING)); + List infos = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (infos != null && infos.size() > 0) { info = infos.get(0); backupId = info.getBackupId(); @@ -695,7 +698,8 @@ public void execute() throws IOException { final BackupSystemTable sysTable = new BackupSystemTable(conn)) { // Failed backup BackupInfo backupInfo; - List list = sysTable.getBackupInfos(withState(BackupState.RUNNING)); + List list = + sysTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (list.size() == 0) { // No failed sessions found System.out.println("REPAIR status: no failed sessions found." diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java index 66e3463f5b58..851dab67c37d 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.backup.impl; import static org.apache.hadoop.hbase.backup.BackupInfo.withState; +import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; import java.io.Closeable; import java.io.IOException; @@ -269,7 +270,8 @@ public BackupInfo createBackupInfo(String backupId, BackupType type, List sessions = systemTable.getBackupInfos(withState(BackupState.RUNNING)); + List sessions = + systemTable.getBackupHistory(NEW_TO_OLD, 1, withState(BackupState.RUNNING)); if (sessions.size() == 0) { return null; } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index a9847f5c77ea..c89b93d32d9c 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -82,6 +81,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.collect.Iterators; @@ -597,25 +597,13 @@ public void writeRegionServerLastLogRollResult(String server, Long ts, String ba } } - /** - * Get all backup information passing the given filters, ordered by descending start time. I.e. - * from newest to oldest. - */ - public List getBackupHistory(BackupInfo.Filter... toInclude) throws IOException { - LOG.trace("get backup history from backup system table"); - - List list = getBackupInfos(toInclude); - list.sort(Comparator.comparing(BackupInfo::getStartTs).reversed()); - return list; - } - /** * Retrieve all table names that are part of any known completed backup */ public Set getTablesIncludedInBackups() throws IOException { // Incremental backups have the same tables as the preceding full backups List infos = - getBackupInfos(withState(BackupState.COMPLETE), withType(BackupType.FULL)); + getBackupHistory(withState(BackupState.COMPLETE), withType(BackupType.FULL)); return infos.stream().flatMap(info -> info.getTableNames().stream()) .collect(Collectors.toSet()); } @@ -647,26 +635,31 @@ public Map> getBackupHistoryForTableSet(Set getBackupInfos(BackupInfo.Filter... toInclude) throws IOException { - return getBackupInfos(Integer.MAX_VALUE, toInclude); + public List getBackupHistory(BackupInfo.Filter... toInclude) throws IOException { + return getBackupHistory(Order.NEW_TO_OLD, Integer.MAX_VALUE, toInclude); } /** - * Get the first n backup infos passing the given filters (ordered by ascending backup id) + * Retrieves the first n entries of the sorted, filtered list of backup infos. + * @param order desired ordering of the results. + * @param n number of entries to return */ - public List getBackupInfos(int n, BackupInfo.Filter... toInclude) throws IOException { + public List getBackupHistory(Order order, int n, BackupInfo.Filter... toInclude) + throws IOException { + Preconditions.checkArgument(n >= 0, "n should be >= 0"); LOG.trace("get backup infos from backup system table"); - if (n <= 0) { + if (n == 0) { return Collections.emptyList(); } Predicate combinedPredicate = Stream.of(toInclude) .map(filter -> (Predicate) filter).reduce(Predicate::and).orElse(x -> true); - Scan scan = createScanForBackupHistory(); + Scan scan = createScanForBackupHistory(order); List list = new ArrayList<>(); try (Table table = connection.getTable(tableName); @@ -852,21 +845,17 @@ public void deleteIncrementalBackupTableSet(String backupRoot) throws IOExceptio /** * Checks if we have at least one backup session in backup system table This API is used by * BackupLogCleaner - * @return true, if - at least one session exists in backup system table table + * @return true, if at least one session exists in backup system table * @throws IOException exception */ public boolean hasBackupSessions() throws IOException { LOG.trace("Has backup sessions from backup system table"); - boolean result = false; - Scan scan = createScanForBackupHistory(); + Scan scan = createScanForBackupHistory(Order.OLD_TO_NEW); scan.setCaching(1); try (Table table = connection.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) { - if (scanner.next() != null) { - result = true; - } - return result; + return scanner.next() != null; } } @@ -1190,15 +1179,23 @@ private Delete createDeleteForIncrBackupTableSet(String backupRoot) { /** * Creates Scan operation to load backup history + * @param order order of the scan results * @return scan operation */ - private Scan createScanForBackupHistory() { + private Scan createScanForBackupHistory(Order order) { Scan scan = new Scan(); byte[] startRow = Bytes.toBytes(BACKUP_INFO_PREFIX); - byte[] stopRow = Arrays.copyOf(startRow, startRow.length); - stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); - scan.withStartRow(startRow); - scan.withStopRow(stopRow); + if (order == Order.NEW_TO_OLD) { + byte[] stopRow = Arrays.copyOf(startRow, startRow.length); + stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1); + scan.setReversed(true); + scan.withStartRow(stopRow, false); + scan.withStopRow(startRow); + } else if (order == Order.OLD_TO_NEW) { + scan.setStartStopRowForPrefixScan(startRow); + } else { + throw new IllegalArgumentException("Unsupported order: " + order); + } scan.addFamily(BackupSystemTable.SESSIONS_FAMILY); scan.readVersions(1); return scan; @@ -1653,4 +1650,15 @@ private static void ensureTableEnabled(Admin admin, TableName tableName) throws } } } + + public enum Order { + /** + * Old backups first, most recents last. I.e. sorted by ascending backupId. + */ + OLD_TO_NEW, + /** + * New backups first, oldest last. I.e. sorted by descending backupId. + */ + NEW_TO_OLD + } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index a301a3e3734f..ebc8ad13be29 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -533,12 +533,21 @@ public static String getLogBackupDir(String backupRootDir, String backupId) { + HConstants.HREGION_LOGDIR_NAME; } + /** + * Loads all backup history as stored in files on the given backup root path. + * @return all backup history, from newest (most recent) to oldest (least recent) + */ private static List getHistory(Configuration conf, Path backupRootPath) throws IOException { // Get all (n) history from backup root destination FileSystem fs = FileSystem.get(backupRootPath.toUri(), conf); - RemoteIterator it = fs.listLocatedStatus(backupRootPath); + RemoteIterator it; + try { + it = fs.listLocatedStatus(backupRootPath); + } catch (FileNotFoundException e) { + return Collections.emptyList(); + } List infos = new ArrayList<>(); while (it.hasNext()) { @@ -557,26 +566,15 @@ private static List getHistory(Configuration conf, Path backupRootPa } } // Sort - Collections.sort(infos, new Comparator() { - @Override - public int compare(BackupInfo o1, BackupInfo o2) { - long ts1 = getTimestamp(o1.getBackupId()); - long ts2 = getTimestamp(o2.getBackupId()); - - if (ts1 == ts2) { - return 0; - } - - return ts1 < ts2 ? 1 : -1; - } - - private long getTimestamp(String backupId) { - return Long.parseLong(Iterators.get(Splitter.on('_').split(backupId).iterator(), 1)); - } - }); + infos.sort(Comparator. naturalOrder().reversed()); return infos; } + /** + * Loads all backup history as stored in files on the given backup root path, and returns the + * first n entries matching all given filters. + * @return (subset of) backup history, from newest (most recent) to oldest (least recent) + */ public static List getHistory(Configuration conf, int n, Path backupRootPath, BackupInfo.Filter... filters) throws IOException { List infos = getHistory(conf, backupRootPath); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index 165003fdb5cb..40c4874e40c0 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -17,14 +17,15 @@ */ package org.apache.hadoop.hbase.backup; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; import org.apache.hadoop.fs.Path; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; @@ -34,8 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; - @Category(LargeTests.class) public class TestBackupShowHistory extends TestBackupBase { @@ -45,96 +44,100 @@ public class TestBackupShowHistory extends TestBackupBase { private static final Logger LOG = LoggerFactory.getLogger(TestBackupShowHistory.class); - private boolean findBackup(List history, String backupId) { - assertTrue(history.size() > 0); - boolean success = false; - for (BackupInfo info : history) { - if (info.getBackupId().equals(backupId)) { - success = true; - break; - } - } - return success; - } - /** - * Verify that full backup is created on a single table with data correctly. Verify that history - * works as expected. - * @throws Exception if doing the backup or an operation on the tables fails + * Verify that backup history retrieval works as expected. */ @Test public void testBackupHistory() throws Exception { - LOG.info("test backup history on a single table with data"); - List tableList = Lists.newArrayList(table1); - String backupId = fullTableBackup(tableList); + // Test without backups present + List history = getBackupAdmin().getHistory(10); + assertEquals(0, history.size()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); + assertEquals(0, history.size()); + + // Create first backup + String backupId = fullTableBackup(Lists.newArrayList(table1)); assertTrue(checkSucceeded(backupId)); LOG.info("backup complete"); - List history = getBackupAdmin().getHistory(10); - assertTrue(findBackup(history, backupId)); + // Tests with one backup present + history = getBackupAdmin().getHistory(10); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); - assertTrue(findBackup(history, backupId)); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + String output = runHistoryCommand(10); + assertTrue(output.indexOf(backupId) > 0); + // Create second backup + String backupId2 = fullTableBackup(Lists.newArrayList(table2)); + assertTrue(checkSucceeded(backupId2)); + LOG.info("backup complete: " + table2); + + // Test with multiple backups + history = getBackupAdmin().getHistory(10); + assertEquals(2, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + assertEquals(backupId, history.get(1).getBackupId()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR)); + assertEquals(2, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + assertEquals(backupId, history.get(1).getBackupId()); + + output = runHistoryCommand(10); + int idx1 = output.indexOf(backupId); + int idx2 = output.indexOf(backupId2); + assertTrue(idx1 >= 0); // Backup 1 is listed + assertTrue(idx2 >= 0); // Backup 2 is listed + assertTrue(idx2 < idx1); // Newest backup (Backup 2) comes first + + // Test with multiple backups & n == 1 + history = getBackupAdmin().getHistory(1); + assertEquals(1, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + + history = BackupUtils.getHistory(conf1, 1, new Path(BACKUP_ROOT_DIR)); + assertEquals(1, history.size()); + assertEquals(backupId2, history.get(0).getBackupId()); + + output = runHistoryCommand(1); + idx1 = output.indexOf(backupId); + idx2 = output.indexOf(backupId2); + assertTrue(idx2 > 0); // most recent backup is listed + assertEquals(-1, idx1); // second most recent backup isn't listed + + // Test with multiple backups & filtering + BackupInfo.Filter tableNameFilter = i -> i.getTableNames().contains(table1); + + history = getBackupAdmin().getHistory(10, tableNameFilter); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + history = BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), tableNameFilter); + assertEquals(1, history.size()); + assertEquals(backupId, history.get(0).getBackupId()); + + } + + private String runHistoryCommand(int n) throws Exception { + String[] args = new String[] { "history", "-n", String.valueOf(n), "-p", BACKUP_ROOT_DIR }; ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.setOut(new PrintStream(baos)); - String[] args = new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR }; - // Run backup + LOG.info("Running history command"); int ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); + assertEquals(0, ret); + String output = baos.toString(); LOG.info(output); baos.close(); - assertTrue(output.indexOf(backupId) > 0); - - tableList = Lists.newArrayList(table2); - String backupId2 = fullTableBackup(tableList); - assertTrue(checkSucceeded(backupId2)); - LOG.info("backup complete: " + table2); - BackupInfo.Filter tableNameFilter = image -> { - if (table1 == null) { - return true; - } - - List names = image.getTableNames(); - return names.contains(table1); - }; - BackupInfo.Filter tableSetFilter = info -> { - String backupId1 = info.getBackupId(); - return backupId1.startsWith("backup"); - }; - - history = getBackupAdmin().getHistory(10, tableNameFilter, tableSetFilter); - assertTrue(history.size() > 0); - boolean success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - history = - BackupUtils.getHistory(conf1, 10, new Path(BACKUP_ROOT_DIR), tableNameFilter, tableSetFilter); - assertTrue(history.size() > 0); - success = true; - for (BackupInfo info : history) { - if (!info.getTableNames().contains(table1)) { - success = false; - break; - } - } - assertTrue(success); - - args = - new String[] { "history", "-n", "10", "-p", BACKUP_ROOT_DIR, "-t", "table1", "-s", "backup" }; - // Run backup - ret = ToolRunner.run(conf1, new BackupDriver(), args); - assertTrue(ret == 0); - LOG.info("show_history"); + return output; } } From de66f5e9f71a81330f6d34cc42eb2737ae6565cc Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Mon, 23 Feb 2026 13:56:57 -0500 Subject: [PATCH 66/78] BackupBoundaries global coverage can allow premature WAL deletion when backup roots have different host coverage (#237) Co-authored-by: Hernan Gelaf-Romer --- .../hbase/backup/master/BackupLogCleaner.java | 17 ++- .../hbase/backup/util/BackupBoundaries.java | 138 +++++++++++------- .../backup/master/TestBackupLogCleaner.java | 28 +++- 3 files changed, 121 insertions(+), 62 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index 03f006948721..e230efb1d00a 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -49,7 +49,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils; import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils; @@ -128,7 +127,8 @@ private BackupBoundaries serverToPreservationBoundaryTs(BackupSystemTable sysTab for (TableName table : backupInfo.getTableSetTimestampMap().keySet()) { for (Map.Entry entry : backupInfo.getTableSetTimestampMap().get(table) .entrySet()) { - builder.addBackupTimestamps(entry.getKey(), entry.getValue(), startCode); + builder.addBackupTimestamps(backupInfo.getBackupId(), entry.getKey(), entry.getValue(), + startCode); } } } @@ -136,10 +136,15 @@ private BackupBoundaries serverToPreservationBoundaryTs(BackupSystemTable sysTab BackupBoundaries boundaries = builder.build(); if (LOG.isDebugEnabled()) { - LOG.debug("Boundaries oldestStartCode: {}", boundaries.getOldestStartCode()); - for (Map.Entry entry : boundaries.getBoundaries().entrySet()) { - LOG.debug("Server: {}, WAL cleanup boundary: {}", entry.getKey().getHostName(), - entry.getValue()); + for (Map.Entry entry : boundaries.getBoundaries() + .entrySet()) { + String backupId = entry.getKey(); + LOG.debug("Backup: {}, Boundaries oldestStartCode: {}", backupId, + entry.getValue().getOldestStartCode()); + for (Map.Entry addressAndTs : entry.getValue().getBoundaries().entrySet()) { + LOG.debug("Backup: {}, Server: {}, WAL cleanup boundary: {}", backupId, + addressAndTs.getKey().getHostName(), addressAndTs.getValue()); + } } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java index b38c1bdb68d7..93878df83f94 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupBoundaries.java @@ -36,21 +36,20 @@ public class BackupBoundaries { private static final Logger LOG = LoggerFactory.getLogger(BackupBoundaries.class); private static final BackupBoundaries EMPTY_BOUNDARIES = - new BackupBoundaries(Collections.emptyMap(), Long.MAX_VALUE); - - // This map tracks, for every RegionServer, the least recent (= oldest / lowest timestamp) - // inclusion in any backup. In other words, it is the timestamp boundary up to which all backup - // roots have included the WAL in their backup. - private final Map boundaries; - - // The minimum WAL roll timestamp from the most recent backup of each backup root, used as a - // fallback cleanup boundary for RegionServers without explicit backup boundaries (e.g., servers - // that joined after backups began) - private final long oldestStartCode; - - private BackupBoundaries(Map boundaries, long oldestStartCode) { + new BackupBoundaries(Collections.emptyMap()); + + // Tracks WAL cleanup boundaries separately for each backup root to ensure WALs are only deleted + // when ALL backup roots no longer need them. The map key is the backup ID of the most recent + // backup from each backup root. Each BoundaryInfo contains: (1) a map of WAL timestamp + // boundaries per RegionServer (the oldest WAL timestamp included in that backup root's most + // recent backup), and (2) an oldestStartCode used as a fallback boundary for RegionServers not + // explicitly tracked (e.g., servers that joined after the backup began). A WAL file can only be + // deleted if it's older than the boundary for ALL backup roots, protecting WALs needed by any + // backup root even when other roots have already backed up that host at a later timestamp. + private final Map boundaries; + + private BackupBoundaries(Map boundaries) { this.boundaries = boundaries; - this.oldestStartCode = oldestStartCode; } public boolean isDeletable(Path walLogPath) { @@ -67,32 +66,19 @@ public boolean isDeletable(Path walLogPath) { Address address = Address.fromString(hostname); long pathTs = WAL.getTimestamp(walLogPath.getName()); - if (!boundaries.containsKey(address)) { - boolean isDeletable = pathTs <= oldestStartCode; - if (LOG.isDebugEnabled()) { - LOG.debug( - "Boundary for {} not found. isDeletable = {} based on oldestStartCode = {} and WAL ts of {}", - walLogPath, isDeletable, oldestStartCode, pathTs); + for (Map.Entry entry : boundaries.entrySet()) { + String backupId = entry.getKey(); + BoundaryInfo boundary = entry.getValue(); + DeleteStatus status = boundary.getStatus(address, pathTs); + if (!status.isDeletable()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Backup {} preventing deletion of {} with ts of {} due to {}", backupId, + walLogPath, pathTs, status); + } + return false; } - return isDeletable; } - - long backupTs = boundaries.get(address); - if (pathTs <= backupTs) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "WAL cleanup time-boundary found for server {}: {}. Ok to delete older file: {}", - address.getHostName(), pathTs, walLogPath); - } - return true; - } - - if (LOG.isDebugEnabled()) { - LOG.debug("WAL cleanup time-boundary found for server {}: {}. Keeping younger file: {}", - address.getHostName(), backupTs, walLogPath); - } - - return false; + return true; } catch (Exception e) { LOG.warn("Error occurred while filtering file: {}. Ignoring cleanup of this log", walLogPath, e); @@ -100,31 +86,56 @@ public boolean isDeletable(Path walLogPath) { } } - public Map getBoundaries() { + public Map getBoundaries() { return boundaries; } - public long getOldestStartCode() { - return oldestStartCode; - } - public static BackupBoundariesBuilder builder(long tsCleanupBuffer) { return new BackupBoundariesBuilder(tsCleanupBuffer); } public static class BackupBoundariesBuilder { - private final Map boundaries = new HashMap<>(); + private final Map boundaries = new HashMap<>(); private final long tsCleanupBuffer; - private long oldestStartCode = Long.MAX_VALUE; - private BackupBoundariesBuilder(long tsCleanupBuffer) { this.tsCleanupBuffer = tsCleanupBuffer; } - public BackupBoundariesBuilder addBackupTimestamps(String host, long hostLogRollTs, - long backupStartCode) { + public BackupBoundariesBuilder addBackupTimestamps(String backupId, String host, + long hostLogRollTs, long backupStartCode) { + BoundaryInfo boundary = boundaries.computeIfAbsent(backupId, ignore -> new BoundaryInfo()); Address address = Address.fromString(host); + boundary.add(address, hostLogRollTs, backupStartCode); + return this; + } + + public BackupBoundaries build() { + if (boundaries.isEmpty()) { + return EMPTY_BOUNDARIES; + } + + for (BoundaryInfo boundary : boundaries.values()) { + boundary.oldestStartCode -= tsCleanupBuffer; + } + + return new BackupBoundaries(boundaries); + } + } + + public static class BoundaryInfo { + private final Map boundaries = new HashMap<>(); + private long oldestStartCode = Long.MAX_VALUE; + + public Map getBoundaries() { + return boundaries; + } + + public long getOldestStartCode() { + return oldestStartCode; + } + + public void add(Address address, long hostLogRollTs, long backupStartCode) { Long storedTs = boundaries.get(address); if (storedTs == null || hostLogRollTs < storedTs) { boundaries.put(address, hostLogRollTs); @@ -133,17 +144,34 @@ public BackupBoundariesBuilder addBackupTimestamps(String host, long hostLogRoll if (oldestStartCode > backupStartCode) { oldestStartCode = backupStartCode; } - - return this; } - public BackupBoundaries build() { - if (boundaries.isEmpty()) { - return EMPTY_BOUNDARIES; + private DeleteStatus getStatus(Address address, long hostLogRollTs) { + Long storedTs = boundaries.get(address); + + if (storedTs == null) { + return hostLogRollTs <= oldestStartCode + ? DeleteStatus.OK + : DeleteStatus.NOT_DELETABLE_START_CODE; } - oldestStartCode -= tsCleanupBuffer; - return new BackupBoundaries(boundaries, oldestStartCode); + return hostLogRollTs <= storedTs ? DeleteStatus.OK : DeleteStatus.NOT_DELETABLE_BOUNDARY; + } + } + + private enum DeleteStatus { + OK(true), + NOT_DELETABLE_START_CODE(false), + NOT_DELETABLE_BOUNDARY(false); + + private final boolean deletable; + + DeleteStatus(boolean deletable) { + this.deletable = deletable; + } + + public boolean isDeletable() { + return deletable; } } } diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java index 57e067148f30..67744fcd41e7 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java @@ -306,7 +306,7 @@ public void testCanDeleteFileWithNewServerWALs() { Path oldWAL = new Path("/hbase/oldWALs/server1%2C60020%2C12345.500000"); String host = BackupUtils.parseHostNameFromLogFile(oldWAL); BackupBoundaries boundaries = BackupBoundaries.builder(0L) - .addBackupTimestamps(host, backupStartCode, backupStartCode).build(); + .addBackupTimestamps("backup1", host, backupStartCode, backupStartCode).build(); assertTrue("WAL older than backup should be deletable", BackupLogCleaner.canDeleteFile(boundaries, oldWAL)); @@ -322,6 +322,32 @@ public void testCanDeleteFileWithNewServerWALs() { BackupLogCleaner.canDeleteFile(boundaries, newServerWAL)); } + @Test + public void testBackupBoundariesProtectsWALsNeededByAnyRoot() { + long t1 = 1000L; + long t2 = 2000L; + + Path walBetweenBoundaries = new Path("/hbase/oldWALs/server1%2C60020%2C12345.1500"); + String host = BackupUtils.parseHostNameFromLogFile(walBetweenBoundaries); + + BackupBoundaries boundaries = + BackupBoundaries.builder(0L).addBackupTimestamps("backup-B1", host, t1, t1) + .addBackupTimestamps("backup-B2", host, t2, t2).build(); + + assertFalse( + "WAL at 1500 should NOT be deletable because backup B1 " + "(boundary=" + t1 + + ") still needs it, even though backup B2 (boundary=" + t2 + ") doesn't", + BackupLogCleaner.canDeleteFile(boundaries, walBetweenBoundaries)); + + Path walBeforeBoth = new Path("/hbase/oldWALs/server1%2C60020%2C12345.500"); + assertTrue("WAL at 500 should be deletable because it's before both boundaries", + BackupLogCleaner.canDeleteFile(boundaries, walBeforeBoth)); + + Path walAfterBoth = new Path("/hbase/oldWALs/server1%2C60020%2C12345.2500"); + assertFalse("WAL at 2500 should NOT be deletable because it's after both boundaries", + BackupLogCleaner.canDeleteFile(boundaries, walAfterBoth)); + } + @Test public void testCleansUpHMasterWal() { Path path = new Path("/hbase/MasterData/WALs/hmaster,60000,1718808578163"); From 29f83bbe37303429f6a3088ae1a4c01cfa9c3c4b Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Fri, 6 Mar 2026 16:38:00 -0500 Subject: [PATCH 67/78] Speed up backups HFile cleaner (#240) --- .../hbase/backup/BackupHFileCleaner.java | 48 +++++++++++-------- .../hbase/backup/impl/BackupSystemTable.java | 14 +++--- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java index bbbae2d631fe..51b60e5bb3f8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; @@ -52,12 +51,37 @@ public class BackupHFileCleaner extends BaseHFileCleanerDelegate implements Abor private boolean stopped = false; private boolean aborted = false; private Connection connection; - // timestamp of most recent completed cleaning run private volatile long previousCleaningCompletionTimestamp = 0; + private volatile Set bulkLoadedHFilesNeedingBackup = null; + + @Override + public void preClean() { + if (stopped) { + return; + } + try (BackupSystemTable tbl = new BackupSystemTable(connection)) { + Set tablesIncludedInBackups = fetchFullyBackedUpTables(tbl); + Set hfileFilenames = new HashSet<>(); + for (BulkLoad bulkLoad : tbl.readBulkloadRows(tablesIncludedInBackups)) { + String path = bulkLoad.getHfilePath(); + hfileFilenames.add(path.substring(path.lastIndexOf('/') + 1)); + } + bulkLoadedHFilesNeedingBackup = hfileFilenames; + LOG.debug("Cached {} unique HFile filenames registered as bulk loads.", + hfileFilenames.size()); + } catch (IOException ioe) { + LOG.error( + "Failed to read registered bulk load references from backup system table, " + + "marking all files as non-deletable.", + ioe); + bulkLoadedHFilesNeedingBackup = null; + } + } @Override public void postClean() { previousCleaningCompletionTimestamp = EnvironmentEdgeManager.currentTime(); + bulkLoadedHFilesNeedingBackup = null; } @Override @@ -66,34 +90,20 @@ public Iterable getDeletableFiles(Iterable files) { return Collections.emptyList(); } - // We use filenames because the HFile will have been moved to the archive since it - // was registered. - final Set hfileFilenames = new HashSet<>(); - try (BackupSystemTable tbl = new BackupSystemTable(connection)) { - Set tablesIncludedInBackups = fetchFullyBackedUpTables(tbl); - for (BulkLoad bulkLoad : tbl.readBulkloadRows(tablesIncludedInBackups)) { - hfileFilenames.add(new Path(bulkLoad.getHfilePath()).getName()); - } - LOG.debug("Found {} unique HFile filenames registered as bulk loads.", hfileFilenames.size()); - } catch (IOException ioe) { - LOG.error( - "Failed to read registered bulk load references from backup system table, marking all files as non-deletable.", - ioe); + final Set hfilesNeedingBackup = bulkLoadedHFilesNeedingBackup; + if (hfilesNeedingBackup == null) { return Collections.emptyList(); } - // Pin the threshold, we don't want the result to change depending on evaluation time. final long recentFileThreshold = previousCleaningCompletionTimestamp; return Iterables.filter(files, file -> { - // If the file is recent, be conservative and wait for one more scan of the bulk loads if (file.getModificationTime() > recentFileThreshold) { LOG.debug("Preventing deletion due to timestamp: {}", file.getPath().toString()); return false; } - // A file can be deleted if it is not registered as a backup bulk load. String hfile = file.getPath().getName(); - if (hfileFilenames.contains(hfile)) { + if (hfilesNeedingBackup.contains(hfile)) { LOG.debug("Preventing deletion due to bulk load registration in backup system table: {}", file.getPath().toString()); return false; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java index c89b93d32d9c..ccadca010562 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSystemTable.java @@ -408,12 +408,10 @@ private List processBulkLoadRowScan(Scan scan) throws IOException { TableName table = null; String fam = null; String path = null; - String region = null; - byte[] row = null; + byte[] row = CellUtil.cloneRow(res.current()); + String rowStr = Bytes.toString(row); + String region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr); for (Cell cell : res.listCells()) { - row = CellUtil.cloneRow(cell); - String rowStr = Bytes.toString(row); - region = BackupSystemTable.getRegionNameFromOrigBulkLoadRow(rowStr); if ( CellUtil.compareQualifiers(cell, BackupSystemTable.TBL_COL, 0, BackupSystemTable.TBL_COL.length) == 0 @@ -1521,12 +1519,12 @@ static Scan createScanForOrigBulkLoadedFiles(@Nullable TableName table) { static String getTableNameFromOrigBulkLoadRow(String rowStr) { // format is bulk : namespace : table : region : file - return Iterators.get(Splitter.onPattern(BLK_LD_DELIM).split(rowStr).iterator(), 1); + return Iterators.get(Splitter.on(BLK_LD_DELIM).split(rowStr).iterator(), 1); } static String getRegionNameFromOrigBulkLoadRow(String rowStr) { // format is bulk : namespace : table : region : file - List parts = Splitter.onPattern(BLK_LD_DELIM).splitToList(rowStr); + List parts = Splitter.on(BLK_LD_DELIM).splitToList(rowStr); Iterator i = parts.iterator(); int idx = 3; if (parts.size() == 4) { @@ -1534,7 +1532,7 @@ static String getRegionNameFromOrigBulkLoadRow(String rowStr) { idx = 2; } String region = Iterators.get(i, idx); - LOG.debug("bulk row string " + rowStr + " region " + region); + LOG.debug("bulk row string {} region {}", rowStr, region); return region; } From c66114ea91f6c5bf16b102dd9421247c275f9713 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Tue, 10 Mar 2026 07:30:28 -0700 Subject: [PATCH 68/78] (Not yet upstream) Reset to last successful cell on EOF with WAL Compression (#236) * Initial changes to allow partial cell read in WAL * Add tests * More tests * Add back comments * Remove redundant tests and simplify * Get tag value even if tag is out of bounds due to addition * Clean up a bit * Add explanatory comment * Initialize to empty * Resolve PR comment issue regarding LRU eviction for tags * Address PR review: two-phase rollback and remove unused dictType param Rollback in UndoableLRUDictionary previously restored nodes one at a time, doing remove/setContents/put on the content-based nodeToIndex HashMap. This could clobber entries when two nodes shared the same content during the restore (e.g., an evicted value re-added to a different slot). The fix restores all node state first, then rebuilds nodeToIndex from scratch. Also removes the unused dictType parameter from TagCompressionContext since every caller hardcodes LRUDictionary.class and we always need UndoableLRUDictionary for correctness. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: skhillon Co-authored-by: Claude Opus 4.6 --- .../hbase/io/TagCompressionContext.java | 24 +- .../io/encoding/BufferedDataBlockEncoder.java | 9 +- .../hadoop/hbase/io/util/LRUDictionary.java | 36 +- .../hbase/io/util/UndoableLRUDictionary.java | 137 +++++++ .../hbase/io/TestTagCompressionContext.java | 9 +- .../io/util/TestUndoableLRUDictionary.java | 359 ++++++++++++++++++ .../regionserver/wal/CompressionContext.java | 2 +- .../wal/ProtobufWALTailingReader.java | 111 ++++-- .../hbase/regionserver/wal/WALCellCodec.java | 68 +++- ...ompressedKvDecoderDeferredDictUpdates.java | 247 ++++++++++++ ...TestWALTailingReaderPartialCellResume.java | 270 +++++++++++++ 11 files changed, 1210 insertions(+), 62 deletions(-) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressedKvDecoderDeferredDictUpdates.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALTailingReaderPartialCellResume.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java index f938fdaab35b..09f8c23a8e51 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java @@ -20,12 +20,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.io.util.Dictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.io.util.UndoableLRUDictionary; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -38,13 +37,10 @@ */ @InterfaceAudience.Private public class TagCompressionContext { - private final Dictionary tagDict; + private final UndoableLRUDictionary tagDict; - public TagCompressionContext(Class dictType, int dictCapacity) - throws SecurityException, NoSuchMethodException, InstantiationException, IllegalAccessException, - InvocationTargetException { - Constructor dictConstructor = dictType.getConstructor(); - tagDict = dictConstructor.newInstance(); + public TagCompressionContext(int dictCapacity) { + tagDict = new UndoableLRUDictionary(); tagDict.init(dictCapacity); } @@ -52,6 +48,18 @@ public void clear() { tagDict.clear(); } + public void checkpoint() { + tagDict.checkpoint(); + } + + public void commit() { + tagDict.commit(); + } + + public void rollback() { + tagDict.rollback(); + } + /** * Compress tags one by one and writes to the OutputStream. * @param out Stream to which the compressed tags to be written diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index 0f15151fe88b..2b11f24578e0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.io.TagCompressionContext; -import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -72,8 +71,7 @@ public ByteBuffer decodeKeyValues(DataInputStream source, decodingCtx.getTagCompressionContext().clear(); } else { try { - TagCompressionContext tagCompressionContext = - new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); @@ -815,7 +813,7 @@ public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) { super(decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { - tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } @@ -1232,8 +1230,7 @@ public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOut encodingCtx.getTagCompressionContext().clear(); } else { try { - TagCompressionContext tagCompressionContext = - new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext tagCompressionContext = new TagCompressionContext(Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java index 4089863d4387..41eb7297294a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/LRUDictionary.java @@ -62,7 +62,7 @@ public short addEntry(byte[] data, int offset, int length) { return addEntryInternal(data, offset, length, true); } - private short addEntryInternal(byte[] data, int offset, int length, boolean copy) { + short addEntryInternal(byte[] data, int offset, int length, boolean copy) { if (length <= 0) return NOT_IN_DICTIONARY; return backingStore.put(data, offset, length, copy); } @@ -77,22 +77,22 @@ public void clear() { * thread safe. Don't use in multi-threaded applications. */ static class BidirectionalLRUMap { - private int currSize = 0; + int currSize = 0; // Head and tail of the LRU list. - private Node head; - private Node tail; + Node head; + Node tail; - private HashMap nodeToIndex = new HashMap<>(); - private Node[] indexToNode; - private int initSize = 0; + HashMap nodeToIndex = new HashMap<>(); + Node[] indexToNode; + int initSize = 0; public BidirectionalLRUMap(int initialSize) { initSize = initialSize; indexToNode = new Node[initialSize]; } - private short put(byte[] array, int offset, int length, boolean copy) { + short put(byte[] array, int offset, int length, boolean copy) { if (copy) { // We copy the bytes we want, otherwise we might be holding references to // massive arrays in our dictionary (or those arrays might change) @@ -104,7 +104,7 @@ private short put(byte[] array, int offset, int length, boolean copy) { } } - private short putInternal(byte[] stored) { + short putInternal(byte[] stored) { if (currSize < initSize) { // There is space to add without evicting. if (indexToNode[currSize] == null) { @@ -125,7 +125,7 @@ private short putInternal(byte[] stored) { } } - private short findIdx(byte[] array, int offset, int length) { + short findIdx(byte[] array, int offset, int length) { Short s; final Node comparisonNode = new ByteArrayBackedNode(); comparisonNode.setContents(array, offset, length); @@ -137,7 +137,7 @@ private short findIdx(byte[] array, int offset, int length) { } } - private short findIdx(ByteBuffer buf, int offset, int length) { + short findIdx(ByteBuffer buf, int offset, int length) { Short s; final ByteBufferBackedNode comparisonNode = new ByteBufferBackedNode(); comparisonNode.setContents(buf, offset, length); @@ -149,13 +149,13 @@ private short findIdx(ByteBuffer buf, int offset, int length) { } } - private byte[] get(short idx) { + byte[] get(short idx) { Preconditions.checkElementIndex(idx, currSize); moveToHead(indexToNode[idx]); return indexToNode[idx].getContents(); } - private void moveToHead(Node n) { + void moveToHead(Node n) { if (head == n) { // no-op -- it's already the head. return; @@ -176,7 +176,7 @@ private void moveToHead(Node n) { setHead(n); } - private void setHead(Node n) { + void setHead(Node n) { // assume it's already unlinked from the list at this point. n.prev = null; n.next = head; @@ -193,7 +193,7 @@ private void setHead(Node n) { } } - private void clear() { + void clear() { for (int i = 0; i < currSize; i++) { indexToNode[i].next = null; indexToNode[i].prev = null; @@ -205,7 +205,7 @@ private void clear() { head = null; } - private static abstract class Node { + static abstract class Node { int offset; int length; Node next; // link towards the tail @@ -219,7 +219,7 @@ private static abstract class Node { } // The actual contents of the LRUDictionary are of ByteArrayBackedNode type - private static class ByteArrayBackedNode extends Node { + static class ByteArrayBackedNode extends Node { private byte[] container; @Override @@ -259,7 +259,7 @@ public boolean equals(Object other) { // Currently only used for finding the index and hence this node acts // as a temporary holder to look up in the indexToNode map // which is formed by ByteArrayBackedNode - private static class ByteBufferBackedNode extends Node { + static class ByteBufferBackedNode extends Node { private ByteBuffer container; public ByteBufferBackedNode() { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java new file mode 100644 index 000000000000..bc1428ded80a --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/UndoableLRUDictionary.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import java.util.IdentityHashMap; +import java.util.Map; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * An LRUDictionary that supports checkpoint and rollback. Used for tag dictionary compression in + * WAL decoding, where dictionary updates within a single cell must be rolled back if the cell read + * fails (e.g., due to EOF on a WAL being tailed). + *

+ * On {@link #checkpoint()}, saves the current LRU state. Operations proceed normally against the + * real dictionary. On {@link #rollback()}, restores the dictionary to the checkpointed state. On + * {@link #commit()}, discards the saved state. + */ +@InterfaceAudience.Private +public class UndoableLRUDictionary extends LRUDictionary { + + private boolean tracking = false; + private int savedCurrSize; + private BidirectionalLRUMap.Node savedHead; + private BidirectionalLRUMap.Node savedTail; + private final Map snapshots = new IdentityHashMap<>(); + + private static class NodeSnapshot { + final BidirectionalLRUMap.Node savedPrev; + final BidirectionalLRUMap.Node savedNext; + final byte[] savedContents; + final int savedOffset; + final int savedLength; + + NodeSnapshot(BidirectionalLRUMap.Node node) { + this.savedPrev = node.prev; + this.savedNext = node.next; + this.savedContents = node.getContents(); + this.savedOffset = node.offset; + this.savedLength = node.length; + } + } + + public void checkpoint() { + tracking = true; + savedCurrSize = backingStore.currSize; + savedHead = backingStore.head; + savedTail = backingStore.tail; + snapshots.clear(); + } + + public void commit() { + snapshots.clear(); + tracking = false; + } + + public void rollback() { + if (!tracking) { + return; + } + for (Map.Entry entry : snapshots.entrySet()) { + BidirectionalLRUMap.Node node = entry.getKey(); + NodeSnapshot snap = entry.getValue(); + node.prev = snap.savedPrev; + node.next = snap.savedNext; + if (snap.savedContents != null) { + node.setContents(snap.savedContents, snap.savedOffset, snap.savedLength); + } + } + backingStore.head = savedHead; + backingStore.tail = savedTail; + backingStore.currSize = savedCurrSize; + backingStore.nodeToIndex.clear(); + for (short i = 0; i < savedCurrSize; i++) { + backingStore.nodeToIndex.put(backingStore.indexToNode[i], i); + } + snapshots.clear(); + tracking = false; + } + + private void saveIfNeeded(BidirectionalLRUMap.Node node) { + if (tracking && node != null && !snapshots.containsKey(node)) { + snapshots.put(node, new NodeSnapshot(node)); + } + } + + @Override + public byte[] getEntry(short idx) { + if (tracking) { + BidirectionalLRUMap.Node node = backingStore.indexToNode[idx]; + saveIfNeeded(node); + if (node.prev != null) { + saveIfNeeded(node.prev); + } + if (node.next != null) { + saveIfNeeded(node.next); + } + saveIfNeeded(backingStore.head); + } + return backingStore.get(idx); + } + + @Override + public short addEntry(byte[] data, int offset, int length) { + if (tracking) { + if (backingStore.currSize < backingStore.initSize) { + BidirectionalLRUMap.Node node = backingStore.indexToNode[backingStore.currSize]; + if (node != null) { + saveIfNeeded(node); + } + saveIfNeeded(backingStore.head); + } else { + BidirectionalLRUMap.Node tail = backingStore.tail; + saveIfNeeded(tail); + if (tail != null && tail.prev != null) { + saveIfNeeded(tail.prev); + } + saveIfNeeded(backingStore.head); + } + } + return addEntryInternal(data, offset, length, true); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java index 4c80d0de0413..447f7af82e75 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.nio.SingleByteBuff; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -56,7 +55,7 @@ public class TestTagCompressionContext { @Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); @@ -83,7 +82,7 @@ public void testCompressUncompressTags1() throws Exception { public void testCompressUncompressTagsWithOffheapKeyValue1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell) createOffheapKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); @@ -107,7 +106,7 @@ public void testCompressUncompressTagsWithOffheapKeyValue1() throws Exception { @Test public void testCompressUncompressTags2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(baos, kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1); @@ -132,7 +131,7 @@ public void testCompressUncompressTags2() throws Exception { public void testCompressUncompressTagsWithOffheapKeyValue2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); - TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); + TagCompressionContext context = new TagCompressionContext(Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell) createOffheapKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java new file mode 100644 index 000000000000..4f6bef827a6b --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestUndoableLRUDictionary.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MiscTests.class, SmallTests.class }) +public class TestUndoableLRUDictionary { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestUndoableLRUDictionary.class); + + private UndoableLRUDictionary dict; + private static final int CAPACITY = 4; + + @Before + public void setUp() { + dict = new UndoableLRUDictionary(); + dict.init(CAPACITY); + } + + private static byte[] entry(String s) { + return Bytes.toBytes(s); + } + + private short add(String s) { + byte[] data = entry(s); + return dict.addEntry(data, 0, data.length); + } + + private void assertNoEntry(short idx) { + assertThrows(IndexOutOfBoundsException.class, () -> dict.getEntry(idx)); + } + + @Test + public void itPreservesAdditionsOnCommit() { + add("a"); + add("b"); + + dict.checkpoint(); + add("c"); + dict.commit(); + + assertArrayEquals(entry("a"), dict.getEntry((short) 0)); + assertArrayEquals(entry("b"), dict.getEntry((short) 1)); + assertArrayEquals(entry("c"), dict.getEntry((short) 2)); + } + + @Test + public void itRevertsAdditionsOnRollback() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertNoEntry((short) 2); + } + + @Test + public void itIsNoOpWhenRollingBackWithoutCheckpoint() { + short idxA = add("a"); + short idxB = add("b"); + + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itRestoresEvictedEntryOnRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAddGetAddPatternAtCapacityThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + + short idxE = add("e"); + byte[] gotC = dict.getEntry(idxC); + assertArrayEquals(entry("c"), gotC); + short idxF = add("f"); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + assertArrayEquals(entry("f"), dict.getEntry(idxF)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAddGetAddPatternAtCapacityThenCommit() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + dict.getEntry(idxC); + short idxF = add("f"); + dict.commit(); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + assertArrayEquals(entry("f"), dict.getEntry(idxF)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itRestoresLruOrderAfterGetEntryOnRollback() { + short idxA = add("a"); + add("b"); + add("c"); + + dict.checkpoint(); + dict.getEntry(idxA); + add("d"); + dict.rollback(); + + assertNoEntry((short) 3); + add("d"); + short idxE = add("e"); + assertEquals(idxA, idxE); + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + } + + @Test + public void itSupportsMultipleCheckpointRollbackCycles() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.rollback(); + + assertNoEntry((short) 2); + + dict.checkpoint(); + add("d"); + dict.rollback(); + + assertNoEntry((short) 2); + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itSupportsMultipleCheckpointCommitCycles() { + dict.checkpoint(); + short idxA = add("a"); + dict.commit(); + + dict.checkpoint(); + short idxB = add("b"); + dict.commit(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itPreservesEvictionsOnCommit() { + short idxA = add("a"); + add("b"); + add("c"); + add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + dict.commit(); + + assertArrayEquals(entry("e"), dict.getEntry(idxE)); + } + + @Test + public void itHandlesMultipleEvictionsDuringTrackingThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + add("e"); + add("f"); + add("g"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesEvictionThenGetEvictedIndexThenRollback() { + short idxA = add("a"); + add("b"); + add("c"); + add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + byte[] gotE = dict.getEntry(idxE); + assertArrayEquals(entry("e"), gotE); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + } + + @Test + public void itHandlesRollbackAfterAddBelowCapacity() { + dict.checkpoint(); + add("a"); + add("b"); + dict.rollback(); + + assertNoEntry((short) 0); + } + + @Test + public void itHandlesCommitThenRollbackSequence() { + short idxA = add("a"); + short idxB = add("b"); + + dict.checkpoint(); + add("c"); + dict.commit(); + + assertArrayEquals(entry("c"), dict.getEntry((short) 2)); + + dict.checkpoint(); + add("d"); + dict.rollback(); + + assertNoEntry((short) 3); + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + } + + @Test + public void itHandlesGetEntryForEverySlotDuringTrackingThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + dict.getEntry(idxA); + dict.getEntry(idxB); + dict.getEntry(idxC); + dict.getEntry(idxD); + add("e"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesAtCapacityEvictAllThenRollback() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + add("e"); + add("f"); + add("g"); + add("h"); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } + + @Test + public void itHandlesRollbackWhenEvictedValueReaddedToAnotherSlot() { + short idxA = add("a"); + short idxB = add("b"); + short idxC = add("c"); + short idxD = add("d"); + + dict.checkpoint(); + short idxE = add("e"); + assertEquals(idxA, idxE); + short idxA2 = add("a"); + assertEquals(idxB, idxA2); + dict.rollback(); + + assertArrayEquals(entry("a"), dict.getEntry(idxA)); + assertArrayEquals(entry("b"), dict.getEntry(idxB)); + assertArrayEquals(entry("c"), dict.getEntry(idxC)); + assertArrayEquals(entry("d"), dict.getEntry(idxD)); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java index 0c5d6047ceec..7856fcb0794b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java @@ -223,7 +223,7 @@ public CompressionContext(Class dictType, boolean recovere getDictionary(DictionaryIndex.QUALIFIER).init(Byte.MAX_VALUE); if (hasTagCompression) { - tagCompressionContext = new TagCompressionContext(dictType, Short.MAX_VALUE); + tagCompressionContext = new TagCompressionContext(Short.MAX_VALUE); } if (hasValueCompression && valueCompressionType != null) { valueCompressor = new ValueCompressor(valueCompressionType); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java index 6cf141d7053e..a3184b6685ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java @@ -30,11 +30,9 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; import org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream; import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; - import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; @@ -50,6 +48,10 @@ public class ProtobufWALTailingReader extends AbstractProtobufWALReader private DelegatingInputStream delegatingInput; + private Entry pendingEntry = null; + private int pendingRemainingCells = 0; + private long pendingResumePosition = -1; + private static final class ReadWALKeyResult { final State state; final Entry entry; @@ -184,6 +186,10 @@ private Result editError() { } private Result readWALEdit(Entry entry, int followingKvCount) { + return readCellsIntoEntry(entry, followingKvCount, false); + } + + private Result readCellsIntoEntry(Entry entry, int remainingCells, boolean isResume) { long posBefore; try { posBefore = inputStream.getPos(); @@ -191,30 +197,44 @@ private Result readWALEdit(Entry entry, int followingKvCount) { LOG.warn("failed to get position", e); return State.ERROR_AND_RESET.getResult(); } - if (followingKvCount == 0) { - LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset={}", - posBefore); + if (remainingCells == 0) { + if (!isResume) { + LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset={}", + posBefore); + } return new Result(State.NORMAL, entry, posBefore); } - int actualCells; - try { - actualCells = entry.getEdit().readFromCells(cellDecoder, followingKvCount); - } catch (Exception e) { - String message = " while reading " + followingKvCount + " WAL KVs; started reading at " - + posBefore + " and read up to " + getPositionQuietly(); - IOException realEofEx = extractHiddenEof(e); - if (realEofEx != null) { - LOG.warn("EOF " + message, realEofEx); - return editEof(); - } else { - LOG.warn("Error " + message, e); + long lastGoodPos = posBefore; + int cellsRead = 0; + for (int i = 0; i < remainingCells; i++) { + try { + lastGoodPos = inputStream.getPos(); + } catch (IOException e) { + LOG.warn("failed to get position before cell read", e); return editError(); } - } - if (actualCells != followingKvCount) { - LOG.warn("Only read {} cells, expected {}; started reading at {} and read up to {}", - actualCells, followingKvCount, posBefore, getPositionQuietly()); - return editEof(); + boolean advanced; + try { + advanced = cellDecoder.advance(); + } catch (Exception e) { + IOException realEofEx = extractHiddenEof(e); + if (realEofEx != null) { + LOG.debug("EOF after reading {} of {} cells; started reading at {}, last good pos={}", + cellsRead, remainingCells, posBefore, lastGoodPos, realEofEx); + return savePendingAndReturnEof(entry, remainingCells - cellsRead, lastGoodPos); + } else { + LOG.warn("Error after reading {} of {} cells; started reading at {}, read up to {}", + cellsRead, remainingCells, posBefore, getPositionQuietly(), e); + return editError(); + } + } + if (!advanced) { + LOG.debug("EOF (advance returned false) after reading {} of {} cells; started at {}," + + " last good pos={}", cellsRead, remainingCells, posBefore, lastGoodPos); + return savePendingAndReturnEof(entry, remainingCells - cellsRead, lastGoodPos); + } + entry.getEdit().add(cellDecoder.current()); + cellsRead++; } long posAfter; try { @@ -231,8 +251,45 @@ private Result readWALEdit(Entry entry, int followingKvCount) { return new Result(State.NORMAL, entry, posAfter); } + private Result savePendingAndReturnEof(Entry entry, int remaining, long resumePos) { + if (hasCompression) { + pendingEntry = entry; + pendingRemainingCells = remaining; + pendingResumePosition = resumePos; + return new Result(State.EOF_AND_RESET, null, resumePos); + } + return editEof(); + } + + private void clearPendingState() { + pendingEntry = null; + pendingRemainingCells = 0; + pendingResumePosition = -1; + } + @Override public Result next(long limit) { + if (pendingEntry != null) { + long originalPosition; + try { + originalPosition = inputStream.getPos(); + } catch (IOException e) { + LOG.warn("failed to get position", e); + clearPendingState(); + return State.EOF_AND_RESET.getResult(); + } + if (limit < 0) { + delegatingInput.setDelegate(inputStream); + } else if (limit <= originalPosition) { + return State.EOF_AND_RESET.getResult(); + } else { + delegatingInput.setDelegate(ByteStreams.limit(inputStream, limit - originalPosition)); + } + Entry entry = pendingEntry; + int remaining = pendingRemainingCells; + clearPendingState(); + return readCellsIntoEntry(entry, remaining, true); + } long originalPosition; try { originalPosition = inputStream.getPos(); @@ -268,6 +325,13 @@ private void skipHeader(FSDataInputStream stream) throws IOException { @Override public void resetTo(long position, boolean resetCompression) throws IOException { + if (resetCompression) { + clearPendingState(); + } + long seekPosition = position; + if (!resetCompression && pendingResumePosition > 0) { + seekPosition = pendingResumePosition; + } close(); Pair pair = open(); boolean resetSucceed = false; @@ -283,6 +347,7 @@ public void resetTo(long position, boolean resetCompression) throws IOException if (compressionCtx != null) { compressionCtx.clear(); } + clearPendingState(); skipHeader(inputStream); } else if (resetCompression && compressionCtx != null) { // clear compressCtx and skip to the expected position, to fill up the dictionary @@ -293,7 +358,7 @@ public void resetTo(long position, boolean resetCompression) throws IOException } } else { // just seek to the expected position - inputStream.seek(position); + inputStream.seek(seekPosition); } resetSucceed = true; } finally { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java index 84709cbc58dd..8c34fbfe5022 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; @@ -279,6 +281,32 @@ static class CompressedKvDecoder extends BaseDecoder { private final boolean hasValueCompression; private final boolean hasTagCompression; + // When the WAL tailing reader hits EOF mid-cell, the compression dictionaries must remain + // in the state they were after the last fully-read cell. Otherwise the reader would need + // an expensive O(n) reset (re-read from the start of the file to rebuild dictionary state). + // To achieve this, dictionary additions for ROW, FAMILY, and QUALIFIER are buffered here + // and only flushed on successful cell parse. On failure, they are discarded. + // Tag dictionary uses checkpoint/rollback via TagCompressionContext instead. + private final List pendingDictAdditions = new ArrayList<>(); + + // Tracks whether we are in the value decompression phase of parseCellInner(), so that on + // IOException we know whether the ValueCompressor's internal state needs to be reset. + private boolean readingValue = false; + + private static class PendingDictAddition { + final Dictionary dict; + final byte[] data; + final int offset; + final int length; + + PendingDictAddition(Dictionary dict, byte[] data, int offset, int length) { + this.dict = dict; + this.data = data; + this.offset = offset; + this.length = length; + } + } + public CompressedKvDecoder(InputStream in, CompressionContext compression) { super(in); this.compression = compression; @@ -286,8 +314,44 @@ public CompressedKvDecoder(InputStream in, CompressionContext compression) { this.hasTagCompression = compression.hasTagCompression(); } + private void commitPendingAdditions() { + for (PendingDictAddition pending : pendingDictAdditions) { + pending.dict.addEntry(pending.data, pending.offset, pending.length); + } + pendingDictAdditions.clear(); + if (hasTagCompression) { + compression.tagCompressionContext.commit(); + } + } + + private void clearPendingAdditions() { + pendingDictAdditions.clear(); + if (hasTagCompression) { + compression.tagCompressionContext.rollback(); + } + } + @Override protected Cell parseCell() throws IOException { + clearPendingAdditions(); + if (hasTagCompression) { + compression.tagCompressionContext.checkpoint(); + } + readingValue = false; + try { + Cell cell = parseCellInner(); + commitPendingAdditions(); + return cell; + } catch (IOException e) { + clearPendingAdditions(); + if (readingValue && hasValueCompression) { + compression.getValueCompressor().clear(); + } + throw e; + } + } + + private Cell parseCellInner() throws IOException { int keylength = StreamUtils.readRawVarint32(in); int vlength = StreamUtils.readRawVarint32(in); int tagsLength = StreamUtils.readRawVarint32(in); @@ -333,7 +397,9 @@ protected Cell parseCell() throws IOException { pos = Bytes.putByte(backingArray, pos, (byte) in.read()); int valLen = typeValLen - 1; if (hasValueCompression) { + readingValue = true; readCompressedValue(in, backingArray, pos, valLen); + readingValue = false; pos += valLen; } else { IOUtils.readFully(in, backingArray, pos, valLen); @@ -358,7 +424,7 @@ private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOExcep // if this isn't in the dictionary, we need to add to the dictionary. int length = StreamUtils.readRawVarint32(in); IOUtils.readFully(in, to, offset, length); - dict.addEntry(to, offset, length); + pendingDictAdditions.add(new PendingDictAddition(dict, to, offset, length)); return length; } else { // the status byte also acts as the higher order byte of the dictionary entry. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressedKvDecoderDeferredDictUpdates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressedKvDecoderDeferredDictUpdates.java new file mode 100644 index 000000000000..39361d035ff2 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCompressedKvDecoderDeferredDictUpdates.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ArrayBackedTag; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.codec.Codec; +import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.util.Dictionary; +import org.apache.hadoop.hbase.io.util.LRUDictionary; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, SmallTests.class }) +public class TestCompressedKvDecoderDeferredDictUpdates { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCompressedKvDecoderDeferredDictUpdates.class); + + private static final byte[] FAMILY = Bytes.toBytes("cf"); + + private static final Set CELL_DICTIONARIES = + EnumSet.of(CompressionContext.DictionaryIndex.ROW, CompressionContext.DictionaryIndex.FAMILY, + CompressionContext.DictionaryIndex.QUALIFIER); + + private static byte[] safeGetEntry(Dictionary dict, short idx) { + try { + return dict.getEntry(idx); + } catch (IndexOutOfBoundsException e) { + return null; + } + } + + private KeyValue createKV(String row, String qualifier, String value, int numTags) { + List tags = new ArrayList<>(numTags); + for (int i = 1; i <= numTags; i++) { + tags.add(new ArrayBackedTag((byte) i, Bytes.toBytes("tag" + row + "-" + i))); + } + return new KeyValue(Bytes.toBytes(row), FAMILY, Bytes.toBytes(qualifier), + HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value), tags); + } + + private byte[] encodeCells(List cells, CompressionContext ctx, Configuration conf) + throws IOException { + WALCellCodec codec = new WALCellCodec(conf, ctx); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + Codec.Encoder encoder = codec.getEncoder(bos); + for (KeyValue kv : cells) { + encoder.write(kv); + } + encoder.flush(); + return bos.toByteArray(); + } + + private int readUntilEof(Codec.Decoder decoder) throws IOException { + int count = 0; + boolean hitEof = false; + while (!hitEof) { + try { + if (!decoder.advance()) { + hitEof = true; + } else { + count++; + } + } catch (Exception e) { + hitEof = true; + } + } + return count; + } + + private void assertDictionariesMatch(CompressionContext actual, CompressionContext expected, + int truncLen, int successfulCells) { + for (CompressionContext.DictionaryIndex idx : CELL_DICTIONARIES) { + Dictionary actualDict = actual.getDictionary(idx); + Dictionary expectedDict = expected.getDictionary(idx); + for (short s = 0; s < Short.MAX_VALUE; s++) { + byte[] actualEntry = safeGetEntry(actualDict, s); + byte[] expectedEntry = safeGetEntry(expectedDict, s); + if (actualEntry == null && expectedEntry == null) { + break; + } + assertArrayEquals( + String.format("Dictionary %s entry %d mismatch at truncLen=%d, successfulCells=%d", idx, + s, truncLen, successfulCells), + expectedEntry, actualEntry); + } + } + } + + private void verifyDictsMatchAfterTruncation(List cells, boolean hasTagCompression, + boolean hasValueCompression, Compression.Algorithm valueAlgo) throws Exception { + Configuration conf = new Configuration(false); + if (hasTagCompression) { + conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true); + } + CompressionContext writeCtx = + new CompressionContext(LRUDictionary.class, false, hasTagCompression, hasValueCompression, + hasValueCompression ? valueAlgo : Compression.Algorithm.NONE); + byte[] fullData = encodeCells(cells, writeCtx, conf); + + for (int truncLen = 1; truncLen < fullData.length; truncLen++) { + byte[] truncated = Arrays.copyOf(fullData, truncLen); + CompressionContext readCtx = + new CompressionContext(LRUDictionary.class, false, hasTagCompression, hasValueCompression, + hasValueCompression ? valueAlgo : Compression.Algorithm.NONE); + Codec.Decoder decoder = + new WALCellCodec(conf, readCtx).getDecoder(new ByteArrayInputStream(truncated)); + int successfulCells = readUntilEof(decoder); + + CompressionContext verifyCtx = + new CompressionContext(LRUDictionary.class, false, hasTagCompression, hasValueCompression, + hasValueCompression ? valueAlgo : Compression.Algorithm.NONE); + Codec.Decoder verifyDecoder = + new WALCellCodec(conf, verifyCtx).getDecoder(new ByteArrayInputStream(fullData)); + for (int i = 0; i < successfulCells; i++) { + assertTrue("verifyDecoder.advance() should return true for cell " + i, + verifyDecoder.advance()); + } + + assertDictionariesMatch(readCtx, verifyCtx, truncLen, successfulCells); + } + } + + @Test + public void itPreservesDictionaryStateOnTruncatedStream() throws Exception { + List cells = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + cells.add(createKV("row-" + i, "qual-" + i, "value-" + i, 0)); + } + verifyDictsMatchAfterTruncation(cells, false, false, null); + } + + @Test + public void itPreservesDictionaryStateWithTagCompression() throws Exception { + List cells = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + cells.add(createKV("row-" + i, "qual-" + i, "value-" + i, 2)); + } + verifyDictsMatchAfterTruncation(cells, true, false, null); + } + + @Test + public void itPreservesDictionaryStateWithValueCompression() throws Exception { + List cells = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + byte[] value = new byte[64]; + Bytes.random(value); + cells.add(createKV("row-" + i, "qual-" + i, Bytes.toString(value), 0)); + } + verifyDictsMatchAfterTruncation(cells, false, true, Compression.Algorithm.GZ); + } + + @Test + public void itCanResumeAfterTruncation() throws Exception { + List cells = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + cells.add(createKV("row-" + i, "qual-" + i, "value-" + i, 0)); + } + + Configuration conf = new Configuration(false); + CompressionContext writeCtx = new CompressionContext(LRUDictionary.class, false, false); + byte[] fullData = encodeCells(cells, writeCtx, conf); + + int[] cellEndOffsets = new int[cells.size()]; + { + CompressionContext scanCtx = new CompressionContext(LRUDictionary.class, false, false); + WALCellCodec scanCodec = new WALCellCodec(conf, scanCtx); + for (int i = 0; i < cells.size(); i++) { + ByteArrayOutputStream cellBos = new ByteArrayOutputStream(); + Codec.Encoder cellEncoder = scanCodec.getEncoder(cellBos); + cellEncoder.write(cells.get(i)); + cellEncoder.flush(); + cellEndOffsets[i] = (i == 0) ? cellBos.size() : cellEndOffsets[i - 1] + cellBos.size(); + } + } + + for (int cellIdx = 0; cellIdx < cells.size() - 1; cellIdx++) { + int truncPoint = cellEndOffsets[cellIdx] + 1; + if (truncPoint >= fullData.length) { + continue; + } + byte[] truncated = Arrays.copyOf(fullData, truncPoint); + + CompressionContext readCtx = new CompressionContext(LRUDictionary.class, false, false); + WALCellCodec readCodec = new WALCellCodec(conf, readCtx); + Codec.Decoder decoder = readCodec.getDecoder(new ByteArrayInputStream(truncated)); + int successfulCells = readUntilEof(decoder); + assertEquals("successfulCells at cellIdx=" + cellIdx, cellIdx + 1, successfulCells); + + int resumeOffset = cellEndOffsets[cellIdx]; + Codec.Decoder resumeDecoder = readCodec.getDecoder( + new ByteArrayInputStream(fullData, resumeOffset, fullData.length - resumeOffset)); + + CompressionContext verifyCtx = new CompressionContext(LRUDictionary.class, false, false); + Codec.Decoder verifyDecoder = + new WALCellCodec(conf, verifyCtx).getDecoder(new ByteArrayInputStream(fullData)); + for (int i = 0; i < successfulCells; i++) { + assertTrue(verifyDecoder.advance()); + } + + for (int i = successfulCells; i < cells.size(); i++) { + assertTrue("resume should advance for cell " + i, resumeDecoder.advance()); + assertTrue("verify should advance for cell " + i, verifyDecoder.advance()); + assertArrayEquals(String.format("cell %d content mismatch at cellIdx=%d", i, cellIdx), + ((KeyValue) verifyDecoder.current()).getBuffer(), + ((KeyValue) resumeDecoder.current()).getBuffer()); + } + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALTailingReaderPartialCellResume.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALTailingReaderPartialCellResume.java new file mode 100644 index 000000000000..39681ed3fbef --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALTailingReaderPartialCellResume.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.wal; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderFactory; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ RegionServerTests.class, MediumTests.class }) +public class TestWALTailingReaderPartialCellResume { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestWALTailingReaderPartialCellResume.class); + + private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility(); + + private static FileSystem FS; + + private static final TableName TN = TableName.valueOf("test"); + private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TN).build(); + private static final byte[] FAMILY = Bytes.toBytes("family"); + + @BeforeClass + public static void setUp() throws IOException { + UTIL.getConfiguration().setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); + UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + FS = FileSystem.getLocal(UTIL.getConfiguration()); + if (!FS.mkdirs(UTIL.getDataTestDir())) { + throw new IOException("can not create " + UTIL.getDataTestDir()); + } + } + + @AfterClass + public static void tearDown() { + UTIL.cleanupTestDir(); + } + + private WAL.Entry createEntry(int index, int numCells) { + WALKeyImpl key = new WALKeyImpl(RI.getEncodedNameAsBytes(), TN, index, + EnvironmentEdgeManager.currentTime(), HConstants.DEFAULT_CLUSTER_ID); + WALEdit edit = new WALEdit(); + for (int c = 0; c < numCells; c++) { + edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put) + .setRow(Bytes.toBytes("row-" + index)).setFamily(FAMILY) + .setQualifier(Bytes.toBytes("qual-" + index + "-" + c)) + .setValue(Bytes.toBytes("value-" + index + "-" + c)).build()); + } + return new WAL.Entry(key, edit); + } + + @Test + public void itReturnsEofAndResetNotCompressionOnPartialEntry() throws Exception { + Path walFile = UTIL.getDataTestDir("wal-partial"); + List endOffsets = new ArrayList<>(); + try (WALProvider.Writer writer = + WALFactory.createWALWriter(FS, walFile, UTIL.getConfiguration())) { + for (int i = 0; i < 5; i++) { + writer.append(createEntry(i, 1)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + writer.append(createEntry(5, 3)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + + long fileLength = FS.getFileStatus(walFile).getLen(); + byte[] content = new byte[(int) fileLength]; + try (FSDataInputStream in = FS.open(walFile)) { + in.readFully(content); + } + + long lastSingleCellEnd = endOffsets.get(4); + int truncPoint = (int) (lastSingleCellEnd + (endOffsets.get(5) - lastSingleCellEnd) / 2); + Path truncFile = UTIL.getDataTestDir("wal-trunc"); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, truncPoint); + } + + try (WALTailingReader reader = + WALFactory.createTailingReader(FS, truncFile, UTIL.getConfiguration(), -1)) { + for (int i = 0; i < 5; i++) { + WALTailingReader.Result result = reader.next(-1); + assertEquals("State should be NORMAL for entry " + i, WALTailingReader.State.NORMAL, + result.getState()); + assertArrayEquals("Row should match for entry " + i, Bytes.toBytes("row-" + i), + CellUtil.cloneRow(result.getEntry().getEdit().getCells().get(0))); + } + + WALTailingReader.Result eofResult = reader.next(-1); + assertEquals( + "With deferred dict updates, EOF mid-cell should return EOF_AND_RESET," + + " not EOF_AND_RESET_COMPRESSION", + WALTailingReader.State.EOF_AND_RESET, eofResult.getState()); + } + } + + @Test + public void itResumesPartialEntryAfterReset() throws Exception { + Path walFile = UTIL.getDataTestDir("wal-resume"); + List endOffsets = new ArrayList<>(); + try (WALProvider.Writer writer = + WALFactory.createWALWriter(FS, walFile, UTIL.getConfiguration())) { + for (int i = 0; i < 3; i++) { + writer.append(createEntry(i, 1)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + writer.append(createEntry(3, 2)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + + long fileLength = FS.getFileStatus(walFile).getLen(); + byte[] content = new byte[(int) fileLength]; + try (FSDataInputStream in = FS.open(walFile)) { + in.readFully(content); + } + + long lastSingleEnd = endOffsets.get(2); + int truncPoint = (int) (lastSingleEnd + (endOffsets.get(3) - lastSingleEnd) / 2); + Path truncFile = UTIL.getDataTestDir("wal-resume-trunc"); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, truncPoint); + } + + try (WALTailingReader reader = + WALFactory.createTailingReader(FS, truncFile, UTIL.getConfiguration(), -1)) { + for (int i = 0; i < 3; i++) { + WALTailingReader.Result result = reader.next(-1); + assertEquals("State should be NORMAL for entry " + i, WALTailingReader.State.NORMAL, + result.getState()); + } + + WALTailingReader.Result eofResult = reader.next(-1); + assertEquals(WALTailingReader.State.EOF_AND_RESET, eofResult.getState()); + long eofPos = eofResult.getEntryEndPos(); + + FS.delete(truncFile, false); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, content.length); + } + + reader.resetTo(eofPos, false); + WALTailingReader.Result resumeResult = reader.next(-1); + assertEquals(WALTailingReader.State.NORMAL, resumeResult.getState()); + WAL.Entry entry = resumeResult.getEntry(); + assertEquals(2, entry.getEdit().getCells().size()); + assertArrayEquals(Bytes.toBytes("row-3"), + CellUtil.cloneRow(entry.getEdit().getCells().get(0))); + assertArrayEquals(Bytes.toBytes("qual-3-0"), + CellUtil.cloneQualifier(entry.getEdit().getCells().get(0))); + assertArrayEquals(Bytes.toBytes("qual-3-1"), + CellUtil.cloneQualifier(entry.getEdit().getCells().get(1))); + } + } + + @Test + public void itHandlesMultipleConsecutivePartialReads() throws Exception { + Path walFile = UTIL.getDataTestDir("wal-multi-partial"); + List endOffsets = new ArrayList<>(); + try (WALProvider.Writer writer = + WALFactory.createWALWriter(FS, walFile, UTIL.getConfiguration())) { + for (int i = 0; i < 2; i++) { + writer.append(createEntry(i, 1)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + writer.append(createEntry(2, 3)); + writer.sync(true); + endOffsets.add(writer.getLength()); + } + + long fileLength = FS.getFileStatus(walFile).getLen(); + byte[] content = new byte[(int) fileLength]; + try (FSDataInputStream in = FS.open(walFile)) { + in.readFully(content); + } + + long multiCellStart = endOffsets.get(1); + long multiCellEnd = endOffsets.get(2); + int midRange = (int) (multiCellEnd - multiCellStart); + + int truncPoint1 = (int) multiCellStart + midRange / 4; + int truncPoint2 = (int) multiCellStart + midRange / 2; + + Path truncFile = UTIL.getDataTestDir("wal-multi-partial-trunc"); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, truncPoint1); + } + + try (WALTailingReader reader = + WALFactory.createTailingReader(FS, truncFile, UTIL.getConfiguration(), -1)) { + for (int i = 0; i < 2; i++) { + WALTailingReader.Result result = reader.next(-1); + assertEquals(WALTailingReader.State.NORMAL, result.getState()); + } + + WALTailingReader.Result eof1 = reader.next(-1); + assertEquals(WALTailingReader.State.EOF_AND_RESET, eof1.getState()); + long pos1 = eof1.getEntryEndPos(); + + FS.delete(truncFile, false); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, truncPoint2); + } + reader.resetTo(pos1, false); + + WALTailingReader.Result eof2 = reader.next(-1); + assertEquals(WALTailingReader.State.EOF_AND_RESET, eof2.getState()); + long pos2 = eof2.getEntryEndPos(); + + FS.delete(truncFile, false); + try (FSDataOutputStream out = FS.create(truncFile)) { + out.write(content, 0, content.length); + } + reader.resetTo(pos2, false); + + WALTailingReader.Result finalResult = reader.next(-1); + assertEquals(WALTailingReader.State.NORMAL, finalResult.getState()); + WAL.Entry entry = finalResult.getEntry(); + assertEquals(3, entry.getEdit().getCells().size()); + assertArrayEquals(Bytes.toBytes("row-2"), + CellUtil.cloneRow(entry.getEdit().getCells().get(0))); + } + } +} From e8c168eaa6c1315c6ced5383eef852ffcb6c02db Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Thu, 19 Mar 2026 07:40:57 -0700 Subject: [PATCH 69/78] HubSpot Backport HBASE-29987 Replication position corruption when WAL file switch detected in ReplicationSourceWALReader run loop (#7909) (#242) When ReplicationSourceWALReader.run() detects a WAL file switch via the switched() check, it enqueues an EOF batch but does not update currentPosition. If the outer loop restarts (e.g., due to WALEntryFilterRetryableException), the new WALEntryStream is created with the stale position from the old file, applied to the new file. This causes an infinite retry loop (EOFException: Cannot seek after EOF) and the corrupted position may be persisted to ZK, surviving restarts. The fix resets currentPosition to entryStream.getPosition() (which returns 0 after dequeueCurrentLog()) before enqueuing the EOF batch. Includes a regression test that reproduces the bug by using nb.capacity=1 to force EOF detection at line 153 (not inside readWALEntries), combined with a WALEntryFilterRetryableException on the first entry of the new file to trigger the outer loop restart. (cherry picked from commit e4f9c65e80dd7d5ffb103f782f054e0cc18f5878) Signed-off-by: Duo Zhang Co-authored-by: skhillon --- .../ReplicationSourceWALReader.java | 1 + .../regionserver/TestBasicWALEntryStream.java | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java index e617fe6d0162..89635ad000c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java @@ -158,6 +158,7 @@ public void run() { // first, check if we have switched a file, if so, we need to manually add an EOF entry // batch to the queue if (currentPath != null && switched(entryStream, currentPath)) { + currentPosition = entryStream.getPosition(); entryBatchQueue.put(WALEntryBatch.endOfFile(currentPath)); continue; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java index b1e2f2c16341..b3265da3887a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java @@ -881,6 +881,58 @@ public void testWALEntryStreamEOFRightAfterHeader() throws Exception { } } + /** + * Verify that when a WAL file switch is detected via the switched() check in + * ReplicationSourceWALReader.run(), currentPosition is reset so that a subsequent + * WALEntryFilterRetryableException does not cause the new file to be opened at the old file's + * position. + */ + @Test + public void testPositionResetOnFileSwitchWithRetryableFilter() throws Exception { + appendEntriesToLogAndSync(3); + log.rollWriter(); + AbstractFSWAL abstractWAL = (AbstractFSWAL) log; + Waiter.waitFor(CONF, 5000, + (Waiter.Predicate) () -> abstractWAL.getInflightWALCloseCount() == 0); + appendEntriesToLogAndSync(3); + + // Batch capacity of 1 ensures EOF on WAL A is detected by hasNext() in the run loop + // (not inside readWALEntries), which triggers the switched() path. + Configuration conf = new Configuration(CONF); + conf.setInt("replication.source.nb.capacity", 1); + + AtomicInteger totalFilterCalls = new AtomicInteger(0); + AtomicBoolean threwOnce = new AtomicBoolean(false); + WALEntryFilter filter = entry -> { + int callNum = totalFilterCalls.incrementAndGet(); + if (callNum > 3 && !threwOnce.get()) { + threwOnce.set(true); + throw new WALEntryFilterRetryableException("simulated filter failure after file switch"); + } + return entry; + }; + + ReplicationSource source = mockReplicationSource(false, conf); + when(source.isPeerEnabled()).thenReturn(true); + ReplicationSourceWALReader reader = + new ReplicationSourceWALReader(fs, conf, logQueue, 0, filter, source, fakeWalGroupId); + reader.start(); + + int totalEntries = 0; + long deadline = System.currentTimeMillis() + 30000; + while (totalEntries < 6) { + long remaining = deadline - System.currentTimeMillis(); + assertTrue("Reader appears stuck - likely position corruption. Only got " + totalEntries + + " of 6 entries", remaining > 0); + WALEntryBatch batch = reader.poll(1); + if (batch != null && batch != WALEntryBatch.NO_MORE_DATA) { + totalEntries += batch.getNbEntries(); + } + } + assertEquals(6, totalEntries); + assertTrue("Filter should have thrown at least once", threwOnce.get()); + } + private static class PartialWALEntryFailingWALEntryFilter implements WALEntryFilter { private int filteredWALEntryCount = -1; private int walEntryCount = 0; From 81f90da2cba74fe9950338edd1a64fab05cbd60c Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Wed, 1 Apr 2026 23:06:44 -0400 Subject: [PATCH 70/78] Inefficient HFile format check in incremental backup bulkload copy (#243) Co-authored-by: Hernan Gelaf-Romer --- .../org/apache/hadoop/hbase/backup/BackupHFileCleaner.java | 6 ++---- .../org/apache/hadoop/hbase/backup/impl/BackupCommands.java | 2 ++ .../hbase/backup/impl/IncrementalTableBackupClient.java | 4 ++-- .../apache/hadoop/hbase/backup/master/BackupLogCleaner.java | 1 + .../apache/hadoop/hbase/backup/TestBackupShowHistory.java | 3 ++- .../hbase/regionserver/wal/ProtobufWALTailingReader.java | 2 ++ 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java index 51b60e5bb3f8..289e09ff9df8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupHFileCleaner.java @@ -70,10 +70,8 @@ public void preClean() { LOG.debug("Cached {} unique HFile filenames registered as bulk loads.", hfileFilenames.size()); } catch (IOException ioe) { - LOG.error( - "Failed to read registered bulk load references from backup system table, " - + "marking all files as non-deletable.", - ioe); + LOG.error("Failed to read registered bulk load references from backup system table, " + + "marking all files as non-deletable.", ioe); bulkLoadedHFilesNeedingBackup = null; } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index ef3a3bcaebb3..edd693954c3a 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -45,6 +45,7 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC; import static org.apache.hadoop.hbase.backup.impl.BackupSystemTable.Order.NEW_TO_OLD; + import java.io.IOException; import java.net.URI; import java.util.List; @@ -69,6 +70,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.yetus.audience.InterfaceAudience; + import org.apache.hbase.thirdparty.com.google.common.base.Splitter; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 5cc68fd43b86..b114e83d3ca8 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -47,9 +47,9 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2; import org.apache.hadoop.hbase.mapreduce.WALPlayer; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.snapshot.SnapshotRegionLocator; @@ -466,7 +466,7 @@ private void incrementalCopyBulkloadHFiles(FileSystem tgtFs, TableName tn) throw List files = new ArrayList<>(); while (locatedFiles.hasNext()) { LocatedFileStatus file = locatedFiles.next(); - if (file.isFile() && HFile.isHFileFormat(tgtFs, file.getPath())) { + if (file.isFile() && StoreFileInfo.isHFile(file.getPath())) { files.add(file.getPath().toString()); } } diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java index e230efb1d00a..dcd56c6291c9 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java @@ -49,6 +49,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.org.apache.commons.collections4.IterableUtils; import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils; diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java index 40c4874e40c0..6195171d90ce 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupShowHistory.java @@ -24,7 +24,6 @@ import java.io.PrintStream; import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -35,6 +34,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + @Category(LargeTests.class) public class TestBackupShowHistory extends TestBackupBase { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java index a3184b6685ac..d4c1fe20e5db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufWALTailingReader.java @@ -30,9 +30,11 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams; import org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream; import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; From d4083d5510576d0d10dc687f0536ca8f04ed45a0 Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Thu, 2 Apr 2026 14:47:23 -0400 Subject: [PATCH 71/78] Deleting a FAILED backup cascades and deletes subsequent COMPLETE backups (#244) Co-authored-by: Hernan Gelaf-Romer --- .../hbase/backup/impl/BackupAdminImpl.java | 7 + .../TestDeleteFailedBackupDoesNotCascade.java | 133 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java index 66b0479ec687..9d1b318a1bf3 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java @@ -306,6 +306,13 @@ private void removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSys private List getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, BackupSystemTable table) throws IOException { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); + // A FAILED backup was never part of the backup chain — its state was rolled back + // via snapshot restore. No subsequent backup depends on it, so there are no + // affected sessions. + if (backupInfo.getState() == BackupState.FAILED) { + LOG.debug("Backup {} is in FAILED state, skipping cascade", backupInfo.getBackupId()); + return new ArrayList<>(); + } long ts = backupInfo.getStartTs(); List list = new ArrayList<>(); List history = table.getBackupHistory(withRoot(backupInfo.getBackupRootDir())); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java new file mode 100644 index 000000000000..2a9580004e7e --- /dev/null +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestDeleteFailedBackupDoesNotCascade.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; + +/** + * Verify that deleting a FAILED backup does not cascade-delete subsequent COMPLETE backups. A + * FAILED backup's state was rolled back via snapshot restore, so no subsequent backup depends on + * it. + */ +@Category(LargeTests.class) +public class TestDeleteFailedBackupDoesNotCascade extends TestBackupBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDeleteFailedBackupDoesNotCascade.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestDeleteFailedBackupDoesNotCascade.class); + + @Test + public void testDeleteFailedBackupDoesNotCascadeToCompletedIncremental() throws Exception { + LOG.info("Test that deleting a FAILED backup does not cascade to COMPLETE incrementals"); + + List tableList = Lists.newArrayList(table1); + + // Step 1: Create a full backup + String fullBackupId = fullTableBackup(tableList); + assertTrue(checkSucceeded(fullBackupId)); + LOG.info("Full backup {} succeeded", fullBackupId); + + // Step 2: Insert data so the incremental has something to back up + try (Connection conn = ConnectionFactory.createConnection(conf1); + Table t1 = conn.getTable(table1)) { + for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { + Put p = new Put(Bytes.toBytes("row-incr1-" + i)); + p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); + t1.put(p); + } + } + + // Step 3: Create a successful incremental backup + String successIncrBackupId = incrementalTableBackup(tableList); + assertTrue(checkSucceeded(successIncrBackupId)); + LOG.info("Successful incremental backup: {}", successIncrBackupId); + + // Step 4: Directly insert a FAILED backup record into the system table. + // This simulates a backup that failed and was recorded as FAILED. Its timestamp + // is between the full and the successful incremental, so the cascade logic + // would find the successful incremental as "affected" if not guarded. + BackupInfo fullInfo = getBackupInfo(fullBackupId); + BackupInfo successInfo = getBackupInfo(successIncrBackupId); + long failedTs = (fullInfo.getStartTs() + successInfo.getStartTs()) / 2; + + String failedBackupId = "backup_" + failedTs; + BackupInfo failedInfo = new BackupInfo(failedBackupId, BackupType.INCREMENTAL, + tableList.toArray(new TableName[0]), BACKUP_ROOT_DIR); + failedInfo.setStartTs(failedTs); + failedInfo.setCompleteTs(failedTs + 1); + failedInfo.setState(BackupState.FAILED); + failedInfo.setFailedMsg("Simulated failure for test"); + + try (BackupSystemTable sysTable = new BackupSystemTable(TEST_UTIL.getConnection())) { + sysTable.updateBackupInfo(failedInfo); + } + assertTrue("FAILED backup should exist", checkFailed(failedBackupId)); + LOG.info("Inserted FAILED backup record: {}", failedBackupId); + + // Step 5: Delete the FAILED backup + int deleted = getBackupAdmin().deleteBackups(new String[] { failedBackupId }); + assertEquals(1, deleted); + LOG.info("Deleted FAILED backup {}", failedBackupId); + + // Step 6: Verify the FAILED backup is gone + assertNull("FAILED backup should be deleted", getBackupInfo(failedBackupId)); + + // Step 7: Verify the successful incremental is still present + successInfo = getBackupInfo(successIncrBackupId); + assertNotNull("COMPLETE incremental backup should NOT be cascade-deleted", successInfo); + assertEquals(BackupState.COMPLETE, successInfo.getState()); + + // Step 8: Verify the full backup is still present + fullInfo = getBackupInfo(fullBackupId); + assertNotNull("Full backup should still exist", fullInfo); + assertEquals(BackupState.COMPLETE, fullInfo.getState()); + + LOG.info("Test passed: deleting FAILED backup did not cascade to COMPLETE incremental"); + } + + private BackupInfo getBackupInfo(String backupId) throws Exception { + try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) { + return table.readBackupInfo(backupId); + } + } +} From 5d1b34180fe389e41498237b13788d25f68b28d7 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Fri, 3 Apr 2026 13:42:49 -0400 Subject: [PATCH 72/78] Package a bundle for the MiniCluster (#245) --- .../hbase-testing-bundle/pom.xml | 316 ++++++++++++++++++ hubspot-client-bundles/pom.xml | 1 + 2 files changed, 317 insertions(+) create mode 100644 hubspot-client-bundles/hbase-testing-bundle/pom.xml diff --git a/hubspot-client-bundles/hbase-testing-bundle/pom.xml b/hubspot-client-bundles/hbase-testing-bundle/pom.xml new file mode 100644 index 000000000000..67cfb5d9dc8c --- /dev/null +++ b/hubspot-client-bundles/hbase-testing-bundle/pom.xml @@ -0,0 +1,316 @@ + + + 4.0.0 + + + com.hubspot.hbase + hubspot-client-bundles + ${revision} + + + hbase-testing-bundle + HBase testing utilities bundle (MiniCluster, HBaseTestingUtility) + + + + + + + org.apache.hbase + hbase-testing-util + ${project.version} + + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-logging + + + org.apache.hbase + hbase-protocol-shaded + + + commons-logging + commons-logging + + + org.slf4j + slf4j-log4j12 + + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-test-jars + generate-resources + + unpack + + + + + org.apache.hbase + hbase-server + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-zookeeper + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-common + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-annotations + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-asyncfs + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-hadoop-compat + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + org.apache.hbase + hbase-hadoop2-compat + ${project.version} + tests + test-jar + ${project.build.outputDirectory} + + + + org.apache.hadoop + hadoop-hdfs + 3.3.6-hubspot-SNAPSHOT + tests + test-jar + ${project.build.outputDirectory} + + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.6.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + create-bundle-with-relocations + + + + org.apache.hbase:* + + + org.apache.hbase.thirdparty:* + + + com.google.protobuf:protobuf-java + + + io.opentelemetry:opentelemetry-api + io.opentelemetry:opentelemetry-context + + + io.dropwizard.metrics:metrics-core + + + com.lmax:disruptor + com.github.ben-manes.caffeine:caffeine + org.agrona:agrona + + org.apache.commons:commons-lang3 + org.apache.commons:commons-math3 + + + org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-hdfs + org.apache.hadoop:hadoop-auth + javax.servlet:javax.servlet-api + org.eclipse.jetty:* + + + + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-logging + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-openssl + org.apache.hbase:hbase-endpoint + + + org.apache.hadoop:hadoop-hdfs-client + + + + + + com.lmax.disruptor + org.apache.hadoop.testing.disruptor + + + com.github.benmanes.caffeine + org.apache.hadoop.testing.caffeine + + + org.agrona + org.apache.hadoop.testing.agrona + + + org.apache.commons.math3 + org.apache.hadoop.testing.commons.math3 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/hubspot-client-bundles/pom.xml b/hubspot-client-bundles/pom.xml index 0105ffd28c43..dd7f81f13ab4 100644 --- a/hubspot-client-bundles/pom.xml +++ b/hubspot-client-bundles/pom.xml @@ -13,6 +13,7 @@ hbase-mapreduce-bundle hbase-backup-restore-bundle hbase-server-it-bundle + hbase-testing-bundle From deb067e7986bfd6976b3e31e4f35cb776b3e1200 Mon Sep 17 00:00:00 2001 From: Kodey Converse Date: Fri, 3 Apr 2026 14:16:54 -0400 Subject: [PATCH 73/78] Fix blazar build (#246) --- .../hbase-testing-bundle/.blazar.yaml | 26 +++++++++++++++++++ .../hbase-testing-bundle/.build-jdk17 | 0 2 files changed, 26 insertions(+) create mode 100644 hubspot-client-bundles/hbase-testing-bundle/.blazar.yaml create mode 100644 hubspot-client-bundles/hbase-testing-bundle/.build-jdk17 diff --git a/hubspot-client-bundles/hbase-testing-bundle/.blazar.yaml b/hubspot-client-bundles/hbase-testing-bundle/.blazar.yaml new file mode 100644 index 000000000000..d16f16de3bfd --- /dev/null +++ b/hubspot-client-bundles/hbase-testing-bundle/.blazar.yaml @@ -0,0 +1,26 @@ +buildpack: + name: Blazar-Buildpack-Java-oss-fork + +env: + PRE_RUN_BEFORE_STEPS: false + # Below variables are generated in prepare_environment.sh. + # The build environment requires environment variables to be explicitly defined before they may + # be modified by the `write-build-env-var` utilty script to persist changes to an environment variable + # throughout a build + REPO_NAME: "" + SET_VERSION: "" + HBASE_VERSION: "" + PKG_RELEASE: "" + FULL_BUILD_VERSION: "" + MAVEN_BUILD_ARGS: "" + +before: + - description: "Prepare build environment" + commands: + - $WORKSPACE/build-scripts/prepare_environment.sh + +depends: + - hubspot-client-bundles +provides: + - hbase-testing-bundle + diff --git a/hubspot-client-bundles/hbase-testing-bundle/.build-jdk17 b/hubspot-client-bundles/hbase-testing-bundle/.build-jdk17 new file mode 100644 index 000000000000..e69de29bb2d1 From 45b717bcfb23ca1ddb1722ce000024c663283af0 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Thu, 23 Apr 2026 10:44:35 -0700 Subject: [PATCH 74/78] [Do not upstream] Filter temporary attributes produced by internal coprocessors in region replication (#247) * Filter cell. extended attributes and split oversized batches in region replica replication Region replica replication was forwarding all extended attributes including large "cell."-prefixed ones added by CDC coprocessors that have no use for replicas. Strip these in replicate() before buffering to reduce memory and network overhead. Also add RPC size limit enforcement (matching HBaseInterClusterReplicationEndpoint) to split batches that exceed hbase.ipc.max.request.size, preventing permanently stalled replication. Co-Authored-By: Claude Opus 4.6 (1M context) * Fix test compilation: add replicationRpcLimit parameter to test call sites Co-Authored-By: Claude Opus 4.6 (1M context) * Add unit tests for splitBatches and extract as package-private static method Co-Authored-By: Claude Opus 4.6 (1M context) * Remove controller.reset() between batch RPCs reset() clears priority, callTimeout, tableName, and regionInfo which were set by the superclass before call() was entered. Since setCellScanner is called at the top of each loop iteration with the new batch's scanner, no cleanup is needed between iterations. Co-Authored-By: Claude Opus 4.6 (1M context) * Revert batch-splitting changes, keep only cell. attribute filtering Scoping down to just the extended attribute filtering. The batch-splitting for oversized RPCs will be addressed separately. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: skhillon Co-authored-by: Claude Opus 4.6 (1M context) --- .../regionserver/RegionReplicaReplicationEndpoint.java | 5 +++++ .../main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java index bf8316626dd3..cb775e9e4a19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.wal.EntryBuffers.RegionEntryBuffer; import org.apache.hadoop.hbase.wal.OutputSink; import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController; import org.apache.hadoop.util.StringUtils; import org.apache.yetus.audience.InterfaceAudience; @@ -228,6 +229,9 @@ public boolean replicate(ReplicateContext replicateContext) { while (this.isRunning()) { try { for (Entry entry : replicateContext.getEntries()) { + if (entry.getKey() instanceof WALKeyImpl) { + ((WALKeyImpl) entry.getKey()).removeExtendedAttributesWithPrefix("cell."); + } entryBuffers.appendEntry(entry); } outputSink.flush(); // make sure everything is flushed @@ -647,5 +651,6 @@ public ReplicateWALEntryResponse call(HBaseRpcController controller) throws Exce } return ReplicateWALEntryResponse.newBuilder().build(); } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java index aff98431e564..5933da1220de 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java @@ -450,6 +450,12 @@ public Map getExtendedAttributes() { : new HashMap(); } + public void removeExtendedAttributesWithPrefix(String prefix) { + if (extendedAttributes != null) { + extendedAttributes.entrySet().removeIf(e -> e.getKey().startsWith(prefix)); + } + } + @Override public String toString() { return tablename + "/" + Bytes.toString(encodedRegionName) + "/" + sequenceId; From eaf422ca7c3f678100417dcfda1611bc516fb7aa Mon Sep 17 00:00:00 2001 From: Hernan Romer Date: Tue, 23 Jun 2026 09:58:09 -0400 Subject: [PATCH 75/78] exclude reload4j (#249) Co-authored-by: Hernan Gelaf-Romer --- .../hbase-client-bundle/pom.xml | 8 ++++++++ .../hbase-mapreduce-bundle/pom.xml | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/hubspot-client-bundles/hbase-client-bundle/pom.xml b/hubspot-client-bundles/hbase-client-bundle/pom.xml index 159949c1cd16..71129d1d8aa2 100644 --- a/hubspot-client-bundles/hbase-client-bundle/pom.xml +++ b/hubspot-client-bundles/hbase-client-bundle/pom.xml @@ -50,6 +50,14 @@ org.slf4j slf4j-log4j12 + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + diff --git a/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml index 233e33750fe1..8e222e7cacdd 100644 --- a/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml +++ b/hubspot-client-bundles/hbase-mapreduce-bundle/pom.xml @@ -102,6 +102,14 @@ org.slf4j slf4j-log4j12 + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + @@ -191,6 +199,14 @@ org.slf4j slf4j-log4j12 + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + org.glassfish.hk2.external jakarta.inject From 42c1effee3a13124d311d959af548d8b27408b73 Mon Sep 17 00:00:00 2001 From: Siddharth Khillon Date: Mon, 27 Jul 2026 07:42:37 -0700 Subject: [PATCH 76/78] [Not yet upstream] Serial replication can get stuck on empty ranges (#252) --- .../SerialReplicationChecker.java | 7 + ...estSerialReplicationMultipleRSCrashes.java | 273 ++++++++++++++++++ .../TestSerialReplicationChecker.java | 15 + 3 files changed, 295 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationMultipleRSCrashes.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationChecker.java index 40ecf2f2f01f..ecf28c687b8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SerialReplicationChecker.java @@ -202,6 +202,13 @@ private boolean canPush(Entry entry, byte[] row) throws IOException { // start from 1. Here we choose the latter one. if (index < 0) { index = -index - 1; + } else if (index > 0 && barriers[index] - barriers[index - 1] == 1) { + // The range [barriers[index-1], barriers[index]) is exactly one seqId wide, containing only + // the openSeqNum which is never a real WAL entry (mvcc.advanceTo sets the base, mvcc.begin + // increments before the first write). This range is guaranteed empty, so don't increment — + // check isRangeFinished against the range before the empty one. + LOG.debug("{} matches barrier {} with gap-1 empty range, checking prior range", entry, + barriers[index]); } else { index++; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationMultipleRSCrashes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationMultipleRSCrashes.java new file mode 100644 index 000000000000..1dda9771ed11 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationMultipleRSCrashes.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.wal.NoEOFWALStreamReader; +import org.apache.hadoop.hbase.wal.WAL.Entry; +import org.apache.hadoop.hbase.wal.WALStreamReader; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests that serial replication completes after consecutive RS crashes, including when a region is + * moved onto an RS whose WAL reader is stuck. + */ +@Category({ ReplicationTests.class, LargeTests.class }) +public class TestSerialReplicationMultipleRSCrashes extends SerialReplicationTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSerialReplicationMultipleRSCrashes.class); + + @Before + public void setUp() throws Exception { + setupWALWriter(); + addPeer(false); + while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() < 3) { + UTIL.getMiniHBaseCluster().startRegionServer(); + } + } + + @Test + public void testTwoConsecutiveRSCrashes() throws Exception { + TableName tableName = createTable(); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 0; i < 100; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 100; i < 200; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 200; i < 300; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + enablePeerAndWaitUntilReplicationDone(300); + checkOrder(300); + } + + @Test + public void testTwoConsecutiveRSCrashesNoWritesBetween() throws Exception { + TableName tableName = createTable(); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 0; i < 100; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 100; i < 200; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + enablePeerAndWaitUntilReplicationDone(200); + checkOrder(200); + } + + @Test + public void testThreeConsecutiveRSCrashes() throws Exception { + TableName tableName = createTable(); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 0; i < 50; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 50; i < 100; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + UTIL.getMiniHBaseCluster().startRegionServer(); + UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() >= 2); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 100; i < 150; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(tableName); + UTIL.waitTableAvailable(tableName); + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 150; i < 200; i++) { + table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + enablePeerAndWaitUntilReplicationDone(200); + checkOrder(200); + } + + @Test + public void testRegionMovedOntoStuckRSIsAlsoStuck() throws Exception { + TableName tableName = TableName.valueOf(name.getMethodName()); + byte[] splitKey = Bytes.toBytes("m"); + UTIL.getAdmin().createTable( + TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF) + .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) + .build(), + new byte[][] { splitKey }); + UTIL.waitTableAvailable(tableName); + + RegionInfo regionA = + UTIL.getConnection().getRegionLocator(tableName).getAllRegionLocations().stream() + .filter(loc -> loc.getRegion().getStartKey().length == 0).findFirst().get().getRegion(); + RegionInfo regionB = + UTIL.getConnection().getRegionLocator(tableName).getAllRegionLocations().stream() + .filter(loc -> loc.getRegion().getStartKey().length > 0).findFirst().get().getRegion(); + + HRegionServer rsForA = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(RegionServerThread::getRegionServer) + .filter(rs -> rs.getRegion(regionA.getEncodedName()) != null).findFirst().get(); + HRegionServer rsForB = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(RegionServerThread::getRegionServer) + .filter(rs -> rs.getRegion(regionB.getEncodedName()) != null).findFirst().get(); + + if (rsForA.getServerName().equals(rsForB.getServerName())) { + HRegionServer otherRS = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(RegionServerThread::getRegionServer) + .filter(rs -> !rs.getServerName().equals(rsForA.getServerName())).findFirst().get(); + moveRegion(regionB, otherRS); + } + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 0; i < 100; i++) { + table.put( + new Put(Bytes.toBytes(String.format("a%04d", i))).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + abortRSHostingRegion(regionA); + UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .anyMatch(t -> t.getRegionServer().getRegion(regionA.getEncodedName()) != null)); + + abortRSHostingRegion(regionA); + UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .anyMatch(t -> t.getRegionServer().getRegion(regionA.getEncodedName()) != null)); + + HRegionServer stuckRS = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .map(RegionServerThread::getRegionServer) + .filter(rs -> rs.getRegion(regionA.getEncodedName()) != null).findFirst().get(); + if (stuckRS.getRegion(regionB.getEncodedName()) == null) { + moveRegion(regionB, stuckRS); + } + + try (Table table = UTIL.getConnection().getTable(tableName)) { + for (int i = 0; i < 100; i++) { + table.put( + new Put(Bytes.toBytes(String.format("z%04d", i))).addColumn(CF, CQ, Bytes.toBytes(i))); + } + } + + enablePeerAndWaitUntilReplicationDone(200); + checkOrderPerRegion(200); + } + + private void checkOrderPerRegion(int expectedEntries) throws IOException { + try (WALStreamReader reader = + NoEOFWALStreamReader.create(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) { + Map lastSeqIdByRegion = new HashMap<>(); + int count = 0; + for (Entry entry;;) { + entry = reader.next(); + if (entry == null) { + break; + } + String region = Bytes.toString(entry.getKey().getEncodedRegionName()); + long seqId = entry.getKey().getSequenceId(); + Long prev = lastSeqIdByRegion.get(region); + assertTrue( + "Sequence id goes backwards for region " + region + " from " + prev + " to " + seqId, + prev == null || seqId >= prev); + lastSeqIdByRegion.put(region, seqId); + count++; + } + assertEquals(expectedEntries, count); + } + } + + private void abortRSHostingRegion(RegionInfo region) throws Exception { + RegionServerThread rsThread = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .filter(t -> t.getRegionServer().getRegion(region.getEncodedName()) != null).findFirst() + .orElseThrow(() -> new RuntimeException("No live RS hosting " + region.getEncodedName())); + rsThread.getRegionServer().abort("for testing"); + rsThread.join(); + } + + private void abortRSHostingRegion(TableName tableName) throws Exception { + RegionServerThread rsThread = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().stream() + .filter(t -> !t.getRegionServer().getRegions(tableName).isEmpty()).findFirst() + .orElseThrow(() -> new RuntimeException("No live RS hosting " + tableName)); + rsThread.getRegionServer().abort("for testing"); + rsThread.join(); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java index 54a67ce4d32d..0f2d00a01bc2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java @@ -300,4 +300,19 @@ public void testCanPushEqualsToBarrier() throws IOException, ReplicationExceptio updatePushedSeqId(region, 99); assertTrue(checker.canPush(createEntry(region, 100), cell)); } + + @Test + public void testCanPushEqualsToBarrierWithGapOne() throws IOException, ReplicationException { + RegionInfo region = RegionInfoBuilder.newBuilder(tableName).build(); + Cell cell = createCell(region); + + addStateAndBarrier(region, RegionState.State.OPEN, 10, 100, 101); + assertFalse(checker.canPush(createEntry(region, 101), cell)); + updatePushedSeqId(region, 99); + assertTrue(checker.canPush(createEntry(region, 101), cell)); + + addStateAndBarrier(region, RegionState.State.OPEN, 9, 17, 25, 28, 31, 34, 38, 39); + updatePushedSeqId(region, 37); + assertTrue(checker.canPush(createEntry(region, 39), cell)); + } } From 4fc277358f71854a6d656e9952e877c6102d29c4 Mon Sep 17 00:00:00 2001 From: sahmadsabri Date: Mon, 27 Jul 2026 23:45:09 -0500 Subject: [PATCH 77/78] Add metrics for HDFS checksum and no-checksum bytes read --- .../MetricsRegionServerSource.java | 10 +++++++ .../MetricsRegionServerWrapper.java | 12 ++++++++ .../MetricsRegionServerSourceImpl.java | 6 ++++ .../apache/hadoop/hbase/fs/HFileSystem.java | 7 +++++ .../hbase/io/FSDataInputStreamWrapper.java | 29 +++++++++++++++---- .../MetricsRegionServerWrapperImpl.java | 15 ++++++++++ 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index c23c222edc54..4318455e350f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -533,6 +533,16 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; String ZEROCOPY_BYTES_READ_DESC = "The number of bytes read through HDFS zero copy"; + String CHECKSUM_BYTES_READ = "checksumBytesRead"; + String CHECKSUM_BYTES_READ_DESC = + "The number of bytes read using HDFS FS-level checksum verification (non-short-circuit path)"; + String NO_CHECKSUM_BYTES_READ = "noChecksumBytesRead"; + String NO_CHECKSUM_BYTES_READ_DESC = + "The number of bytes read with HDFS FS-level checksum disabled (HBase checksum or short circuit path)"; + String DATANODE_MAX_TRANSFER_THREADS = "datanodeMaxTransferThreads"; + String DATANODE_MAX_TRANSFER_THREADS_DESC = + "Configured value of dfs.datanode.max.transfer.threads; used to track the impact of changing this setting"; + String LOCAL_RACK_BYTES_READ = "localRackBytesRead"; String LOCAL_RACK_BYTES_READ_DESC = "The number of bytes read from the same rack of the RegionServer, but not the local HDFS DataNode"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 67d31ffe64c4..fd3b84f9657e 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -554,6 +554,18 @@ public interface MetricsRegionServerWrapper { /** Returns Number of bytes read locally through HDFS zero copy. */ long getZeroCopyBytesRead(); + /** Returns Number of bytes read with HDFS FS-level checksum enabled. */ + long getChecksumBytesRead(); + + /** + * Returns Number of bytes read with HDFS FS-level checksum disabled (HBase checksum or short + * circuit). + */ + long getNoChecksumBytesRead(); + + /** Returns Configured value of dfs.datanode.max.transfer.threads. */ + long getDatanodeMaxTransferThreads(); + /** * Returns Count of requests blocked because the memstore size is larger than blockingMemStoreSize */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index b42a02d0e659..779cd834e59f 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -568,6 +568,12 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde rsWrap.getShortCircuitBytesRead()) .addGauge(Interns.info(ZEROCOPY_BYTES_READ, ZEROCOPY_BYTES_READ_DESC), rsWrap.getZeroCopyBytesRead()) + .addGauge(Interns.info(CHECKSUM_BYTES_READ, CHECKSUM_BYTES_READ_DESC), + rsWrap.getChecksumBytesRead()) + .addGauge(Interns.info(NO_CHECKSUM_BYTES_READ, NO_CHECKSUM_BYTES_READ_DESC), + rsWrap.getNoChecksumBytesRead()) + .addGauge(Interns.info(DATANODE_MAX_TRANSFER_THREADS, DATANODE_MAX_TRANSFER_THREADS_DESC), + rsWrap.getDatanodeMaxTransferThreads()) .addGauge(Interns.info(SPLIT_QUEUE_LENGTH, SPLIT_QUEUE_LENGTH_DESC), rsWrap.getSplitQueueSize()) .addGauge(Interns.info(COMPACTION_QUEUE_LENGTH, COMPACTION_QUEUE_LENGTH_DESC), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java index 1a8e5cc4c776..beffc980840e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java @@ -106,9 +106,16 @@ public HFileSystem(Configuration conf, boolean useHBaseChecksum) throws IOExcept if (useHBaseChecksum && !(fs instanceof LocalFileSystem)) { conf = new Configuration(conf); conf.setBoolean("dfs.client.read.shortcircuit.skip.checksum", true); + LOG.info("HBase checksum enabled: set dfs.client.read.shortcircuit.skip.checksum=true; " + + "HDFS will skip checksum verification during short circuit reads"); this.noChecksumFs = maybeWrapFileSystem(newInstanceFileSystem(conf), conf); this.noChecksumFs.setVerifyChecksum(false); + LOG.info("Opened noChecksumFs with FS-level checksum verification disabled"); } else { + LOG.info( + "HBase checksum not enabled (useHBaseChecksum={}, localFs={}): " + + "HDFS checksum verification is active", + useHBaseChecksum, (fs instanceof LocalFileSystem)); this.noChecksumFs = maybeWrapFileSystem(fs, conf); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java index 33eace47d632..5051496ca0d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java @@ -86,6 +86,8 @@ private static class ReadStatistics { long totalLocalBytesRead; long totalShortCircuitBytesRead; long totalZeroCopyBytesRead; + long totalChecksumBytesRead; + long totalNoChecksumBytesRead; } protected Path readerPath; @@ -230,7 +232,7 @@ public void checksumOk() { } } - private void updateInputStreamStatistics(FSDataInputStream stream) { + private void updateInputStreamStatistics(FSDataInputStream stream, boolean fsChecksumEnabled) { // If the underlying file system is HDFS, update read statistics upon close. if (stream instanceof HdfsDataInputStream) { /** @@ -240,14 +242,19 @@ private void updateInputStreamStatistics(FSDataInputStream stream) { */ HdfsDataInputStream hdfsDataInputStream = (HdfsDataInputStream) stream; synchronized (readStatistics) { - readStatistics.totalBytesRead += - hdfsDataInputStream.getReadStatistics().getTotalBytesRead(); + long bytesRead = hdfsDataInputStream.getReadStatistics().getTotalBytesRead(); + readStatistics.totalBytesRead += bytesRead; readStatistics.totalLocalBytesRead += hdfsDataInputStream.getReadStatistics().getTotalLocalBytesRead(); readStatistics.totalShortCircuitBytesRead += hdfsDataInputStream.getReadStatistics().getTotalShortCircuitBytesRead(); readStatistics.totalZeroCopyBytesRead += hdfsDataInputStream.getReadStatistics().getTotalZeroCopyBytesRead(); + if (fsChecksumEnabled) { + readStatistics.totalChecksumBytesRead += bytesRead; + } else { + readStatistics.totalNoChecksumBytesRead += bytesRead; + } } } } @@ -276,17 +283,29 @@ public static long getZeroCopyBytesRead() { } } + public static long getChecksumBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalChecksumBytesRead; + } + } + + public static long getNoChecksumBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalNoChecksumBytesRead; + } + } + /** CloseClose stream(s) if necessary. */ @Override public void close() { if (!doCloseStreams) { return; } - updateInputStreamStatistics(this.streamNoFsChecksum); + updateInputStreamStatistics(this.streamNoFsChecksum, false); // we do not care about the close exception as it is for reading, no data loss issue. Closeables.closeQuietly(streamNoFsChecksum); - updateInputStreamStatistics(stream); + updateInputStreamStatistics(stream, true); Closeables.closeQuietly(stream); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index a256e8827a39..81a228d4d713 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -1087,6 +1087,21 @@ public long getZeroCopyBytesRead() { return FSDataInputStreamWrapper.getZeroCopyBytesRead(); } + @Override + public long getChecksumBytesRead() { + return FSDataInputStreamWrapper.getChecksumBytesRead(); + } + + @Override + public long getNoChecksumBytesRead() { + return FSDataInputStreamWrapper.getNoChecksumBytesRead(); + } + + @Override + public long getDatanodeMaxTransferThreads() { + return regionServer.getConfiguration().getInt("dfs.datanode.max.transfer.threads", 4096); + } + @Override public long getBlockedRequestsCount() { return aggregate.blockedRequestsCount; From d9992659d68614933ed67be1b3f319b94eef57fc Mon Sep 17 00:00:00 2001 From: sahmadsabri Date: Tue, 28 Jul 2026 00:13:37 -0500 Subject: [PATCH 78/78] Add methods to retrieve checksum and no-checksum bytes read metrics --- .../MetricsRegionServerWrapperStub.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 84654784c58d..2635adaf840b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -557,6 +557,21 @@ public long getZeroCopyBytesRead() { return 0; } + @Override + public long getChecksumBytesRead() { + return 0; + } + + @Override + public long getNoChecksumBytesRead() { + return 0; + } + + @Override + public long getDatanodeMaxTransferThreads() { + return 4096; + } + @Override public long getBlockedRequestsCount() { return 0;