diff --git a/cleaner/README.md b/cleaner/README.md index cdbea48..d3c8ae1 100644 --- a/cleaner/README.md +++ b/cleaner/README.md @@ -17,12 +17,22 @@ To determine what to delete we will use information stored in /var/lib/docker/di IMAGE_RETAIN_PERIOD - we will not delete images if they have events since `current_timestamp - IMAGE_RETAIN_PERIOD` (default 3d) VOLUMES_RETAIN_PERIOD - we will not delete volumes if they have events since `current_timestamp - IMAGE_RETAIN_PERIOD` (default 3d) - + + DISK_USAGE_THRESHOLD - docker-clean.sh (pre-build) launches a clean if current disk usage exceeds this (default 0.8) + INODES_USAGE_THRESHOLD - docker-clean.sh (pre-build) launches a clean if current inode usage exceeds this (default 0.8) + + CLEANER_AGENT_DISK_USAGE_THRESHOLD - cleaner-agent.sh's continuous, mid-build loop launches a clean if current disk usage exceeds this (default 0.9) + CLEANER_AGENT_INODES_USAGE_THRESHOLD - cleaner-agent.sh's continuous, mid-build loop launches a clean if current inode usage exceeds this (default 0.9) + ####### defaults: CLEAN_PERIOD_SECONDS=${CLEAN_PERIOD_SECONDS:-21600} # 6 hours CLEAN_PERIOD_BUILDS=${CLEAN_PERIOD_BUILDS:-10} IMAGE_RETAIN_PERIOD=${IMAGE_RETAIN_PERIOD:-259200} VOLUMES_RETAIN_PERIOD=${VOLUMES_RETAIN_PERIOD:-259200} + DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-0.8} + INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD:-0.8} + CLEANER_AGENT_DISK_USAGE_THRESHOLD=${CLEANER_AGENT_DISK_USAGE_THRESHOLD:-0.9} + CLEANER_AGENT_INODES_USAGE_THRESHOLD=${CLEANER_AGENT_INODES_USAGE_THRESHOLD:-0.9} ##### Logic: - save current docker events by `docker events --until 0s -f ${EVENT_FORMAT} > /var/lib/docker/dind-volume/events/$(date +%s)` diff --git a/cleaner/cleaner-agent.sh b/cleaner/cleaner-agent.sh index 11a1082..8a9410d 100755 --- a/cleaner/cleaner-agent.sh +++ b/cleaner/cleaner-agent.sh @@ -4,8 +4,6 @@ # examining usage threshold # echo "$0: - Entering at $(date) " -START_DISK_USAGE_THRESHOLD=0.9 -START_INODES_USAGE_THRESHOLD=0.9 DIR=$(dirname ${BASH_SOURCE}) @@ -33,8 +31,8 @@ sigterm_trap(){ trap sigterm_trap SIGTERM SIGINT need_to_clean() { - IS_DISK_USAGE_THRESHOLD=$(check_disk_usage_threshold ${START_DISK_USAGE_THRESHOLD}) - IS_INODES_USAGE_THRESHOLD=$(check_inodes_usage_threshold ${START_INODES_USAGE_THRESHOLD}) + IS_DISK_USAGE_THRESHOLD=$(check_disk_usage_threshold ${CLEANER_AGENT_DISK_USAGE_THRESHOLD}) + IS_INODES_USAGE_THRESHOLD=$(check_inodes_usage_threshold ${CLEANER_AGENT_INODES_USAGE_THRESHOLD}) if [[ ${IS_DISK_USAGE_THRESHOLD} == 1 || ${IS_INODES_USAGE_THRESHOLD} == 1 ]]; then echo 1 fi diff --git a/cleaner/config b/cleaner/config index 0724631..9db8f23 100644 --- a/cleaner/config +++ b/cleaner/config @@ -7,6 +7,13 @@ VOLUMES_RETAIN_PERIOD=${VOLUMES_RETAIN_PERIOD:-259200} DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-0.8} INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD:-0.8} +# Thresholds used by cleaner-agent.sh's continuous, mid-build loop. Kept separate (and higher by +# default) from DISK_USAGE_THRESHOLD/INODES_USAGE_THRESHOLD above so it only steps in once the +# pre-build docker-clean.sh pass wasn't enough, but still overridable via the Runtime Environment +# spec like every other threshold here. +CLEANER_AGENT_DISK_USAGE_THRESHOLD=${CLEANER_AGENT_DISK_USAGE_THRESHOLD:-0.9} +CLEANER_AGENT_INODES_USAGE_THRESHOLD=${CLEANER_AGENT_INODES_USAGE_THRESHOLD:-0.9} + #### Defining DIND_VOLUME_STAT dir and stat files DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker} DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume} diff --git a/cleaner/functions.sh b/cleaner/functions.sh index d140b4e..f430658 100644 --- a/cleaner/functions.sh +++ b/cleaner/functions.sh @@ -40,13 +40,13 @@ display_df(){ } check_disk_usage_threshold(){ - local THRESHOLD=${1:-${DISK_USAGE_THRESHOLD}} - df -P ${DOCKERD_DATA_ROOT} | awk -v T=${THRESHOLD} 'NR==2 {print ( $3 / $2 > T ) ? "1": "0" }' + local DISK_THRESHOLD=${1:-${DISK_USAGE_THRESHOLD}} + df -P ${DOCKERD_DATA_ROOT} | awk -v T=${DISK_THRESHOLD} 'NR==2 {print ( $3 / $2 > T ) ? "1": "0" }' } check_inodes_usage_threshold(){ - local THRESHOLD=${1:-${DISK_USAGE_THRESHOLD}} - df -iP ${DOCKERD_DATA_ROOT} | awk -v T=${THRESHOLD} 'NR==2 {print ( $3 / $2 > T ) ? "1": "0" }' + local INODES_THRESHOLD=${1:-${INODES_USAGE_THRESHOLD}} + df -iP ${DOCKERD_DATA_ROOT} | awk -v T=${INODES_THRESHOLD} 'NR==2 {print ( $3 / $2 > T ) ? "1": "0" }' } clean_temporary_objects(){