Skip to content

fix: restore quay prune tags and resync IS on promotion-quay failure #5240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deepsm007 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: restore quay prune tags and resync IS on promotion-quay failure #5240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 87 additions & 16 deletions pkg/steps/release/promote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,11 @@ func getTagCommand(tagSpecs []string, loglevel int) string {
loglevel, strings.Join(tagSpecs, " "))
}

type quayOfficialISTag struct {
quayProxy string
isTag string
}

// quayProxyTagFromISKey derives the quay-proxy floating tag from an IS tag key.
// Handles "namespace/stream-quay:tag" (4.23+) and consolidated "ocp/4.13:tag" (4.11–4.22).
// Example: "ocp/4.13:cli" → "quay-proxy.ci.openshift.org/openshift/ci:ocp_4.13_cli".
Expand Down Expand Up @@ -393,9 +398,7 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp
var images []string
var pruneImages []string
var tags []string
// resolveAndTagPairs holds [quayProxyTag, isTag] for official ocp IS targets (consolidated
// ocp/4.x:tag and legacy *-quay). Resolved post-mirror via oc image info + oc tag.
var resolveAndTagPairs [][2]string
var officialQuayISTags []quayOfficialISTag

isQuayStep := name == api.PromotionQuayStepName

Expand All @@ -415,7 +418,7 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp
if ok {
ns := k[:strings.Index(k, "/")]
if api.RefersToOfficialImage(ns, api.WithOKD) {
resolveAndTagPairs = append(resolveAndTagPairs, [2]string{quayProxyTag, k})
officialQuayISTags = append(officialQuayISTags, quayOfficialISTag{quayProxy: quayProxyTag, isTag: k})
continue
}
}
Expand All @@ -432,17 +435,16 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp
registryConfig := filepath.Join(api.RegistryPushCredentialsCICentralSecretMountPath, coreapi.DockerConfigJsonKey)
command := []string{"/bin/sh", "-c"}

var commands []string
var mirrorCommands []string
var postMirrorCommands []string

// Generate mirror commands if there are images to mirror
if len(images) > 0 {
commands = append(commands, getMirrorRetryShell(registryConfig, images, 2))
mirrorCommands = append(mirrorCommands, getMirrorRetryShell(registryConfig, images, 2))
}

if isQuayStep {
for _, pair := range resolveAndTagPairs {
quayProxyTag, isTag := pair[0], pair[1]
commands = append(commands, getResolveAndTagRetryShell(registryConfig, quayProxyTag, isTag, 2, promotionCLIImageInfoFilterOS(nodeArchitectures)))
for _, tag := range officialQuayISTags {
postMirrorCommands = append(postMirrorCommands, getResolveAndTagRetryShell(registryConfig, tag.quayProxy, tag.isTag, 2, promotionCLIImageInfoFilterOS(nodeArchitectures)))
}
}

Expand All @@ -458,24 +460,93 @@ func getPromotionPod(imageMirrorTarget map[string]string, timeStr string, namesp
}

tagCommands = append(tagCommands, "set -e")
commands = append(commands, strings.Join(tagCommands, "\n"))
postMirrorCommands = append(postMirrorCommands, strings.Join(tagCommands, "\n"))
} else if len(tags) > 0 {
// For regular promotion, use the original retry logic
tagCommand := fmt.Sprintf(retryLoopTemplate, 5, "Tag attempt $r", getTagCommand(tags, 2), retryLoopWithBackoff)
commands = append(commands, tagCommand)
postMirrorCommands = append(postMirrorCommands, tagCommand)
}

var args []string
var pruneMirror string
if len(pruneImages) > 0 {
// See https://github.com/openshift/release/blob/2080ec4a49337c27577a4b2ff08a538e96436e65/hack/qci_registry_pruner.py for details.
// Note that we don't retry here and we ignore failures because (a) it may be the first time an image tag is
// being promoted to and trying to add a pruning tag to the existing image is doomed to fail. (b) pruning tags
// help eliminate a rare race condition. The cost of an occasional failure in establishing them is very low.
args = append(args, fmt.Sprintf("%s || true", getMirrorCommand(registryConfig, pruneImages, 2)))
pruneMirror = fmt.Sprintf("%s || true", getMirrorCommand(registryConfig, pruneImages, 2))
}

args = append(args, commands...)
args = []string{strings.Join(args, "\n")}
var args []string
if isQuayStep {
filterByOS := promotionCLIImageInfoFilterOS(nodeArchitectures)
var mirrorFailureRollback, postFailureRollback string
if len(pruneImages) > 0 {
restoreSpecs := make([]string, 0, len(pruneImages))
for _, spec := range pruneImages {
live, prune, ok := strings.Cut(spec, "=")
if ok && live != "" && prune != "" {
restoreSpecs = append(restoreSpecs, fmt.Sprintf("%s=%s", prune, live))
}
}
if len(restoreSpecs) > 0 {
mirrorFailureRollback = fmt.Sprintf(`echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
%s || true`, getMirrorCommand(registryConfig, restoreSpecs, 2))
postFailureRollback = mirrorFailureRollback
if len(officialQuayISTags) > 0 {
postFailureRollback += "\necho \"promotion-quay: re-syncing app.ci imagestreams from restored quay tags\" >&2"
for _, tag := range officialQuayISTags {
repo := tag.quayProxy[:strings.LastIndex(tag.quayProxy, ":")]
quayIOTag := strings.Replace(tag.quayProxy, api.QCIAPPCIDomain, "quay.io", 1)
postFailureRollback += fmt.Sprintf(`
_digest=$(oc image info --registry-config=%s --filter-by-os=%s %s | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference %s@${_digest} %s || true`,
registryConfig, filterByOS, quayIOTag, repo, tag.isTag)
}
}
}
}

var parts []string
if pruneMirror != "" {
parts = append(parts, pruneMirror)
}
parts = append(parts, "_mirror_exit=0")
if len(mirrorCommands) > 0 {
parts = append(parts, "set +e")
parts = append(parts, mirrorCommands...)
parts = append(parts, "_mirror_exit=$?", "set -e")
if mirrorFailureRollback != "" {
parts = append(parts, fmt.Sprintf(`if [ "${_mirror_exit}" -ne 0 ]; then
%s
exit "${_mirror_exit}"
fi`, mirrorFailureRollback))
} else {
parts = append(parts, `if [ "${_mirror_exit}" -ne 0 ]; then exit "${_mirror_exit}"; fi`)
}
}
if len(postMirrorCommands) > 0 {
parts = append(parts, "set +e")
parts = append(parts, postMirrorCommands...)
parts = append(parts, "_promotion_exit=$?", "set -e")
if postFailureRollback != "" {
parts = append(parts, fmt.Sprintf(`if [ "${_promotion_exit}" -ne 0 ]; then
%s
fi`, postFailureRollback))
}
parts = append(parts, `exit "${_promotion_exit}"`)
}
args = []string{strings.Join(parts, "\n")}
} else {
var promotionCommands []string
promotionCommands = append(promotionCommands, mirrorCommands...)
promotionCommands = append(promotionCommands, postMirrorCommands...)
var parts []string
if pruneMirror != "" {
parts = append(parts, pruneMirror)
}
parts = append(parts, strings.Join(promotionCommands, "\n"))
args = []string{strings.Join(parts, "\n")}
}

image := cliImage
nodeSelector := map[string]string{"kubernetes.io/arch": "amd64"}
Expand Down
17 changes: 17 additions & 0 deletions pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ spec:
- args:
- |-
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ci_a_latest=quay.io/openshift/ci:20240603235401_prune_ci_a_latest quay.io/openshift/ci:ci_c_latest=quay.io/openshift/ci:20240603235401_prune_ci_c_latest || true
_mirror_exit=0
set +e
for r in {1..5}; do
echo Mirror attempt $r
if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ci_a_latest registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ddd=quay.io/openshift/ci:ci_c_latest; then break; fi
Expand All @@ -19,11 +21,26 @@ spec:
echo Sleeping randomized $backoff before retry
sleep $backoff
done
_mirror_exit=$?
set -e
if [ "${_mirror_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ci_a_latest=quay.io/openshift/ci:ci_a_latest quay.io/openshift/ci:20240603235401_prune_ci_c_latest=quay.io/openshift/ci:ci_c_latest || true
exit "${_mirror_exit}"
fi
set +e
set +e
for r in {1..2}; do echo "Tag attempt $r (all together)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:${component} && break; :; done
for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:ddd ci/${component}-quay:c && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done
for r in {1..3}; do echo "Tag attempt $r (individual)"; oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@sha256:bbb ci/ci-quay:${component} && break; backoff=$(($RANDOM % 120))s; echo Sleeping randomized $backoff before retry; sleep $backoff; done
set -e
_promotion_exit=$?
set -e
if [ "${_promotion_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ci_a_latest=quay.io/openshift/ci:ci_a_latest quay.io/openshift/ci:20240603235401_prune_ci_c_latest=quay.io/openshift/ci:ci_c_latest || true
fi
exit "${_promotion_exit}"
command:
- /bin/sh
- -c
Expand Down
25 changes: 24 additions & 1 deletion pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_4.12.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ metadata:
spec:
containers:
- args:
- |
- |-
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base || true
_mirror_exit=0
set +e
for r in {1..5}; do
echo Mirror attempt $r
if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base; then break; fi
Expand All @@ -19,6 +21,14 @@ spec:
echo Sleeping randomized $backoff before retry
sleep $backoff
done
_mirror_exit=$?
set -e
if [ "${_mirror_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base || true
exit "${_mirror_exit}"
fi
set +e
for r in {1..5}; do
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes; then
Expand Down Expand Up @@ -46,6 +56,19 @@ spec:
backoff=$(($RANDOM % 120))s
sleep "${backoff}"
done

_promotion_exit=$?
set -e
if [ "${_promotion_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base || true
echo "promotion-quay: re-syncing app.ci imagestreams from restored quay tags" >&2
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes || true
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.12_ovn-kubernetes-base | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.12:ovn-kubernetes-base || true
fi
exit "${_promotion_exit}"
command:
- /bin/sh
- -c
Expand Down
27 changes: 26 additions & 1 deletion pkg/steps/release/testdata/zz_fixture_TestGetPromotionPod_promotion_quay_multiple_tags.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ metadata:
spec:
containers:
- args:
- |
- |-
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift=quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift || true
_mirror_exit=0
set +e
for r in {1..5}; do
echo Mirror attempt $r
if oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:aaa=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:bbb=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base registry.build02.ci.openshift.org/ci-op-y2n8rsh3/pipeline@sha256:ccc=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift; then break; fi
Expand All @@ -19,6 +21,14 @@ spec:
echo Sleeping randomized $backoff before retry
sleep $backoff
done
_mirror_exit=$?
set -e
if [ "${_mirror_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift || true
exit "${_mirror_exit}"
fi
set +e
for r in {1..5}; do
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
if [ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes; then
Expand Down Expand Up @@ -60,6 +70,21 @@ spec:
backoff=$(($RANDOM % 120))s
sleep "${backoff}"
done

_promotion_exit=$?
set -e
if [ "${_promotion_exit}" -ne 0 ]; then
echo "promotion-quay: restoring quay floating tags from prune backups after promotion failure" >&2
oc image mirror --loglevel=2 --keep-manifest-list --registry-config=/etc/push-secret/.dockerconfigjson --max-per-registry=10 quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-base=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base quay.io/openshift/ci:20240603235401_prune_ovn-kubernetes-microshift=quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift || true
echo "promotion-quay: re-syncing app.ci imagestreams from restored quay tags" >&2
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes || true
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-base | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes-base || true
_digest=$(oc image info --registry-config=/etc/push-secret/.dockerconfigjson --filter-by-os=linux/amd64 quay.io/openshift/ci:ocp_4.21_ovn-kubernetes-microshift | sed -n '/^Digest:[[:space:]]/s/^Digest:[[:space:]]*//p' | head -n1)
[ -n "${_digest}" ] && oc tag --source=docker --loglevel=2 --reference-policy='source' --import-mode='PreserveOriginal' --reference quay-proxy.ci.openshift.org/openshift/ci@${_digest} ocp/4.21:ovn-kubernetes-microshift || true
fi
exit "${_promotion_exit}"
command:
- /bin/sh
- -c
Expand Down
Loading