diff --git a/README.md b/README.md index 9d8e513..b8be22b 100644 --- a/README.md +++ b/README.md @@ -51,4 +51,4 @@ To play with the FCM, you can also run this controller inside the [mini-lab](htt 1. Deploy the FCM into the mini-lab with `make deploy` 1. Adapt the example [firewalldeployment.yaml](config/examples/firewalldeployment.yaml) and apply with `kubectl apply -f config/examples/firewalldeployment.yaml` 1. Note that the firewall-controller will not be able to connect to the mini-lab due to network restrictions, so the firewall will not get ready. - - You can make the firewall become ready anyway by setting the annotation `kubectl annotate fw firewall.metal-stack.io/no-controller-connection=true` + - You can make the firewall become ready anyway by setting the annotation `kubectl annotate fw -n firewall firewall.metal-stack.io/no-controller-connection=true` diff --git a/api/v2/config/controller.go b/api/v2/config/controller.go index f12a185..8a607e3 100644 --- a/api/v2/config/controller.go +++ b/api/v2/config/controller.go @@ -182,10 +182,10 @@ func (c *NewControllerConfig) validate() error { if c.ProgressDeadline <= 0 { return fmt.Errorf("progress deadline must be specified") } - if c.FirewallHealthTimeout <= 0 { + if c.FirewallHealthTimeout < 0 { return fmt.Errorf("firewall health timeout must be specified") } - if c.CreateTimeout <= 0 { + if c.CreateTimeout < 0 { return fmt.Errorf("create timeout must be specified") } diff --git a/api/v2/types_firewall.go b/api/v2/types_firewall.go index 34b18e8..c8cf401 100644 --- a/api/v2/types_firewall.go +++ b/api/v2/types_firewall.go @@ -1,8 +1,10 @@ package v2 import ( + "fmt" "sort" "strconv" + "time" "github.com/metal-stack/metal-lib/pkg/pointer" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -183,6 +185,9 @@ const ( FirewallMonitorDeployed ConditionType = "MonitorDeployed" // FirewallDistanceConfigured indicates that the firewall-controller has configured the given firewall distance. FirewallDistanceConfigured ConditionType = "Distance" + // FirewallProvisioned indicates that all health conditions have been met at least once. + // Once set to true, it stays true and is used to detect condition degradation. + FirewallProvisioned ConditionType = "Provisioned" ) // ShootAccess contains secret references to construct a shoot client in the firewall-controller to update its firewall monitor. @@ -351,3 +356,169 @@ func SortFirewallsByImportance(fws []*Firewall) { return !a.CreationTimestamp.Before(&b.CreationTimestamp) }) } + +type ( + FirewallStatusResult string + + FirewallStatusEvalResult struct { + Result FirewallStatusResult + Reason string + TimeoutIn *time.Duration + } +) + +const ( + FirewallStatusReady FirewallStatusResult = "ready" + FirewallStatusProgressing FirewallStatusResult = "progressing" + FirewallStatusUnhealthy FirewallStatusResult = "unhealthy" + FirewallStatusHealthTimeout FirewallStatusResult = "health-timeout" + FirewallStatusCreateTimeout FirewallStatusResult = "create-timeout" +) + +func EvaluateFirewallStatus(fw *Firewall, createTimeout, healthTimeout time.Duration) *FirewallStatusEvalResult { + var ( + checkForTimeout = func(fw *Firewall, condition ConditionType, timeout time.Duration) (time.Duration, bool) { + if timeout == 0 { + return 0, false + } + + var ( + cond = pointer.SafeDeref(fw.Status.Conditions.Get(condition)) + transitionTime = cond.LastTransitionTime.Time + deadline = time.Until(transitionTime.Add(timeout)) + ) + + if deadline < 0 { + return 0, true + } + + return deadline, false + } + + collectUnhealthyConditions = func(cts ...ConditionType) []*Condition { + var res []*Condition + + for _, ct := range cts { + cond := fw.Status.Conditions.Get(ct) + if cond == nil { + res = append(res, &Condition{Type: ct}) + } else if cond.Status != ConditionTrue { + res = append(res, cond) + } + } + + return res + } + + unhealthyTypes []string + timeoutIn *time.Duration + ) + + switch fw.Status.Phase { + case FirewallPhaseCreating, FirewallPhaseCrashing: + unhealthyConds := collectUnhealthyConditions( + FirewallCreated, + FirewallReady, + FirewallProvisioned, + ) + + if len(unhealthyConds) == 0 { + return &FirewallStatusEvalResult{ + Result: FirewallStatusReady, + Reason: "", + } + } + + if createTimeout > 0 { + if t, ok := checkForTimeout(fw, FirewallReady, createTimeout); ok { + return &FirewallStatusEvalResult{ + Result: FirewallStatusCreateTimeout, + Reason: fmt.Sprintf("%s create timeout exceeded, firewall not provisioned in time", createTimeout.String()), + } + } else if createTimeout != 0 { + timeoutIn = &t + } + } + + for _, c := range unhealthyConds { + unhealthyTypes = append(unhealthyTypes, string(c.Type)) + } + + return &FirewallStatusEvalResult{ + Result: FirewallStatusProgressing, + Reason: fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes), + TimeoutIn: timeoutIn, + } + + case FirewallPhaseRunning: + fallthrough + + default: + unhealthyConds := collectUnhealthyConditions( + FirewallCreated, + FirewallReady, + FirewallProvisioned, + FirewallControllerConnected, + FirewallControllerSeedConnected, + FirewallDistanceConfigured, + ) + + if len(unhealthyConds) == 0 { + return &FirewallStatusEvalResult{ + Result: FirewallStatusReady, + Reason: "", + } + } + + var ( + ready = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallReady)).Status == ConditionTrue + provisioned = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallProvisioned)).Status == ConditionTrue + connected = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerConnected)).Status == ConditionTrue + seedConnected = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerSeedConnected)).Status == ConditionTrue + ) + + if provisioned { + switch { + case !seedConnected: + if t, ok := checkForTimeout(fw, FirewallControllerSeedConnected, healthTimeout); ok { + return &FirewallStatusEvalResult{ + Result: FirewallStatusHealthTimeout, + Reason: fmt.Sprintf("%s health timeout exceeded, seed connection lost", healthTimeout.String()), + } + } else if healthTimeout != 0 { + timeoutIn = &t + } + + case !connected: + if t, ok := checkForTimeout(fw, FirewallControllerConnected, healthTimeout); ok { + return &FirewallStatusEvalResult{ + Result: FirewallStatusHealthTimeout, + Reason: fmt.Sprintf("%s health timeout exceeded, firewall monitor not reconciled anymore", healthTimeout.String()), + } + } else if healthTimeout != 0 { + timeoutIn = &t + } + + case !ready: + if t, ok := checkForTimeout(fw, FirewallReady, healthTimeout); ok { + return &FirewallStatusEvalResult{ + Result: FirewallStatusHealthTimeout, + Reason: fmt.Sprintf("%s health timeout exceeded, firewall is not ready from perspective of the metal-api", healthTimeout.String()), + } + } else if healthTimeout != 0 { + timeoutIn = &t + } + } + } + + for _, c := range unhealthyConds { + unhealthyTypes = append(unhealthyTypes, string(c.Type)) + } + + return &FirewallStatusEvalResult{ + Result: FirewallStatusUnhealthy, + Reason: fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes), + TimeoutIn: timeoutIn, + } + } +} diff --git a/api/v2/types_firewall_test.go b/api/v2/types_firewall_test.go index e145730..142a912 100644 --- a/api/v2/types_firewall_test.go +++ b/api/v2/types_firewall_test.go @@ -6,7 +6,10 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/metal-stack/metal-lib/pkg/pointer" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "testing/synctest" ) func Test_SortFirewallsByImportance(t *testing.T) { @@ -107,3 +110,183 @@ func Test_SortFirewallsByImportance(t *testing.T) { }) } } + +func Test_EvaluateFirewallStatus(t *testing.T) { + tests := []struct { + name string + modFn func(fw *Firewall) + healthTimeout time.Duration + createTimeout time.Duration + want *FirewallStatusEvalResult + wantReason string + }{ + { + name: "ready firewall in running phase", + modFn: nil, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusReady, + }, + }, + { + name: "unhealthy firewall in running phase due to firewall monitor not reconciling", + modFn: func(fw *Firewall) { + fw.Status.Conditions.Set(Condition{ + Type: FirewallControllerConnected, + Status: ConditionFalse, + }) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusUnhealthy, + Reason: "not all health conditions are true: [Connected]", + }, + }, + { + name: "unhealthy firewall in running phase due to firewall not reconciling", + modFn: func(fw *Firewall) { + fw.Status.Conditions.Set(Condition{ + Type: FirewallControllerSeedConnected, + Status: ConditionFalse, + }) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusUnhealthy, + Reason: "not all health conditions are true: [SeedConnected]", + }, + }, + { + name: "unhealthy firewall in running phase due to readiness condition false", + modFn: func(fw *Firewall) { + fw.Status.Conditions.Set(Condition{ + Type: FirewallReady, + Status: ConditionFalse, + }) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusUnhealthy, + Reason: "not all health conditions are true: [Ready]", + }, + }, + { + name: "health timeout reached because seed not connected", + healthTimeout: 5 * time.Minute, + modFn: func(fw *Firewall) { + cond := fw.Status.Conditions.Get(FirewallControllerSeedConnected) + cond.Status = ConditionFalse + fw.Status.Conditions.Set(*cond) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusHealthTimeout, + Reason: "5m0s health timeout exceeded, seed connection lost", + }, + }, + { + name: "health timeout not yet reached", + healthTimeout: 15 * time.Minute, + modFn: func(fw *Firewall) { + cond := fw.Status.Conditions.Get(FirewallControllerSeedConnected) + cond.Status = ConditionFalse + fw.Status.Conditions.Set(*cond) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusUnhealthy, + Reason: "not all health conditions are true: [SeedConnected]", + TimeoutIn: pointer.Pointer(5 * time.Minute), + }, + }, + { + name: "create timeout reached because not provisioned", + createTimeout: 5 * time.Minute, + modFn: func(fw *Firewall) { + fw.Status.Phase = FirewallPhaseCreating + cond := fw.Status.Conditions.Get(FirewallProvisioned) + cond.Status = ConditionFalse + fw.Status.Conditions.Set(*cond) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusCreateTimeout, + Reason: "5m0s create timeout exceeded, firewall not provisioned in time", + }, + }, + { + name: "create timeout not yet reached", + createTimeout: 15 * time.Minute, + modFn: func(fw *Firewall) { + fw.Status.Phase = FirewallPhaseCreating + cond := fw.Status.Conditions.Get(FirewallProvisioned) + cond.Status = ConditionFalse + fw.Status.Conditions.Set(*cond) + }, + want: &FirewallStatusEvalResult{ + Result: FirewallStatusProgressing, + Reason: "not all health conditions are true: [Provisioned]", + TimeoutIn: pointer.Pointer(5 * time.Minute), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + tenMinutesAgo := time.Now().Add(-10 * time.Minute) + + fw := &Firewall{ + Status: FirewallStatus{ + Phase: FirewallPhaseRunning, + Conditions: Conditions{ + { + Type: FirewallControllerConnected, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallControllerSeedConnected, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallCreated, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallReady, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallProvisioned, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallDistanceConfigured, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + { + Type: FirewallMonitorDeployed, + Status: ConditionTrue, + LastTransitionTime: metav1.NewTime(tenMinutesAgo), + LastUpdateTime: metav1.NewTime(tenMinutesAgo), + }, + }, + }, + } + + if tt.modFn != nil { + tt.modFn(fw) + } + + got := EvaluateFirewallStatus(fw, tt.createTimeout, tt.healthTimeout) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("diff = %s", diff) + } + }) + }) + } +} diff --git a/api/v2/zz_generated.deepcopy.go b/api/v2/zz_generated.deepcopy.go index b9d9399..393c86c 100644 --- a/api/v2/zz_generated.deepcopy.go +++ b/api/v2/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v2 import ( runtime "k8s.io/apimachinery/pkg/runtime" + timex "time" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -742,6 +743,26 @@ func (in *FirewallStatus) DeepCopy() *FirewallStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FirewallStatusEvalResult) DeepCopyInto(out *FirewallStatusEvalResult) { + *out = *in + if in.TimeoutIn != nil { + in, out := &in.TimeoutIn, &out.TimeoutIn + *out = new(timex.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FirewallStatusEvalResult. +func (in *FirewallStatusEvalResult) DeepCopy() *FirewallStatusEvalResult { + if in == nil { + return nil + } + out := new(FirewallStatusEvalResult) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FirewallTemplateSpec) DeepCopyInto(out *FirewallTemplateSpec) { *out = *in diff --git a/config/examples/certs/ca-key.pem b/config/examples/certs/ca-key.pem index 8f5fb66..542f656 100644 --- a/config/examples/certs/ca-key.pem +++ b/config/examples/certs/ca-key.pem @@ -1,5 +1,5 @@ -----BEGIN EC PRIVATE KEY----- -MHcCAQEEIBRabFggNFg6LUPxY5AeplDzeqZQmnsnFY9OmWQW2eGBoAoGCCqGSM49 -AwEHoUQDQgAEkP91tJGv5pIytEgKOlwTeksfWC1MczdEmj8ouOiaQfFvCkLl5NB/ -uRLrjoR8vDamER2UM+BumDy1XfM849aIww== +MHcCAQEEIMdzRnQT5XJYI5YdllH2IC4TDpkkoswIUSPxVggCmz8uoAoGCCqGSM49 +AwEHoUQDQgAEzPBxsUSwbxKnyOHzLBxJtne4EKF2dktJ7cgiq88H4i2QWvH8Eu5f +WlSuos1/tjF7NdnZwdR3F09M3FWN2z32vw== -----END EC PRIVATE KEY----- diff --git a/config/examples/certs/ca.pem b/config/examples/certs/ca.pem index dbb0bc8..01cbd20 100644 --- a/config/examples/certs/ca.pem +++ b/config/examples/certs/ca.pem @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- -MIIBvTCCAWSgAwIBAgIUY2eiJLpYQK4h35iDJbGsUPZlsAcwCgYIKoZIzj0EAwIw +MIIBvTCCAWSgAwIBAgIUK74MlGBl5v/PxcvYR1gX/4ZahecwCgYIKoZIzj0EAwIw PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp -YTELMAkGA1UEAxMCY2EwHhcNMjMwNDE4MDc1NDAwWhcNMjgwNDE2MDc1NDAwWjA9 +YTELMAkGA1UEAxMCY2EwHhcNMjQxMDI1MTI0MDAwWhcNMjkxMDI0MTI0MDAwWjA9 MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh -MQswCQYDVQQDEwJjYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJD/dbSRr+aS -MrRICjpcE3pLH1gtTHM3RJo/KLjomkHxbwpC5eTQf7kS646EfLw2phEdlDPgbpg8 -tV3zPOPWiMOjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G -A1UdDgQWBBRL7+6t0aYt/vvqePoDdyJsQ6DQ5jAKBggqhkjOPQQDAgNHADBEAiB5 -4nITXzq23b7HZWf/TN22DQX+9Ajc2xOws2lwlx8TpQIgSP0zTa3yGeabqBgjmANZ -GTYZaSABLBAoQ1Lt5E6sCVs= +MQswCQYDVQQDEwJjYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABMzwcbFEsG8S +p8jh8ywcSbZ3uBChdnZLSe3IIqvPB+ItkFrx/BLuX1pUrqLNf7YxezXZ2cHUdxdP +TNxVjds99r+jQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBRmKUtHhVtOaft2ka15nfnH6agg8zAKBggqhkjOPQQDAgNHADBEAiAz +dCfM0jLlTDzaEXz5z1XEg8LhJWQV5YYoF+DUlJiU/gIgfSvcno9zARAKNNH06qF0 +XCzKTrC60QhD+N1wFN7X2og= -----END CERTIFICATE----- diff --git a/config/examples/certs/tls.crt b/config/examples/certs/tls.crt index 58f1a0f..8df5e63 100644 --- a/config/examples/certs/tls.crt +++ b/config/examples/certs/tls.crt @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE----- -MIICRDCCAeqgAwIBAgIUHwoSR0+noLCqqJ10vEJkTAng4GowCgYIKoZIzj0EAwIw +MIICQzCCAeqgAwIBAgIUZtyTg/sZOeE2HL7hDL6lVCo+QBcwCgYIKoZIzj0EAwIw PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp -YTELMAkGA1UEAxMCY2EwHhcNMjMwNDE4MDc1NDAwWhcNMjQwNDE3MDc1NDAwWjBE +YTELMAkGA1UEAxMCY2EwHhcNMjQxMDI1MTI0MDAwWhcNMjUxMDI1MTI0MDAwWjBE MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh -MRIwEAYDVQQDEwlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARf -FNJn/7dufCbR0AC+BnTyvhn98yvOiD+ASWXaVYeBgsuB9GfUWlVyp+fjdAkgWNZd -4S4uNz6aD1G/KlE6GBFQo4HAMIG9MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUj7eN -RLAK/BYVXeJfk6iS1xyJZRowHwYDVR0jBBgwFoAUS+/urdGmLf776nj6A3cibEOg -0OYwPgYDVR0RBDcwNYIJbG9jYWxob3N0gihmaXJld2FsbC1jb250cm9sbGVyLW1h -bmFnZXIuZmlyZXdhbGwuc3ZjMAoGCCqGSM49BAMCA0gAMEUCIQDsfaRwE5W901yK -JAQfSYlT+txLN8cdseHeDLXTwBo2IAIgV0g9f6F8KbyY6dvPHkoArRbZMIa3PFyL -/rflwrZzrPY= +MRIwEAYDVQQDEwlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARN +eruOjegpfrIkOew6QNy5HsOXzL+Oie/ubpUxphleQhX7/pLjGNvo8ueWDyN0ZZ0G +vxexgYUDZkXh19dg9RzQo4HAMIG9MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUyxBq +6HMZNcJlyn+b0GRQqPwvepgwHwYDVR0jBBgwFoAUZilLR4VbTmn7dpGteZ35x+mo +IPMwPgYDVR0RBDcwNYIJbG9jYWxob3N0gihmaXJld2FsbC1jb250cm9sbGVyLW1h +bmFnZXIuZmlyZXdhbGwuc3ZjMAoGCCqGSM49BAMCA0cAMEQCIEIHZ3Uj6fNvYgKv +JbI28i8nsdF3PbCGhLW6XnFABwqBAiAP9KPZf9zAAN8DHum2s1sOYTVOHGm4drkq +NLAFeNNXbg== -----END CERTIFICATE----- diff --git a/config/examples/certs/tls.key b/config/examples/certs/tls.key index 6af5725..7e37484 100644 --- a/config/examples/certs/tls.key +++ b/config/examples/certs/tls.key @@ -1,5 +1,5 @@ -----BEGIN EC PRIVATE KEY----- -MHcCAQEEIGkp4UEW0A/611PSa/ryMg+7c2yB11ZqtA/GR1yMaeq+oAoGCCqGSM49 -AwEHoUQDQgAEXxTSZ/+3bnwm0dAAvgZ08r4Z/fMrzog/gEll2lWHgYLLgfRn1FpV -cqfn43QJIFjWXeEuLjc+mg9RvypROhgRUA== +MHcCAQEEIJZT9vmyYJDxyP3gyJpkeS02M0hgXlrrrjTCmlmUOcQ0oAoGCCqGSM49 +AwEHoUQDQgAETXq7jo3oKX6yJDnsOkDcuR7Dl8y/jonv7m6VMaYZXkIV+/6S4xjb +6PLnlg8jdGWdBr8XsYGFA2ZF4dfXYPUc0A== -----END EC PRIVATE KEY----- diff --git a/config/examples/kustomize/patch-webhooks.yaml b/config/examples/kustomize/patch-webhooks.yaml index 5a5d127..9ab2d62 100644 --- a/config/examples/kustomize/patch-webhooks.yaml +++ b/config/examples/kustomize/patch-webhooks.yaml @@ -4,45 +4,45 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration webhooks: -- name: firewall.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall -- name: firewallset.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall -- name: firewalldeployment.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall + - name: firewall.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall + - name: firewallset.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall + - name: firewalldeployment.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: validating-webhook-configuration webhooks: -- name: firewall.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall -- name: firewallset.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall -- name: firewalldeployment.metal-stack.io - clientConfig: - caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVWTJlaUpMcFlRSzRoMzVpREpiR3NVUFpsc0Fjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05Nak13TkRFNE1EYzFOREF3V2hjTk1qZ3dOREUyTURjMU5EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJKRC9kYlNScithUwpNclJJQ2pwY0UzcExIMWd0VEhNM1JKby9LTGpvbWtIeGJ3cEM1ZVRRZjdrUzY0NkVmTHcycGhFZGxEUGdicGc4CnRWM3pQT1BXaU1PalFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJMNys2dDBhWXQvdnZxZVBvRGR5SnNRNkRRNWpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCNQo0bklUWHpxMjNiN0haV2YvVE4yMkRRWCs5QWpjMnhPd3MybHdseDhUcFFJZ1NQMHpUYTN5R2VhYnFCZ2ptQU5aCkdUWVphU0FCTEJBb1ExTHQ1RTZzQ1ZzPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - service: - name: firewall-controller-manager - namespace: firewall + - name: firewall.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall + - name: firewallset.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall + - name: firewalldeployment.metal-stack.io + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ2VENDQVdTZ0F3SUJBZ0lVSzc0TWxHQmw1di9QeGN2WVIxZ1gvNFphaGVjd0NnWUlLb1pJemowRUF3SXcKUFRFTE1Ba0dBMVVFQmhNQ1JFVXhEekFOQmdOVkJBZ1RCazExYm1samFERVFNQTRHQTFVRUJ4TUhRbUYyWVhKcApZVEVMTUFrR0ExVUVBeE1DWTJFd0hoY05NalF4TURJMU1USTBNREF3V2hjTk1qa3hNREkwTVRJME1EQXdXakE5Ck1Rc3dDUVlEVlFRR0V3SkVSVEVQTUEwR0ExVUVDQk1HVFhWdWFXTm9NUkF3RGdZRFZRUUhFd2RDWVhaaGNtbGgKTVFzd0NRWURWUVFERXdKallUQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJNendjYkZFc0c4UwpwOGpoOHl3Y1NiWjN1QkNoZG5aTFNlM0lJcXZQQitJdGtGcngvQkx1WDFwVXJxTE5mN1l4ZXpYWjJjSFVkeGRQClROeFZqZHM5OXIralFqQkFNQTRHQTFVZER3RUIvd1FFQXdJQkJqQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEcKQTFVZERnUVdCQlJtS1V0SGhWdE9hZnQya2ExNW5mbkg2YWdnOHpBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlBegpkQ2ZNMGpMbFREemFFWHo1ejFYRWc4TGhKV1FWNVlZb0YrRFVsSmlVL2dJZ2ZTdmNubzl6QVJBS05OSDA2cUYwClhDektUckM2MFFoRCtOMXdGTjdYMm9nPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + service: + name: firewall-controller-manager + namespace: firewall diff --git a/controllers/deployment/delete.go b/controllers/deployment/delete.go index 7d0298c..5e77bd4 100644 --- a/controllers/deployment/delete.go +++ b/controllers/deployment/delete.go @@ -6,6 +6,7 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" + corev1 "k8s.io/api/core/v1" ) func (c *controller) Delete(r *controllers.Ctx[*v2.FirewallDeployment]) error { @@ -33,7 +34,7 @@ func (c *controller) deleteFirewallSets(r *controllers.Ctx[*v2.FirewallDeploymen r.Log.Info("set deletion timestamp on firewall set", "set-name", set.Name) - c.recorder.Eventf(set, nil, "Normal", "Delete", "deleted firewallset %s", set.Name) + c.recorder.Eventf(set, nil, corev1.EventTypeNormal, "Delete", "deleting set", "deleted firewall set %s", set.Name) } if len(sets) > 0 { diff --git a/controllers/deployment/reconcile.go b/controllers/deployment/reconcile.go index ef9c981..7ab6c5c 100644 --- a/controllers/deployment/reconcile.go +++ b/controllers/deployment/reconcile.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" @@ -217,7 +218,7 @@ func (c *controller) syncFirewallSet(r *controllers.Ctx[*v2.FirewallDeployment], cond := v2.NewCondition(v2.FirewallDeplomentProgressing, v2.ConditionTrue, "FirewallSetUpdated", fmt.Sprintf("Updated firewall set %q.", set.Name)) r.Target.Status.Conditions.Set(cond) - c.recorder.Eventf(set, nil, "Normal", "Update", "updated firewallset %s", set.Name) + c.recorder.Eventf(set, nil, corev1.EventTypeNormal, "Update", "updating set", "updated firewall set %s", set.Name) return nil } diff --git a/controllers/deployment/recreate.go b/controllers/deployment/recreate.go index da483c1..f5de6ba 100644 --- a/controllers/deployment/recreate.go +++ b/controllers/deployment/recreate.go @@ -6,6 +6,8 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" + + corev1 "k8s.io/api/core/v1" ) // recreateStrategy first deletes the existing firewall sets and then creates a new one @@ -20,7 +22,7 @@ func (c *controller) recreateStrategy(r *controllers.Ctx[*v2.FirewallDeployment] return err } - c.recorder.Eventf(set, nil, "Normal", "Recreate", "recreated firewallset old: %s new: %s", latestSet.Name, set.Name) + c.recorder.Eventf(set, nil, corev1.EventTypeNormal, "Recreate", "recreating set", "recreated firewall set, old: %s new: %s", latestSet.Name, set.Name) latestSet = set } diff --git a/controllers/deployment/rolling.go b/controllers/deployment/rolling.go index 07de9f6..61c48a5 100644 --- a/controllers/deployment/rolling.go +++ b/controllers/deployment/rolling.go @@ -6,6 +6,7 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" + corev1 "k8s.io/api/core/v1" ) // rollingUpdateStrategy first creates a new set and deletes the old one's when the new one becomes ready @@ -20,7 +21,7 @@ func (c *controller) rollingUpdateStrategy(r *controllers.Ctx[*v2.FirewallDeploy return err } - c.recorder.Eventf(newSet, nil, "Normal", "Create", "created firewallset %s", newSet.Name) + c.recorder.Eventf(newSet, nil, corev1.EventTypeNormal, "Create", "creating set", "created firewall set %s", newSet.Name) ownedSets = append(ownedSets, newSet) diff --git a/controllers/firewall/delete.go b/controllers/firewall/delete.go index 191584c..20f1de1 100644 --- a/controllers/firewall/delete.go +++ b/controllers/firewall/delete.go @@ -9,6 +9,8 @@ import ( "github.com/metal-stack/firewall-controller-manager/controllers" "github.com/metal-stack/metal-go/api/client/machine" apierrors "k8s.io/apimachinery/pkg/api/errors" + + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -46,7 +48,7 @@ func (c *controller) Delete(r *controllers.Ctx[*v2.Firewall]) error { r.Log.Info("deleted firewall", "firewall-name", f.Name, "id", *resp.Payload.ID) - c.recorder.Eventf(r.Target, nil, "Normal", "Delete", "deleted firewall %s id %s", r.Target.Name, *resp.Payload.ID) + c.recorder.Eventf(r.Target, nil, corev1.EventTypeNormal, "Delete", "deleting firewall", "deleted firewall %s id %s", r.Target.Name, *resp.Payload.ID) } return nil diff --git a/controllers/firewall/reconcile.go b/controllers/firewall/reconcile.go index 26170ae..13f25e6 100644 --- a/controllers/firewall/reconcile.go +++ b/controllers/firewall/reconcile.go @@ -12,6 +12,8 @@ import ( "github.com/metal-stack/metal-go/api/client/machine" "github.com/metal-stack/metal-go/api/models" "github.com/metal-stack/metal-lib/pkg/pointer" + + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" ) @@ -54,6 +56,11 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.Firewall]) error { cond := v2.NewCondition(v2.FirewallCreated, v2.ConditionTrue, "Created", fmt.Sprintf("Firewall %q created successfully.", pointer.SafeDeref(pointer.SafeDeref(f.Allocation).Name))) r.Target.Status.Conditions.Set(cond) + // this is mainly for tests when the firewall is already present + if r.Target.Status.Phase == v2.FirewallPhase("") { + r.Target.Status.Phase = v2.FirewallPhaseCreating + } + var currentStatus *v2.MachineStatus currentStatus, err = getMachineStatus(f) if err != nil { @@ -169,7 +176,7 @@ func (c *controller) createFirewall(r *controllers.Ctx[*v2.Firewall]) (*models.V cond := v2.NewCondition(v2.FirewallCreated, v2.ConditionTrue, "Created", fmt.Sprintf("Firewall %q created successfully.", pointer.SafeDeref(pointer.SafeDeref(resp.Payload.Allocation).Name))) r.Target.Status.Conditions.Set(cond) - c.recorder.Eventf(r.Target, nil, "Normal", "Create", "created firewall %s id %s", r.Target.Name, pointer.SafeDeref(resp.Payload.ID)) + c.recorder.Eventf(r.Target, nil, corev1.EventTypeNormal, "Create", "created firewall %s id %s", "creating firewall", r.Target.Name, pointer.SafeDeref(resp.Payload.ID)) return resp.Payload, nil } @@ -189,6 +196,7 @@ func isFirewallProgressing(status *v2.MachineStatus) bool { if status.LastEvent.Event != "Phoned Home" { return true } + return false } @@ -207,6 +215,7 @@ func isFirewallReady(status *v2.MachineStatus) bool { if status.LastEvent.Event == "Phoned Home" { return true } + return false } diff --git a/controllers/firewall/status.go b/controllers/firewall/status.go index cb8c564..38a1e09 100644 --- a/controllers/firewall/status.go +++ b/controllers/firewall/status.go @@ -129,6 +129,11 @@ func SetFirewallStatusFromMonitor(fw *v2.Firewall, mon *v2.FirewallMonitor) { cond = v2.NewCondition(v2.FirewallDistanceConfigured, v2.ConditionTrue, "NotChecking", "Not checking distance due to firewall annotation.") fw.Status.Conditions.Set(cond) + if isProvisioned(fw) { + cond := v2.NewCondition(v2.FirewallProvisioned, v2.ConditionTrue, "Provisioned", "All firewall conditions have been met.") + fw.Status.Conditions.Set(cond) + } + return } @@ -155,11 +160,30 @@ func SetFirewallStatusFromMonitor(fw *v2.Firewall, mon *v2.FirewallMonitor) { fw.Status.ControllerStatus = connection + var ( + // currently, the firewall-controller writes the reconcile time hard-coded every three minutes + // the FCM reconciles the firewall hard-coded at least every two minutes + // + // this can be visualized as: + // + // fc (write) w w w w + // | | | | + // t (minutes) 0--1--2--3--4--5--6--7--8--9--10-- + // | | | | | | + // FCM (read) r r r r r r + // + // so, read out data will contain t={0, 0, 3, 6, 6, 9}, which shows that the maximum distance is three minutes + maximumSeedUpdateDrift = 3 * time.Minute + // in this case, the firewall-controller almost permanently updates this value (fw.Spec.Interval, by default 10s) + // so we can assume the read out interval from the fcm firewall reconcile, which is maximum two minutes as described above + maximumShootUpdateDrift = 2 * time.Minute + ) + // Check if the firewall-controller has reconciled the shoot if connection.Updated.Time.IsZero() { cond := v2.NewCondition(v2.FirewallControllerConnected, v2.ConditionFalse, "NotConnected", "Controller has not yet connected to shoot.") fw.Status.Conditions.Set(cond) - } else if time.Since(connection.Updated.Time) > 5*time.Minute { + } else if time.Since(connection.Updated.Time) > maximumShootUpdateDrift { cond := v2.NewCondition(v2.FirewallControllerConnected, v2.ConditionFalse, "StoppedReconciling", fmt.Sprintf("Controller has stopped reconciling since %s to shoot.", connection.Updated.String())) fw.Status.Conditions.Set(cond) } else { @@ -171,7 +195,7 @@ func SetFirewallStatusFromMonitor(fw *v2.Firewall, mon *v2.FirewallMonitor) { if connection.SeedUpdated.Time.IsZero() { cond := v2.NewCondition(v2.FirewallControllerSeedConnected, v2.ConditionFalse, "NotConnected", "Controller has not yet connected to seed.") fw.Status.Conditions.Set(cond) - } else if time.Since(connection.SeedUpdated.Time) > 5*time.Minute { + } else if time.Since(connection.SeedUpdated.Time) > maximumSeedUpdateDrift { cond := v2.NewCondition(v2.FirewallControllerSeedConnected, v2.ConditionFalse, "StoppedReconciling", fmt.Sprintf("Controller has stopped reconciling since %s to seed.", connection.SeedUpdated.String())) fw.Status.Conditions.Set(cond) } else { @@ -190,4 +214,25 @@ func SetFirewallStatusFromMonitor(fw *v2.Firewall, mon *v2.FirewallMonitor) { cond := v2.NewCondition(v2.FirewallDistanceConfigured, v2.ConditionFalse, "NotConfigured", fmt.Sprintf("Controller has configured distance %d, but %d is specified.", connection.ActualDistance, fw.Distance)) fw.Status.Conditions.Set(cond) } + + if isProvisioned(fw) { + cond := v2.NewCondition(v2.FirewallProvisioned, v2.ConditionTrue, "Provisioned", "All firewall conditions have been met.") + fw.Status.Conditions.Set(cond) + } +} + +func isProvisioned(fw *v2.Firewall) bool { + for _, ct := range []v2.ConditionType{ + v2.FirewallCreated, + v2.FirewallReady, + v2.FirewallControllerConnected, + v2.FirewallControllerSeedConnected, + v2.FirewallDistanceConfigured, + } { + cond := fw.Status.Conditions.Get(ct) + if cond == nil || cond.Status != v2.ConditionTrue { + return false + } + } + return true } diff --git a/controllers/generic_controller.go b/controllers/generic_controller.go index 5a585e2..73408e3 100644 --- a/controllers/generic_controller.go +++ b/controllers/generic_controller.go @@ -103,8 +103,7 @@ func (g GenericController[O]) Reconcile(ctx context.Context, req ctrl.Request) ( log.Info("reconciling resource deletion flow") err := g.reconciler.Delete(rctx) if err != nil { - var requeueErr *requeueError - if errors.As(err, &requeueErr) { + if requeueErr, ok := errors.AsType[*requeueError](err); ok { log.Info(requeueErr.Error()) return ctrl.Result{RequeueAfter: requeueErr.after}, nil //nolint:nilerr we need to return nil such that the requeue works } @@ -192,16 +191,13 @@ func (g GenericController[O]) Reconcile(ctx context.Context, req ctrl.Request) ( err := g.reconciler.Reconcile(rctx) if err != nil { - var requeueErr *requeueError - - switch { - case errors.As(err, &requeueErr): + if requeueErr, ok := errors.AsType[*requeueError](err); ok { log.Info(requeueErr.Error()) return ctrl.Result{RequeueAfter: requeueErr.after}, nil //nolint:nilerr we need to return nil such that the requeue works - default: - log.Error(err, "error during reconcile") - return ctrl.Result{}, err } + + log.Error(err, "error during reconcile") + return ctrl.Result{}, err } return ctrl.Result{}, statusErr diff --git a/controllers/set/delete.go b/controllers/set/delete.go index 8d0e1ce..1ab22a4 100644 --- a/controllers/set/delete.go +++ b/controllers/set/delete.go @@ -6,7 +6,8 @@ import ( v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" - "github.com/metal-stack/metal-lib/pkg/pointer" + + corev1 "k8s.io/api/core/v1" ) func (c *controller) Delete(r *controllers.Ctx[*v2.FirewallSet]) error { @@ -34,7 +35,7 @@ func (c *controller) deleteFirewalls(r *controllers.Ctx[*v2.FirewallSet], fws .. r.Log.Info("set deletion timestamp on firewall", "firewall-name", fw.Name) - c.recorder.Eventf(fw, nil, "Normal", "Delete", "deleted firewall %s", fw.Name) + c.recorder.Eventf(fw, nil, corev1.EventTypeNormal, "Delete", "deleting firewall", "deleted firewall %s", fw.Name) } if len(fws) > 0 { @@ -43,28 +44,3 @@ func (c *controller) deleteFirewalls(r *controllers.Ctx[*v2.FirewallSet], fws .. return nil } - -func (c *controller) deleteAfterTimeout(r *controllers.Ctx[*v2.FirewallSet], fws ...*v2.Firewall) ([]*v2.Firewall, error) { - var result []*v2.Firewall - - for _, fw := range fws { - if fw.Status.Phase != v2.FirewallPhaseCreating { - continue - } - - connected := pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallControllerConnected)).Status == v2.ConditionTrue - - if !connected && time.Since(fw.CreationTimestamp.Time) > c.c.GetCreateTimeout() { - r.Log.Info("firewall not getting ready, deleting from set", "firewall-name", fw.Name) - - err := c.deleteFirewalls(r, fw) - if err != nil { - return nil, err - } - - result = append(result, fw) - } - } - - return result, nil -} diff --git a/controllers/set/reconcile.go b/controllers/set/reconcile.go index b557f5e..3f0aa53 100644 --- a/controllers/set/reconcile.go +++ b/controllers/set/reconcile.go @@ -8,8 +8,10 @@ import ( "github.com/google/uuid" v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallSet]) error { @@ -90,7 +92,7 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallSet]) error { r.Log.Info("firewall created", "firewall-name", fw.Name) - c.recorder.Eventf(r.Target, nil, "Normal", "Create", "created firewall %s", fw.Name) + c.recorder.Eventf(r.Target, nil, corev1.EventTypeNormal, "Create", "creating firewall", "created firewall %s", fw.Name) ownedFirewalls = append(ownedFirewalls, fw) } @@ -110,13 +112,6 @@ func (c *controller) Reconcile(r *controllers.Ctx[*v2.FirewallSet]) error { } } - deletedFws, err := c.deleteAfterTimeout(r, ownedFirewalls...) - if err != nil { - return err - } - - ownedFirewalls = controllers.Except(ownedFirewalls, deletedFws...) - err = c.setStatus(r, ownedFirewalls) if err != nil { return err diff --git a/controllers/set/status.go b/controllers/set/status.go index a71b8d2..7b05812 100644 --- a/controllers/set/status.go +++ b/controllers/set/status.go @@ -1,42 +1,32 @@ package set import ( - "time" - v2 "github.com/metal-stack/firewall-controller-manager/api/v2" "github.com/metal-stack/firewall-controller-manager/controllers" - "github.com/metal-stack/metal-lib/pkg/pointer" ) func (c *controller) setStatus(r *controllers.Ctx[*v2.FirewallSet], ownedFirewalls []*v2.Firewall) error { r.Target.Status.TargetReplicas = r.Target.Spec.Replicas - r.Target.Status.ReadyReplicas = 0 r.Target.Status.ProgressingReplicas = 0 r.Target.Status.UnhealthyReplicas = 0 for _, fw := range ownedFirewalls { - var ( - fw = fw + status := v2.EvaluateFirewallStatus(fw, c.c.GetCreateTimeout(), c.c.GetFirewallHealthTimeout()) - created = pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallCreated)).Status == v2.ConditionTrue - ready = pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallReady)).Status == v2.ConditionTrue - connected = pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallControllerConnected)).Status == v2.ConditionTrue - seedConnected = pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallControllerSeedConnected)).Status == v2.ConditionTrue - distance = pointer.SafeDeref(fw.Status.Conditions.Get(v2.FirewallDistanceConfigured)).Status == v2.ConditionTrue - ) - - if created && ready && connected && seedConnected && distance { + switch status.Result { + case v2.FirewallStatusReady: r.Target.Status.ReadyReplicas++ continue - } - - if created && time.Since(pointer.SafeDeref(fw.Status.MachineStatus).AllocationTimestamp.Time) < c.c.GetFirewallHealthTimeout() { + case v2.FirewallStatusProgressing: r.Target.Status.ProgressingReplicas++ continue + case v2.FirewallStatusUnhealthy, v2.FirewallStatusCreateTimeout, v2.FirewallStatusHealthTimeout: + fallthrough + default: + r.Target.Status.UnhealthyReplicas++ + continue } - - r.Target.Status.UnhealthyReplicas++ } revision, err := controllers.Revision(r.Target) diff --git a/controllers/timeout/controller.go b/controllers/timeout/controller.go new file mode 100644 index 0000000..a7bbc31 --- /dev/null +++ b/controllers/timeout/controller.go @@ -0,0 +1,44 @@ +package timeout + +import ( + "github.com/go-logr/logr" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + v2 "github.com/metal-stack/firewall-controller-manager/api/v2" + "github.com/metal-stack/firewall-controller-manager/api/v2/config" + "github.com/metal-stack/firewall-controller-manager/controllers" +) + +type controller struct { + c *config.ControllerConfig + client client.Client + namespace string + log logr.Logger + recorder events.EventRecorder +} + +func SetupWithManager(log logr.Logger, recorder events.EventRecorder, mgr ctrl.Manager, c *config.ControllerConfig) error { + if c.GetFirewallHealthTimeout() <= 0 && c.GetCreateTimeout() <= 0 { + log.Info("not registering timeout controller because neither create nor health timeout configured") + return nil + } + + g := &controller{ + c: c, + log: log, + client: c.GetSeedClient(), + namespace: c.GetSeedNamespace(), + recorder: recorder, + } + + return ctrl.NewControllerManagedBy(mgr). + For( + &v2.FirewallSet{}, + ). + Named("FirewallHealthTimeout"). + WithEventFilter(predicate.NewPredicateFuncs(controllers.SkipOtherNamespace(c.GetSeedNamespace()))). + Complete(g) +} diff --git a/controllers/timeout/reconcile.go b/controllers/timeout/reconcile.go new file mode 100644 index 0000000..b072efb --- /dev/null +++ b/controllers/timeout/reconcile.go @@ -0,0 +1,102 @@ +package timeout + +import ( + "context" + "fmt" + "sort" + + v2 "github.com/metal-stack/firewall-controller-manager/api/v2" + "github.com/metal-stack/firewall-controller-manager/controllers" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" +) + +func (c *controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if req.Namespace != c.namespace { // should already be filtered out through predicate, but we will check anyway + return ctrl.Result{}, nil + } + + set := &v2.FirewallSet{} + if err := c.client.Get(ctx, req.NamespacedName, set, &client.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + c.log.Info("resource no longer exists") + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("error retrieving resource: %w", err) + } + + if !set.GetDeletionTimestamp().IsZero() { + return ctrl.Result{}, nil + } + + ownedFirewalls, _, err := controllers.GetOwnedResources(ctx, c.c.GetSeedClient(), set.Spec.Selector, set, &v2.FirewallList{}, func(fl *v2.FirewallList) []*v2.Firewall { + return fl.GetItems() + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("unable to get owned firewalls: %w", err) + } + + return c.deleteIfUnhealthyOrTimeout(ctx, ownedFirewalls...) +} + +func (c *controller) deleteIfUnhealthyOrTimeout(ctx context.Context, fws ...*v2.Firewall) (ctrl.Result, error) { + type fwWithStatus struct { + firewall *v2.Firewall + status *v2.FirewallStatusEvalResult + } + + var nextTimeouts []*fwWithStatus + + for _, fw := range fws { + status := v2.EvaluateFirewallStatus(fw, c.c.GetCreateTimeout(), c.c.GetFirewallHealthTimeout()) + + switch status.Result { + case v2.FirewallStatusCreateTimeout, v2.FirewallStatusHealthTimeout: + c.log.Info("firewall timeout exceeded, deleting from set", "reason", status.Reason, "firewall-name", fw.Name) + + if fw.DeletionTimestamp != nil { + c.log.Info("deletion timestamp on firewall already set", "firewall-name", fw.Name) + continue + } + + err := c.c.GetSeedClient().Delete(ctx, fw) + if err != nil { + return ctrl.Result{}, err + } + + c.recorder.Eventf(fw, nil, corev1.EventTypeNormal, "Delete", "deleting firewall", "deleted firewall %s due to %s", fw.Name, status) + + case v2.FirewallStatusUnhealthy: + if status.TimeoutIn != nil { + nextTimeouts = append(nextTimeouts, &fwWithStatus{ + firewall: fw, + status: status, + }) + } + } + } + + if len(nextTimeouts) > 0 { + sort.SliceStable(nextTimeouts, func(i, j int) bool { + return *nextTimeouts[i].status.TimeoutIn < *nextTimeouts[j].status.TimeoutIn + }) + + var ( + nextTimeout = nextTimeouts[0] + in = *nextTimeout.status.TimeoutIn + ) + + c.log.Info("scheduled check for next health timeout", "firewall-name", nextTimeout.firewall.Name, "reason", nextTimeout.status.Reason, "in", in.String()) + + return ctrl.Result{ + RequeueAfter: in, + }, nil + } + + return ctrl.Result{}, nil +} diff --git a/integration/integration_test.go b/integration/integration_test.go index 147aded..1b51a7f 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -176,6 +176,7 @@ var _ = Context("integration test", Ordered, func() { }, Machine: func(m *mock.Mock) { m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall1.ID}}, nil).Maybe() }, Image: func(m *mock.Mock) { m.On("FindLatestImage", mock.Anything, nil).Return(&image.FindLatestImageOK{Payload: image1}, nil).Maybe() @@ -314,6 +315,7 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Connected")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled shoot at %s.", mon.ControllerStatus.Updated.String()))) }) + It("should have the firewall-controller connected to seed condition true", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -324,6 +326,7 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Connected")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", mon.ControllerStatus.SeedUpdated.String()))) }) + It("should have configured the distance", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -334,6 +337,13 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Configured")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallShortestDistance))) }) + + It("should be in the running phase", func() { + Eventually(func() v2.FirewallPhase { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + return fw.Status.Phase + }, 5*time.Second, interval).Should(Equal(v2.FirewallPhaseRunning)) + }) }) Context("the firewall set resource", func() { @@ -579,6 +589,13 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.LastTransitionTime).NotTo(BeZero()) }) + It("should be in the creating phase", func() { + Eventually(func() v2.FirewallPhase { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + return fw.Status.Phase + }, 5*time.Second, interval).Should(Equal(v2.FirewallPhaseCreating)) + }) + It("should have firewall networks populated", func() { var nws []v2.FirewallNetwork var fw = fw.DeepCopy() @@ -1005,6 +1022,17 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *firewall1.Allocation.Name))) }) + It("should have the provisioned condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallProvisioned, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("All firewall conditions have been met.")) + }) + It("should have the monitor condition true", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -1026,6 +1054,7 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Connected")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled shoot at %s.", mon.ControllerStatus.Updated.String()))) }) + It("should have the firewall-controller connected to seed condition true", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -1036,6 +1065,7 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Connected")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller reconciled firewall at %s.", mon.ControllerStatus.SeedUpdated.String()))) }) + It("should have configured the distance", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -1046,6 +1076,13 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("Configured")) Expect(cond.Message).To(Equal(fmt.Sprintf("Controller has configured the specified distance %d.", v2.FirewallShortestDistance))) }) + + It("should be in the running phase", func() { + Eventually(func() v2.FirewallPhase { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + return fw.Status.Phase + }, 5*time.Second, interval).Should(Equal(v2.FirewallPhaseRunning)) + }) }) Context("the firewall set resource", func() { @@ -1373,6 +1410,17 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *readyFirewall.Allocation.Name))) }) + It("should have the provisioned condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallProvisioned, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("All firewall conditions have been met.")) + }) + It("should have the monitor condition true", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, newFw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -1767,6 +1815,17 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Message).To(Equal(fmt.Sprintf("Firewall %q is phoning home and alive.", *firewall1.Allocation.Name))) }) + It("should have the provisioned condition true", func() { + cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { + return fd.Status.Conditions + }, v2.FirewallProvisioned, v2.ConditionTrue, 15*time.Second) + + Expect(cond.LastTransitionTime).NotTo(BeZero()) + Expect(cond.LastUpdateTime).NotTo(BeZero()) + Expect(cond.Reason).To(Equal("Provisioned")) + Expect(cond.Message).To(Equal("All firewall conditions have been met.")) + }) + It("should have the monitor condition true", func() { cond := testcommon.WaitForCondition(k8sClient, ctx, fw.DeepCopy(), func(fd *v2.Firewall) v2.Conditions { return fd.Status.Conditions @@ -1788,6 +1847,13 @@ var _ = Context("integration test", Ordered, func() { Expect(cond.Reason).To(Equal("NotChecking")) Expect(cond.Message).To(Equal("Not checking controller connection due to firewall annotation.")) }) + + It("should be in the running phase", func() { + Eventually(func() v2.FirewallPhase { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fw), fw)).To(Succeed()) + return fw.Status.Phase + }, 5*time.Second, interval).Should(Equal(v2.FirewallPhaseRunning)) + }) }) Context("the firewall set resource", func() { @@ -1910,5 +1976,88 @@ var _ = Context("integration test", Ordered, func() { }) }) }) + }) + + When("creating a firewall set that simulates unhealthiness", Ordered, func() { + var firewallSet *v2.FirewallSet + + BeforeAll(func() { + swapMetalClient(&metalclient.MetalMockFns{ + Firewall: func(m *mock.Mock) { + m.On("AllocateFirewall", mock.Anything, nil).Return(&metalfirewall.AllocateFirewallOK{Payload: firewall3}, nil).Maybe() + m.On("FindFirewall", mock.Anything, nil).Return(&metalfirewall.FindFirewallOK{Payload: firewall3}, nil).Maybe() + m.On("FindFirewalls", mock.Anything, nil).Return(&metalfirewall.FindFirewallsOK{Payload: []*models.V1FirewallResponse{firewall3}}, nil).Maybe() + }, + Network: func(m *mock.Mock) { + m.On("FindNetwork", mock.Anything, nil).Return(&network.FindNetworkOK{Payload: network1}, nil).Maybe() + }, + Machine: func(m *mock.Mock) { + m.On("UpdateMachine", mock.Anything, nil).Return(&machine.UpdateMachineOK{Payload: &models.V1MachineResponse{}}, nil).Maybe() + m.On("FreeMachine", mock.Anything, nil).Return(&machine.FreeMachineOK{Payload: &models.V1MachineResponse{ID: firewall3.ID}}, nil).Maybe() + }, + Image: func(m *mock.Mock) { + m.On("FindLatestImage", mock.Anything, nil).Return(&image.FindLatestImageOK{Payload: image1}, nil).Maybe() + }, + }) + + Expect(k8sClient.Create(ctx, deployment())).To(Succeed()) + + Eventually(func() error { + firewallSetList := &v2.FirewallSetList{} + err := k8sClient.List(ctx, firewallSetList, client.InNamespace(namespaceName)) + if err != nil { + return err + } + if len(firewallSetList.Items) == 0 { + return fmt.Errorf("no firewall sets found") + } + firewallSet = &firewallSetList.Items[0] + return nil + }, 15*time.Second, interval).Should(Succeed(), "FirewallSet should be created") + }) + + It("should simulate unhealthiness and trigger deletion", func() { + firewallList := &v2.FirewallList{} + Eventually(func() int { + err := k8sClient.List(ctx, firewallList, client.InNamespace(firewallSet.Namespace)) + if err != nil { + return 0 + } + return len(firewallList.Items) + }, 15*time.Second, interval).Should(BeNumerically(">", 0), "Should have at least one firewall") + + Eventually(func() error { + for _, item := range firewallList.Items { + var fw v2.Firewall + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(&item), &fw) + if err != nil { + fmt.Printf("Failed to get firewall: %v\n", err) + return err + } + + if fw.Status.ControllerStatus == nil { + fw.Status.ControllerStatus = &v2.ControllerConnection{} + } + //add a fake concile so the unhealty firewall gets deleted + fw.Status.ControllerStatus.SeedUpdated.Time = time.Now().Add(-(firewallHealthTimeout + time.Minute)) + err = k8sClient.Status().Update(ctx, &fw) + if err != nil { + return fmt.Errorf("failed to update firewall status: %w", err) + } + } + + return nil + }, 10*time.Second, interval).Should(Succeed(), "All Firewalls should be deleted") + + By("verifying that a new firewall has been created") + Eventually(func() int { + newFirewallList := &v2.FirewallList{} + Expect(k8sClient.List(ctx, newFirewallList, client.InNamespace(firewallSet.Namespace))).To(Succeed()) + return len(newFirewallList.Items) + }, 10*time.Second, interval).Should(Equal(1), "A new firewall should be created") + }) + + }) + }) diff --git a/integration/metal_resources_test.go b/integration/metal_resources_test.go index a8c9605..24e7925 100644 --- a/integration/metal_resources_test.go +++ b/integration/metal_resources_test.go @@ -310,6 +310,92 @@ var ( Vrf: 50, Vrfshared: true, } + firewall3 = &models.V1FirewallResponse{ + Allocation: &models.V1MachineAllocation{ + BootInfo: &models.V1BootInfo{ + Bootloaderid: new("bootloaderid"), + Cmdline: new("cmdline"), + ImageID: new("imageid"), + Initrd: new("initrd"), + Kernel: new("kernel"), + OsPartition: new("ospartition"), + PrimaryDisk: new("primarydisk"), + }, + Created: new(strfmt.DateTime(testTime.Add(-20 * 24 * time.Hour))), + Creator: new("creator"), + Description: "firewall allocation 3", + Filesystemlayout: fsl1, + Hostname: new("firewall-hostname-3"), + Image: image1, + Name: new("firewall-3"), + Networks: []*models.V1MachineNetwork{ + { + Asn: new(int64(200)), + Destinationprefixes: []string{"2.2.2.2"}, + Ips: []string{"1.1.1.1"}, + Nat: new(false), + Networkid: new("private"), + Networktype: pointer.Pointer(net.PrivatePrimaryUnshared), + Prefixes: []string{"prefixes"}, + Private: new(true), + Underlay: new(false), + Vrf: new(int64(100)), + }, + }, + Project: new("project-1"), + Reinstall: new(false), + Role: pointer.Pointer(models.V1MachineAllocationRoleFirewall), + SSHPubKeys: []string{"sshpubkey"}, + Succeeded: new(true), + UserData: "---userdata---", + }, + Bios: &models.V1MachineBIOS{ + Date: new("biosdata"), + Vendor: new("biosvendor"), + Version: new("biosversion"), + }, + Description: "firewall 1", + Events: &models.V1MachineRecentProvisioningEvents{ + CrashLoop: new(true), + FailedMachineReclaim: new(true), + LastErrorEvent: &models.V1MachineProvisioningEvent{ + Event: new("Crashed"), + Message: "crash", + Time: strfmt.DateTime(testTime.Add(-10 * 24 * time.Hour)), + }, + LastEventTime: strfmt.DateTime(testTime.Add(-7 * 24 * time.Hour)), + Log: []*models.V1MachineProvisioningEvent{ + { + Event: new("Phoned Home"), + Message: "phoning home", + Time: strfmt.DateTime(testTime.Add(-7 * 24 * time.Hour)), + }, + }, + }, + Hardware: &models.V1MachineHardware{ + CPUCores: new(int32(16)), + Disks: []*models.V1MachineBlockDevice{}, + Memory: new(int64(32)), + Nics: []*models.V1MachineNic{}, + }, + ID: new("3"), + Ledstate: &models.V1ChassisIdentifyLEDState{ + Description: new(""), + Value: new(""), + }, + Liveliness: new("Unhealthy"), + Name: "firewall-3", + Partition: partition1, + Rackid: "rack-1", + Size: size1, + State: &models.V1MachineState{ + Description: new("state"), + Issuer: "issuer", + MetalHammerVersion: new("version"), + Value: new(""), + }, + Tags: []string{"a"}, + } ) // we are sharing a client for the tests, so we need to make sure we do not run contradicting tests in parallel diff --git a/integration/suite_test.go b/integration/suite_test.go index 46dcf94..8db2fed 100644 --- a/integration/suite_test.go +++ b/integration/suite_test.go @@ -16,6 +16,7 @@ import ( "github.com/metal-stack/firewall-controller-manager/controllers/firewall" "github.com/metal-stack/firewall-controller-manager/controllers/monitor" "github.com/metal-stack/firewall-controller-manager/controllers/set" + "github.com/metal-stack/firewall-controller-manager/controllers/timeout" "github.com/metal-stack/firewall-controller-manager/controllers/update" metalclient "github.com/metal-stack/metal-go/test/client" "github.com/metal-stack/metal-lib/pkg/tag" @@ -32,7 +33,9 @@ import ( ) const ( - namespaceName = "test" + namespaceName = "test" + firewallHealthTimeout = 19 * 24 * time.Hour + firewallCreateTimeout = 19 * 24 * time.Hour ) var ( @@ -130,8 +133,8 @@ var _ = BeforeSuite(func() { ClusterTag: fmt.Sprintf("%s=%s", tag.ClusterID, "cluster-a"), SafetyBackoff: 10 * time.Second, ProgressDeadline: 10 * time.Minute, - FirewallHealthTimeout: 20 * time.Minute, - CreateTimeout: 10 * time.Minute, + FirewallHealthTimeout: firewallHealthTimeout, + CreateTimeout: firewallCreateTimeout, }) Expect(err).ToNot(HaveOccurred()) @@ -167,6 +170,14 @@ var _ = BeforeSuite(func() { ) Expect(err).ToNot(HaveOccurred()) + err = timeout.SetupWithManager( + ctrl.Log.WithName("controllers").WithName("timeout"), + mgr.GetEventRecorder("timeout-controller"), + mgr, + cc, + ) + Expect(err).ToNot(HaveOccurred()) + err = deployment.SetupWebhookWithManager(ctrl.Log.WithName("defaulting-webhook"), mgr, cc) Expect(err).ToNot(HaveOccurred()) err = set.SetupWebhookWithManager(ctrl.Log.WithName("defaulting-webhook"), mgr, cc) diff --git a/main.go b/main.go index e68aad6..96e1893 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,7 @@ import ( "github.com/metal-stack/firewall-controller-manager/controllers/firewall" "github.com/metal-stack/firewall-controller-manager/controllers/monitor" "github.com/metal-stack/firewall-controller-manager/controllers/set" + "github.com/metal-stack/firewall-controller-manager/controllers/timeout" "github.com/metal-stack/firewall-controller-manager/controllers/update" ) @@ -75,8 +76,8 @@ func main() { "Enabling this will ensure there is only one active controller manager") flag.StringVar(&namespace, "namespace", "", "the namespace this controller is running") flag.DurationVar(&reconcileInterval, "reconcile-interval", 10*time.Minute, "duration after which a resource is getting reconciled at minimum") - flag.DurationVar(&firewallHealthTimeout, "firewall-health-timeout", 20*time.Minute, "duration after a created firewall not getting ready is considered dead") - flag.DurationVar(&createTimeout, "create-timeout", 10*time.Minute, "duration after which a firewall in the creation phase will be recreated") + flag.DurationVar(&firewallHealthTimeout, "firewall-health-timeout", 0*time.Minute, "duration after a created firewall not getting ready is considered dead") + flag.DurationVar(&createTimeout, "create-timeout", 0*time.Minute, "duration after which a firewall in the creation phase will be recreated") flag.DurationVar(&safetyBackoff, "safety-backoff", 10*time.Second, "duration after which a resource is getting reconciled at minimum") flag.DurationVar(&progressDeadline, "progress-deadline", 15*time.Minute, "time after which a deployment is considered unhealthy instead of progressing (informational)") flag.DurationVar(&gracefulShutdownTimeout, "graceful-shutdown-timeout", -1, "grace period after which the controller shuts down") @@ -282,6 +283,9 @@ func main() { if err := update.SetupWithManager(ctrl.Log.WithName("controllers").WithName("update"), seedMgr.GetEventRecorder("update-controller"), seedMgr, cc); err != nil { log.Fatalf("unable to setup update controller: %v", err) } + if err := timeout.SetupWithManager(ctrl.Log.WithName("controllers").WithName("timeout"), seedMgr.GetEventRecorder("timeout-controller"), seedMgr, cc); err != nil { + log.Fatalf("unable to setup timeout controller: %v", err) + } if err := deployment.SetupWebhookWithManager(ctrl.Log.WithName("defaulting-webhook"), seedMgr, cc); err != nil { log.Fatalf("unable to setup webhook, controller deployment %v", err)