[Feature] Early connections support (#1102)

Author: informalict

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - (Feature) Add new field to DeploymentReplicationStatus with details on DC2DC sync status
+- (Feature) Early connections support
 
 ## [1.2.16](https://github.com/arangodb/kube-arangodb/tree/1.2.16) (2022-09-14)
 - (Feature) Add ArangoDeployment ServerGroupStatus

cmd/lifecycle_probes.go

@@ -22,44 +22,80 @@ package cmd
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
+	"strconv"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
+	"github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/jwt"
 
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 )
 
 var (
 	cmdLifecycleProbe = &cobra.Command{
 		Use: "probe",
-		Run: cmdLifecycleProbeCheck,
+		Run: cmdLifecycleProbeRun,
+	}
+	cmdLifecycleProbeLiveness = &cobra.Command{
+		Use: "liveness",
+		Run: cmdLifecycleProbeRun,
+	}
+	cmdLifecycleProbeReadiness = &cobra.Command{
+		Use: "readiness",
+		Run: cmdLifecycleProbeRun,
+	}
+	cmdLifecycleProbeStartUp = &cobra.Command{
+		Use: "startup",
+		Run: cmdLifecycleProbeRun,
 	}
 
 	probeInput struct {
-		SSL      bool
-		Auth     bool
-		Endpoint string
-		JWTPath  string
+		Endpoint        string
+		JWTPath         string
+		ArangoDBVersion string
+		ServerGroup     string
+		DeploymentMode  string
+		SSL             bool
+		Auth            bool
+		Enterprise      bool
 	}
 )
 
 func init() {
 	f := cmdLifecycleProbe.PersistentFlags()
 
+	cmdLifecycleProbe.AddCommand(cmdLifecycleProbeLiveness)
+	cmdLifecycleProbe.AddCommand(cmdLifecycleProbeReadiness)
+	cmdLifecycleProbe.AddCommand(cmdLifecycleProbeStartUp)
+
 	f.BoolVarP(&probeInput.SSL, "ssl", "", false, "Determines if SSL is enabled")
 	f.BoolVarP(&probeInput.Auth, "auth", "", false, "Determines if authentication is enabled")
-	f.StringVarP(&probeInput.Endpoint, "endpoint", "", "/_api/version", "Endpoint (path) to call for lifecycle probe")
+	f.StringVarP(&probeInput.Endpoint, "endpoint", "", client.ServerApiVersionEndpoint, "Endpoint (path) to call for lifecycle probe")
+	f.MarkDeprecated("endpoint", "Endpoint is chosen automatically by the lifecycle process")
 	f.StringVarP(&probeInput.JWTPath, "jwt", "", shared.ClusterJWTSecretVolumeMountDir, "Path to the JWT tokens")
+	f.StringVar(&probeInput.ArangoDBVersion, "arangodb-version", os.Getenv(resources.ArangoDBOverrideVersionEnv),
+		"Version of the ArangoDB")
+	f.StringVar(&probeInput.ServerGroup, "serverGroup", os.Getenv(resources.ArangoDBOverrideServerGroupEnv),
+		"Name of the group where a server belongs to")
+	f.StringVar(&probeInput.DeploymentMode, "deploymentMode", os.Getenv(resources.ArangoDBOverrideDeploymentModeEnv),
+		"A deployment mode (Cluster, Single, ActiveFailover)")
+	enterprise, _ := strconv.ParseBool(os.Getenv(resources.ArangoDBOverrideEnterpriseEnv))
+	f.BoolVar(&probeInput.Enterprise, "enterprise", enterprise, "Determines if ArangoDB is enterprise")
 }
 
 func probeClient() *http.Client {
@@ -147,10 +183,10 @@ func addAuthHeader(req *http.Request) error {
 	return nil
 }
 
-func doRequest() (*http.Response, error) {
+func doRequest(endpoint string) (*http.Response, error) {
 	client := probeClient()
 
-	req, err := http.NewRequest(http.MethodGet, probeEndpoint(probeInput.Endpoint), nil)
+	req, err := http.NewRequest(http.MethodGet, probeEndpoint(endpoint), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -162,22 +198,25 @@ func doRequest() (*http.Response, error) {
 	return client.Do(req)
 }
 
-func cmdLifecycleProbeCheck(cmd *cobra.Command, args []string) {
-	if err := cmdLifecycleProbeCheckE(); err != nil {
+func cmdLifecycleProbeRun(cmd *cobra.Command, _ []string) {
+	if err := cmdLifecycleProbeRunE(cmd); err != nil {
 		log.Error().Err(err).Msgf("Fatal")
 		os.Exit(1)
 	}
 }
 
-func cmdLifecycleProbeCheckE() error {
-	resp, err := doRequest()
+func cmdLifecycleProbeRunE(cmd *cobra.Command) error {
+	endpoint := getEndpoint(api.ProbeType(cmd.Use))
+	resp, err := doRequest(endpoint)
 	if err != nil {
 		return err
 	}
+	if resp.Body != nil {
+		defer resp.Body.Close()
+	}
 
 	if resp.StatusCode != http.StatusOK {
 		if resp.Body != nil {
-			defer resp.Body.Close()
 			if data, err := ioutil.ReadAll(resp.Body); err == nil {
 				return errors.Errorf("Unexpected code: %d - %s", resp.StatusCode, string(data))
 			}
@@ -186,7 +225,51 @@ func cmdLifecycleProbeCheckE() error {
 		return errors.Errorf("Unexpected code: %d", resp.StatusCode)
 	}
 
+	if endpoint == client.ServerStatusEndpoint {
+		// When server status endpoint is used then HTTP status code 200 is not enough.
+		// The progress should be also checked.
+		if resp.Body == nil {
+			return errors.Errorf("Expected body from the \"%s\" endpoint", endpoint)
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return errors.Errorf("Failed to read body from the \"%s\" endpoint", endpoint)
+		}
+
+		status := client.ServerStatus{}
+		if err = json.Unmarshal(data, &status); err != nil {
+			return errors.Errorf("Failed to unmarshal %s into server status", string(data))
+		}
+
+		if progress, ok := status.GetProgress(); !ok {
+			return errors.Errorf("server not ready: %s", progress)
+		}
+	}
+
 	log.Info().Msgf("Check passed")
 
 	return nil
 }
+
+// getEndpoint returns endpoint to the ArangoDB instance where readiness should be checked.
+func getEndpoint(probeType api.ProbeType) string {
+	if probeType == api.ProbeTypeReadiness {
+		if probeInput.DeploymentMode == string(api.DeploymentModeActiveFailover) {
+			v := driver.Version(probeInput.ArangoDBVersion)
+			if features.FailoverLeadership().Supported(v, probeInput.Enterprise) {
+				return client.ServerApiVersionEndpoint
+			}
+		}
+
+		return client.ServerAvailabilityEndpoint
+	}
+
+	if probeInput.ServerGroup == api.ServerGroupDBServersString {
+		v := driver.Version(probeInput.ArangoDBVersion)
+		if features.Version310().Supported(v, probeInput.Enterprise) {
+			return client.ServerStatusEndpoint
+		}
+	}
+
+	return client.ServerApiVersionEndpoint
+}

pkg/apis/deployment/v1/probes.go

@@ -0,0 +1,29 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+type ProbeType string
+
+const (
+	ProbeTypeLiveness  = "liveness"
+	ProbeTypeReadiness = "readiness"
+	ProbeTypeStartUp   = "startup"
+)

pkg/apis/deployment/v2alpha1/probes.go

@@ -0,0 +1,29 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+type ProbeType string
+
+const (
+	ProbeTypeLiveness  = "liveness"
+	ProbeTypeReadiness = "readiness"
+	ProbeTypeStartUp   = "startup"
+)

pkg/deployment/client/status.go

@@ -0,0 +1,76 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package client
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	// ServerProgressPhaseInWait describes success progress state of a server.
+	ServerProgressPhaseInWait = "in wait"
+	// ServerStatusEndpoint describes endpoint of a server status.
+	ServerStatusEndpoint = "/_admin/status"
+	// ServerApiVersionEndpoint describes endpoint of a server version.
+	ServerApiVersionEndpoint = "/_api/version"
+	// ServerAvailabilityEndpoint describes endpoint of a server availability.
+	ServerAvailabilityEndpoint = "/_admin/server/availability"
+)
+
+// ServerProgress describes server progress.
+type ServerProgress struct {
+	// Phase is a name of the lifecycle phase the instance is currently in.
+	Phase string `json:"phase,omitempty"`
+	// Feature is internal name of the feature that is currently being prepared
+	Feature string `json:"feature,omitempty"`
+	// Current recovery sequence number value, if the instance is currently recovering.
+	// If the instance is already past the recovery, this attribute contains the last handled recovery sequence number.
+	RecoveryTick int `json:"recoveryTick,omitempty"`
+}
+
+// ServerInfo describes server information.
+type ServerInfo struct {
+	ServerProgress ServerProgress `json:"progress,omitempty"`
+}
+
+// ServerStatus describes server status.
+type ServerStatus struct {
+	ServerInfo ServerInfo `json:"serverInfo,omitempty"`
+}
+
+// GetProgress returns human-readable progress status of the server, and true if server is ready.
+func (s ServerStatus) GetProgress() (string, bool) {
+	p := s.ServerInfo.ServerProgress
+	var result strings.Builder
+
+	if len(p.Feature) > 0 {
+		result.WriteString("feature: " + p.Feature + ", ")
+	}
+
+	result.WriteString("phase: " + p.Phase)
+
+	if p.RecoveryTick > 0 {
+		result.WriteString(", recoveryTick: " + fmt.Sprintf("%d", p.RecoveryTick))
+	}
+
+	return result.String(), p.Phase == ServerProgressPhaseInWait
+}

pkg/deployment/deployment_encryption_test.go

@@ -52,7 +52,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
 							firstAgentStatus,
 						},
 					},
-					Images: createTestImagesWithVersion(false, "3.7.0"),
+					Images: createTestImagesWithVersion(false, testVersion),
 				}
 
 				testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus)
@@ -117,7 +117,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
 							firstDBServerStatus,
 						},
 					},
-					Images: createTestImagesWithVersion(false, "3.7.0"),
+					Images: createTestImagesWithVersion(false, testVersion),
 				}
 
 				testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus)
@@ -200,7 +200,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
 							firstAgentStatus,
 						},
 					},
-					Images: createTestImagesWithVersion(true, "3.7.0"),
+					Images: createTestImagesWithVersion(true, testVersion),
 				}
 
 				testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus)
@@ -263,7 +263,7 @@ func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
 							firstAgentStatus,
 						},
 					},
-					Images: createTestImagesWithVersion(true, "3.7.0"),
+					Images: createTestImagesWithVersion(true, testVersion),
 				}
 
 				testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus)