WIP first e2e test for projection

On-behalf-of: @SAP christoph.mewes@sap.com

Author: xrstf

deploy/crd/kcp.io/syncagent.kcp.io_publishedresources.yaml

@@ -351,8 +351,10 @@ spec:
                 related:
                   items:
                     properties:
-                      apiVersion:
-                        description: APIVersion is the API group and version of the related resource.
+                      group:
+                        description: |-
+                          Group is the API group of the related resource. This should be left blank for resources
+                          in the core API group.
                         type: string
                       identifier:
                         description: |-
@@ -367,7 +369,7 @@ spec:
                           provided by an APIExport and not Kube-native.
                         type: string
                       kind:
-                        description: ConfigMap or Secret
+                        description: Kind is the object kind of the related resource (for example "Secret").
                         type: string
                       mutation:
                         description: |-
@@ -694,8 +696,29 @@ spec:
                           - service
                           - kcp
                         type: string
+                      projection:
+                        description: |-
+                          Projection is used to change the GVK of a related resource on the opposite side of
+                          its origin.
+                          All fields in the projection are optional. If a field is set, it will overwrite
+                          that field in the GVK.
+                        properties:
+                          group:
+                            description: The API group, for example "myservice.example.com". Leave empty to not modify the API group.
+                            type: string
+                          kind:
+                            description: The resource Kind, for example "Database". Leave empty to not modify the kind.
+                            type: string
+                          version:
+                            description: The API version, for example "v1beta1". Leave empty to not modify the version.
+                            type: string
+                        type: object
+                      version:
+                        description: |-
+                          Version is the API version of the related resource. This can be left blank to automatically
+                          use the preferred version.
+                        type: string
                     required:
-                      - apiVersion
                       - identifier
                       - kind
                       - object

internal/controller/apiexport/controller.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/kcp-dev/api-syncagent/internal/controllerutil"
 	predicateutil "github.com/kcp-dev/api-syncagent/internal/controllerutil/predicate"
+	"github.com/kcp-dev/api-syncagent/internal/projection"
 	"github.com/kcp-dev/api-syncagent/internal/resources/reconciling"
 	syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1"
 
@@ -128,7 +129,7 @@ func (r *Reconciler) reconcile(ctx context.Context) error {
 
 	// for each PR, we note down the created ARS and also the GVKs of related resources
 	arsList := sets.New[string]()
-	claimedResources := sets.New[kcpdevv1alpha1.GroupResource]()
+	claimedResources := sets.New[permissionClaim]()
 
 	// PublishedResources use kinds, but the PermissionClaims use resource names (plural),
 	// so we must translate accordingly
@@ -139,45 +140,57 @@ func (r *Reconciler) reconcile(ctx context.Context) error {
 
 		// to evaluate the namespace filter, the agent needs to fetch the namespace
 		if filter := pubResource.Spec.Filter; filter != nil && filter.Namespace != nil {
-			claimedResources.Insert(kcpdevv1alpha1.GroupResource{
+			claimedResources.Insert(permissionClaim{
 				Group:    "",
 				Resource: "namespaces",
 			})
 		}
 
 		if pubResource.Spec.EnableWorkspacePaths {
-			claimedResources.Insert(kcpdevv1alpha1.GroupResource{
+			claimedResources.Insert(permissionClaim{
 				Group:    "core.kcp.io",
 				Resource: "logicalclusters",
 			})
 		}
 
+		// Add a claim for every related resource, but make sure to project the GVK
+		// of related resources to their kcp-side equivalent first if they originate
+		// on the service cluster.
 		for _, rr := range pubResource.Spec.Related {
-			resource, err := mapper.ResourceFor(schema.GroupVersionResource{
-				Resource: rr.Kind,
-			})
+			kcpGVK := projection.RelatedResourceKcpGVK(&rr)
+
+			versions := []string{}
+			if kcpGVK.Version != "" {
+				versions = append(versions, kcpGVK.Version)
+			}
+
+			mapping, err := mapper.RESTMapping(schema.GroupKind{
+				Group: kcpGVK.Group,
+				Kind:  kcpGVK.Kind,
+			}, versions...)
 			if err != nil {
 				return fmt.Errorf("unknown related resource kind %q: %w", rr.Kind, err)
 			}
 
-			claimedResources.Insert(kcpdevv1alpha1.GroupResource{
-				Group:    "",
-				Resource: resource.Resource,
+			claimedResources.Insert(permissionClaim{
+				Group:        kcpGVK.Group,
+				Resource:     mapping.Resource.Resource,
+				IdentityHash: rr.IdentityHash,
 			})
 		}
 	}
 
-	// Related resources (Secrets, ConfigMaps) are namespaced and so the Sync Agent will
-	// always need to be able to see and manage namespaces.
+	// We always want to see namespaces, for simplicity. Technically if we knew the scope
+	// of every related resource, we could determine if a namespace claim is truly necessary.
 	if claimedResources.Len() > 0 {
-		claimedResources.Insert(kcpdevv1alpha1.GroupResource{
+		claimedResources.Insert(permissionClaim{
 			Group:    "",
 			Resource: "namespaces",
 		})
 	}
 
 	// We always want to create events.
-	claimedResources.Insert(kcpdevv1alpha1.GroupResource{
+	claimedResources.Insert(permissionClaim{
 		Group:    "",
 		Resource: "events",
 	})

internal/controller/apiexport/reconciler.go

@@ -32,12 +32,29 @@ import (
 	"k8s.io/client-go/tools/record"
 )
 
+// permissionClaim is the same as kcp's PermissionClaim, just trimmed down to
+// kind, group and identity and more importantly, without all the other fields
+// that makes this struct not comparable (i.e. not suitable for a Set).
+type permissionClaim struct {
+	Group        string
+	Resource     string
+	IdentityHash string
+}
+
+func (c permissionClaim) String() string {
+	if c.Group == "" {
+		return c.Resource
+	}
+
+	return fmt.Sprintf("%s/%s", c.Group, c.Resource)
+}
+
 // createAPIExportReconciler creates the reconciler for the APIExport.
 // WARNING: The APIExport in this is NOT created by the Sync Agent, it's created
 // by a controller in kcp. Make sure you don't create a reconciling conflict!
 func (r *Reconciler) createAPIExportReconciler(
 	availableResourceSchemas sets.Set[string],
-	claimedResourceKinds sets.Set[kcpdevv1alpha1.GroupResource],
+	claimedResourceKinds sets.Set[permissionClaim],
 	agentName string,
 	apiExportName string,
 	recorder record.EventRecorder,
@@ -60,17 +77,21 @@ func (r *Reconciler) createAPIExportReconciler(
 			// only ensure the ones originating from the published resources;
 			// step 1 is to collect all existing claims with the same properties
 			// as ours.
-			existingClaims := sets.New[kcpdevv1alpha1.GroupResource]()
+			existingClaims := sets.New[permissionClaim]()
 			for _, claim := range existing.Spec.PermissionClaims {
 				if claim.All && len(claim.ResourceSelector) == 0 {
-					existingClaims.Insert(claim.GroupResource)
+					existingClaims.Insert(permissionClaim{
+						Group:        claim.Group,
+						Resource:     claim.Resource,
+						IdentityHash: claim.IdentityHash,
+					})
 				}
 			}
 
 			missingClaims := claimedResourceKinds.Difference(existingClaims)
 
 			claimsToAdd := missingClaims.UnsortedList()
-			slices.SortStableFunc(claimsToAdd, func(a, b kcpdevv1alpha1.GroupResource) int {
+			slices.SortStableFunc(claimsToAdd, func(a, b permissionClaim) int {
 				if a.Group != b.Group {
 					return strings.Compare(a.Group, b.Group)
 				}
@@ -81,15 +102,19 @@ func (r *Reconciler) createAPIExportReconciler(
 			// add our missing claims
 			for _, claimed := range claimsToAdd {
 				existing.Spec.PermissionClaims = append(existing.Spec.PermissionClaims, kcpdevv1alpha1.PermissionClaim{
-					GroupResource: claimed,
-					All:           true,
+					GroupResource: kcpdevv1alpha1.GroupResource{
+						Group:    claimed.Group,
+						Resource: claimed.Resource,
+					},
+					All:          true,
+					IdentityHash: claimed.IdentityHash,
 				})
 			}
 
 			if missingClaims.Len() > 0 {
 				claims := make([]string, 0, len(claimsToAdd))
 				for _, claimed := range claimsToAdd {
-					claims = append(claims, groupResourceToString(claimed))
+					claims = append(claims, claimed.String())
 				}
 				recorder.Eventf(existing, corev1.EventTypeNormal, "AddingPermissionClaims", "Added new permission claim(s) for all %s.", strings.Join(claims, ", "))
 			}
@@ -112,14 +137,6 @@ func (r *Reconciler) createAPIExportReconciler(
 	}
 }
 
-func groupResourceToString(gr kcpdevv1alpha1.GroupResource) string {
-	if gr.Group == "" {
-		return gr.Resource
-	}
-
-	return fmt.Sprintf("%s/%s", gr.Group, gr.Resource)
-}
-
 func mergeResourceSchemas(existing []string, configured sets.Set[string]) []string {
 	var result []string
 

internal/projection/projection.go

@@ -113,6 +113,59 @@ func ProjectCRD(crd *apiextensionsv1.CustomResourceDefinition, pubRes *syncagent
 	return result, nil
 }
 
+// RelatedResourceProjectedGVK returns the effective GVK on the destination side,
+// after the projection rules for the related resource have been applied.
+func RelatedResourceProjectedGVK(rr *syncagentv1alpha1.RelatedResourceSpec) schema.GroupVersionKind {
+	resultGVK := schema.GroupVersionKind{
+		Group:   rr.Group,
+		Version: rr.Version,
+		Kind:    rr.Kind,
+	}
+
+	projection := rr.Projection
+	if projection == nil {
+		return resultGVK
+	}
+
+	if projection.Group != "" {
+		resultGVK.Group = projection.Group
+	}
+
+	if projection.Version != "" {
+		resultGVK.Version = projection.Version
+	}
+
+	if projection.Kind != "" {
+		resultGVK.Kind = projection.Kind
+	}
+
+	return resultGVK
+}
+
+func RelatedResourceKcpGVK(rr *syncagentv1alpha1.RelatedResourceSpec) schema.GroupVersionKind {
+	if rr.Origin == syncagentv1alpha1.RelatedResourceOriginKcp {
+		return schema.GroupVersionKind{
+			Group:   rr.Group,
+			Version: rr.Version,
+			Kind:    rr.Kind,
+		}
+	}
+
+	return RelatedResourceProjectedGVK(rr)
+}
+
+func RelatedResourceServiceGVK(rr *syncagentv1alpha1.RelatedResourceSpec) schema.GroupVersionKind {
+	if rr.Origin == syncagentv1alpha1.RelatedResourceOriginService {
+		return schema.GroupVersionKind{
+			Group:   rr.Group,
+			Version: rr.Version,
+			Kind:    rr.Kind,
+		}
+	}
+
+	return RelatedResourceProjectedGVK(rr)
+}
+
 func stripUnwantedVersions(crd *apiextensionsv1.CustomResourceDefinition, pubRes *syncagentv1alpha1.PublishedResource) (*apiextensionsv1.CustomResourceDefinition, error) {
 	src := pubRes.Spec.Resource
 

internal/sync/syncer_related.go

@@ -28,13 +28,15 @@ import (
 	"github.com/tidwall/gjson"
 	"go.uber.org/zap"
 
+	"github.com/kcp-dev/api-syncagent/internal/projection"
 	"github.com/kcp-dev/api-syncagent/internal/sync/templating"
 	syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -97,11 +99,13 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
 		return strings.Compare(aKey, bKey)
 	})
 
-	// Synchronize objects the same way the parent object was synchronized.
+	// Synchronize related objects the same way the parent object was synchronized.
+	projectedAPIVersion, projectedKind := projection.RelatedResourceProjectedGVK(&relRes).ToAPIVersionAndKind()
+
 	for idx, resolved := range resolvedObjects {
 		destObject := &unstructured.Unstructured{}
-		destObject.SetAPIVersion("v1") // we only support ConfigMaps and Secrets, both are in core/v1
-		destObject.SetKind(relRes.Kind)
+		destObject.SetAPIVersion(projectedAPIVersion)
+		destObject.SetKind(projectedKind)
 
 		if err = dest.client.Get(ctx, resolved.destination, destObject); err != nil {
 			destObject = nil
@@ -119,6 +123,8 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
 			object:      destObject,
 		}
 
+		fmt.Printf("destobjct: %+v\n", destObject)
+
 		syncer := objectSyncer{
 			// Related objects within kcp are not labelled with the agent name because it's unnecessary.
 			// agentName: "",
@@ -127,13 +133,16 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
 			stateStore: stateStore,
 			// how to create a new destination object
 			destCreator: func(source *unstructured.Unstructured) (*unstructured.Unstructured, error) {
+				fmt.Printf("source: %+v\n", *source)
 				dest := source.DeepCopy()
+				dest.SetAPIVersion(projectedAPIVersion)
+				dest.SetKind(projectedKind)
 				dest.SetName(resolved.destination.Name)
 				dest.SetNamespace(resolved.destination.Namespace)
 
 				return dest, nil
 			},
-			// ConfigMaps and Secrets have no subresources
+			// reloated resources have no subresources
 			subresources: nil,
 			// only sync the status back if the object originates in kcp,
 			// as the service side should never have to rely on new status infos coming
@@ -171,8 +180,8 @@ func (s *ResourceSyncer) processRelatedResource(ctx context.Context, log *zap.Su
 			value, err := json.Marshal(relatedObjectAnnotation{
 				Namespace:  resolved.destination.Namespace,
 				Name:       resolved.destination.Name,
-				APIVersion: "v1", // we only support ConfigMaps and Secrets
-				Kind:       relRes.Kind,
+				APIVersion: resolved.original.GetAPIVersion(),
+				Kind:       resolved.original.GetKind(),
 			})
 			if err != nil {
 				return false, fmt.Errorf("failed to encode related object annotation: %w", err)
@@ -355,8 +364,10 @@ func resolveRelatedResourceObjectsInNamespaces(ctx context.Context, relatedOrigi
 		}
 
 		for originName, destName := range nameMap {
+			apiVersion := schema.GroupVersion{Group: relRes.Group, Version: relRes.Version}.String()
+
 			originObj := &unstructured.Unstructured{}
-			originObj.SetAPIVersion("v1") // we only support ConfigMaps and Secrets, both are in core/v1
+			originObj.SetAPIVersion(apiVersion)
 			originObj.SetKind(relRes.Kind)
 
 			err = relatedOrigin.client.Get(ctx, types.NamespacedName{Name: originName, Namespace: originNamespace}, originObj)
@@ -407,8 +418,10 @@ func resolveRelatedResourceObjectsInNamespace(ctx context.Context, relatedOrigin
 		return mapSlices(originNames, destNames), nil
 
 	case spec.Selector != nil:
+		apiVersion := schema.GroupVersion{Group: relRes.Group, Version: relRes.Version}.String()
+
 		originObjects := &unstructured.UnstructuredList{}
-		originObjects.SetAPIVersion("v1") // we only support ConfigMaps and Secrets, both are in core/v1
+		originObjects.SetAPIVersion(apiVersion)
 		originObjects.SetKind(relRes.Kind)
 
 		labelSelector, err := templateLabelSelector(relatedOrigin, relatedDest, relRes.Origin, &spec.Selector.LabelSelector)

internal/sync/syncer_related_test.go

@@ -216,6 +216,8 @@ func TestResolveRelatedResourceObjects(t *testing.T) {
 				Identifier: "test",
 				Origin:     syncagentv1alpha1.RelatedResourceOriginService,
 				Kind:       "Secret",
+				Group:      "",
+				Version:    "v1",
 				Object:     testcase.objectSpec,
 			}