diff --git a/Makefile b/Makefile index e3501c38..ecc6c2e0 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ OS ?= $(shell uname -s) OS := $(shell echo $(OS) | tr '[:upper:]' '[:lower:]') K8S_LATEST_VER ?= $(shell curl -s https://dl.k8s.io/release/stable.txt) export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= v1.14.0 +TAG ?= main .PHONY: all all: build diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index ae5338bc..c6e84b84 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,12 +21,11 @@ limitations under the License. package v1beta1 import ( + apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - - apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index c6847165..8f8dea58 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -35,7 +35,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - "--v=5" - - "--version=v1.14.0" + - "--version=main" - "--agent-in-mgmt-cluster=false" env: - name: GOMEMLIMIT diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 0d4e141d..22ae3140 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -7,8 +7,8 @@ spec: template: spec: initContainers: - - image: docker.io/projectsveltos/addon-controller:v1.14.0 + - image: docker.io/projectsveltos/addon-controller:main name: initialization containers: - - image: docker.io/projectsveltos/addon-controller:v1.14.0 + - image: docker.io/projectsveltos/addon-controller:main name: controller diff --git a/controllers/drift_detection_upgrade.go b/controllers/drift_detection_upgrade.go index 08672ada..37d5d252 100644 --- a/controllers/drift_detection_upgrade.go +++ b/controllers/drift_detection_upgrade.go @@ -194,6 +194,7 @@ func upgradeDriftDetectionDeploymentsInMgmtCluster(ctx context.Context, logger l } var patches []libsveltosv1beta1.Patch + var watchNamespaces []string clusterNamespace, clusterName, clusterType, found := getClusterDataFromDriftDetectionManagerDeployment(&driftDetectionDeployments.Items[i]) if found { @@ -207,10 +208,19 @@ func upgradeDriftDetectionDeploymentsInMgmtCluster(ctx context.Context, logger l } else if patches == nil { patches = globalPatches } + + watchNamespaces, err = getAgentWatchNamespaces(ctx, mgmtClient, + clusterNamespace, clusterName, clusterType, logger) + if err != nil { + logger.V(logs.LogInfo).Info( + fmt.Sprintf("cluster %s %s/%s failed to get drift detection watch namespaces: %v", + clusterType, clusterNamespace, clusterName, err)) + allProcessed = false + } } err = upgradeDriftDetectionDeployment(ctx, config, mgmtClient, &driftDetectionDeployments.Items[i], - patches, logger) + patches, watchNamespaces, logger) if err != nil { logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to upgrade driftDetection deployment: %v", err)) allProcessed = false @@ -226,14 +236,15 @@ func upgradeDriftDetectionDeploymentsInMgmtCluster(ctx context.Context, logger l } func upgradeDriftDetectionDeployment(ctx context.Context, config *rest.Config, c client.Client, - depl *appsv1.Deployment, patches []libsveltosv1beta1.Patch, logger logr.Logger) error { + depl *appsv1.Deployment, patches []libsveltosv1beta1.Patch, watchNamespaces []string, + logger logr.Logger) error { exist, clusterNs, clusterName, clusterType := deplAssociatedClusterExist(ctx, c, depl, logger) if exist { logger.V(logs.LogDebug).Info(fmt.Sprintf("Upgrade drift detection deployment for cluster %s %s/%s", clusterType, clusterNs, clusterName)) return deployDriftDetectionManagerInManagementCluster(ctx, config, clusterNs, clusterName, - "do-not-send-updates", clusterType, patches, logger) + "do-not-send-updates", clusterType, patches, watchNamespaces, logger) } return nil diff --git a/controllers/export_test.go b/controllers/export_test.go index 83758a40..d7e82d39 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -208,6 +208,11 @@ var ( RemoveDriftDetectionManagerFromManagementCluster = removeDriftDetectionManagerFromManagementCluster GetDriftDetectionNamespaceInMgmtCluster = getDriftDetectionNamespaceInMgmtCluster GetGlobalDriftDetectionManagerPatches = getGlobalDriftDetectionManagerPatches + GetAgentWatchNamespaces = getAgentWatchNamespaces +) + +const ( + AgentWatchNamespacesAnnotation = agentWatchNamespacesAnnotation ) var ( diff --git a/controllers/handlers_resources_test.go b/controllers/handlers_resources_test.go index f4464481..489af729 100644 --- a/controllers/handlers_resources_test.go +++ b/controllers/handlers_resources_test.go @@ -515,4 +515,85 @@ var _ = Describe("Hash methods", func() { Expect(reflect.DeepEqual(hash, expectedHash)).To(BeTrue()) } }) + + It("ResourcesHash changes when the target Cluster's watch-namespaces annotation changes, "+ + "only in ContinuousWithDriftDetection mode", func() { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + }, + } + Expect(testEnv.Create(context.TODO(), namespace)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv, namespace)).To(Succeed()) + + cluster := &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace.Name, + }, + } + Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv, cluster)).To(Succeed()) + + newClusterSummary := func(syncMode configv1beta1.SyncMode) *configv1beta1.ClusterSummary { + cs := &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace.Name, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: cluster.Namespace, + ClusterName: cluster.Name, + ClusterType: libsveltosv1beta1.ClusterTypeSveltos, + ClusterProfileSpec: configv1beta1.Spec{ + SyncMode: syncMode, + }, + }, + } + Expect(testEnv.Create(context.TODO(), cs)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv, cs)).To(Succeed()) + return cs + } + + driftClusterSummary := newClusterSummary(configv1beta1.SyncModeContinuousWithDriftDetection) + beforeHash, err := controllers.ResourcesHash(context.TODO(), testEnv, driftClusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + + continuousClusterSummary := newClusterSummary(configv1beta1.SyncModeContinuous) + continuousBeforeHash, err := controllers.ResourcesHash(context.TODO(), testEnv, continuousClusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + + updatedWatchNamespaces := "ns-a,ns-b" + cluster.Annotations = map[string]string{ + controllers.AgentWatchNamespacesAnnotation: updatedWatchNamespaces, + } + Expect(testEnv.Update(context.TODO(), cluster)).To(Succeed()) + + // waitForObject only confirms the object still exists, which for an Update (unlike a + // Create) is trivially already true: it says nothing about whether this specific + // annotation value has propagated through testEnv's cache yet. getClusterProfileSpecHash + // reads through that same cache (getManagementClusterClient), so poll for the real value + // there instead, or afterHash below could still observe the pre-update annotations. + Eventually(func() bool { + current := &libsveltosv1beta1.SveltosCluster{} + if err := testEnv.Get(context.TODO(), client.ObjectKeyFromObject(cluster), current); err != nil { + return false + } + return current.Annotations[controllers.AgentWatchNamespacesAnnotation] == updatedWatchNamespaces + }, timeout, pollingInterval).Should(BeTrue()) + + afterHash, err := controllers.ResourcesHash(context.TODO(), testEnv, driftClusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(reflect.DeepEqual(beforeHash, afterHash)).To(BeFalse()) + + // Same annotation change must NOT affect the hash outside ContinuousWithDriftDetection: + // nothing reads --watch-namespaces in that mode, so it would just be wasted cost. + continuousAfterHash, err := controllers.ResourcesHash(context.TODO(), testEnv, continuousClusterSummary, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(reflect.DeepEqual(continuousBeforeHash, continuousAfterHash)).To(BeTrue()) + }) }) diff --git a/controllers/handlers_utils.go b/controllers/handlers_utils.go index efa060be..9cfc0999 100644 --- a/controllers/handlers_utils.go +++ b/controllers/handlers_utils.go @@ -1104,6 +1104,21 @@ func processDeployedGVKs(ctx context.Context, isMgmtCluster bool, remoteConfig * LabelSelector: labels.Set(labelSelector.MatchLabels).String(), } + // Only the remote (managed) cluster is ever reached with a credential restricted by + // agentWatchNamespacesAnnotation; local (management-cluster) deploys use addon-controller's + // own full-privilege client and are unaffected. Unlike drift-detection-manager, this is + // honored unconditionally: no license check, agentless or not (see the annotation's doc + // comment). Without it, a genuinely RBAC-restricted credential can't deploy at all. + var watchNamespaces []string + if !isMgmtCluster { + watchNamespaces, err = getAgentWatchNamespaces(ctx, getManagementClusterClient(), + clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + clusterSummary.Spec.ClusterType, logger) + if err != nil { + return nil, err + } + } + for i := range deployedGVKs { // TODO: move this to separate method logger.V(logs.LogDebug).Info(fmt.Sprintf("removing stale resources for GVK %s", deployedGVKs[i].String())) @@ -1125,7 +1140,8 @@ func processDeployedGVKs(ctx context.Context, isMgmtCluster bool, remoteConfig * Resource: mapping.Resource.Resource, } - list, err := d.Resource(resourceId).List(ctx, listOptions) + namespaced := mapping.Scope.Name() == meta.RESTScopeNameNamespace + items, err := listDeployedResources(ctx, d, resourceId, &listOptions, namespaced, watchNamespaces) if err != nil { return nil, err } @@ -1139,8 +1155,8 @@ func processDeployedGVKs(ctx context.Context, isMgmtCluster bool, remoteConfig * expectedDeploymentType = string(configv1beta1.DeploymentTypeLocal) } - for j := range list.Items { - r := list.Items[j] + for j := range items { + r := items[j] // Protective signal, same reasoning as the clustersummary annotation check below: // skip a resource here when it is explicitly annotated for the *other* deployment @@ -1183,6 +1199,33 @@ func processDeployedGVKs(ctx context.Context, isMgmtCluster bool, remoteConfig * return undeployed, nil } +// listDeployedResources lists resources of resourceId matching listOptions. The dynamic client +// has no single call to list across only a chosen subset of namespaces, only across all of them +// (used here when the resource is cluster-scoped, or watchNamespaces is empty) or within exactly +// one, so a non-empty watchNamespaces for a namespaced resource is handled by fanning out one +// List per configured namespace and aggregating the results. +func listDeployedResources(ctx context.Context, d dynamic.Interface, resourceId schema.GroupVersionResource, + listOptions *metav1.ListOptions, namespaced bool, watchNamespaces []string) ([]unstructured.Unstructured, error) { + + if !namespaced || len(watchNamespaces) == 0 { + list, err := d.Resource(resourceId).List(ctx, *listOptions) + if err != nil { + return nil, err + } + return list.Items, nil + } + + items := make([]unstructured.Unstructured, 0) + for _, ns := range watchNamespaces { + list, err := d.Resource(resourceId).Namespace(ns).List(ctx, *listOptions) + if err != nil { + return nil, err + } + items = append(items, list.Items...) + } + return items, nil +} + func undeployStaleResources(ctx context.Context, isMgmtCluster bool, remoteConfig *rest.Config, remoteClient client.Client, featureID libsveltosv1beta1.FeatureID, clusterSummary *configv1beta1.ClusterSummary, deployedGVKs []schema.GroupVersionKind, @@ -1527,6 +1570,18 @@ func getClusterProfileSpecHash(ctx context.Context, clusterSummary *configv1beta // the upgrade via ClusterSummary redeployment. v1.0.1 is still added here to make // sure hash does not change config += "v1.0.1" + + // A change to the operator's agentWatchNamespacesAnnotation on the target Cluster must + // redeploy drift-detection-manager with the new --watch-namespaces value. Without this, + // shouldRedeploy would see an unchanged hash and skip the redeploy entirely, leaving the + // annotation change unapplied until the next periodic drift-detection upgrade sweep. + watchNamespaces, err := getAgentWatchNamespaces(ctx, getManagementClusterClient(), + clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + clusterSummary.Spec.ClusterType, logger) + if err != nil { + return "", err + } + config += strings.Join(watchNamespaces, ",") } mgmtResourceHash, err := getTemplateResourceRefHash(ctx, clusterSummary) diff --git a/controllers/handlers_utils_test.go b/controllers/handlers_utils_test.go index eb243890..40de83b4 100644 --- a/controllers/handlers_utils_test.go +++ b/controllers/handlers_utils_test.go @@ -1144,6 +1144,115 @@ var _ = Describe("HandlersUtils", func() { }, timeout, pollingInterval).Should(BeNil()) }) + It(`undeployStaleResources honors agent.projectsveltos.io/watch-namespaces, ignoring stale resources outside the configured namespaces`, func() { + watchedNamespace := randomString() + otherNamespace := randomString() + for _, ns := range []string{watchedNamespace, otherNamespace} { + Expect(testEnv.Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + Expect(waitForObject(ctx, testEnv.Client, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})).To(Succeed()) + } + + // Restrict this Cluster to watchedNamespace only. + currentCluster := &clusterv1.Cluster{} + Expect(testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: clusterSummary.Spec.ClusterNamespace, Name: clusterSummary.Spec.ClusterName}, + currentCluster)).To(Succeed()) + if currentCluster.Annotations == nil { + currentCluster.Annotations = map[string]string{} + } + currentCluster.Annotations[controllers.AgentWatchNamespacesAnnotation] = watchedNamespace + Expect(testEnv.Update(context.TODO(), currentCluster)).To(Succeed()) + + // Neither ServiceAccount is in currentPolicies (nil), so both are candidates for + // deletion were it not for the namespace scoping. + watchedServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: watchedNamespace, + Name: randomString(), + Labels: map[string]string{ + deployer.ReasonLabel: string(libsveltosv1beta1.FeatureResources), + }, + Annotations: map[string]string{ + deployer.ReferenceKindAnnotation: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + deployer.ReferenceNamespaceAnnotation: randomString(), + deployer.ReferenceNameAnnotation: randomString(), + }, + }, + } + otherServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: otherNamespace, + Name: randomString(), + Labels: map[string]string{ + deployer.ReasonLabel: string(libsveltosv1beta1.FeatureResources), + }, + Annotations: map[string]string{ + deployer.ReferenceKindAnnotation: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + deployer.ReferenceNamespaceAnnotation: randomString(), + deployer.ReferenceNameAnnotation: randomString(), + }, + }, + } + Expect(testEnv.Create(context.TODO(), watchedServiceAccount)).To(Succeed()) + Expect(waitForObject(ctx, testEnv.Client, watchedServiceAccount)).To(Succeed()) + Expect(testEnv.Create(context.TODO(), otherServiceAccount)).To(Succeed()) + Expect(waitForObject(ctx, testEnv.Client, otherServiceAccount)).To(Succeed()) + + currentClusterProfile := &configv1beta1.ClusterProfile{} + Expect(testEnv.Get(context.TODO(), + types.NamespacedName{Name: clusterProfile.Name}, currentClusterProfile)).To(Succeed()) + addOwnerReference(context.TODO(), testEnv.Client, watchedServiceAccount, currentClusterProfile) + addOwnerReference(context.TODO(), testEnv.Client, otherServiceAccount, currentClusterProfile) + + currentClusterSummary := &configv1beta1.ClusterSummary{} + Expect(testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: clusterSummary.Namespace, Name: clusterSummary.Name}, + currentClusterSummary)).To(Succeed()) + currentClusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + { + FeatureID: libsveltosv1beta1.FeatureResources, + Status: libsveltosv1beta1.FeatureStatusProvisioned, + }, + } + currentClusterSummary.Status.DeployedGVKs = []libsveltosv1beta1.FeatureDeploymentInfo{ + { + FeatureID: libsveltosv1beta1.FeatureResources, + DeployedGroupVersionKind: []string{ + testServiceAccountKindV1, + }, + }, + } + Expect(testEnv.Status().Update(context.TODO(), currentClusterSummary)).To(Succeed()) + + deployedGKVs := controllers.GetDeployedGroupVersionKinds(currentClusterSummary, libsveltosv1beta1.FeatureResources) + Expect(deployedGKVs).ToNot(BeEmpty()) + + _, err := controllers.UndeployStaleResources(context.TODO(), false, testEnv.Config, testEnv.Client, + libsveltosv1beta1.FeatureResources, currentClusterSummary, deployedGKVs, nil, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + + // Stale and inside the configured namespace: found and removed. + Eventually(func() bool { + currentServiceAccount := &corev1.ServiceAccount{} + err = testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: watchedNamespace, Name: watchedServiceAccount.Name}, + currentServiceAccount) + return err != nil && apierrors.IsNotFound(err) + }, timeout, pollingInterval).Should(BeTrue()) + + // Also stale, but outside the configured namespace: the scoped search never reaches + // it, so it must never be touched. + Consistently(func() error { + currentServiceAccount := &corev1.ServiceAccount{} + return testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: otherNamespace, Name: otherServiceAccount.Name}, + currentServiceAccount) + }, timeout, pollingInterval).Should(BeNil()) + }) + It(`undeployStaleResources does not remove resources deployed by the other deployment type of the same ClusterSummary`, func() { // Further self-managed SveltosCluster scenario: this time both resources were deployed // by the SAME ClusterSummary, one via deploymentType Local and one via deploymentType diff --git a/controllers/resourcesummary.go b/controllers/resourcesummary.go index 175f606a..660a2280 100644 --- a/controllers/resourcesummary.go +++ b/controllers/resourcesummary.go @@ -75,6 +75,27 @@ const ( // * **Strategic Merge Patch** // * **JSON Patch (RFC6902)** driftDetectionOverrideAnnotation = "driftdetection.projectsveltos.io/config-override-ref" + + // This optional annotation restricts namespaced-resource access to the listed namespaces: a + // comma-separated list, e.g. "ns-a,ns-b,ns-c". Named under agent.projectsveltos.io, not + // driftdetection.projectsveltos.io, since it has more than one consumer, each with its own + // rule for when it applies: + // + // - drift-detection-manager (and sveltos-agent, not wired up here yet) only honors it for + // clusters deployed in agentless (management-cluster) mode, and only with a valid Sveltos + // Enterprise license granting NamespaceScopedAgents. It checks that directly against the + // sveltos-license Secret and falls back to watching everything, cluster-wide, if the + // license is missing or invalid. This is the paid differentiator: continuous deployment + // under restricted RBAC works either way (see below), but continuous drift detection + // under restricted RBAC requires a license. + // - addon-controller's own search for stale resources to remove (processDeployedGVKs) + // honors it unconditionally: no license check, agentless or not. Unlike watching for + // drift, this isn't a premium capability — without it, a genuinely RBAC-restricted + // credential just can't deploy at all, license or no license. + // + // Cluster-scoped resources are unaffected in both cases, since they have no namespace for + // this restriction to apply to. + agentWatchNamespacesAnnotation = "agent.projectsveltos.io/watch-namespaces" ) func getDriftDetectionNamespaceInMgmtCluster(sveltosNamespace string) string { @@ -104,6 +125,12 @@ func deployDriftDetectionManagerInCluster(ctx context.Context, c client.Client, } } + watchNamespaces, err := getAgentWatchNamespaces(ctx, c, clusterNamespace, clusterName, + clusterType, logger) + if err != nil { + return err + } + err = deployDriftDetectionCRDs(ctx, clusterNamespace, clusterName, applicant, featureID, clusterType, isPullMode, startInMgmtCluster, logger) if err != nil { @@ -115,13 +142,61 @@ func deployDriftDetectionManagerInCluster(ctx context.Context, c client.Client, if startInMgmtCluster && !isPullMode { restConfig := getManagementClusterConfig() return deployDriftDetectionManagerInManagementCluster(ctx, restConfig, clusterNamespace, clusterName, - "do-not-send-updates", clusterType, patches, logger) + "do-not-send-updates", clusterType, patches, watchNamespaces, logger) + } + + if len(watchNamespaces) > 0 { + logger.V(logs.LogInfo).Info(fmt.Sprintf( + "%s is only supported for agentless (management-cluster) deploys, ignoring it for this push-mode cluster", + agentWatchNamespacesAnnotation)) } return deployDriftDetectionManagerInManagedCluster(ctx, clusterNamespace, clusterName, applicant, featureID, "do-not-send-updates", clusterType, patches, logger) } +// getAgentWatchNamespaces reads agentWatchNamespacesAnnotation off the Cluster/SveltosCluster +// instance, the same way getPerClusterDriftDetectionManagerPatches reads its own annotation. +// Returns nil, not an error, when the Cluster instance, the annotation, or its value is missing: +// all of which mean "no restriction configured", the same as an empty --watch-namespaces on +// drift-detection-manager itself. See agentWatchNamespacesAnnotation's doc comment for how its +// two callers (this file, and processDeployedGVKs) differ in when they apply it. +func getAgentWatchNamespaces(ctx context.Context, c client.Client, + clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, + logger logr.Logger) ([]string, error) { + + cluster, err := clusterproxy.GetCluster(ctx, c, clusterNamespace, clusterName, clusterType) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + annotations := cluster.GetAnnotations() + if annotations == nil { + return nil, nil + } + + value, ok := annotations[agentWatchNamespacesAnnotation] + if !ok || value == "" { + return nil, nil + } + + namespaces := make([]string, 0) + for _, ns := range strings.Split(value, ",") { + ns = strings.TrimSpace(ns) + if ns != "" { + namespaces = append(namespaces, ns) + } + } + + logger.V(logs.LogDebug).Info(fmt.Sprintf("drift-detection-manager watch namespaces: %v", namespaces), + "annotation", agentWatchNamespacesAnnotation) + + return namespaces, nil +} + func deployDriftDetectionCRDs(ctx context.Context, clusterNamespace, clusterName, applicant, featureID string, clusterType libsveltosv1beta1.ClusterType, isPullMode, startInMgmtCluster bool, logger logr.Logger) error { @@ -274,7 +349,7 @@ func deployDriftDetectionManagerInManagedCluster(ctx context.Context, // Those instances are all running in the namespace where projectsveltos is deployed func deployDriftDetectionManagerInManagementCluster(ctx context.Context, restConfig *rest.Config, clusterNamespace, clusterName, mode string, clusterType libsveltosv1beta1.ClusterType, - patches []libsveltosv1beta1.Patch, logger logr.Logger) error { + patches []libsveltosv1beta1.Patch, watchNamespaces []string, logger logr.Logger) error { logger.V(logs.LogDebug).Info("deploy drift-detection-manager in management cluster") driftDetectionManagerYAML := string(driftdetection.GetDriftDetectionManagerInMgmtClusterYAML()) @@ -282,6 +357,13 @@ func deployDriftDetectionManagerInManagementCluster(ctx context.Context, restCon driftDetectionManagerYAML = prepareDriftDetectionManagerYAML(driftDetectionManagerYAML, clusterNamespace, clusterName, mode, clusterType) + // Unlike cluster-namespace/cluster-name/cluster-type above, this substitution is a no-op + // (empty in, empty out) when watchNamespaces is empty, keeping today's cluster-wide default. + if len(watchNamespaces) > 0 { + driftDetectionManagerYAML = strings.ReplaceAll(driftDetectionManagerYAML, "watch-namespaces=", + fmt.Sprintf("watch-namespaces=%s", strings.Join(watchNamespaces, ","))) + } + // Following labels are added on the objects representing the drift-detection-manager // for this cluster. lbls := getDriftDetectionManagerLabels(clusterNamespace, clusterName, clusterType) diff --git a/controllers/resourcesummary_test.go b/controllers/resourcesummary_test.go index 7879abea..a0656fea 100644 --- a/controllers/resourcesummary_test.go +++ b/controllers/resourcesummary_test.go @@ -96,7 +96,7 @@ var _ = Describe("ResourceSummary Deployer", func() { clusterType := libsveltosv1beta1.ClusterTypeSveltos Expect(controllers.DeployDriftDetectionManagerInManagementCluster(context.TODO(), testEnv.Config, - clusterNamespace, clusterName, "", clusterType, nil, + clusterNamespace, clusterName, "", clusterType, nil, nil, textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) expectedLabels := controllers.GetDriftDetectionManagerLabels(clusterNamespace, clusterName, clusterType) @@ -144,6 +144,68 @@ var _ = Describe("ResourceSummary Deployer", func() { }, timeout, pollingInterval).Should(BeTrue()) }) + It("deployDriftDetectionManagerInManagementCluster passes --watch-namespaces through when configured", func() { + clusterNamespace := randomString() + clusterName := randomString() + clusterType := libsveltosv1beta1.ClusterTypeSveltos + watchNamespace1 := randomString() + watchNamespace2 := randomString() + watchNamespaces := []string{watchNamespace1, watchNamespace2} + + Expect(controllers.DeployDriftDetectionManagerInManagementCluster(context.TODO(), testEnv.Config, + clusterNamespace, clusterName, "", clusterType, nil, watchNamespaces, + textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) + + expectedLabels := controllers.GetDriftDetectionManagerLabels(clusterNamespace, clusterName, clusterType) + listOptions := []client.ListOption{ + client.InNamespace(controllers.GetDriftDetectionNamespaceInMgmtCluster(sveltosNamespace)), + } + + Eventually(func() bool { + deployments := &appsv1.DeploymentList{} + err := testEnv.List(context.TODO(), deployments, listOptions...) + if err != nil { + return false + } + + for i := range deployments.Items { + d := &deployments.Items[i] + if !verifyLabels(d.Labels, expectedLabels) { + continue + } + expectedArg := fmt.Sprintf("--watch-namespaces=%s,%s", watchNamespace1, watchNamespace2) + for _, arg := range d.Spec.Template.Spec.Containers[0].Args { + if arg == expectedArg { + return true + } + } + } + return false + }, timeout, pollingInterval).Should(BeTrue()) + + Expect(controllers.RemoveDriftDetectionManagerFromManagementCluster(context.TODO(), clusterNamespace, clusterName, + clusterType, textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) + }) + + It("deployDriftDetectionManagerInCluster ignores the watch-namespaces annotation for push-mode clusters", func() { + cluster := prepareCluster() + clusterSummaryName := randomString() + + cluster.Annotations = map[string]string{ + controllers.AgentWatchNamespacesAnnotation: fmt.Sprintf("%s,%s", randomString(), randomString()), + } + Expect(testEnv.Update(context.TODO(), cluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) + + // Succeeding at all, for a push-mode (agentless=false) deploy, is the assertion: the + // annotation only ever gets read and passed through for agentless deploys (see + // deployDriftDetectionManagerInCluster), so a push-mode deploy with it set must behave + // exactly as if it were never set, not fail or behave differently. + Expect(controllers.DeployDriftDetectionManagerInCluster(context.TODO(), testEnv.Client, cluster.Namespace, + cluster.Name, clusterSummaryName, string(libsveltosv1beta1.FeatureHelm), libsveltosv1beta1.ClusterTypeCapi, + false, false, textlogger.NewLogger(textlogger.NewConfig()))).To(Succeed()) + }) + It("getGlobalDriftDetectionManagerPatches reads old post render patches from ConfigMap", func() { cmYAML := fmt.Sprintf(`apiVersion: v1 data: @@ -315,6 +377,63 @@ metadata: }) }) +var _ = Describe("getAgentWatchNamespaces", func() { + It("returns nil when the Cluster has no watch-namespaces annotation", func() { + namespace := randomString() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + Expect(testEnv.Create(context.TODO(), ns)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed()) + + sveltosCluster := &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace, + }, + } + Expect(testEnv.Create(context.TODO(), sveltosCluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, sveltosCluster)).To(Succeed()) + + watchNamespaces, err := controllers.GetAgentWatchNamespaces(context.TODO(), testEnv.Client, + sveltosCluster.Namespace, sveltosCluster.Name, libsveltosv1beta1.ClusterTypeSveltos, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(watchNamespaces).To(BeNil()) + }) + + It("parses the comma-separated annotation, trimming whitespace", func() { + namespace := randomString() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + Expect(testEnv.Create(context.TODO(), ns)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed()) + + sveltosCluster := &libsveltosv1beta1.SveltosCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomString(), + Namespace: namespace, + Annotations: map[string]string{ + controllers.AgentWatchNamespacesAnnotation: "ns-a, ns-b ,ns-c", + }, + }, + } + Expect(testEnv.Create(context.TODO(), sveltosCluster)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, sveltosCluster)).To(Succeed()) + + watchNamespaces, err := controllers.GetAgentWatchNamespaces(context.TODO(), testEnv.Client, + sveltosCluster.Namespace, sveltosCluster.Name, libsveltosv1beta1.ClusterTypeSveltos, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(watchNamespaces).To(Equal([]string{"ns-a", "ns-b", "ns-c"})) + }) + + It("returns nil, not an error, when the Cluster instance does not exist", func() { + watchNamespaces, err := controllers.GetAgentWatchNamespaces(context.TODO(), testEnv.Client, + randomString(), randomString(), libsveltosv1beta1.ClusterTypeSveltos, + textlogger.NewLogger(textlogger.NewConfig())) + Expect(err).To(BeNil()) + Expect(watchNamespaces).To(BeNil()) + }) +}) + func prepareCluster() *clusterv1.Cluster { namespace := randomString() ns := &corev1.Namespace{ diff --git a/controllers/test_constants_test.go b/controllers/test_constants_test.go index 508bb7c3..1dd4e1ad 100644 --- a/controllers/test_constants_test.go +++ b/controllers/test_constants_test.go @@ -111,7 +111,8 @@ const ( testRegionLabelTemplate = `{{ index .Cluster.metadata.labels "region" }}` // Misc - testClusterRoleKindV1 = "ClusterRole.v1.rbac.authorization.k8s.io" + testClusterRoleKindV1 = "ClusterRole.v1.rbac.authorization.k8s.io" + testServiceAccountKindV1 = "ServiceAccount.v1." // Multiline patch testEnvLabelPatch = `- op: add diff --git a/manifest/deployment-agentless.yaml b/manifest/deployment-agentless.yaml index 39d5e5fb..5504187b 100644 --- a/manifest/deployment-agentless.yaml +++ b/manifest/deployment-agentless.yaml @@ -26,7 +26,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - --v=5 - - --version=v1.14.0 + - --version=main - --agent-in-mgmt-cluster=true command: - /manager @@ -43,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -101,7 +101,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/deployment-shard.yaml b/manifest/deployment-shard.yaml index 64b7c8a3..c17ef18d 100644 --- a/manifest/deployment-shard.yaml +++ b/manifest/deployment-shard.yaml @@ -26,7 +26,7 @@ spec: - --shard-key={{.SHARD}} - --capi-onboard-annotation= - --v=5 - - --version=v1.14.0 + - --version=main - --agent-in-mgmt-cluster=false command: - /manager @@ -43,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -101,7 +101,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 83713a4a..0f1a6c72 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -11987,7 +11987,7 @@ spec: - --shard-key= - --capi-onboard-annotation= - --v=5 - - --version=v1.14.0 + - --version=main - --agent-in-mgmt-cluster=false command: - /manager @@ -12004,7 +12004,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main livenessProbe: failureThreshold: 3 httpGet: @@ -12062,7 +12062,7 @@ spec: fieldPath: metadata.namespace - name: IS_INITIALIZATION value: "true" - image: docker.io/projectsveltos/addon-controller:v1.14.0 + image: docker.io/projectsveltos/addon-controller:main name: initialization securityContext: allowPrivilegeEscalation: false diff --git a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go index 6a55b8c4..de79c98f 100644 --- a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go +++ b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.go @@ -42,9 +42,10 @@ spec: - --cluster-namespace= - --cluster-name= - --cluster-type= + - --watch-namespaces= - --current-cluster=management-cluster - --run-mode=do-not-send-updates - - --version=v1.14.0 + - --version=main command: - /manager env: @@ -60,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:485cb5d78fc701347d06539302eda642694902acebab268cd528eaf59424c0d3 + image: docker.io/projectsveltos/drift-detection-manager@sha256:9929c55d34a317ea0516bc5f578058614778c2ddf95d6373fa2aad0b69c201cf livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml index 22159feb..93281746 100644 --- a/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml +++ b/pkg/drift-detection/drift-detection-manager-in-mgmt-cluster.yaml @@ -24,9 +24,10 @@ spec: - --cluster-namespace= - --cluster-name= - --cluster-type= + - --watch-namespaces= - --current-cluster=management-cluster - --run-mode=do-not-send-updates - - --version=v1.14.0 + - --version=main command: - /manager env: @@ -42,7 +43,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:485cb5d78fc701347d06539302eda642694902acebab268cd528eaf59424c0d3 + image: docker.io/projectsveltos/drift-detection-manager@sha256:9929c55d34a317ea0516bc5f578058614778c2ddf95d6373fa2aad0b69c201cf livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager.go b/pkg/drift-detection/drift-detection-manager.go index 0cdcb8b1..e2018ee8 100644 --- a/pkg/drift-detection/drift-detection-manager.go +++ b/pkg/drift-detection/drift-detection-manager.go @@ -146,7 +146,8 @@ spec: - --cluster-type= - --current-cluster=managed-cluster - --run-mode=do-not-send-updates - - --version=v1.14.0 + - --version=main + - --watch-namespaces= command: - /manager env: @@ -162,7 +163,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:485cb5d78fc701347d06539302eda642694902acebab268cd528eaf59424c0d3 + image: docker.io/projectsveltos/drift-detection-manager@sha256:9929c55d34a317ea0516bc5f578058614778c2ddf95d6373fa2aad0b69c201cf livenessProbe: failureThreshold: 3 httpGet: diff --git a/pkg/drift-detection/drift-detection-manager.yaml b/pkg/drift-detection/drift-detection-manager.yaml index e56bc1ae..440f6430 100644 --- a/pkg/drift-detection/drift-detection-manager.yaml +++ b/pkg/drift-detection/drift-detection-manager.yaml @@ -128,7 +128,8 @@ spec: - --cluster-type= - --current-cluster=managed-cluster - --run-mode=do-not-send-updates - - --version=v1.14.0 + - --version=main + - --watch-namespaces= command: - /manager env: @@ -144,7 +145,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectsveltos/drift-detection-manager@sha256:485cb5d78fc701347d06539302eda642694902acebab268cd528eaf59424c0d3 + image: docker.io/projectsveltos/drift-detection-manager@sha256:9929c55d34a317ea0516bc5f578058614778c2ddf95d6373fa2aad0b69c201cf livenessProbe: failureThreshold: 3 httpGet: