AppCred fixes and enhancing the AppCred kuttl tests

internal/openstack/applicationcredential.go

@@ -59,6 +59,34 @@ func isACEnabled(globalAC corev1beta1.ApplicationCredentialSection, serviceAC *c
 	return serviceAC != nil && serviceAC.Enabled
 }
 
+// CleanupApplicationCredentialForService deletes the AC CR for a service if it exists.
+// Used when a service or its AC is disabled - deletes the AC CR if it exists regardless
+// of the AC enabled flag.
+func CleanupApplicationCredentialForService(
+	ctx context.Context,
+	helper *helper.Helper,
+	instance *corev1beta1.OpenStackControlPlane,
+	serviceName string,
+) error {
+	acName := keystonev1.GetACCRName(serviceName)
+	acCR := &keystonev1.KeystoneApplicationCredential{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      acName,
+			Namespace: instance.Namespace,
+		},
+	}
+	Log := GetLogger(ctx)
+	err := helper.GetClient().Delete(ctx, acCR)
+	if k8s_errors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	Log.Info("Service disabled, deleted existing KeystoneApplicationCredential CR", "service", serviceName, "acName", acName)
+	return nil
+}
+
 // EnsureApplicationCredentialForService handles AC creation for a single service.
 // If service is not ready, AC creation is deferred
 // If AC already exists and is ready, it's used immediately
@@ -125,6 +153,11 @@ func EnsureApplicationCredentialForService(
 
 	// Check if AC CR exists and is ready
 	if acExists {
+		// We want to run reconcileApplicationCredential to update the AC CR if it exists and is ready and AC config fields changed
+		err = reconcileApplicationCredential(ctx, helper, instance, acName, serviceUser, secretName, passwordSelector, merged)
+		if err != nil {
+			return "", ctrl.Result{}, err
+		}
 		if acCR.IsReady() {
 			Log.Info("Application Credential is ready", "service", serviceName, "acName", acName, "secretName", acCR.Status.SecretName)
 			return acCR.Status.SecretName, ctrl.Result{}, nil

internal/openstack/barbican.go

@@ -37,6 +37,10 @@ func ReconcileBarbican(ctx context.Context, instance *corev1beta1.OpenStackContr
 		instance.Status.ContainerImages.BarbicanAPIImage = nil
 		instance.Status.ContainerImages.BarbicanWorkerImage = nil
 		instance.Status.ContainerImages.BarbicanKeystoneListenerImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, barbican.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -73,8 +77,8 @@ func ReconcileBarbican(ctx context.Context, instance *corev1beta1.OpenStackContr
 		barbicanSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Barbican.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Barbican.ApplicationCredential != nil ||
 		instance.Spec.Barbican.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/cinder.go

@@ -59,6 +59,10 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl
 		instance.Status.ContainerImages.CinderSchedulerImage = nil
 		instance.Status.ContainerImages.CinderBackupImage = nil
 		instance.Status.ContainerImages.CinderVolumeImages = make(map[string]*string)
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, cinder.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -96,8 +100,8 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl
 		cinderSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Cinder.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Cinder.ApplicationCredential != nil ||
 		instance.Spec.Cinder.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/designate.go

@@ -41,6 +41,10 @@ func ReconcileDesignate(ctx context.Context, instance *corev1beta1.OpenStackCont
 		instance.Status.ContainerImages.DesignateBackendbind9Image = nil
 		instance.Status.ContainerImages.DesignateUnboundImage = nil
 		instance.Status.ContainerImages.NetUtilsImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, designate.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -85,8 +89,8 @@ func ReconcileDesignate(ctx context.Context, instance *corev1beta1.OpenStackCont
 		designateSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Designate.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Designate.ApplicationCredential != nil ||
 		instance.Spec.Designate.Template.DesignateAPI.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/glance.go

@@ -64,6 +64,10 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneGlanceReadyCondition)
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposeGlanceReadyCondition)
 		instance.Status.ContainerImages.GlanceAPIImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, glance.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -128,9 +132,8 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl
 		}
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Glance.ApplicationCredential) || hasACConfigured {
-
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Glance.ApplicationCredential != nil || hasACConfigured {
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(
 			ctx,
 			helper,

internal/openstack/heat.go

@@ -40,6 +40,10 @@ func ReconcileHeat(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		instance.Status.ContainerImages.HeatAPIImage = nil
 		instance.Status.ContainerImages.HeatCfnapiImage = nil
 		instance.Status.ContainerImages.HeatEngineImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, heat.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -120,8 +124,8 @@ func ReconcileHeat(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		heatSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Heat.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Heat.ApplicationCredential != nil ||
 		instance.Spec.Heat.Template.Auth.ApplicationCredentialSecret != "" {
 
 		heatACSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/ironic.go

@@ -40,6 +40,13 @@ func ReconcileIronic(ctx context.Context, instance *corev1beta1.OpenStackControl
 		instance.Status.ContainerImages.IronicNeutronAgentImage = nil
 		instance.Status.ContainerImages.IronicPxeImage = nil
 		instance.Status.ContainerImages.IronicPythonAgentImage = nil
+		// Clean up AC CRs when service is disabled (ironic has two: ironic and ironic-inspector)
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, ironic.Name); err != nil {
+			return ctrl.Result{}, err
+		}
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, "ironic-inspector"); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -124,8 +131,8 @@ func ReconcileIronic(ctx context.Context, instance *corev1beta1.OpenStackControl
 		ironicSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Ironic.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Ironic.ApplicationCredential != nil ||
 		instance.Spec.Ironic.Template.Auth.ApplicationCredentialSecret != "" ||
 		instance.Spec.Ironic.Template.IronicInspector.Auth.ApplicationCredentialSecret != "" {
 

internal/openstack/manila.go

@@ -38,6 +38,10 @@ func ReconcileManila(ctx context.Context, instance *corev1beta1.OpenStackControl
 		instance.Status.ContainerImages.ManilaAPIImage = nil
 		instance.Status.ContainerImages.ManilaSchedulerImage = nil
 		instance.Status.ContainerImages.ManilaShareImages = make(map[string]*string)
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, manila.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -75,8 +79,8 @@ func ReconcileManila(ctx context.Context, instance *corev1beta1.OpenStackControl
 		manilaSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Manila.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Manila.ApplicationCredential != nil ||
 		instance.Spec.Manila.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/neutron.go

@@ -39,6 +39,10 @@ func ReconcileNeutron(ctx context.Context, instance *corev1beta1.OpenStackContro
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneNeutronReadyCondition)
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposeNeutronReadyCondition)
 		instance.Status.ContainerImages.NeutronAPIImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, neutronAPI.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -119,8 +123,8 @@ func ReconcileNeutron(ctx context.Context, instance *corev1beta1.OpenStackContro
 		neutronSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Neutron.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Neutron.ApplicationCredential != nil ||
 		instance.Spec.Neutron.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/nova.go

@@ -61,6 +61,10 @@ func ReconcileNova(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		instance.Status.ContainerImages.NovaConductorImage = nil
 		instance.Status.ContainerImages.NovaNovncImage = nil
 		instance.Status.ContainerImages.NovaSchedulerImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, nova.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -191,8 +195,8 @@ func ReconcileNova(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		novaSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Nova.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Nova.ApplicationCredential != nil ||
 		instance.Spec.Nova.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/octavia.go

@@ -59,6 +59,10 @@ func ReconcileOctavia(ctx context.Context, instance *corev1beta1.OpenStackContro
 		instance.Status.ContainerImages.OctaviaHousekeepingImage = nil
 		instance.Status.ContainerImages.OctaviaApacheImage = nil
 		instance.Status.ContainerImages.OctaviaRsyslogImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, octavia.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -167,9 +171,9 @@ func ReconcileOctavia(ctx context.Context, instance *corev1beta1.OpenStackContro
 		octaviaSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Octavia.ApplicationCredential) ||
-		instance.Spec.Octavia.Template.OctaviaAPI.Auth.ApplicationCredentialSecret != "" {
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Octavia.ApplicationCredential != nil ||
+		instance.Spec.Octavia.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(
 			ctx,
@@ -194,7 +198,7 @@ func ReconcileOctavia(ctx context.Context, instance *corev1beta1.OpenStackContro
 		// Set ApplicationCredentialSecret based on what the helper returned:
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
-		instance.Spec.Octavia.Template.OctaviaAPI.Auth.ApplicationCredentialSecret = acSecretName
+		instance.Spec.Octavia.Template.Auth.ApplicationCredentialSecret = acSecretName
 	}
 
 	svcs, err := service.GetServicesListWithLabel(

internal/openstack/placement.go

@@ -34,6 +34,10 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC
 		}
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlanePlacementAPIReadyCondition)
 		instance.Status.Conditions.Remove(corev1beta1.OpenStackControlPlaneExposePlacementAPIReadyCondition)
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, placementAPI.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -79,8 +83,8 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC
 		placementSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Placement.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Placement.ApplicationCredential != nil ||
 		instance.Spec.Placement.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(

internal/openstack/swift.go

@@ -40,6 +40,10 @@ func ReconcileSwift(ctx context.Context, instance *corev1beta1.OpenStackControlP
 		instance.Status.ContainerImages.SwiftContainerImage = nil
 		instance.Status.ContainerImages.SwiftObjectImage = nil
 		instance.Status.ContainerImages.SwiftProxyImage = nil
+		// Clean up AC CRs when service is disabled
+		if err := CleanupApplicationCredentialForService(ctx, helper, instance, swift.Name); err != nil {
+			return ctrl.Result{}, err
+		}
 		return ctrl.Result{}, nil
 	}
 
@@ -109,8 +113,8 @@ func ReconcileSwift(ctx context.Context, instance *corev1beta1.OpenStackControlP
 		swiftSecret = instance.Spec.Secret
 	}
 
-	// Only call if AC enabled or currently configured
-	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Swift.ApplicationCredential) ||
+	// Always reconcile AC - EnsureApplicationCredentialForService checks cluster state and handles the full AC lifecycle.
+	if instance.Spec.Swift.ApplicationCredential != nil ||
 		instance.Spec.Swift.Template.SwiftProxy.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(