From a150a970d19a639ed47149e6fc7a4a243eb2dbc9 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Wed, 4 Feb 2026 09:57:51 +0100 Subject: [PATCH] Use kvm filters pipeline with configured experimental projects --- cmd/main.go | 3 +- helm/bundles/cortex-nova/values.yaml | 2 + .../scheduling/nova/external_scheduler_api.go | 24 ++++++++--- .../nova/external_scheduler_api_test.go | 40 +++++++++++++------ 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index cad93b9a1..9ceb90bf0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -298,7 +298,8 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler") os.Exit(1) } - nova.NewAPI(config, decisionController).Init(mux) + httpAPIConf := conf.GetConfigOrDie[nova.HTTPAPIConfig]() + nova.NewAPI(httpAPIConf, decisionController).Init(mux) } if slices.Contains(config.EnabledControllers, "nova-deschedulings-pipeline-controller") { // Deschedulings controller diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index fcbce1cfd..3635ba5b0 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -96,6 +96,8 @@ cortex-scheduling-controllers: conf: <<: *cortexConf leaderElectionID: cortex-nova-scheduling + # OpenStack projects that use experimental features. + experimentalProjectIDs: [] monitoring: labels: <<: *cortexMonitoringLabels diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 26083c481..0367f9d27 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -12,11 +12,11 @@ import ( "io" "log/slog" "net/http" + "slices" "strings" api "github.com/cobaltcore-dev/cortex/api/delegation/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/pkg/conf" scheduling "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" corev1 "k8s.io/api/core/v1" @@ -26,6 +26,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" ) +// Custom configuration for the Nova external scheduler api. +type HTTPAPIConfig struct { + // OpenStack projects that use experimental features. + ExperimentalProjectIDs []string `json:"experimentalProjectIDs,omitempty"` +} + type HTTPAPIDelegate interface { // Process the decision from the API. Should create and return the updated decision. ProcessNewDecisionFromAPI(ctx context.Context, decision *v1alpha1.Decision) error @@ -37,12 +43,12 @@ type HTTPAPI interface { } type httpAPI struct { - config conf.Config + config HTTPAPIConfig monitor scheduling.APIMonitor delegate HTTPAPIDelegate } -func NewAPI(config conf.Config, delegate HTTPAPIDelegate) HTTPAPI { +func NewAPI(config HTTPAPIConfig, delegate HTTPAPIDelegate) HTTPAPI { return &httpAPI{ config: config, monitor: scheduling.NewSchedulerMonitor(), @@ -88,11 +94,19 @@ func (httpAPI *httpAPI) inferPipelineName(requestData api.ExternalSchedulerReque } switch strings.ToLower(hvType) { case "qemu", "ch": + enableAllFilters := false + // If the nova request matches a configurable openstack project, + // use a different pipeline that has all filters enabled. + if slices.Contains(httpAPI.config.ExperimentalProjectIDs, requestData.Spec.Data.ProjectID) { + enableAllFilters = true + } if requestData.Reservation { + enableAllFilters = true + } + if enableAllFilters { return "nova-external-scheduler-kvm-all-filters-enabled", nil - } else { - return "nova-external-scheduler-kvm", nil } + return "nova-external-scheduler-kvm", nil case "vmware vcenter server": if requestData.Reservation { return "", errors.New("reservations are not supported on vmware hypervisors") diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index 6ac7706ed..3fb735954 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -15,7 +15,6 @@ import ( novaapi "github.com/cobaltcore-dev/cortex/api/delegation/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/pkg/conf" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -32,8 +31,8 @@ func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, dec } func TestNewAPI(t *testing.T) { - config := conf.Config{SchedulingDomain: "test-operator"} delegate := &mockHTTPAPIDelegate{} + config := HTTPAPIConfig{} api := NewAPI(config, delegate) @@ -46,10 +45,6 @@ func TestNewAPI(t *testing.T) { t.Fatal("NewAPI did not return httpAPI type") } - if httpAPI.config.SchedulingDomain != "test-operator" { - t.Errorf("Expected scheduling domain 'test-operator', got %s", httpAPI.config.SchedulingDomain) - } - if httpAPI.delegate != delegate { t.Error("Delegate not set correctly") } @@ -60,8 +55,8 @@ func TestNewAPI(t *testing.T) { } func TestHTTPAPI_Init(t *testing.T) { - config := conf.Config{SchedulingDomain: "test-operator"} delegate := &mockHTTPAPIDelegate{} + config := HTTPAPIConfig{} api := NewAPI(config, delegate) mux := http.NewServeMux() @@ -79,8 +74,8 @@ func TestHTTPAPI_Init(t *testing.T) { } func TestHTTPAPI_canRunScheduler(t *testing.T) { - config := conf.Config{SchedulingDomain: "test-operator"} delegate := &mockHTTPAPIDelegate{} + config := HTTPAPIConfig{} api := NewAPI(config, delegate).(*httpAPI) tests := []struct { @@ -271,7 +266,6 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - config := conf.Config{SchedulingDomain: "test-operator"} delegate := &mockHTTPAPIDelegate{ processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { if tt.processDecisionErr != nil { @@ -285,6 +279,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { }, } + config := HTTPAPIConfig{} api := NewAPI(config, delegate).(*httpAPI) var body *strings.Reader @@ -324,8 +319,6 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) { } func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { - config := conf.Config{SchedulingDomain: v1alpha1.SchedulingDomainNova} - var capturedDecision *v1alpha1.Decision delegate := &mockHTTPAPIDelegate{ processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { @@ -338,6 +331,7 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { }, } + config := HTTPAPIConfig{} api := NewAPI(config, delegate).(*httpAPI) requestData := novaapi.ExternalSchedulerRequest{ @@ -395,8 +389,10 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { } func TestHTTPAPI_inferPipelineName(t *testing.T) { - config := conf.Config{SchedulingDomain: "test-operator"} delegate := &mockHTTPAPIDelegate{} + config := HTTPAPIConfig{ + ExperimentalProjectIDs: []string{"my-experimental-project-id"}, + } api := NewAPI(config, delegate).(*httpAPI) tests := []struct { @@ -406,6 +402,26 @@ func TestHTTPAPI_inferPipelineName(t *testing.T) { expectErr bool errContains string }{ + { + name: "experimental project ID requesting kvm vm", + requestData: novaapi.ExternalSchedulerRequest{ + Spec: novaapi.NovaObject[novaapi.NovaSpec]{ + Data: novaapi.NovaSpec{ + ProjectID: "my-experimental-project-id", + Flavor: novaapi.NovaObject[novaapi.NovaFlavor]{ + Data: novaapi.NovaFlavor{ + ExtraSpecs: map[string]string{ + "capabilities:hypervisor_type": "qemu", + }, + }, + }, + }, + }, + Reservation: false, + }, + expectedResult: "nova-external-scheduler-kvm-all-filters-enabled", + expectErr: false, + }, { name: "qemu hypervisor without reservation", requestData: novaapi.ExternalSchedulerRequest{