diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfiguration.java
index 1072fb823d..ca777bd2cc 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfiguration.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfiguration.java
@@ -37,6 +37,10 @@ public class LeaderElectionConfiguration {
   private final LeaderCallbacks leaderCallbacks;
   private final boolean exitOnStopLeading;
 
+  /**
+   * @deprecated Use {@link LeaderElectionConfigurationBuilder} instead
+   */
+  @Deprecated(forRemoval = true)
   public LeaderElectionConfiguration(String leaseName, String leaseNamespace, String identity) {
     this(
         leaseName,
@@ -49,30 +53,26 @@ public LeaderElectionConfiguration(String leaseName, String leaseNamespace, Stri
         true);
   }
 
+  /**
+   * @deprecated Use {@link LeaderElectionConfigurationBuilder} instead
+   */
+  @Deprecated(forRemoval = true)
   public LeaderElectionConfiguration(String leaseName, String leaseNamespace) {
-    this(
-        leaseName,
-        leaseNamespace,
-        LEASE_DURATION_DEFAULT_VALUE,
-        RENEW_DEADLINE_DEFAULT_VALUE,
-        RETRY_PERIOD_DEFAULT_VALUE,
-        null,
-        null,
-        true);
+    this(leaseName, leaseNamespace, null);
   }
 
+  /**
+   * @deprecated Use {@link LeaderElectionConfigurationBuilder} instead
+   */
+  @Deprecated(forRemoval = true)
   public LeaderElectionConfiguration(String leaseName) {
-    this(
-        leaseName,
-        null,
-        LEASE_DURATION_DEFAULT_VALUE,
-        RENEW_DEADLINE_DEFAULT_VALUE,
-        RETRY_PERIOD_DEFAULT_VALUE,
-        null,
-        null,
-        true);
+    this(leaseName, null);
   }
 
+  /**
+   * @deprecated Use {@link LeaderElectionConfigurationBuilder} instead
+   */
+  @Deprecated(forRemoval = true)
   public LeaderElectionConfiguration(
       String leaseName,
       String leaseNamespace,
@@ -82,6 +82,10 @@ public LeaderElectionConfiguration(
     this(leaseName, leaseNamespace, leaseDuration, renewDeadline, retryPeriod, null, null, true);
   }
 
+  /**
+   * @deprecated Use {@link LeaderElectionConfigurationBuilder} instead
+   */
+  @Deprecated // this will be made package-only
   public LeaderElectionConfiguration(
       String leaseName,
       String leaseNamespace,
diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfigurationBuilder.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfigurationBuilder.java
index 74f2c81cba..51ee40d84c 100644
--- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfigurationBuilder.java
+++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/LeaderElectionConfigurationBuilder.java
@@ -31,7 +31,6 @@ public final class LeaderElectionConfigurationBuilder {
   private Duration renewDeadline = RENEW_DEADLINE_DEFAULT_VALUE;
   private Duration retryPeriod = RETRY_PERIOD_DEFAULT_VALUE;
   private LeaderCallbacks leaderCallbacks;
-  private boolean exitOnStopLeading = true;
 
   private LeaderElectionConfigurationBuilder(String leaseName) {
     this.leaseName = leaseName;
@@ -71,12 +70,22 @@ public LeaderElectionConfigurationBuilder withLeaderCallbacks(LeaderCallbacks le
     return this;
   }
 
+  /**
+   * @deprecated Use {@link #buildForTest(boolean)} instead as setting this to false should only be
+   *     used for testing purposes
+   */
+  @Deprecated(forRemoval = true)
   public LeaderElectionConfigurationBuilder withExitOnStopLeading(boolean exitOnStopLeading) {
-    this.exitOnStopLeading = exitOnStopLeading;
-    return this;
+    throw new UnsupportedOperationException(
+        "Setting exitOnStopLeading should only be used for testing purposes, use buildForTest"
+            + " instead");
   }
 
   public LeaderElectionConfiguration build() {
+    return buildForTest(false);
+  }
+
+  public LeaderElectionConfiguration buildForTest(boolean exitOnStopLeading) {
     return new LeaderElectionConfiguration(
         leaseName,
         leaseNamespace,