diff --git a/google/auth/crypt/rsa.py b/google/auth/crypt/rsa.py
index 4b2fb39ff..639be9069 100644
--- a/google/auth/crypt/rsa.py
+++ b/google/auth/crypt/rsa.py
@@ -24,7 +24,6 @@
 
 from google.auth import _helpers
 from google.auth.crypt import _cryptography_rsa
-from google.auth.crypt import _python_rsa
 from google.auth.crypt import base
 
 RSA_KEY_MODULE_PREFIX = "rsa.key"
@@ -37,6 +36,7 @@ class RSAVerifier(base.Verifier):
         public_key (Union["rsa.key.PublicKey", cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey]):
             The public key used to verify signatures.
     Raises:
+        ImportError: if called with an rsa.key.PublicKey, when the rsa library is not installed
         ValueError: if an unrecognized public key is provided
     """
 
@@ -45,6 +45,8 @@ def __init__(self, public_key):
         if isinstance(public_key, RSAPublicKey):
             impl_lib = _cryptography_rsa
         elif module_str.startswith(RSA_KEY_MODULE_PREFIX):
+            from google.auth.crypt import _python_rsa
+
             impl_lib = _python_rsa
         else:
             raise ValueError(f"unrecognized public key type: {type(public_key)}")
@@ -85,6 +87,7 @@ class RSASigner(base.Signer, base.FromServiceAccountMixin):
             public key or certificate.
 
     Raises:
+        ImportError: if called with an rsa.key.PrivateKey, when the rsa library is not installed
         ValueError: if an unrecognized public key is provided
     """
 
@@ -93,6 +96,8 @@ def __init__(self, private_key, key_id=None):
         if isinstance(private_key, RSAPrivateKey):
             impl_lib = _cryptography_rsa
         elif module_str.startswith(RSA_KEY_MODULE_PREFIX):
+            from google.auth.crypt import _python_rsa
+
             impl_lib = _python_rsa
         else:
             raise ValueError(f"unrecognized private key type: {type(private_key)}")
diff --git a/setup.py b/setup.py
index ba9e214b1..718a6e585 100644
--- a/setup.py
+++ b/setup.py
@@ -25,9 +25,6 @@
 DEPENDENCIES = (
     "pyasn1-modules>=0.2.1",
     cryptography_base_require,
-    # TODO: remove rsa from dependencies in next release (replaced with cryptography)i
-    # https://github.com/googleapis/google-auth-library-python/issues/1810
-    "rsa>=3.1.4,<5",
 )
 
 requests_extra_require = ["requests >= 2.20.0, < 3.0.0"]
@@ -73,6 +70,9 @@
     # TODO(https://github.com/googleapis/google-auth-library-python/issues/1722): `test_aiohttp_requests` depend on
     # aiohttp < 3.10.0 which is a bug. Investigate and remove the pinned aiohttp version.
     "aiohttp < 3.10.0",
+    # rsa library was removed as a dependency, but we still have some code paths that support it
+    # TODO: remove dependency when google.auth.crypt._python_rsa is removed
+    "rsa>=3.1.4,<5",
 ]
 
 extras = {
diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt
index 52ad3af91..d9655a360 100644
--- a/testing/constraints-3.7.txt
+++ b/testing/constraints-3.7.txt
@@ -7,7 +7,6 @@
 # Then this file should have foo==1.14.0
 pyasn1-modules==0.2.1
 setuptools==40.3.0
-rsa==3.1.4
 aiohttp==3.6.2
 requests==2.20.0
 pyjwt==2.0
\ No newline at end of file
diff --git a/tests/crypt/test_rsa.py b/tests/crypt/test_rsa.py
index 6f7aa2691..6ed822ad9 100644
--- a/tests/crypt/test_rsa.py
+++ b/tests/crypt/test_rsa.py
@@ -18,7 +18,7 @@
 from cryptography.hazmat import backends
 from cryptography.hazmat.primitives import serialization
 import pytest
-import rsa as rsa_lib
+import rsa as rsa_lib  # type: ignore
 
 from google.auth.crypt import _cryptography_rsa
 from google.auth.crypt import _python_rsa