diff --git a/HISTORY.md b/HISTORY.md index 6265ac0f..e56822da 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,9 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ## 25.4.0 (UNRELEASED) +- Add the {mod}`tomllib ` preconf converter. + See [here](https://catt.rs/en/latest/preconf.html#tomllib) for details. + ([#716](https://github.com/python-attrs/cattrs/pull/716)) - Fix structuring of nested generic classes with stringified annotations. ([#688](https://github.com/python-attrs/cattrs/pull/688)) - Python 3.9 is no longer supported, as it is end-of-life. Use previous versions on this Python version. diff --git a/docs/cattrs.preconf.rst b/docs/cattrs.preconf.rst index 6b8f9312..05c27bbf 100644 --- a/docs/cattrs.preconf.rst +++ b/docs/cattrs.preconf.rst @@ -73,6 +73,14 @@ cattrs.preconf.tomlkit module :undoc-members: :show-inheritance: +cattrs.preconf.tomllib module +----------------------------- + +.. automodule:: cattrs.preconf.tomllib + :members: + :undoc-members: + :show-inheritance: + cattrs.preconf.ujson module --------------------------- diff --git a/docs/preconf.md b/docs/preconf.md index 6b07a0cb..9ce58d45 100644 --- a/docs/preconf.md +++ b/docs/preconf.md @@ -132,6 +132,24 @@ _msgspec_ doesn't support PyPy. ``` + +## _tomllib_ + +Found at {mod}`cattrs.preconf.tomllib`. + +Bytes are serialized as base 85 strings. Sets are serialized as lists, and deserialized back into sets. +Tuples are serialized as lists, and deserialized back into tuples. +_tomllib_ only supports mappings with string keys so mappings will have their keys stringified before serialization, and destringified during deserialization. + +Writing is supported via the [_tomli-w_](https://pypi.org/project/tomli-w/) library; this needs to be installed separately (or depend on the `cattrs[tomllib]` extra). + +On Python 3.10, the [_tomli_](https://pypi.org/project/tomli/) library is required. + +```{versionadded} NEXT + +``` + + ## _ujson_ Found at {mod}`cattrs.preconf.ujson`. @@ -208,3 +226,4 @@ Tuples are serialized as lists, and deserialized back into tuples. _tomlkit_ only supports mappings with string keys so mappings will have their keys stringified before serialization, and destringified during deserialization. [`date`](https://docs.python.org/3/library/datetime.html#datetime.date) and [`datetime`](https://docs.python.org/3/library/datetime.html#datetime.datetime) objects are passed through to be unstructured by _tomlkit_ itself. + diff --git a/pyproject.toml b/pyproject.toml index c1e5bf9c..33a29ce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,10 @@ bson = [ msgspec = [ "msgspec>=0.19.0; implementation_name == \"cpython\"", ] +tomllib = [ + "tomli>=1.1.0; python_version < '3.11'", + "tomli-w>=1.1.0", +] [tool.pytest.ini_options] addopts = "-l --benchmark-sort=fullname --benchmark-warmup=true --benchmark-warmup-iterations=5 --benchmark-group-by=fullname" diff --git a/src/cattrs/preconf/tomllib.py b/src/cattrs/preconf/tomllib.py new file mode 100644 index 00000000..2a264ed6 --- /dev/null +++ b/src/cattrs/preconf/tomllib.py @@ -0,0 +1,94 @@ +"""Preconfigured converters for tomllib.""" + +from base64 import b85decode, b85encode +from collections.abc import Set +from datetime import date, datetime +from enum import Enum +from operator import attrgetter +from typing import Any, TypeVar, Union + +try: + from tomllib import loads +except ImportError: + from tomli import loads + +try: + from tomli_w import dumps +except ImportError: # pragma: nocover + dumps = None + +from .._compat import is_mapping, is_subclass +from ..converters import BaseConverter, Converter +from ..fns import identity +from ..strategies import configure_union_passthrough +from . import validate_datetime, wrap + +__all__ = ["TomllibConverter", "configure_converter", "make_converter"] + +T = TypeVar("T") +_enum_value_getter = attrgetter("_value_") + + +class TomllibConverter(Converter): + """A converter subclass specialized for tomllib.""" + + if dumps is not None: + + def dumps(self, obj: Any, unstructure_as: Any = None, **kwargs: Any) -> str: + return dumps(self.unstructure(obj, unstructure_as=unstructure_as), **kwargs) + + def loads(self, data: str, cl: type[T], **kwargs: Any) -> T: + return self.structure(loads(data, **kwargs), cl) + + +def configure_converter(converter: BaseConverter): + """ + Configure the converter for use with the tomllib library. + + * bytes are serialized as base85 strings + * sets are serialized as lists + * tuples are serializas as lists + * mapping keys are coerced into strings when unstructuring + * dates and datetimes are left for tomllib to handle + """ + converter.register_structure_hook(bytes, lambda v, _: b85decode(v)) + converter.register_unstructure_hook( + bytes, lambda v: (b85encode(v) if v else b"").decode("utf8") + ) + + @converter.register_unstructure_hook_factory(is_mapping) + def gen_unstructure_mapping(cl: Any, unstructure_to=None): + key_handler = str + args = getattr(cl, "__args__", None) + if args: + if is_subclass(args[0], str): + key_handler = _enum_value_getter if is_subclass(args[0], Enum) else None + elif is_subclass(args[0], bytes): + + def key_handler(k: bytes): + return b85encode(k).decode("utf8") + + return converter.gen_unstructure_mapping( + cl, unstructure_to=unstructure_to, key_handler=key_handler + ) + + converter.register_unstructure_hook(datetime, identity) + converter.register_structure_hook(datetime, validate_datetime) + converter.register_unstructure_hook(date, identity) + converter.register_structure_hook( + date, lambda v, _: v if isinstance(v, date) else date.fromisoformat(v) + ) + configure_union_passthrough(Union[str, int, float, bool], converter) + + +@wrap(TomllibConverter) +def make_converter(*args: Any, **kwargs: Any) -> TomllibConverter: + kwargs["unstruct_collection_overrides"] = { + Set: list, + tuple: list, + **kwargs.get("unstruct_collection_overrides", {}), + } + res = TomllibConverter(*args, **kwargs) + configure_converter(res) + + return res diff --git a/tests/test_preconf.py b/tests/test_preconf.py index 7ee47d36..a6d25299 100644 --- a/tests/test_preconf.py +++ b/tests/test_preconf.py @@ -50,6 +50,7 @@ from cattrs.preconf.msgpack import make_converter as msgpack_make_converter from cattrs.preconf.pyyaml import make_converter as pyyaml_make_converter from cattrs.preconf.tomlkit import make_converter as tomlkit_make_converter +from cattrs.preconf.tomllib import make_converter as tomllib_make_converter from cattrs.preconf.ujson import make_converter as ujson_make_converter NO_MSGSPEC: Final = python_implementation() == "PyPy" @@ -127,6 +128,10 @@ def everythings( allow_datetime_microseconds=True, key_blacklist_characters=[], ): + """ + Args: + min_key_length: The minimum key length for keys in dicts. + """ key_text = text( characters( blacklist_categories=("Cs",) if allow_null_bytes_in_keys else ("Cs", "Cc"), @@ -784,6 +789,51 @@ class A: assert converter.loads(data, A) == A(date(2023, 1, 1)) +@given( + everythings( + allow_null_bytes_in_keys=False, + key_blacklist_characters=['"', "\\"], + allow_control_characters_in_values=False, + ), + booleans(), +) +def test_tomllib_converter(everything: Everything, detailed_validation: bool): + + converter = tomllib_make_converter(detailed_validation=detailed_validation) + raw = converter.dumps(everything) + + assert converter.loads(raw, Everything) == everything + + +@given( + everythings( + allow_null_bytes_in_keys=False, + key_blacklist_characters=['"', "\\"], + allow_control_characters_in_values=False, + ), + booleans(), +) +def test_tomllib_converter_dumps(everything: Everything, detailed_validation: bool): + converter = tomllib_make_converter(detailed_validation=detailed_validation) + raw = converter.dumps(everything) + assert converter.loads(raw, Everything) == everything + + +@given( + everythings( + allow_null_bytes_in_keys=False, + key_blacklist_characters=['"', "\\"], + allow_control_characters_in_values=False, + ) +) +def test_tomllib_converter_unstruct_collection_overrides(everything: Everything): + converter = tomllib_make_converter(unstruct_collection_overrides={Set: sorted}) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) def test_cbor2(everything: Everything): from cbor2 import dumps as cbor2_dumps @@ -951,3 +1001,8 @@ def test_literal_dicts_msgspec(): from cattrs.preconf.msgspec import make_converter as msgspec_make_converter test_literal_dicts(msgspec_make_converter) + + +def test_literal_dicts_tomllib(): + """Dicts with keys that aren't subclasses of `type` work.""" + test_literal_dicts(tomllib_make_converter) diff --git a/uv.lock b/uv.lock index 0b9c442b..954ab926 100644 --- a/uv.lock +++ b/uv.lock @@ -126,6 +126,10 @@ pyyaml = [ tomlkit = [ { name = "tomlkit" }, ] +tomllib = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli-w" }, +] ujson = [ { name = "ujson" }, ] @@ -166,11 +170,13 @@ requires-dist = [ { name = "orjson", marker = "implementation_name == 'cpython' and extra == 'orjson'", specifier = ">=3.11.3" }, { name = "pymongo", marker = "extra == 'bson'", specifier = ">=4.4.0" }, { name = "pyyaml", marker = "extra == 'pyyaml'", specifier = ">=6.0" }, + { name = "tomli", marker = "python_full_version < '3.11' and extra == 'tomllib'", specifier = ">=1.1.0" }, + { name = "tomli-w", marker = "extra == 'tomllib'", specifier = ">=1.1.0" }, { name = "tomlkit", marker = "extra == 'tomlkit'", specifier = ">=0.11.8" }, { name = "typing-extensions", specifier = ">=4.14.0" }, { name = "ujson", marker = "extra == 'ujson'", specifier = ">=5.10.0" }, ] -provides-extras = ["bson", "cbor2", "msgpack", "msgspec", "orjson", "pyyaml", "tomlkit", "ujson"] +provides-extras = ["bson", "cbor2", "msgpack", "msgspec", "orjson", "pyyaml", "tomlkit", "tomllib", "ujson"] [package.metadata.requires-dev] bench = [{ name = "pyperf", specifier = ">=2.6.1" }] @@ -431,7 +437,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -1446,6 +1452,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, ] +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + [[package]] name = "tomlkit" version = "0.13.3"