Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for custom JSON encoders/decoders. #3300

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions docs/advanced/json.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# JSON encoding/decoding

You can set a custom json encoder and decoder, e.g. if you want to use `msgspec` instead or python's `json` implementation.

Your custom function needs to return `bytes`.


**Custom orjson implementation:**
```python
import httpx
import orjson
import typing


def custom_json_encoder(json_data: typing.Any) -> bytes:
return orjson.dumps(json_data)

httpx.register_json_encoder(custom_json_encoder)


def custom_json_decoder(json_data: bytes, **kwargs: typing.Any) -> bytes:
return orjson.loads(json_data)

httpx.register_json_decoder(custom_json_decoder)
```


**Custom msgspec implementation:**
```python
import httpx
import msgspec
import typing


encoder = msgspec.json.Encoder()
def custom_json_encoder(json_data: typing.Any) -> bytes:
return encoder.encode(json_data)

httpx.register_json_encoder(custom_json_encoder)


decoder = msgspec.json.Decoder()
def custom_json_decoder(json_data: bytes, **kwargs: typing.Any) -> bytes:
return decoder.decode(json_data)

httpx.register_json_decoder(custom_json_decoder)
```

2 changes: 2 additions & 0 deletions httpx/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ def main() -> None: # type: ignore
"QueryParams",
"ReadError",
"ReadTimeout",
"register_json_decoder",
"register_json_encoder",
"RemoteProtocolError",
"request",
"Request",
Expand Down
36 changes: 33 additions & 3 deletions httpx/_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

import inspect
import warnings
from json import dumps as json_dumps
from json import dumps as json_dumps, loads as json_loads
from typing import (
Any,
AsyncIterable,
AsyncIterator,
Callable,
Iterable,
Iterator,
Mapping,
Expand All @@ -25,7 +26,32 @@
)
from ._utils import peek_filelike_length, primitive_value_to_str

__all__ = ["ByteStream"]
__all__ = ["ByteStream", "register_json_decoder", "register_json_encoder"]


def json_encoder(json_data: Any) -> bytes:
return json_dumps(json_data).encode("utf-8")


def json_decoder(json_data: bytes, **kwargs: Any) -> Any:
return json_loads(json_data, **kwargs)


def register_json_encoder(
json_encode_callable: Callable[[Any], bytes],
) -> None:
global json_encoder
json_encoder = json_encode_callable # type: ignore


def register_json_decoder(
json_decode_callable: Callable[
[bytes, Any],
Any,
],
) -> None:
global json_decoder
json_decoder = json_decode_callable # type: ignore


class ByteStream(AsyncByteStream, SyncByteStream):
Expand Down Expand Up @@ -133,6 +159,10 @@ def encode_content(
raise TypeError(f"Unexpected type for 'content', {type(content)!r}")


def decode_json(json_data: bytes, **kwargs: Any) -> Any:
return json_decoder(json_data, **kwargs)


def encode_urlencoded_data(
data: RequestData,
) -> tuple[dict[str, str], ByteStream]:
Expand Down Expand Up @@ -174,7 +204,7 @@ def encode_html(html: str) -> tuple[dict[str, str], ByteStream]:


def encode_json(json: Any) -> tuple[dict[str, str], ByteStream]:
body = json_dumps(json).encode("utf-8")
body = json_encoder(json)
content_length = str(len(body))
content_type = "application/json"
headers = {"Content-Length": content_length, "Content-Type": content_type}
Expand Down
11 changes: 8 additions & 3 deletions httpx/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@

import datetime
import email.message
import json as jsonlib
import typing
import urllib.request
from collections.abc import Mapping
from http.cookiejar import Cookie, CookieJar

from ._content import ByteStream, UnattachedStream, encode_request, encode_response
from ._content import (
ByteStream,
UnattachedStream,
decode_json,
encode_request,
encode_response,
)
from ._decoders import (
SUPPORTED_DECODERS,
ByteChunker,
Expand Down Expand Up @@ -763,7 +768,7 @@ def raise_for_status(self) -> Response:
raise HTTPStatusError(message, request=request, response=self)

def json(self, **kwargs: typing.Any) -> typing.Any:
return jsonlib.loads(self.content, **kwargs)
return decode_json(self.content, **kwargs)

@property
def cookies(self) -> Cookies:
Expand Down
33 changes: 33 additions & 0 deletions tests/test_content.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import io
import json
import typing

import pytest
Expand Down Expand Up @@ -180,6 +181,38 @@ async def test_json_content():
assert async_content == b'{"Hello": "world!"}'


def test_json_content_register_custom_encoder():
try:

def test_encoder(json_content):
raise Exception("Encoder raise")

httpx.register_json_encoder(test_encoder)
with pytest.raises(Exception, match="Encoder raise"):
httpx.Request(method, url, json={"Hello": "world!"})
finally:
httpx.register_json_encoder(
lambda json_content: json.dumps(json_content).encode("utf-8")
)


def test_json_content_register_custom_decoder():
try:

def test_decoder(json_content: bytes, **kwargs: typing.Any) -> typing.Any:
raise Exception("Decoder raise")

httpx.register_json_decoder(test_decoder) # type: ignore

with pytest.raises(Exception, match="Decoder raise"):
response = httpx.Response(200, content='{"Hello": "world!"}')
response.json()
finally:
httpx.register_json_decoder(
lambda json_data, **kwargs: json.loads(json_data, **kwargs)
)


@pytest.mark.anyio
async def test_urlencoded_content():
request = httpx.Request(method, url, data={"Hello": "world!"})
Expand Down
2 changes: 1 addition & 1 deletion tests/test_exported_members.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@


def test_all_imports_are_exported() -> None:
included_private_members = ["__description__", "__title__", "__version__"]
included_private_members = {"__description__", "__title__", "__version__"}
assert httpx.__all__ == sorted(
(
member
Expand Down