Add support for pydantic v2 via pydantic.v1 compat module

This adds support for pydantic v2 by using the `pydantic.v1` compat module
when pydantic v2 is installed and otherwise falling back to the `pydantic` module.

In order to satisfy mypy, we only import `pydantic.v1` during type
checking. This allows proper type checking without needing to clutter
the code with `type: ignores`.

Once some time is passed, we can entirely drop compatibility for pydantic v1 and
start using the actual pydantic v2 code to reap the additional
performance benefits and other improvements of pydantic v2.

Signed-off-by: Maxwell G <[email protected]>

changelog.d/16332.feature

@@ -0,0 +1,2 @@
+Added support for pydantic v2 in addition to pydantic v1.
+Contributed by Maxwell G (@gotmax23).

pyproject.toml

@@ -209,9 +209,9 @@ ijson = ">=3.1.4"
 matrix-common = "^1.3.0"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
-# This is the most recent version of Pydantic with available on common distros.
-# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
-pydantic = "^1.7.4"
+# We support pydantic v1 and pydantic v2 via the pydantic.v1 compat module.
+# See https://github.com/matrix-org/synapse/issues/15858
+pydantic = ">=1.7.4, <3"
 
 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -319,6 +319,8 @@ all = [
 isort = ">=5.10.1"
 black = ">=22.7.0"
 ruff = "0.0.286"
+# Type checking only works with the pydantic.v1 compat module from pydantic v2
+pydantic = "^2"
 
 # Typechecking
 lxml-stubs = ">=0.4.0"

scripts-dev/check_pydantic_models.py

@@ -36,11 +36,41 @@
 import traceback
 import unittest.mock
 from contextlib import contextmanager
-from typing import Any, Callable, Dict, Generator, List, Set, Type, TypeVar
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Set,
+    Type,
+    TypeVar,
+)
 
 from parameterized import parameterized
-from pydantic import BaseModel as PydanticBaseModel, conbytes, confloat, conint, constr
-from pydantic.typing import get_args
+
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import (
+        BaseModel as PydanticBaseModel,
+        conbytes,
+        confloat,
+        conint,
+        constr,
+    )
+    from pydantic.v1.typing import get_args
+else:
+    from pydantic import (  # type: ignore
+        BaseModel as PydanticBaseModel,
+        conbytes,
+        confloat,
+        conint,
+        constr,
+    )
+    from pydantic.typing import get_args  # type: ignore
+
 from typing_extensions import ParamSpec
 
 logger = logging.getLogger(__name__)
@@ -251,7 +281,10 @@ def test_expression_without_strict_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import constr
+                try:
+                    from pydantic.v1 import constr
+                except ImportError:
+                    from pydantic import constr
                 constr()
                 """
             )
@@ -269,7 +302,10 @@ def test_wildcard_import_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import *
+                try:
+                    from pydantic.v1 import *
+                except ImportError:
+                    from pydantic import *
                 constr()
                 """
             )
@@ -278,7 +314,10 @@ def test_alternative_import_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic.types import constr
+                try:
+                    from pydantic.v1.types import constr
+                except ImportError:
+                    from pydantic.types import constr
                 constr()
                 """
             )
@@ -287,16 +326,22 @@ def test_alternative_import_attribute_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                import pydantic.types
-                pydantic.types.constr()
+                try:
+                    from pydantic.v1 import types as pydantic_types
+                except ImportError:
+                    from pydantic import types as pydantic_types
+                pydantic_types.constr()
                 """
             )
 
     def test_kwarg_but_no_strict_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import constr
+                try:
+                    from pydantic.v1 import constr
+                except ImportError:
+                    from pydantic import constr
                 constr(min_length=10)
                 """
             )
@@ -305,7 +350,10 @@ def test_kwarg_strict_False_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import constr
+                try:
+                    from pydantic.v1 import constr
+                except ImportError:
+                    from pydantic import constr
                 constr(strict=False)
                 """
             )
@@ -314,7 +362,10 @@ def test_kwarg_strict_True_doesnt_raise(self) -> None:
         with monkeypatch_pydantic():
             run_test_snippet(
                 """
-                from pydantic import constr
+                try:
+                    from pydantic.v1 import constr
+                except ImportError:
+                    from pydantic import constr
                 constr(strict=True)
                 """
             )
@@ -323,7 +374,10 @@ def test_annotation_without_strict_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import constr
+                try:
+                    from pydantic.v1 import constr
+                except ImportError:
+                    from pydantic import constr
                 x: constr()
                 """
             )
@@ -332,7 +386,10 @@ def test_field_annotation_without_strict_raises(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic import BaseModel, conint
+                try:
+                    from pydantic.v1 import BaseModel, conint
+                except ImportError:
+                    from pydantic import BaseModel, conint
                 class C:
                     x: conint()
                 """
@@ -361,7 +418,10 @@ def test_field_holding_unwanted_type_raises(self, annotation: str) -> None:
             run_test_snippet(
                 f"""
                 from typing import *
-                from pydantic import *
+                try:
+                    from pydantic.v1 import *
+                except ImportError:
+                    from pydantic import *
                 class C(BaseModel):
                     f: {annotation}
                 """
@@ -388,7 +448,10 @@ def test_field_holding_accepted_type_doesnt_raise(self, annotation: str) -> None
             run_test_snippet(
                 f"""
                 from typing import *
-                from pydantic import *
+                try:
+                    from pydantic.v1 import *
+                except ImportError:
+                    from pydantic import *
                 class C(BaseModel):
                     f: {annotation}
                 """
@@ -398,7 +461,10 @@ def test_field_holding_str_raises_with_alternative_import(self) -> None:
         with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException):
             run_test_snippet(
                 """
-                from pydantic.main import BaseModel
+                try:
+                    from pydantic.v1.main import BaseModel
+                except ImportError:
+                    from pydantic.main import BaseModel
                 class C(BaseModel):
                     f: str
                 """

synapse/_pydantic_compat.py

@@ -0,0 +1,6 @@
+from packaging.version import Version
+from pydantic import __version__ as pydantic_version
+
+HAS_PYDANTIC_V2: bool = Version(pydantic_version).major == 2
+
+__all__ = ("HAS_PYDANTIC_V2",)

synapse/config/_util.py

@@ -11,10 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict, Type, TypeVar
+from typing import TYPE_CHECKING, Any, Dict, Type, TypeVar
 
 import jsonschema
-from pydantic import BaseModel, ValidationError, parse_obj_as
+
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import BaseModel, ValidationError, parse_obj_as
+else:
+    from pydantic import BaseModel, ValidationError, parse_obj_as
 
 from synapse.config._base import ConfigError
 from synapse.types import JsonDict, StrSequence

synapse/config/workers.py

@@ -15,10 +15,16 @@
 
 import argparse
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import attr
-from pydantic import BaseModel, Extra, StrictBool, StrictInt, StrictStr
+
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import BaseModel, Extra, StrictBool, StrictInt, StrictStr
+else:
+    from pydantic import BaseModel, Extra, StrictBool, StrictInt, StrictStr
 
 from synapse.config._base import (
     Config,

synapse/events/validator.py

@@ -12,10 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import collections.abc
-from typing import List, Type, Union, cast
+from typing import TYPE_CHECKING, List, Type, Union, cast
 
 import jsonschema
-from pydantic import Field, StrictBool, StrictStr
+
+if TYPE_CHECKING:
+    from pydantic.v1 import Field, StrictBool, StrictStr
+else:
+    try:
+        from pydantic.v1 import Field, StrictBool, StrictStr
+    except ImportError:
+        from pydantic import Field, StrictBool, StrictStr  # type: ignore
 
 from synapse.api.constants import (
     MAX_ALIAS_LENGTH,

synapse/http/servlet.py

@@ -28,8 +28,15 @@
     overload,
 )
 
-from pydantic import BaseModel, MissingError, PydanticValueError, ValidationError
-from pydantic.error_wrappers import ErrorWrapper
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import BaseModel, MissingError, PydanticValueError, ValidationError
+    from pydantic.v1.error_wrappers import ErrorWrapper
+else:
+    from pydantic import BaseModel, MissingError, PydanticValueError, ValidationError
+    from pydantic.error_wrappers import ErrorWrapper
+
 from typing_extensions import Literal
 
 from twisted.web.server import Request

synapse/rest/client/account.py

@@ -18,7 +18,12 @@
 from typing import TYPE_CHECKING, List, Optional, Tuple
 from urllib.parse import urlparse
 
-from pydantic import StrictBool, StrictStr, constr
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import StrictBool, StrictStr, constr
+else:
+    from pydantic import StrictBool, StrictStr, constr
 from typing_extensions import Literal
 
 from twisted.web.server import Request

synapse/rest/client/devices.py

@@ -17,7 +17,12 @@
 from http import HTTPStatus
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
-from pydantic import Extra, StrictStr
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import Extra, StrictStr
+else:
+    from pydantic import Extra, StrictStr
 
 from synapse.api import errors
 from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError

synapse/rest/client/directory.py

@@ -15,7 +15,13 @@
 import logging
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
-from pydantic import StrictStr
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import StrictStr
+else:
+    from pydantic import StrictStr
+
 from typing_extensions import Literal
 
 from twisted.web.server import Request

synapse/rest/client/models.py

@@ -13,7 +13,12 @@
 # limitations under the License.
 from typing import TYPE_CHECKING, Dict, Optional
 
-from pydantic import Extra, StrictInt, StrictStr, constr, validator
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import Extra, StrictInt, StrictStr, constr, validator
+else:
+    from pydantic import Extra, StrictInt, StrictStr, constr, validator  # type: ignore
 
 from synapse.rest.models import RequestBodyModel
 from synapse.util.threepids import validate_email

synapse/rest/key/v2/remote_key_resource.py

@@ -16,7 +16,13 @@
 import re
 from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
 
-from pydantic import Extra, StrictInt, StrictStr
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import Extra, StrictInt, StrictStr
+else:
+    from pydantic import StrictInt, StrictStr, Extra
+
 from signedjson.sign import sign_json
 
 from twisted.web.server import Request

synapse/rest/models.py

@@ -1,4 +1,11 @@
-from pydantic import BaseModel, Extra
+from typing import TYPE_CHECKING
+
+from synapse._pydantic_compat import HAS_PYDANTIC_V2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    from pydantic.v1 import BaseModel, Extra
+else:
+    from pydantic import BaseModel, Extra
 
 
 class RequestBodyModel(BaseModel):