You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

401 line
18 KiB

  1. import sys
  2. import types
  3. import typing
  4. from typing import (
  5. TYPE_CHECKING,
  6. Any,
  7. ClassVar,
  8. Dict,
  9. ForwardRef,
  10. Generic,
  11. Iterator,
  12. List,
  13. Mapping,
  14. Optional,
  15. Tuple,
  16. Type,
  17. TypeVar,
  18. Union,
  19. cast,
  20. )
  21. from weakref import WeakKeyDictionary, WeakValueDictionary
  22. from typing_extensions import Annotated, Literal as ExtLiteral
  23. from pydantic.v1.class_validators import gather_all_validators
  24. from pydantic.v1.fields import DeferredType
  25. from pydantic.v1.main import BaseModel, create_model
  26. from pydantic.v1.types import JsonWrapper
  27. from pydantic.v1.typing import display_as_type, get_all_type_hints, get_args, get_origin, typing_base
  28. from pydantic.v1.utils import all_identical, lenient_issubclass
  29. if sys.version_info >= (3, 10):
  30. from typing import _UnionGenericAlias
  31. if sys.version_info >= (3, 8):
  32. from typing import Literal
  33. GenericModelT = TypeVar('GenericModelT', bound='GenericModel')
  34. TypeVarType = Any # since mypy doesn't allow the use of TypeVar as a type
  35. CacheKey = Tuple[Type[Any], Any, Tuple[Any, ...]]
  36. Parametrization = Mapping[TypeVarType, Type[Any]]
  37. # weak dictionaries allow the dynamically created parametrized versions of generic models to get collected
  38. # once they are no longer referenced by the caller.
  39. if sys.version_info >= (3, 9): # Typing for weak dictionaries available at 3.9
  40. GenericTypesCache = WeakValueDictionary[CacheKey, Type[BaseModel]]
  41. AssignedParameters = WeakKeyDictionary[Type[BaseModel], Parametrization]
  42. else:
  43. GenericTypesCache = WeakValueDictionary
  44. AssignedParameters = WeakKeyDictionary
  45. # _generic_types_cache is a Mapping from __class_getitem__ arguments to the parametrized version of generic models.
  46. # This ensures multiple calls of e.g. A[B] return always the same class.
  47. _generic_types_cache = GenericTypesCache()
  48. # _assigned_parameters is a Mapping from parametrized version of generic models to assigned types of parametrizations
  49. # as captured during construction of the class (not instances).
  50. # E.g., for generic model `Model[A, B]`, when parametrized model `Model[int, str]` is created,
  51. # `Model[int, str]`: {A: int, B: str}` will be stored in `_assigned_parameters`.
  52. # (This information is only otherwise available after creation from the class name string).
  53. _assigned_parameters = AssignedParameters()
  54. class GenericModel(BaseModel):
  55. __slots__ = ()
  56. __concrete__: ClassVar[bool] = False
  57. if TYPE_CHECKING:
  58. # Putting this in a TYPE_CHECKING block allows us to replace `if Generic not in cls.__bases__` with
  59. # `not hasattr(cls, "__parameters__")`. This means we don't need to force non-concrete subclasses of
  60. # `GenericModel` to also inherit from `Generic`, which would require changes to the use of `create_model` below.
  61. __parameters__: ClassVar[Tuple[TypeVarType, ...]]
  62. # Setting the return type as Type[Any] instead of Type[BaseModel] prevents PyCharm warnings
  63. def __class_getitem__(cls: Type[GenericModelT], params: Union[Type[Any], Tuple[Type[Any], ...]]) -> Type[Any]:
  64. """Instantiates a new class from a generic class `cls` and type variables `params`.
  65. :param params: Tuple of types the class . Given a generic class
  66. `Model` with 2 type variables and a concrete model `Model[str, int]`,
  67. the value `(str, int)` would be passed to `params`.
  68. :return: New model class inheriting from `cls` with instantiated
  69. types described by `params`. If no parameters are given, `cls` is
  70. returned as is.
  71. """
  72. def _cache_key(_params: Any) -> CacheKey:
  73. args = get_args(_params)
  74. # python returns a list for Callables, which is not hashable
  75. if len(args) == 2 and isinstance(args[0], list):
  76. args = (tuple(args[0]), args[1])
  77. return cls, _params, args
  78. cached = _generic_types_cache.get(_cache_key(params))
  79. if cached is not None:
  80. return cached
  81. if cls.__concrete__ and Generic not in cls.__bases__:
  82. raise TypeError('Cannot parameterize a concrete instantiation of a generic model')
  83. if not isinstance(params, tuple):
  84. params = (params,)
  85. if cls is GenericModel and any(isinstance(param, TypeVar) for param in params):
  86. raise TypeError('Type parameters should be placed on typing.Generic, not GenericModel')
  87. if not hasattr(cls, '__parameters__'):
  88. raise TypeError(f'Type {cls.__name__} must inherit from typing.Generic before being parameterized')
  89. check_parameters_count(cls, params)
  90. # Build map from generic typevars to passed params
  91. typevars_map: Dict[TypeVarType, Type[Any]] = dict(zip(cls.__parameters__, params))
  92. if all_identical(typevars_map.keys(), typevars_map.values()) and typevars_map:
  93. return cls # if arguments are equal to parameters it's the same object
  94. # Create new model with original model as parent inserting fields with DeferredType.
  95. model_name = cls.__concrete_name__(params)
  96. validators = gather_all_validators(cls)
  97. type_hints = get_all_type_hints(cls).items()
  98. instance_type_hints = {k: v for k, v in type_hints if get_origin(v) is not ClassVar}
  99. fields = {k: (DeferredType(), cls.__fields__[k].field_info) for k in instance_type_hints if k in cls.__fields__}
  100. model_module, called_globally = get_caller_frame_info()
  101. created_model = cast(
  102. Type[GenericModel], # casting ensures mypy is aware of the __concrete__ and __parameters__ attributes
  103. create_model(
  104. model_name,
  105. __module__=model_module or cls.__module__,
  106. __base__=(cls,) + tuple(cls.__parameterized_bases__(typevars_map)),
  107. __config__=None,
  108. __validators__=validators,
  109. __cls_kwargs__=None,
  110. **fields,
  111. ),
  112. )
  113. _assigned_parameters[created_model] = typevars_map
  114. if called_globally: # create global reference and therefore allow pickling
  115. object_by_reference = None
  116. reference_name = model_name
  117. reference_module_globals = sys.modules[created_model.__module__].__dict__
  118. while object_by_reference is not created_model:
  119. object_by_reference = reference_module_globals.setdefault(reference_name, created_model)
  120. reference_name += '_'
  121. created_model.Config = cls.Config
  122. # Find any typevars that are still present in the model.
  123. # If none are left, the model is fully "concrete", otherwise the new
  124. # class is a generic class as well taking the found typevars as
  125. # parameters.
  126. new_params = tuple(
  127. {param: None for param in iter_contained_typevars(typevars_map.values())}
  128. ) # use dict as ordered set
  129. created_model.__concrete__ = not new_params
  130. if new_params:
  131. created_model.__parameters__ = new_params
  132. # Save created model in cache so we don't end up creating duplicate
  133. # models that should be identical.
  134. _generic_types_cache[_cache_key(params)] = created_model
  135. if len(params) == 1:
  136. _generic_types_cache[_cache_key(params[0])] = created_model
  137. # Recursively walk class type hints and replace generic typevars
  138. # with concrete types that were passed.
  139. _prepare_model_fields(created_model, fields, instance_type_hints, typevars_map)
  140. return created_model
  141. @classmethod
  142. def __concrete_name__(cls: Type[Any], params: Tuple[Type[Any], ...]) -> str:
  143. """Compute class name for child classes.
  144. :param params: Tuple of types the class . Given a generic class
  145. `Model` with 2 type variables and a concrete model `Model[str, int]`,
  146. the value `(str, int)` would be passed to `params`.
  147. :return: String representing a the new class where `params` are
  148. passed to `cls` as type variables.
  149. This method can be overridden to achieve a custom naming scheme for GenericModels.
  150. """
  151. param_names = [display_as_type(param) for param in params]
  152. params_component = ', '.join(param_names)
  153. return f'{cls.__name__}[{params_component}]'
  154. @classmethod
  155. def __parameterized_bases__(cls, typevars_map: Parametrization) -> Iterator[Type[Any]]:
  156. """
  157. Returns unbound bases of cls parameterised to given type variables
  158. :param typevars_map: Dictionary of type applications for binding subclasses.
  159. Given a generic class `Model` with 2 type variables [S, T]
  160. and a concrete model `Model[str, int]`,
  161. the value `{S: str, T: int}` would be passed to `typevars_map`.
  162. :return: an iterator of generic sub classes, parameterised by `typevars_map`
  163. and other assigned parameters of `cls`
  164. e.g.:
  165. ```
  166. class A(GenericModel, Generic[T]):
  167. ...
  168. class B(A[V], Generic[V]):
  169. ...
  170. assert A[int] in B.__parameterized_bases__({V: int})
  171. ```
  172. """
  173. def build_base_model(
  174. base_model: Type[GenericModel], mapped_types: Parametrization
  175. ) -> Iterator[Type[GenericModel]]:
  176. base_parameters = tuple(mapped_types[param] for param in base_model.__parameters__)
  177. parameterized_base = base_model.__class_getitem__(base_parameters)
  178. if parameterized_base is base_model or parameterized_base is cls:
  179. # Avoid duplication in MRO
  180. return
  181. yield parameterized_base
  182. for base_model in cls.__bases__:
  183. if not issubclass(base_model, GenericModel):
  184. # not a class that can be meaningfully parameterized
  185. continue
  186. elif not getattr(base_model, '__parameters__', None):
  187. # base_model is "GenericModel" (and has no __parameters__)
  188. # or
  189. # base_model is already concrete, and will be included transitively via cls.
  190. continue
  191. elif cls in _assigned_parameters:
  192. if base_model in _assigned_parameters:
  193. # cls is partially parameterised but not from base_model
  194. # e.g. cls = B[S], base_model = A[S]
  195. # B[S][int] should subclass A[int], (and will be transitively via B[int])
  196. # but it's not viable to consistently subclass types with arbitrary construction
  197. # So don't attempt to include A[S][int]
  198. continue
  199. else: # base_model not in _assigned_parameters:
  200. # cls is partially parameterized, base_model is original generic
  201. # e.g. cls = B[str, T], base_model = B[S, T]
  202. # Need to determine the mapping for the base_model parameters
  203. mapped_types: Parametrization = {
  204. key: typevars_map.get(value, value) for key, value in _assigned_parameters[cls].items()
  205. }
  206. yield from build_base_model(base_model, mapped_types)
  207. else:
  208. # cls is base generic, so base_class has a distinct base
  209. # can construct the Parameterised base model using typevars_map directly
  210. yield from build_base_model(base_model, typevars_map)
  211. def replace_types(type_: Any, type_map: Mapping[Any, Any]) -> Any:
  212. """Return type with all occurrences of `type_map` keys recursively replaced with their values.
  213. :param type_: Any type, class or generic alias
  214. :param type_map: Mapping from `TypeVar` instance to concrete types.
  215. :return: New type representing the basic structure of `type_` with all
  216. `typevar_map` keys recursively replaced.
  217. >>> replace_types(Tuple[str, Union[List[str], float]], {str: int})
  218. Tuple[int, Union[List[int], float]]
  219. """
  220. if not type_map:
  221. return type_
  222. type_args = get_args(type_)
  223. origin_type = get_origin(type_)
  224. if origin_type is Annotated:
  225. annotated_type, *annotations = type_args
  226. return Annotated[replace_types(annotated_type, type_map), tuple(annotations)]
  227. if (origin_type is ExtLiteral) or (sys.version_info >= (3, 8) and origin_type is Literal):
  228. return type_map.get(type_, type_)
  229. # Having type args is a good indicator that this is a typing module
  230. # class instantiation or a generic alias of some sort.
  231. if type_args:
  232. resolved_type_args = tuple(replace_types(arg, type_map) for arg in type_args)
  233. if all_identical(type_args, resolved_type_args):
  234. # If all arguments are the same, there is no need to modify the
  235. # type or create a new object at all
  236. return type_
  237. if (
  238. origin_type is not None
  239. and isinstance(type_, typing_base)
  240. and not isinstance(origin_type, typing_base)
  241. and getattr(type_, '_name', None) is not None
  242. ):
  243. # In python < 3.9 generic aliases don't exist so any of these like `list`,
  244. # `type` or `collections.abc.Callable` need to be translated.
  245. # See: https://www.python.org/dev/peps/pep-0585
  246. origin_type = getattr(typing, type_._name)
  247. assert origin_type is not None
  248. # PEP-604 syntax (Ex.: list | str) is represented with a types.UnionType object that does not have __getitem__.
  249. # We also cannot use isinstance() since we have to compare types.
  250. if sys.version_info >= (3, 10) and origin_type is types.UnionType: # noqa: E721
  251. return _UnionGenericAlias(origin_type, resolved_type_args)
  252. return origin_type[resolved_type_args]
  253. # We handle pydantic generic models separately as they don't have the same
  254. # semantics as "typing" classes or generic aliases
  255. if not origin_type and lenient_issubclass(type_, GenericModel) and not type_.__concrete__:
  256. type_args = type_.__parameters__
  257. resolved_type_args = tuple(replace_types(t, type_map) for t in type_args)
  258. if all_identical(type_args, resolved_type_args):
  259. return type_
  260. return type_[resolved_type_args]
  261. # Handle special case for typehints that can have lists as arguments.
  262. # `typing.Callable[[int, str], int]` is an example for this.
  263. if isinstance(type_, (List, list)):
  264. resolved_list = list(replace_types(element, type_map) for element in type_)
  265. if all_identical(type_, resolved_list):
  266. return type_
  267. return resolved_list
  268. # For JsonWrapperValue, need to handle its inner type to allow correct parsing
  269. # of generic Json arguments like Json[T]
  270. if not origin_type and lenient_issubclass(type_, JsonWrapper):
  271. type_.inner_type = replace_types(type_.inner_type, type_map)
  272. return type_
  273. # If all else fails, we try to resolve the type directly and otherwise just
  274. # return the input with no modifications.
  275. new_type = type_map.get(type_, type_)
  276. # Convert string to ForwardRef
  277. if isinstance(new_type, str):
  278. return ForwardRef(new_type)
  279. else:
  280. return new_type
  281. def check_parameters_count(cls: Type[GenericModel], parameters: Tuple[Any, ...]) -> None:
  282. actual = len(parameters)
  283. expected = len(cls.__parameters__)
  284. if actual != expected:
  285. description = 'many' if actual > expected else 'few'
  286. raise TypeError(f'Too {description} parameters for {cls.__name__}; actual {actual}, expected {expected}')
  287. DictValues: Type[Any] = {}.values().__class__
  288. def iter_contained_typevars(v: Any) -> Iterator[TypeVarType]:
  289. """Recursively iterate through all subtypes and type args of `v` and yield any typevars that are found."""
  290. if isinstance(v, TypeVar):
  291. yield v
  292. elif hasattr(v, '__parameters__') and not get_origin(v) and lenient_issubclass(v, GenericModel):
  293. yield from v.__parameters__
  294. elif isinstance(v, (DictValues, list)):
  295. for var in v:
  296. yield from iter_contained_typevars(var)
  297. else:
  298. args = get_args(v)
  299. for arg in args:
  300. yield from iter_contained_typevars(arg)
  301. def get_caller_frame_info() -> Tuple[Optional[str], bool]:
  302. """
  303. Used inside a function to check whether it was called globally
  304. Will only work against non-compiled code, therefore used only in pydantic.generics
  305. :returns Tuple[module_name, called_globally]
  306. """
  307. try:
  308. previous_caller_frame = sys._getframe(2)
  309. except ValueError as e:
  310. raise RuntimeError('This function must be used inside another function') from e
  311. except AttributeError: # sys module does not have _getframe function, so there's nothing we can do about it
  312. return None, False
  313. frame_globals = previous_caller_frame.f_globals
  314. return frame_globals.get('__name__'), previous_caller_frame.f_locals is frame_globals
  315. def _prepare_model_fields(
  316. created_model: Type[GenericModel],
  317. fields: Mapping[str, Any],
  318. instance_type_hints: Mapping[str, type],
  319. typevars_map: Mapping[Any, type],
  320. ) -> None:
  321. """
  322. Replace DeferredType fields with concrete type hints and prepare them.
  323. """
  324. for key, field in created_model.__fields__.items():
  325. if key not in fields:
  326. assert field.type_.__class__ is not DeferredType
  327. # https://github.com/nedbat/coveragepy/issues/198
  328. continue # pragma: no cover
  329. assert field.type_.__class__ is DeferredType, field.type_.__class__
  330. field_type_hint = instance_type_hints[key]
  331. concrete_type = replace_types(field_type_hint, typevars_map)
  332. field.type_ = concrete_type
  333. field.outer_type_ = concrete_type
  334. field.prepare()
  335. created_model.__annotations__[key] = concrete_type