You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

257 lines
8.9 KiB

  1. from collections import OrderedDict, deque
  2. from dataclasses import dataclass, field
  3. from types import TracebackType
  4. from typing import Deque, Generic, List, NamedTuple, Optional, Type, TypeVar
  5. from .. import (
  6. BrokenResourceError, ClosedResourceError, EndOfStream, WouldBlock, get_cancelled_exc_class)
  7. from .._core._compat import DeprecatedAwaitable
  8. from ..abc import Event, ObjectReceiveStream, ObjectSendStream
  9. from ..lowlevel import checkpoint
  10. T_Item = TypeVar('T_Item')
  11. class MemoryObjectStreamStatistics(NamedTuple):
  12. current_buffer_used: int #: number of items stored in the buffer
  13. #: maximum number of items that can be stored on this stream (or :data:`math.inf`)
  14. max_buffer_size: float
  15. open_send_streams: int #: number of unclosed clones of the send stream
  16. open_receive_streams: int #: number of unclosed clones of the receive stream
  17. tasks_waiting_send: int #: number of tasks blocked on :meth:`MemoryObjectSendStream.send`
  18. #: number of tasks blocked on :meth:`MemoryObjectReceiveStream.receive`
  19. tasks_waiting_receive: int
  20. @dataclass(eq=False)
  21. class MemoryObjectStreamState(Generic[T_Item]):
  22. max_buffer_size: float = field()
  23. buffer: Deque[T_Item] = field(init=False, default_factory=deque)
  24. open_send_channels: int = field(init=False, default=0)
  25. open_receive_channels: int = field(init=False, default=0)
  26. waiting_receivers: 'OrderedDict[Event, List[T_Item]]' = field(init=False,
  27. default_factory=OrderedDict)
  28. waiting_senders: 'OrderedDict[Event, T_Item]' = field(init=False, default_factory=OrderedDict)
  29. def statistics(self) -> MemoryObjectStreamStatistics:
  30. return MemoryObjectStreamStatistics(
  31. len(self.buffer), self.max_buffer_size, self.open_send_channels,
  32. self.open_receive_channels, len(self.waiting_senders), len(self.waiting_receivers))
  33. @dataclass(eq=False)
  34. class MemoryObjectReceiveStream(Generic[T_Item], ObjectReceiveStream[T_Item]):
  35. _state: MemoryObjectStreamState[T_Item]
  36. _closed: bool = field(init=False, default=False)
  37. def __post_init__(self) -> None:
  38. self._state.open_receive_channels += 1
  39. def receive_nowait(self) -> T_Item:
  40. """
  41. Receive the next item if it can be done without waiting.
  42. :return: the received item
  43. :raises ~anyio.ClosedResourceError: if this send stream has been closed
  44. :raises ~anyio.EndOfStream: if the buffer is empty and this stream has been
  45. closed from the sending end
  46. :raises ~anyio.WouldBlock: if there are no items in the buffer and no tasks
  47. waiting to send
  48. """
  49. if self._closed:
  50. raise ClosedResourceError
  51. if self._state.waiting_senders:
  52. # Get the item from the next sender
  53. send_event, item = self._state.waiting_senders.popitem(last=False)
  54. self._state.buffer.append(item)
  55. send_event.set()
  56. if self._state.buffer:
  57. return self._state.buffer.popleft()
  58. elif not self._state.open_send_channels:
  59. raise EndOfStream
  60. raise WouldBlock
  61. async def receive(self) -> T_Item:
  62. await checkpoint()
  63. try:
  64. return self.receive_nowait()
  65. except WouldBlock:
  66. # Add ourselves in the queue
  67. receive_event = Event()
  68. container: List[T_Item] = []
  69. self._state.waiting_receivers[receive_event] = container
  70. try:
  71. await receive_event.wait()
  72. except get_cancelled_exc_class():
  73. # Ignore the immediate cancellation if we already received an item, so as not to
  74. # lose it
  75. if not container:
  76. raise
  77. finally:
  78. self._state.waiting_receivers.pop(receive_event, None)
  79. if container:
  80. return container[0]
  81. else:
  82. raise EndOfStream
  83. def clone(self) -> 'MemoryObjectReceiveStream[T_Item]':
  84. """
  85. Create a clone of this receive stream.
  86. Each clone can be closed separately. Only when all clones have been closed will the
  87. receiving end of the memory stream be considered closed by the sending ends.
  88. :return: the cloned stream
  89. """
  90. if self._closed:
  91. raise ClosedResourceError
  92. return MemoryObjectReceiveStream(_state=self._state)
  93. def close(self) -> None:
  94. """
  95. Close the stream.
  96. This works the exact same way as :meth:`aclose`, but is provided as a special case for the
  97. benefit of synchronous callbacks.
  98. """
  99. if not self._closed:
  100. self._closed = True
  101. self._state.open_receive_channels -= 1
  102. if self._state.open_receive_channels == 0:
  103. send_events = list(self._state.waiting_senders.keys())
  104. for event in send_events:
  105. event.set()
  106. async def aclose(self) -> None:
  107. self.close()
  108. def statistics(self) -> MemoryObjectStreamStatistics:
  109. """
  110. Return statistics about the current state of this stream.
  111. .. versionadded:: 3.0
  112. """
  113. return self._state.statistics()
  114. def __enter__(self) -> 'MemoryObjectReceiveStream[T_Item]':
  115. return self
  116. def __exit__(self, exc_type: Optional[Type[BaseException]],
  117. exc_val: Optional[BaseException],
  118. exc_tb: Optional[TracebackType]) -> None:
  119. self.close()
  120. @dataclass(eq=False)
  121. class MemoryObjectSendStream(Generic[T_Item], ObjectSendStream[T_Item]):
  122. _state: MemoryObjectStreamState[T_Item]
  123. _closed: bool = field(init=False, default=False)
  124. def __post_init__(self) -> None:
  125. self._state.open_send_channels += 1
  126. def send_nowait(self, item: T_Item) -> DeprecatedAwaitable:
  127. """
  128. Send an item immediately if it can be done without waiting.
  129. :param item: the item to send
  130. :raises ~anyio.ClosedResourceError: if this send stream has been closed
  131. :raises ~anyio.BrokenResourceError: if the stream has been closed from the
  132. receiving end
  133. :raises ~anyio.WouldBlock: if the buffer is full and there are no tasks waiting
  134. to receive
  135. """
  136. if self._closed:
  137. raise ClosedResourceError
  138. if not self._state.open_receive_channels:
  139. raise BrokenResourceError
  140. if self._state.waiting_receivers:
  141. receive_event, container = self._state.waiting_receivers.popitem(last=False)
  142. container.append(item)
  143. receive_event.set()
  144. elif len(self._state.buffer) < self._state.max_buffer_size:
  145. self._state.buffer.append(item)
  146. else:
  147. raise WouldBlock
  148. return DeprecatedAwaitable(self.send_nowait)
  149. async def send(self, item: T_Item) -> None:
  150. await checkpoint()
  151. try:
  152. self.send_nowait(item)
  153. except WouldBlock:
  154. # Wait until there's someone on the receiving end
  155. send_event = Event()
  156. self._state.waiting_senders[send_event] = item
  157. try:
  158. await send_event.wait()
  159. except BaseException:
  160. self._state.waiting_senders.pop(send_event, None) # type: ignore[arg-type]
  161. raise
  162. if self._state.waiting_senders.pop(send_event, None): # type: ignore[arg-type]
  163. raise BrokenResourceError
  164. def clone(self) -> 'MemoryObjectSendStream[T_Item]':
  165. """
  166. Create a clone of this send stream.
  167. Each clone can be closed separately. Only when all clones have been closed will the
  168. sending end of the memory stream be considered closed by the receiving ends.
  169. :return: the cloned stream
  170. """
  171. if self._closed:
  172. raise ClosedResourceError
  173. return MemoryObjectSendStream(_state=self._state)
  174. def close(self) -> None:
  175. """
  176. Close the stream.
  177. This works the exact same way as :meth:`aclose`, but is provided as a special case for the
  178. benefit of synchronous callbacks.
  179. """
  180. if not self._closed:
  181. self._closed = True
  182. self._state.open_send_channels -= 1
  183. if self._state.open_send_channels == 0:
  184. receive_events = list(self._state.waiting_receivers.keys())
  185. self._state.waiting_receivers.clear()
  186. for event in receive_events:
  187. event.set()
  188. async def aclose(self) -> None:
  189. self.close()
  190. def statistics(self) -> MemoryObjectStreamStatistics:
  191. """
  192. Return statistics about the current state of this stream.
  193. .. versionadded:: 3.0
  194. """
  195. return self._state.statistics()
  196. def __enter__(self) -> 'MemoryObjectSendStream[T_Item]':
  197. return self
  198. def __exit__(self, exc_type: Optional[Type[BaseException]],
  199. exc_val: Optional[BaseException],
  200. exc_tb: Optional[TracebackType]) -> None:
  201. self.close()