You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

278 lines
9.7 KiB

  1. from __future__ import annotations
  2. # This module contains abstractions for the input stream. You don't have to
  3. # looks further, there are no pretty code.
  4. #
  5. # We define two classes here.
  6. #
  7. # Mark(source, line, column)
  8. # It's just a record and its only use is producing nice error messages.
  9. # Parser does not use it for any other purposes.
  10. #
  11. # Reader(source, data)
  12. # Reader determines the encoding of `data` and converts it to unicode.
  13. # Reader provides the following methods and attributes:
  14. # reader.peek(length=1) - return the next `length` characters
  15. # reader.forward(length=1) - move the current position to `length`
  16. # characters.
  17. # reader.index - the number of the current character.
  18. # reader.line, stream.column - the line and the column of the current
  19. # character.
  20. import codecs
  21. from ruamel.yaml.error import YAMLError, FileMark, StringMark, YAMLStreamError
  22. from ruamel.yaml.util import RegExp
  23. if False: # MYPY
  24. from typing import Any, Dict, Optional, List, Union, Text, Tuple, Optional # NOQA
  25. # from ruamel.yaml.compat import StreamTextType # NOQA
  26. __all__ = ['Reader', 'ReaderError']
  27. class ReaderError(YAMLError):
  28. def __init__(
  29. self, name: Any, position: Any, character: Any, encoding: Any, reason: Any,
  30. ) -> None:
  31. self.name = name
  32. self.character = character
  33. self.position = position
  34. self.encoding = encoding
  35. self.reason = reason
  36. def __str__(self) -> Any:
  37. if isinstance(self.character, bytes):
  38. return (
  39. f"'{self.encoding!s}' codec can't decode byte #x{ord(self.character):02x}: "
  40. f'{self.reason!s}\n'
  41. f' in "{self.name!s}", position {self.position:d}'
  42. )
  43. else:
  44. return (
  45. f'unacceptable character #x{self.character:04x}: {self.reason!s}\n'
  46. f' in "{self.name!s}", position {self.position:d}'
  47. )
  48. class Reader:
  49. # Reader:
  50. # - determines the data encoding and converts it to a unicode string,
  51. # - checks if characters are in allowed range,
  52. # - adds '\0' to the end.
  53. # Reader accepts
  54. # - a `bytes` object,
  55. # - a `str` object,
  56. # - a file-like object with its `read` method returning `str`,
  57. # - a file-like object with its `read` method returning `unicode`.
  58. # Yeah, it's ugly and slow.
  59. def __init__(self, stream: Any, loader: Any = None) -> None:
  60. self.loader = loader
  61. if self.loader is not None and getattr(self.loader, '_reader', None) is None:
  62. self.loader._reader = self
  63. self.reset_reader()
  64. self.stream: Any = stream # as .read is called
  65. def reset_reader(self) -> None:
  66. self.name: Any = None
  67. self.stream_pointer = 0
  68. self.eof = True
  69. self.buffer = ""
  70. self.pointer = 0
  71. self.raw_buffer: Any = None
  72. self.raw_decode = None
  73. self.encoding: Optional[Text] = None
  74. self.index = 0
  75. self.line = 0
  76. self.column = 0
  77. @property
  78. def stream(self) -> Any:
  79. try:
  80. return self._stream
  81. except AttributeError:
  82. raise YAMLStreamError('input stream needs to be specified')
  83. @stream.setter
  84. def stream(self, val: Any) -> None:
  85. if val is None:
  86. return
  87. self._stream = None
  88. if isinstance(val, str):
  89. self.name = '<unicode string>'
  90. self.check_printable(val)
  91. self.buffer = val + '\0'
  92. elif isinstance(val, bytes):
  93. self.name = '<byte string>'
  94. self.raw_buffer = val
  95. self.determine_encoding()
  96. else:
  97. if not hasattr(val, 'read'):
  98. raise YAMLStreamError('stream argument needs to have a read() method')
  99. self._stream = val
  100. self.name = getattr(self.stream, 'name', '<file>')
  101. self.eof = False
  102. self.raw_buffer = None
  103. self.determine_encoding()
  104. def peek(self, index: int = 0) -> Text:
  105. try:
  106. return self.buffer[self.pointer + index]
  107. except IndexError:
  108. self.update(index + 1)
  109. return self.buffer[self.pointer + index]
  110. def prefix(self, length: int = 1) -> Any:
  111. if self.pointer + length >= len(self.buffer):
  112. self.update(length)
  113. return self.buffer[self.pointer : self.pointer + length]
  114. def forward_1_1(self, length: int = 1) -> None:
  115. if self.pointer + length + 1 >= len(self.buffer):
  116. self.update(length + 1)
  117. while length != 0:
  118. ch = self.buffer[self.pointer]
  119. self.pointer += 1
  120. self.index += 1
  121. if ch in '\n\x85\u2028\u2029' or (
  122. ch == '\r' and self.buffer[self.pointer] != '\n'
  123. ):
  124. self.line += 1
  125. self.column = 0
  126. elif ch != '\uFEFF':
  127. self.column += 1
  128. length -= 1
  129. def forward(self, length: int = 1) -> None:
  130. if self.pointer + length + 1 >= len(self.buffer):
  131. self.update(length + 1)
  132. while length != 0:
  133. ch = self.buffer[self.pointer]
  134. self.pointer += 1
  135. self.index += 1
  136. if ch == '\n' or (ch == '\r' and self.buffer[self.pointer] != '\n'):
  137. self.line += 1
  138. self.column = 0
  139. elif ch != '\uFEFF':
  140. self.column += 1
  141. length -= 1
  142. def get_mark(self) -> Any:
  143. if self.stream is None:
  144. return StringMark(
  145. self.name, self.index, self.line, self.column, self.buffer, self.pointer,
  146. )
  147. else:
  148. return FileMark(self.name, self.index, self.line, self.column)
  149. def determine_encoding(self) -> None:
  150. while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
  151. self.update_raw()
  152. if isinstance(self.raw_buffer, bytes):
  153. if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
  154. self.raw_decode = codecs.utf_16_le_decode # type: ignore
  155. self.encoding = 'utf-16-le'
  156. elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
  157. self.raw_decode = codecs.utf_16_be_decode # type: ignore
  158. self.encoding = 'utf-16-be'
  159. else:
  160. self.raw_decode = codecs.utf_8_decode # type: ignore
  161. self.encoding = 'utf-8'
  162. self.update(1)
  163. NON_PRINTABLE = RegExp(
  164. '[^\x09\x0A\x0D\x20-\x7E\x85' '\xA0-\uD7FF' '\uE000-\uFFFD' '\U00010000-\U0010FFFF' ']' # NOQA
  165. )
  166. _printable_ascii = ('\x09\x0A\x0D' + "".join(map(chr, range(0x20, 0x7F)))).encode('ascii')
  167. @classmethod
  168. def _get_non_printable_ascii(cls: Text, data: bytes) -> Optional[Tuple[int, Text]]: # type: ignore # NOQA
  169. ascii_bytes = data.encode('ascii') # type: ignore
  170. non_printables = ascii_bytes.translate(None, cls._printable_ascii) # type: ignore
  171. if not non_printables:
  172. return None
  173. non_printable = non_printables[:1]
  174. return ascii_bytes.index(non_printable), non_printable.decode('ascii')
  175. @classmethod
  176. def _get_non_printable_regex(cls, data: Text) -> Optional[Tuple[int, Text]]:
  177. match = cls.NON_PRINTABLE.search(data)
  178. if not bool(match):
  179. return None
  180. return match.start(), match.group()
  181. @classmethod
  182. def _get_non_printable(cls, data: Text) -> Optional[Tuple[int, Text]]:
  183. try:
  184. return cls._get_non_printable_ascii(data) # type: ignore
  185. except UnicodeEncodeError:
  186. return cls._get_non_printable_regex(data)
  187. def check_printable(self, data: Any) -> None:
  188. non_printable_match = self._get_non_printable(data)
  189. if non_printable_match is not None:
  190. start, character = non_printable_match
  191. position = self.index + (len(self.buffer) - self.pointer) + start
  192. raise ReaderError(
  193. self.name,
  194. position,
  195. ord(character),
  196. 'unicode',
  197. 'special characters are not allowed',
  198. )
  199. def update(self, length: int) -> None:
  200. if self.raw_buffer is None:
  201. return
  202. self.buffer = self.buffer[self.pointer :]
  203. self.pointer = 0
  204. while len(self.buffer) < length:
  205. if not self.eof:
  206. self.update_raw()
  207. if self.raw_decode is not None:
  208. try:
  209. data, converted = self.raw_decode(self.raw_buffer, 'strict', self.eof)
  210. except UnicodeDecodeError as exc:
  211. character = self.raw_buffer[exc.start]
  212. if self.stream is not None:
  213. position = self.stream_pointer - len(self.raw_buffer) + exc.start
  214. elif self.stream is not None:
  215. position = self.stream_pointer - len(self.raw_buffer) + exc.start
  216. else:
  217. position = exc.start
  218. raise ReaderError(self.name, position, character, exc.encoding, exc.reason)
  219. else:
  220. data = self.raw_buffer
  221. converted = len(data)
  222. self.check_printable(data)
  223. self.buffer += data
  224. self.raw_buffer = self.raw_buffer[converted:]
  225. if self.eof:
  226. self.buffer += '\0'
  227. self.raw_buffer = None
  228. break
  229. def update_raw(self, size: Optional[int] = None) -> None:
  230. if size is None:
  231. size = 4096
  232. data = self.stream.read(size)
  233. if self.raw_buffer is None:
  234. self.raw_buffer = data
  235. else:
  236. self.raw_buffer += data
  237. self.stream_pointer += len(data)
  238. if not data:
  239. self.eof = True
  240. # try:
  241. # import psyco
  242. # psyco.bind(Reader)
  243. # except ImportError:
  244. # pass