您最多选择25个主题 主题必须以字母或数字开头,可以包含连字符 (-),并且长度不得超过35个字符
 
 
 
 

439 行
15 KiB

  1. import contextlib
  2. import json
  3. import logging
  4. import os
  5. import re
  6. import shlex
  7. import signal
  8. import subprocess
  9. import sys
  10. from importlib import import_module
  11. from multiprocessing import get_context
  12. from multiprocessing.context import SpawnProcess
  13. from pathlib import Path
  14. from time import sleep
  15. from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union
  16. import anyio
  17. from .filters import DefaultFilter
  18. from .main import Change, FileChange, awatch, watch
  19. if TYPE_CHECKING:
  20. from typing import Literal
  21. __all__ = 'run_process', 'arun_process', 'detect_target_type', 'import_string'
  22. logger = logging.getLogger('watchfiles.main')
  23. def run_process(
  24. *paths: Union[Path, str],
  25. target: Union[str, Callable[..., Any]],
  26. args: Tuple[Any, ...] = (),
  27. kwargs: Optional[Dict[str, Any]] = None,
  28. target_type: "Literal['function', 'command', 'auto']" = 'auto',
  29. callback: Optional[Callable[[Set[FileChange]], None]] = None,
  30. watch_filter: Optional[Callable[[Change, str], bool]] = DefaultFilter(),
  31. grace_period: float = 0,
  32. debounce: int = 1_600,
  33. step: int = 50,
  34. debug: Optional[bool] = None,
  35. sigint_timeout: int = 5,
  36. sigkill_timeout: int = 1,
  37. recursive: bool = True,
  38. ignore_permission_denied: bool = False,
  39. ) -> int:
  40. """
  41. Run a process and restart it upon file changes.
  42. `run_process` can work in two ways:
  43. * Using `multiprocessing.Process` † to run a python function
  44. * Or, using `subprocess.Popen` to run a command
  45. !!! note
  46. **†** technically `multiprocessing.get_context('spawn').Process` to avoid forking and improve
  47. code reload/import.
  48. Internally, `run_process` uses [`watch`][watchfiles.watch] with `raise_interrupt=False` so the function
  49. exits cleanly upon `Ctrl+C`.
  50. Args:
  51. *paths: matches the same argument of [`watch`][watchfiles.watch]
  52. target: function or command to run
  53. args: arguments to pass to `target`, only used if `target` is a function
  54. kwargs: keyword arguments to pass to `target`, only used if `target` is a function
  55. target_type: type of target. Can be `'function'`, `'command'`, or `'auto'` in which case
  56. [`detect_target_type`][watchfiles.run.detect_target_type] is used to determine the type.
  57. callback: function to call on each reload, the function should accept a set of changes as the sole argument
  58. watch_filter: matches the same argument of [`watch`][watchfiles.watch]
  59. grace_period: number of seconds after the process is started before watching for changes
  60. debounce: matches the same argument of [`watch`][watchfiles.watch]
  61. step: matches the same argument of [`watch`][watchfiles.watch]
  62. debug: matches the same argument of [`watch`][watchfiles.watch]
  63. sigint_timeout: the number of seconds to wait after sending sigint before sending sigkill
  64. sigkill_timeout: the number of seconds to wait after sending sigkill before raising an exception
  65. recursive: matches the same argument of [`watch`][watchfiles.watch]
  66. Returns:
  67. number of times the function was reloaded.
  68. ```py title="Example of run_process running a function"
  69. from watchfiles import run_process
  70. def callback(changes):
  71. print('changes detected:', changes)
  72. def foobar(a, b):
  73. print('foobar called with:', a, b)
  74. if __name__ == '__main__':
  75. run_process('./path/to/dir', target=foobar, args=(1, 2), callback=callback)
  76. ```
  77. As well as using a `callback` function, changes can be accessed from within the target function,
  78. using the `WATCHFILES_CHANGES` environment variable.
  79. ```py title="Example of run_process accessing changes"
  80. from watchfiles import run_process
  81. def foobar(a, b, c):
  82. # changes will be an empty list "[]" the first time the function is called
  83. changes = os.getenv('WATCHFILES_CHANGES')
  84. changes = json.loads(changes)
  85. print('foobar called due to changes:', changes)
  86. if __name__ == '__main__':
  87. run_process('./path/to/dir', target=foobar, args=(1, 2, 3))
  88. ```
  89. Again with the target as `command`, `WATCHFILES_CHANGES` can be used
  90. to access changes.
  91. ```bash title="example.sh"
  92. echo "changers: ${WATCHFILES_CHANGES}"
  93. ```
  94. ```py title="Example of run_process running a command"
  95. from watchfiles import run_process
  96. if __name__ == '__main__':
  97. run_process('.', target='./example.sh')
  98. ```
  99. """
  100. if target_type == 'auto':
  101. target_type = detect_target_type(target)
  102. logger.debug('running "%s" as %s', target, target_type)
  103. catch_sigterm()
  104. process = start_process(target, target_type, args, kwargs)
  105. reloads = 0
  106. if grace_period:
  107. logger.debug('sleeping for %s seconds before watching for changes', grace_period)
  108. sleep(grace_period)
  109. try:
  110. for changes in watch(
  111. *paths,
  112. watch_filter=watch_filter,
  113. debounce=debounce,
  114. step=step,
  115. debug=debug,
  116. raise_interrupt=False,
  117. recursive=recursive,
  118. ignore_permission_denied=ignore_permission_denied,
  119. ):
  120. callback and callback(changes)
  121. process.stop(sigint_timeout=sigint_timeout, sigkill_timeout=sigkill_timeout)
  122. process = start_process(target, target_type, args, kwargs, changes)
  123. reloads += 1
  124. finally:
  125. process.stop()
  126. return reloads
  127. async def arun_process(
  128. *paths: Union[Path, str],
  129. target: Union[str, Callable[..., Any]],
  130. args: Tuple[Any, ...] = (),
  131. kwargs: Optional[Dict[str, Any]] = None,
  132. target_type: "Literal['function', 'command', 'auto']" = 'auto',
  133. callback: Optional[Callable[[Set[FileChange]], Any]] = None,
  134. watch_filter: Optional[Callable[[Change, str], bool]] = DefaultFilter(),
  135. grace_period: float = 0,
  136. debounce: int = 1_600,
  137. step: int = 50,
  138. debug: Optional[bool] = None,
  139. recursive: bool = True,
  140. ignore_permission_denied: bool = False,
  141. ) -> int:
  142. """
  143. Async equivalent of [`run_process`][watchfiles.run_process], all arguments match those of `run_process` except
  144. `callback` which can be a coroutine.
  145. Starting and stopping the process and watching for changes is done in a separate thread.
  146. As with `run_process`, internally `arun_process` uses [`awatch`][watchfiles.awatch], however `KeyboardInterrupt`
  147. cannot be caught and suppressed in `awatch` so these errors need to be caught separately, see below.
  148. ```py title="Example of arun_process usage"
  149. import asyncio
  150. from watchfiles import arun_process
  151. async def callback(changes):
  152. await asyncio.sleep(0.1)
  153. print('changes detected:', changes)
  154. def foobar(a, b):
  155. print('foobar called with:', a, b)
  156. async def main():
  157. await arun_process('.', target=foobar, args=(1, 2), callback=callback)
  158. if __name__ == '__main__':
  159. try:
  160. asyncio.run(main())
  161. except KeyboardInterrupt:
  162. print('stopped via KeyboardInterrupt')
  163. ```
  164. """
  165. import inspect
  166. if target_type == 'auto':
  167. target_type = detect_target_type(target)
  168. logger.debug('running "%s" as %s', target, target_type)
  169. catch_sigterm()
  170. process = await anyio.to_thread.run_sync(start_process, target, target_type, args, kwargs)
  171. reloads = 0
  172. if grace_period:
  173. logger.debug('sleeping for %s seconds before watching for changes', grace_period)
  174. await anyio.sleep(grace_period)
  175. async for changes in awatch(
  176. *paths,
  177. watch_filter=watch_filter,
  178. debounce=debounce,
  179. step=step,
  180. debug=debug,
  181. recursive=recursive,
  182. ignore_permission_denied=ignore_permission_denied,
  183. ):
  184. if callback is not None:
  185. r = callback(changes)
  186. if inspect.isawaitable(r):
  187. await r
  188. await anyio.to_thread.run_sync(process.stop)
  189. process = await anyio.to_thread.run_sync(start_process, target, target_type, args, kwargs, changes)
  190. reloads += 1
  191. await anyio.to_thread.run_sync(process.stop)
  192. return reloads
  193. # Use spawn context to make sure code run in subprocess
  194. # does not reuse imported modules in main process/context
  195. spawn_context = get_context('spawn')
  196. def split_cmd(cmd: str) -> List[str]:
  197. import platform
  198. posix = platform.uname().system.lower() != 'windows'
  199. return shlex.split(cmd, posix=posix)
  200. def start_process(
  201. target: Union[str, Callable[..., Any]],
  202. target_type: "Literal['function', 'command']",
  203. args: Tuple[Any, ...],
  204. kwargs: Optional[Dict[str, Any]],
  205. changes: Optional[Set[FileChange]] = None,
  206. ) -> 'CombinedProcess':
  207. if changes is None:
  208. changes_env_var = '[]'
  209. else:
  210. changes_env_var = json.dumps([[c.raw_str(), p] for c, p in changes])
  211. os.environ['WATCHFILES_CHANGES'] = changes_env_var
  212. process: Union[SpawnProcess, subprocess.Popen[bytes]]
  213. if target_type == 'function':
  214. kwargs = kwargs or {}
  215. if isinstance(target, str):
  216. args = target, get_tty_path(), args, kwargs
  217. target_ = run_function
  218. kwargs = {}
  219. else:
  220. target_ = target
  221. process = spawn_context.Process(target=target_, args=args, kwargs=kwargs)
  222. process.start()
  223. else:
  224. if args or kwargs:
  225. logger.warning('ignoring args and kwargs for "command" target')
  226. assert isinstance(target, str), 'target must be a string to run as a command'
  227. popen_args = split_cmd(target)
  228. process = subprocess.Popen(popen_args)
  229. return CombinedProcess(process)
  230. def detect_target_type(target: Union[str, Callable[..., Any]]) -> "Literal['function', 'command']":
  231. """
  232. Used by [`run_process`][watchfiles.run_process], [`arun_process`][watchfiles.arun_process]
  233. and indirectly the CLI to determine the target type with `target_type` is `auto`.
  234. Detects the target type - either `function` or `command`. This method is only called with `target_type='auto'`.
  235. The following logic is employed:
  236. * If `target` is not a string, it is assumed to be a function
  237. * If `target` ends with `.py` or `.sh`, it is assumed to be a command
  238. * Otherwise, the target is assumed to be a function if it matches the regex `[a-zA-Z0-9_]+(\\.[a-zA-Z0-9_]+)+`
  239. If this logic does not work for you, specify the target type explicitly using the `target_type` function argument
  240. or `--target-type` command line argument.
  241. Args:
  242. target: The target value
  243. Returns:
  244. either `'function'` or `'command'`
  245. """
  246. if not isinstance(target, str):
  247. return 'function'
  248. elif target.endswith(('.py', '.sh')):
  249. return 'command'
  250. elif re.fullmatch(r'[a-zA-Z0-9_]+(\.[a-zA-Z0-9_]+)+', target):
  251. return 'function'
  252. else:
  253. return 'command'
  254. class CombinedProcess:
  255. def __init__(self, p: 'Union[SpawnProcess, subprocess.Popen[bytes]]'):
  256. self._p = p
  257. assert self.pid is not None, 'process not yet spawned'
  258. def stop(self, sigint_timeout: int = 5, sigkill_timeout: int = 1) -> None:
  259. os.environ.pop('WATCHFILES_CHANGES', None)
  260. if self.is_alive():
  261. logger.debug('stopping process...')
  262. os.kill(self.pid, signal.SIGINT)
  263. try:
  264. self.join(sigint_timeout)
  265. except subprocess.TimeoutExpired:
  266. # Capture this exception to allow the self.exitcode to be reached.
  267. # This will allow the SIGKILL to be sent, otherwise it is swallowed up.
  268. logger.warning('SIGINT timed out after %r seconds', sigint_timeout)
  269. pass
  270. if self.exitcode is None:
  271. logger.warning('process has not terminated, sending SIGKILL')
  272. os.kill(self.pid, signal.SIGKILL)
  273. self.join(sigkill_timeout)
  274. else:
  275. logger.debug('process stopped')
  276. else:
  277. logger.warning('process already dead, exit code: %d', self.exitcode)
  278. def is_alive(self) -> bool:
  279. if isinstance(self._p, SpawnProcess):
  280. return self._p.is_alive()
  281. else:
  282. return self._p.poll() is None
  283. @property
  284. def pid(self) -> int:
  285. # we check the process has always been spawned when CombinedProcess is initialised
  286. return self._p.pid # type: ignore[return-value]
  287. def join(self, timeout: int) -> None:
  288. if isinstance(self._p, SpawnProcess):
  289. self._p.join(timeout)
  290. else:
  291. self._p.wait(timeout)
  292. @property
  293. def exitcode(self) -> Optional[int]:
  294. if isinstance(self._p, SpawnProcess):
  295. return self._p.exitcode
  296. else:
  297. return self._p.returncode
  298. def run_function(function: str, tty_path: Optional[str], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> None:
  299. with set_tty(tty_path):
  300. func = import_string(function)
  301. func(*args, **kwargs)
  302. def import_string(dotted_path: str) -> Any:
  303. """
  304. Stolen approximately from django. Import a dotted module path and return the attribute/class designated by the
  305. last name in the path. Raise ImportError if the import fails.
  306. """
  307. try:
  308. module_path, class_name = dotted_path.strip(' ').rsplit('.', 1)
  309. except ValueError as e:
  310. raise ImportError(f'"{dotted_path}" doesn\'t look like a module path') from e
  311. module = import_module(module_path)
  312. try:
  313. return getattr(module, class_name)
  314. except AttributeError as e:
  315. raise ImportError(f'Module "{module_path}" does not define a "{class_name}" attribute') from e
  316. def get_tty_path() -> Optional[str]: # pragma: no cover
  317. """
  318. Return the path to the current TTY, if any.
  319. Virtually impossible to test in pytest, hence no cover.
  320. """
  321. try:
  322. return os.ttyname(sys.stdin.fileno())
  323. except OSError:
  324. # fileno() always fails with pytest
  325. return '/dev/tty'
  326. except AttributeError:
  327. # on Windows. No idea of a better solution
  328. return None
  329. @contextlib.contextmanager
  330. def set_tty(tty_path: Optional[str]) -> Generator[None, None, None]:
  331. if tty_path:
  332. try:
  333. with open(tty_path) as tty: # pragma: no cover
  334. sys.stdin = tty
  335. yield
  336. except OSError:
  337. # eg. "No such device or address: '/dev/tty'", see https://github.com/samuelcolvin/watchfiles/issues/40
  338. yield
  339. else:
  340. # currently on windows tty_path is None and there's nothing we can do here
  341. yield
  342. def raise_keyboard_interrupt(signum: int, _frame: Any) -> None: # pragma: no cover
  343. logger.warning('received signal %s, raising KeyboardInterrupt', signal.Signals(signum))
  344. raise KeyboardInterrupt
  345. def catch_sigterm() -> None:
  346. """
  347. Catch SIGTERM and raise KeyboardInterrupt instead. This means watchfiles will stop quickly
  348. on `docker compose stop` and other cases where SIGTERM is sent.
  349. Without this the watchfiles process will be killed while a running process will continue uninterrupted.
  350. """
  351. logger.debug('registering handler for SIGTERM on watchfiles process %d', os.getpid())
  352. signal.signal(signal.SIGTERM, raise_keyboard_interrupt)