Changeset View
Changeset View
Standalone View
Standalone View
python_modules/dagster/dagster/daemon/controller.py
| Show First 20 Lines • Show All 206 Lines • ▼ Show 20 Lines | def check_daemon_heartbeats(self): | ||||
| if failed_daemons: | if failed_daemons: | ||||
| raise Exception( | raise Exception( | ||||
| "Stopping dagster-daemon process since the following threads are no longer sending heartbeats: {failed_daemons}".format( | "Stopping dagster-daemon process since the following threads are no longer sending heartbeats: {failed_daemons}".format( | ||||
| failed_daemons=failed_daemons | failed_daemons=failed_daemons | ||||
| ) | ) | ||||
| ) | ) | ||||
| def check_daemon_loop(self): | def check_daemon_loop(self, check_heartbeats=True): | ||||
| start_time = time.time() | start_time = time.time() | ||||
| last_heartbeat_check_time = start_time | last_heartbeat_check_time = start_time if check_heartbeats else None | ||||
| while True: | while True: | ||||
| # Wait until a daemon has been unhealthy for a long period of time | # Wait until a daemon has been unhealthy for a long period of time | ||||
| # before potentially restarting it due to a hanging or failed daemon | # before potentially restarting it due to a hanging or failed daemon | ||||
| with raise_interrupts_as(KeyboardInterrupt): | with raise_interrupts_as(KeyboardInterrupt): | ||||
| time.sleep(THREAD_CHECK_INTERVAL) | time.sleep(THREAD_CHECK_INTERVAL) | ||||
| self.check_daemon_threads() | self.check_daemon_threads() | ||||
| if not check_heartbeats: | |||||
| continue | |||||
| now = time.time() | now = time.time() | ||||
| # Give the daemon enough time to send an initial heartbeat before checking | # Give the daemon enough time to send an initial heartbeat before checking | ||||
| if ( | if ( | ||||
| (now - start_time) < 2 * self._heartbeat_interval_seconds | (now - start_time) < 2 * self._heartbeat_interval_seconds | ||||
| or now - last_heartbeat_check_time < HEARTBEAT_CHECK_INTERVAL | or now - last_heartbeat_check_time < HEARTBEAT_CHECK_INTERVAL | ||||
| ): | ): | ||||
| continue | continue | ||||
| ▲ Show 20 Lines • Show All 142 Lines • Show Last 20 Lines | |||||