launchd に SIGKILL される前に自発終了する：24h タイムアウトの 20h で graceful exit する設計

1
<key>ExitTimeOut</key>
2
<integer>86400</integer>  <!-- 24 時間 -->

1
import os
2
import time
3
import signal
4

5
START_TIME = time.time()
6
MAX_RUNTIME_SECONDS = 20 * 3600  # 20 時間で graceful exit
7
HARD_TIMEOUT_SECONDS = 24 * 3600  # launchd の ExitTimeOut
8

9
def should_graceful_exit() -> bool:
10
    elapsed = time.time() - START_TIME
11
    return elapsed >= MAX_RUNTIME_SECONDS
12

13
def main_loop():
14
    for batch in iter_batches():
15
        if should_graceful_exit():
9 collapsed lines
16
            logger.warning(
17
                f"approaching ExitTimeOut "
18
                f"(elapsed={time.time() - START_TIME:.0f}s), "
19
                f"shutting down gracefully"
20
            )
21
            cleanup_and_exit()
22
            return
23

24
        process_batch(batch)

1
def cleanup_and_exit():
2
    """中途半端な状態を残さず終わる"""
3
    try:
4
        # 1. 書きかけのファイルを完成させる or 削除
5
        flush_pending_files()
6

7
        # 2. 進捗をチェックポイントに書き出し
8
        save_checkpoint({
9
            "last_processed_id": current_id,
10
            "exit_reason": "graceful_shutdown",
11
            "exit_time": datetime.now().isoformat(),
12
        })
13

14
        # 3. DB 接続をクローズ
15
        close_all_connections()
9 collapsed lines
16

17
        # 4. 通知
18
        send_slack(f"job exited gracefully at {time.time() - START_TIME:.0f}s")
19

20
    except Exception:
21
        logger.exception("cleanup failed but exiting anyway")
22
    finally:
23
        # exit 0 でないと launchd が再起動してしまう（KeepAlive 設定次第）
24
        sys.exit(0)

1
def load_checkpoint() -> dict | None:
2
    if not CHECKPOINT_PATH.exists():
3
        return None
4
    return json.loads(CHECKPOINT_PATH.read_text())
5

6
def main():
7
    checkpoint = load_checkpoint()
8
    if checkpoint and checkpoint.get("exit_reason") == "graceful_shutdown":
9
        logger.info(f"resuming from checkpoint: {checkpoint}")
10
        start_id = checkpoint["last_processed_id"]
11
    else:
12
        start_id = 0
13

14
    main_loop(start_id)

1
import signal
2

3
def handle_sigterm(signum, frame):
4
    logger.warning("received SIGTERM, shutting down")
5
    cleanup_and_exit()
6

7
signal.signal(signal.SIGTERM, handle_sigterm)

1
<key>ExitTimeOut</key>
2
<integer>86400</integer>  <!-- 24 時間 = ハードリミット -->

1
<key>KeepAlive</key>
2
<dict>
3
    <key>SuccessfulExit</key>
4
    <true/>  <!-- exit 0 でも再起動する -->
5
</dict>

1
<key>ThrottleInterval</key>
2
<integer>60</integer>  <!-- 再起動の最低間隔: 60 秒 -->

1
HARD_TIMEOUT_SECONDS = int(os.getenv("EXIT_TIMEOUT_SECONDS", 86400))
2
MAX_RUNTIME_SECONDS = int(HARD_TIMEOUT_SECONDS * 0.83)  # 20/24 = 0.83

1
def process_batch(batch):
2
    for i, item in enumerate(batch):
3
        if i % 100 == 0 and should_graceful_exit():
4
            save_partial_progress(batch[:i])
5
            cleanup_and_exit()
6
        process_item(item)

1
def cleanup_and_exit():
2
    save_checkpoint(...)  # これは必須
3
    close_db_connections()  # これも必須
4
    try:
5
        send_slack(...)  # 通知は best-effort
6
    except Exception:
7
        pass  # 失敗しても止めない
8
    sys.exit(0)