Bound daemon startup and shutdown and exit with a useful code

start_daemon and stop_daemon polled "while True" with no bound, and no
seaf-cli exit code was checked. A daemon that failed to start left the
container spinning forever with nothing pointing at the cause, and there was
no signal handling, so "docker stop" waited the full 10 second grace period
and then SIGKILLed the process with the daemon still running.

Readiness polling is now bounded by DAEMON_START_TIMEOUT and
DAEMON_STOP_TIMEOUT. The stop timeout is 8 seconds so a graceful shutdown
fits inside the docker stop grace period, and a test asserts that relation.

SIGTERM and SIGINT raise GracefulShutdown, and the lifecycle moved into
start.run(), which always stops the daemon on the way out and returns the
process exit code: 0 on a clean finish or a stop signal, 1 on a daemon or
configuration failure.

seaf-cli exit codes are checked. A failed sync is reported per library and
does not stop the remaining libraries, since one bad library should not take
the container down.

Verified in the built image: "docker stop" now returns in 0s with exit code 0
and the daemon stopped, against 10s and exit 137 before. A daemon that cannot
start exits 1 with the failing command and its exit code in the log.

Also bumps actions/setup-python to v7, which clears the last Node 20
deprecation warning; v7 only drops the unused pip-install input.

dsc/client.py

@@ -12,6 +12,7 @@ from urllib3.util.retry import Retry
 import seafile
 
 from dsc import const
+from dsc.errors import DaemonError, DaemonTimeout
 from dsc.misc import create_dir, hide_password, user_cmd
 
 _lg = logging.getLogger(__name__)
@@ -59,6 +60,29 @@ class SeafileClient:
     def __gen_cmd(self, argv: list) -> list:
         return user_cmd(argv)
 
+    def __run(self, argv: list, check: bool = True, **kwargs):
+        """Run a seaf-cli command and, unless told otherwise, check its exit
+        code. An unchecked failure here surfaces much later as a daemon that
+        never becomes ready, with nothing pointing at the real cause."""
+        proc = subprocess.run(self.__gen_cmd(argv), **kwargs)
+        if check and proc.returncode != 0:
+            raise DaemonError(
+                f"Command {' '.join(argv)} failed with exit code {proc.returncode}"
+            )
+        return proc
+
+    def __wait_for_daemon(self, ready: bool, timeout: float, action: str):
+        """Wait until the daemon is ready (or stopped), but never forever."""
+        deadline = time.monotonic() + timeout
+        while self.daemon_ready != ready:
+            if time.monotonic() >= deadline:
+                state = "ready" if ready else "stopped"
+                raise DaemonTimeout(
+                    f"Seafile daemon did not become {state} within {timeout}s"
+                    f" while {action}"
+                )
+            time.sleep(const.DAEMON_POLL_PERIOD)
+
     @property
     def token(self):
         if self.__token is None:
@@ -101,32 +125,26 @@ class SeafileClient:
             return
         cmd = ["seaf-cli", "init", "-d", self.app_dir]
         _lg.info("Initializing seafile config: %s", " ".join(cmd))
-        subprocess.run(self.__gen_cmd(cmd))
+        self.__run(cmd)
 
     def start_daemon(self):
         cmd = ["seaf-cli", "start"]
         _lg.info("Starting seafile daemon: %s", " ".join(cmd))
-        subprocess.run(self.__gen_cmd(cmd))
-        _lg.info("Waiting for seafile daemon to start")
-
-        while True:
-            if self.daemon_ready:
-                break
-            time.sleep(5)
-
+        self.__run(cmd)
+        _lg.info("Waiting up to %ss for seafile daemon to start",
+                 const.DAEMON_START_TIMEOUT)
+        self.__wait_for_daemon(True, const.DAEMON_START_TIMEOUT, "starting")
         _lg.info("Seafile daemon is ready")
 
     def stop_daemon(self):
         cmd = ["seaf-cli", "stop"]
         _lg.info("Stopping seafile daemon: %s", " ".join(cmd))
-        subprocess.run(self.__gen_cmd(cmd))
-        _lg.info("Waiting for seafile daemon to stop")
-
-        while True:
-            if not self.daemon_ready:
-                break
-            time.sleep(5)
-
+        # A daemon that is already gone makes "stop" fail; the state below is
+        # what matters, not this exit code.
+        self.__run(cmd, check=False)
+        _lg.info("Waiting up to %ss for seafile daemon to stop",
+                 const.DAEMON_STOP_TIMEOUT)
+        self.__wait_for_daemon(False, const.DAEMON_STOP_TIMEOUT, "stopping")
         _lg.info("Seafile daemon is stopped")
 
     def get_library_id(self, library) -> Optional[str]:
@@ -135,11 +153,14 @@ class SeafileClient:
                 return lib_id
         return None
 
-    def sync_lib(self, lib_id: str, lib_dir: str):
+    def sync_lib(self, lib_id: str, lib_dir: str) -> bool:
         """
         Sync a library into an already resolved directory. The caller decides
         the directory, see dsc.paths.plan_lib_dirs: the name comes from the
         server and must not be turned into a path here.
+
+        Returns whether the library was accepted. One library that cannot be
+        synced must not stop the others, so this reports instead of raising.
         """
         create_dir(lib_dir)
         cmd = [
@@ -155,7 +176,12 @@ class SeafileClient:
             "Syncing library %s into %s: %s", lib_id, lib_dir,
             " ".join(hide_password(cmd, self.password)),
         )
-        subprocess.run(self.__gen_cmd(cmd))
+        proc = self.__run(cmd, check=False)
+        if proc.returncode != 0:
+            _lg.error("Failed to sync library %s into %s: seaf-cli exited with %s",
+                      lib_id, lib_dir, proc.returncode)
+            return False
+        return True
 
     def __print_tx_task(self, tx_task) -> str:
         """ Print transfer task status """
@@ -261,7 +287,7 @@ class SeafileClient:
             # check current value
             cmd = ["seaf-cli", "config", "-k", key]
             _lg.info("Checking seafile client option: %s", " ".join(cmd))
-            proc = subprocess.run(self.__gen_cmd(cmd), stdout=subprocess.PIPE)
+            proc = self.__run(cmd, stdout=subprocess.PIPE)
             # stdout looks like "option = value"
             cur_value = proc.stdout.decode().strip()
             try:
@@ -274,7 +300,7 @@ class SeafileClient:
             # set new value
             cmd = ["seaf-cli", "config", "-k", key, "-v", str(value)]
             _lg.info("Setting seafile client option: %s", " ".join(cmd))
-            subprocess.run(self.__gen_cmd(cmd))
+            self.__run(cmd)
             need_restart = True
 
         if need_restart:

dsc/const.py

@@ -4,6 +4,13 @@ DEPRECATED_LIBS_DIR = "/data"
 DEFAULT_USERNAME = "seafile"
 STATUS_POLL_PERIOD = 1
 
+# Bounds for daemon startup and shutdown, in seconds. The stop timeout has to
+# fit inside the grace period of "docker stop", which sends SIGKILL 10 seconds
+# after SIGTERM by default, otherwise shutdown is never graceful.
+DAEMON_POLL_PERIOD = 1
+DAEMON_START_TIMEOUT = 120
+DAEMON_STOP_TIMEOUT = 8
+
 AVAILABLE_SEAFCLI_OPTIONS = {
     "delete_confirm_threshold",
     "disable_verify_certificate",

dsc/errors.py

@@ -0,0 +1,17 @@
+"""Errors that must end the container with a useful message."""
+
+
+class DscError(Exception):
+    """Base class for failures that stop the client."""
+
+
+class DaemonError(DscError):
+    """A seaf-cli command failed."""
+
+
+class DaemonTimeout(DscError):
+    """The daemon did not reach the expected state in time."""
+
+
+class GracefulShutdown(DscError):
+    """SIGTERM or SIGINT was received; stop the daemon and exit cleanly."""