Add HTTP retry strategy for server connection failures

Implement automatic retry with exponential backoff (up to 30 attempts)
for 502/503/504 errors. Prevents container crash-loop during server
outages, reducing kernel log spam from repeated restarts.

Dockerfile

@@ -13,7 +13,7 @@ RUN curl https://linux-clients.seafile.com/seafile.asc | apt-key add - && \
 
 # Use virtual environment
 ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
+RUN python3 -m venv --system-site-packages $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # Install app requirements

README.md

@@ -1,5 +1,5 @@
 # docker-seafile-client
-Docker image for Seafile terminal client.
+Docker image for [Seafile CLI client](https://help.seafile.com/syncing_client/linux-cli/).
 
 ### Docker-compose example:
 ```yaml
@@ -25,10 +25,20 @@ volumes:
 ```
 
 ### Environment variables:
- - `LIBRARY_ID` - library to sync, ID or name. Multiple libraries could be separated by colon `:`.
- - `SERVER_HOST` - hostname of your Seafile server, eg: `seafile.example.com`. If you're using non-standard port, you can specify it here, eg: `seafile.example.com:8080`.
+ - `LIBRARY_ID` - library to sync, ID or name. Multiple libraries could be
+ separated by colon `:`.
+ - `SERVER_HOST` - hostname of your Seafile server, eg: _seafile.example.com_.
+ If you're using non-standard port, you can specify it here,
+ eg: _seafile.example.com:8080_.
  - `USERNAME` / `PASSWORD` - credentials to access Seafile server.
- - `SEAFILE_UID` / `SEAFILE_GID` - UID/GID of user inside container. You can use it to set permissions on synced files. Default values are `1000`/`1000`.
+ - `SEAFILE_UID` / `SEAFILE_GID` - UID/GID of user inside container. You can
+ use it to set permissions on synced files. Default values are _1000_ / _1000_.
+ - `DELETE_CONFIRM_THRESHOLD` - represents the number of files that require
+ confirmation before being deleted simultaneously. Default value is _500_.
+ - `DISABLE_VERIFY_CERTIFICATE` - set to _true_ to disable server's certificate
+ verification. Default value is _false_.
+ - `UPLOAD_LIMIT` / `DOWNLOAD_LIMIT` -  upload/download speed limit in B/s
+ (bytes per second). Default values are _0_ (unlimited).
 
 ### Volumes:
  - `/dsc/seafile-data`  Seafile client data directory (sync status, etc).
@@ -36,11 +46,23 @@ volumes:
 
 
 ### Some notes
-`LIBRARY_ID` could be library ID or library name. Library ID is a 36-character string, which is a part of URI when you open library in webUI. Library name is a name you gave to library when you created it.
+`LIBRARY_ID` could be library ID or library name. Library ID is a 36-character
+ string, which is a part of URI when you open library in webUI. Library name is
+ a name you gave to library when you created it.
 
-Libraries will be synced in subdirectories of `/dsc/seafile` directory inside container. You can mount it to host directory to access files.
+Libraries will be synced in subdirectories of `/dsc/seafile` directory inside
+ container. You can mount it to host directory to access files.
 
-`hostname` parameter is optional, but it's recommended to set it to some unique value, it will be shown in Seafile webUI as client name (`terminal-dsc` in given example).
+`hostname` parameter is optional, but it's recommended to set it to some unique
+ value, it will be shown in Seafile webUI as client name (`terminal-dsc` in
+ given example).
 
-`sync-data` volume is optional too, but it's recommended to use it. Otherwise, sync status will be lost when container is recreated.
+`sync-data` volume is optional too, but it's recommended to use it. Otherwise,
+ sync status will be lost when container is recreated.
 
+At the moment there is no suitable way to confirm deletion of large number of
+ files. So, if you're going to delete a lot of files, you should set
+ `DELETE_CONFIRM_THRESHOLD` to some larger value.
+
+### Links
+- [Official Seafile CLI client documentation](https://help.seafile.com/syncing_client/linux-cli/)

dsc/client.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 import os
 import subprocess
@@ -7,6 +8,9 @@ from urllib.parse import urlparse
 
 from cached_property import cached_property_with_ttl
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+import seafile
 
 from dsc import const
 from dsc.misc import create_dir, hide_password
@@ -20,13 +24,36 @@ class SeafileClient:
                  user: str,
                  passwd: str,
                  app_dir: str = const.DEFAULT_APP_DIR):
-        up = urlparse(requests.get(f"http://{host}").url)
-        self.url = f"{up.scheme}://{up.netloc}"
         self.user = user
         self.password = passwd
         self.app_dir = os.path.abspath(app_dir)
+        self.rpc = seafile.RpcClient(os.path.join(self.app_dir, 'seafile-data', 'seafile.sock'))
         self.__token = None
 
+        # determine server URL (assume HTTPS unless explicitly specified)
+        if host.startswith('http://') or host.startswith('https://'):
+            self.url = host.rstrip('/')
+        else:
+            self.url = f"https://{host}"
+
+        # configure session with retry strategy
+        # enable urllib3 retry logging at DEBUG level (shows retry attempts)
+        urllib3_logger = logging.getLogger("urllib3.connectionpool")
+        urllib3_logger.setLevel(logging.DEBUG)
+        urllib3_logger.propagate = True
+
+        self.session = requests.Session()
+        retry_strategy = Retry(
+            total=30,
+            backoff_factor=2,
+            backoff_max=60,
+            status_forcelist=[500, 502, 503, 504],
+            allowed_methods=["GET", "POST"]
+        )
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
+
     def __str__(self):
         return f"SeafileClient({self.user}@{self.url})"
 
@@ -38,7 +65,7 @@ class SeafileClient:
         if self.__token is None:
             url = f"{self.url}/api2/auth-token/"
             _lg.info("Fetching token: %s", url)
-            r = requests.post(url, data={"username": self.user, "password": self.password})
+            r = self.session.post(url, data={"username": self.user, "password": self.password})
             if r.status_code != 200:
                 raise RuntimeError(f"Can't get token: {r.text}")
             self.__token = r.json()["token"]
@@ -49,7 +76,7 @@ class SeafileClient:
         url = f"{self.url}/api2/repos/"
         _lg.info("Fetching remote libraries: %s", url)
         auth_header = {"Authorization": f"Token {self.token}"}
-        r = requests.get(url, headers=auth_header)
+        r = self.session.get(url, headers=auth_header)
         if r.status_code != 200:
             raise RuntimeError(r.text)
         r_libs = {lib["id"]: lib["name"] for lib in r.json()}
@@ -90,6 +117,19 @@ class SeafileClient:
 
         _lg.info("Seafile daemon is ready")
 
+    def stop_daemon(self):
+        cmd = "seaf-cli stop"
+        _lg.info("Stopping seafile daemon: %s", cmd)
+        subprocess.run(self.__gen_cmd(cmd))
+        _lg.info("Waiting for seafile daemon to stop")
+
+        while True:
+            if not self.daemon_ready:
+                break
+            time.sleep(5)
+
+        _lg.info("Seafile daemon is stopped")
+
     def get_library_id(self, library) -> Optional[str]:
         for lib_id, lib_name in self.remote_libraries.items():
             if library in (lib_id, lib_name):
@@ -115,31 +155,84 @@ class SeafileClient:
         )
         subprocess.run(self.__gen_cmd(" ".join(cmd)))
 
-    def get_status(self):
-        cmd = "seaf-cli status"
-        _lg.debug("Fetching seafile client status: %s", cmd)
-        out = subprocess.check_output(self.__gen_cmd(cmd))
-        out = out.decode().splitlines()
+    def __print_tx_task(self, tx_task) -> str:
+        """ Print transfer task status """
+        try:
+            percentage = tx_task.block_done / tx_task.block_total * 100
+            tx_rate = tx_task.rate / 1024.0
+            return f" {percentage:.1f}%, {tx_rate:.1f}KB/s"
+        except ZeroDivisionError:
+            return ""
 
+    def get_status(self) -> dict:
+        """ Get status of all libraries """
         statuses = dict()
-        for line in out:
-            if line.startswith("#") or not line.strip():
+
+        # fetch statuses of libraries being cloned
+        tasks = self.rpc.get_clone_tasks()
+        for clone_task in tasks:
+            if clone_task.state == "done":
                 continue
-            lib, status = line.split(sep="\t", maxsplit=1)
-            lib = lib.strip()
-            status = " ".join(status.split())
-            statuses[lib] = status
+
+            elif clone_task.state == "fetch":
+                statuses[clone_task.repo_name] = "downloading"
+                tx_task = self.rpc.find_transfer_task(clone_task.repo_id)
+                statuses[clone_task.repo_name] += self.__print_tx_task(tx_task)
+
+            elif clone_task.state == "error":
+                err = self.rpc.sync_error_id_to_str(clone_task.error)
+                statuses[clone_task.repo_name] = f"error: {err}"
+
+            else:
+                statuses[clone_task.repo_name] = clone_task.state
+
+        # fetch statuses of synced libraries
+        repos = self.rpc.get_repo_list(-1, -1)
+        for repo in repos:
+            auto_sync_enabled = self.rpc.is_auto_sync_enabled()
+            if not auto_sync_enabled or not repo.auto_sync:
+                statuses[repo.name] = "auto sync disabled"
+                continue
+
+            sync_task = self.rpc.get_repo_sync_task(repo.id)
+            if sync_task is None:
+                statuses[repo.name] = "waiting for sync"
+
+            elif sync_task.state in ("uploading", "downloading"):
+                statuses[repo.name] = sync_task.state
+                tx_task = self.rpc.find_transfer_task(repo.id)
+
+                if sync_task.state == "downloading":
+                    if tx_task.rt_state == "data":
+                        statuses[repo.name] += " files"
+                    elif tx_task.rt_state == "fs":
+                        statuses[repo.name] += " file list"
+
+                statuses[repo.name] += self.__print_tx_task(tx_task)
+
+            elif sync_task.state == "error":
+                err = self.rpc.sync_error_id_to_str(sync_task.error)
+                statuses[repo.name] = f"error: {err}"
+
+            else:
+                statuses[repo.name] = sync_task.state
+
         return statuses
 
     def watch_status(self):
         prev_status = dict()
+        max_name_len = 0
+        fmt = "Library {:%ds} {}" % max_name_len
         while True:
             time.sleep(const.STATUS_POLL_PERIOD)
             cur_status = self.get_status()
-            for folder, state in cur_status.items():
-                if state != prev_status.get(folder):
-                    logging.info("Library %s:\t%s", folder, state)
-                prev_status[folder] = cur_status[folder]
+            for library, state in cur_status.items():
+                if state != prev_status.get(library):
+                    if 30 > len(library) > max_name_len:
+                        max_name_len = len(library)
+                        fmt = "Library {:%ds}    {}" % max_name_len
+                    logging.info(fmt.format(library, state))
+                prev_status[library] = cur_status[library]
 
     def get_local_libraries(self) -> set:
         cmd = "seaf-cli list"
@@ -152,3 +245,37 @@ class SeafileClient:
             lib_name, lib_id, lib_path = line.rsplit(maxsplit=3)
             local_libs.add(lib_id)
         return local_libs
+
+    def configure(self, args: argparse.Namespace, check_for_daemon: bool = True):
+        need_restart = False
+        # Options can be fetched or set only when daemon is running
+        if check_for_daemon and not self.daemon_ready:
+            self.start_daemon()
+
+        for key, value in args.__dict__.items():
+            if key not in const.AVAILABLE_SEAFCLI_OPTIONS:
+                continue
+
+            # check current value
+            cmd = f"seaf-cli config -k {key}"
+            _lg.info("Checking seafile client option: %s", cmd)
+            proc = subprocess.run(self.__gen_cmd(cmd), stdout=subprocess.PIPE)
+            # stdout looks like "option = value"
+            cur_value = proc.stdout.decode().strip()
+            try:
+                cur_value = cur_value.split(sep="=")[1].strip()
+            except IndexError:
+                cur_value = None
+            if cur_value == str(value):
+                continue
+
+            # set new value
+            cmd = f"seaf-cli config -k {key} -v {value}"
+            _lg.info("Setting seafile client option: %s", cmd)
+            subprocess.run(self.__gen_cmd(cmd))
+            need_restart = True
+
+        if need_restart:
+            _lg.info("Restarting seafile daemon")
+            self.stop_daemon()
+            self.start_daemon()

dsc/const.py

@@ -3,3 +3,10 @@ DEFAULT_LIBS_DIR = "/dsc/seafile"
 DEPRECATED_LIBS_DIR = "/data"
 DEFAULT_USERNAME = "seafile"
 STATUS_POLL_PERIOD = 1
+
+AVAILABLE_SEAFCLI_OPTIONS = {
+    "delete_confirm_threshold",
+    "disable_verify_certificate",
+    "upload_limit",
+    "download_limit",
+}

dsc/misc.py

@@ -43,6 +43,6 @@ def create_dir(dir_path: str):
 def hide_password(cmd: list, password: str) -> list:
     cmd = cmd.copy()
     for i, arg in enumerate(cmd):
-        if arg == password:
-            cmd[i] = '********'
+        if password in arg:
+            cmd[i] = arg.replace(password, "********")
     return cmd

requirements.txt

@@ -1,2 +1,2 @@
 cached_property==1.5.2
-requests==2.31.0
+requests==2.32.4

start.py

@@ -16,28 +16,54 @@ def main():
     logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--uid", default=os.getenv("SEAFILE_UID", default=1000), type=int)
-    parser.add_argument("--gid", default=os.getenv("SEAFILE_GID", default=1000), type=int)
-    parser.add_argument("--host", default=os.getenv("SERVER_HOST"))
-    parser.add_argument("--username", default=os.getenv("USERNAME"))
-    parser.add_argument("--password", default=os.getenv("PASSWORD"))
-    parser.add_argument("--libs", default=os.getenv("LIBRARY_ID"))
+    parser.add_argument("-s", "--server")
+    parser.add_argument("-u", "--username")
+    parser.add_argument("-p", "--password")
+    parser.add_argument("-l", "--libraries")
+    parser.add_argument("--uid", type=int)
+    parser.add_argument("--gid", type=int)
+    parser.add_argument("--upload-limit", type=int, default=0)
+    parser.add_argument("--download-limit", type=int, default=0)
+    parser.add_argument("--disable-verify-certificate", action="store_true")
+    parser.add_argument("--delete-confirm-threshold", type=int, default=500)
+
+    parser.set_defaults(
+        server=os.getenv("SERVER_HOST"),
+        username=os.getenv("USERNAME"),
+        password=os.getenv("PASSWORD"),
+        libraries=os.getenv("LIBRARY_ID"),
+        uid=os.getenv("SEAFILE_UID", default=1000),
+        gid=os.getenv("SEAFILE_GID", default=1000),
+        upload_limit=os.getenv("UPLOAD_LIMIT"),
+        download_limit=os.getenv("DOWNLOAD_LIMIT"),
+        disable_verify_certificate=os.getenv("DISABLE_VERIFY_CERTIFICATE") in ("true", "1", "True"),
+        delete_confirm_threshold=os.getenv("DELETE_CONFIRM_THRESHOLD"),
+    )
     args = parser.parse_args()
+    if not args.server:
+        parser.error("Seafile server is not specified")
+    if not args.username:
+        parser.error("username is not specified")
+    if not args.password:
+        parser.error("password is not specified")
+    if not args.libraries:
+        parser.error("library is not specified")
 
     setup_uid(args.uid, args.gid)
     create_dir(const.DEFAULT_APP_DIR)
 
-    client = SeafileClient(args.host, args.username, args.password, const.DEFAULT_APP_DIR)
+    client = SeafileClient(args.server, args.username, args.password, const.DEFAULT_APP_DIR)
     client.init_config()
     client.start_daemon()
+    client.configure(args, check_for_daemon=False)
 
     libs_to_sync = set()
-    for arg_lib in args.libs.split(sep=":"):
+    for arg_lib in args.libraries.split(sep=":"):
         lib_id = client.get_library_id(arg_lib)
         if lib_id:
             libs_to_sync.add(lib_id)
         else:
-            _lg.warning("Library %s is not found on server %s", arg_lib, args.host)
+            _lg.warning("Library %s is not found on server %s", arg_lib, args.server)
 
     # don't start to sync libraries already in sync
     libs_to_sync -= client.get_local_libraries()
@@ -55,6 +81,7 @@ def main():
         client.sync_lib(lib_id, libs_dir)
     client.watch_status()
 
+    client.stop_daemon()
     return 0