Initialize seafile client during container start

Bump requests from 2.27.1 to 2.31.0

.github/workflows/docker-publish.yml

@@ -0,0 +1,53 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run tests
+        run: |
+          if [ -f docker-compose.test.yml ]; then
+            docker-compose --file docker-compose.test.yml build
+            docker-compose --file docker-compose.test.yml run sut
+          else
+            docker build . --file Dockerfile
+          fi
+
+  push:
+    needs: test
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: snegov/seafile-client
+          tag_with_ref: true
+          add_git_labels: true
+          push: ${{ startsWith(github.ref, 'refs/tags/') }}
+
+      - uses: docker/build-push-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: snegov/seafile-client
+          tags: latest
+          add_git_labels: true
+          push: ${{ startsWith(github.ref, 'refs/tags/') }}

Dockerfile

@@ -1,23 +1,36 @@
-FROM python:3-slim
+FROM debian:bookworm-slim
 
-RUN apt-get update && apt-get install gnupg -y
+# Install seafile client
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8756C4F765C9AC3CB6B85D62379CE192D401AB61 && \
+RUN apt-get update && \
-    echo deb http://deb.seadrive.org buster main | tee /etc/apt/sources.list.d/seafile.list && \
+    apt-get install gnupg curl python3.11-venv -y && \
+    rm -rf /var/lib/apt/lists/*
+RUN curl https://linux-clients.seafile.com/seafile.asc | apt-key add - && \
+    echo 'deb [arch=amd64] https://linux-clients.seafile.com/seafile-deb/bookworm/ stable main' \
+    > /etc/apt/sources.list.d/seafile.list && \
     apt-get update -y && \
-    apt-get install -y seafile-cli procps curl grep && \
+    apt-get install -y seafile-cli && \
     rm -rf /var/lib/apt/lists/*
 
-WORKDIR /seafile-client
+# Use virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Install app requirements
+WORKDIR /dsc
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY seafile_client ./seafile_client/
+# Copy app
+COPY dsc ./dsc/
 COPY start.py ./start.py
 
-RUN chmod +x /seafile-client/start.py && \
+# Create seafile user and init seafile client
-    useradd -U -d /seafile-client -s /bin/bash seafile && \
+RUN chmod +x /dsc/start.py && \
+    useradd -U -d /dsc -s /bin/bash seafile && \
     usermod -G users seafile && \
-    chown seafile:seafile -R /seafile-client && \
+    mkdir -p /dsc/seafile-data && \
-    su - seafile -c "seaf-cli init -d /seafile-client"
+    chown seafile:seafile -R /dsc
 
+VOLUME /dsc/seafile-data
 CMD ["./start.py"]

LICENSE

@@ -1,7 +1,7 @@
 MIT License
 
 Copyright (c) 2018 Robin Grönberg
-Copyright (c) 2019 Maks Snegov
+Copyright (c) 2019-2023 Maks Snegov
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,35 +1,46 @@
 # docker-seafile-client
-Run a seafile client inside docker witch can sync files from seafile repositories
+Docker image for Seafile terminal client.
 
-See docker-compose how to use.
+### Docker-compose example:
-
-## Docker-compose example:
 ```yaml
-version: '3'
 services:
   seafile-client:
     restart: always
     image: snegov/seafile-client
-    container_name: seafile-client
     environment:
-      - LIBRARY_ID=<your-library-id-here>
+      - LIBRARY_ID="79867cbf-2944-488d-9105-852463ecdf9e:my_library"
-      - SERVER_HOST=<server-host>
+      - SERVER_HOST=seafile.example.com
-      - SERVER_PORT=<server-port>
+      - USERNAME=user
-      - USERNAME=<username>
+      - PASSWORD=password
-      - PASSWORD=<password>
+      - SEAFILE_UID=1000
-      - DATA_DIR=<directory-path-to-sync>
+      - SEAFILE_GID=100
-      - SEAFILE_UID=<your_uid>
+    hostname: dsc
-      - SEAFILE_GID=<your_gid>
     volumes:
-      - <host-volume-path>:<directory-path-to-sync>
+      - /home/johndow/seafile:/dsc/seafile
+      - sync-data:/dsc/seafile-data
+    container_name: seafile-client
+
+volumes:
+  sync-data:
 ```
 
-## Environment variables:
+### Environment variables:
- - `LIBRARY_ID=<your-library-id-here>`  ID of library to sync; multiple libraries could be separated by colon `:`
+ - `LIBRARY_ID` - library to sync, ID or name. Multiple libraries could be separated by colon `:`.
- - `SERVER_HOST=<server-host>`          Hostname of your seafile server, eg: seafile.example.com
+ - `SERVER_HOST` - hostname of your Seafile server, eg: `seafile.example.com`. If you're using non-standard port, you can specify it here, eg: `seafile.example.com:8080`.
- - `SERVER_PORT=<server-port>`          Which port the server is hosted on: usually 443 (https) or 80 (http)
+ - `USERNAME`/ `PASSWORD` - credentials to access Seafile server.
- - `USERNAME=<username>`                Seafile account username
+ - `SEAFILE_UID` / `SEAFILE_GID` - UID/GID of user inside container. You can use it to set permissions on synced files. Default values are `1000`/`1000`.
- - `PASSWORD=<password>`                Seafile account password
+
- - `DATA_DIR=<directory-path-to-sync>`  The path where to put the files
+### Volumes:
- - `SEAFILE_UID=<uid>`                  Downloaded files will have this uid
+ - `/dsc/seafile-data`  Seafile client data directory (sync status, etc).
- - `SEAFILE_GID=<gid>`                  Downloaded files will have this gid
+ - `/dsc/seafile`       Seafile libraries content.
+
+
+### Some notes
+`LIBRARY_ID` could be library ID or library name. Library ID is a 36-character string, which is a part of URI when you open library in webUI. Library name is a name you gave to library when you created it.
+
+Libraries will be synced in subdirectories of `/dsc/seafile` directory inside container. You can mount it to host directory to access files.
+
+`hostname` parameter is optional, but it's recommended to set it to some unique value, it will be shown in Seafile webUI as client name (`terminal-dsc` in given example).
+
+`sync-data` volume is optional too, but it's recommended to use it. Otherwise, sync status will be lost when container is recreated.
+

docker-compose.example.yml

@@ -1,17 +0,0 @@
-version: '3'
-services:
-  seafile-client:
-    restart: always
-    image: snegov/seafile-client
-    container_name: seafile-client
-    environment:
-      - LIBRARY_ID=<your-library-id-here>
-      - SERVER_HOST=<server-host>
-      - SERVER_PORT=<server-port>
-      - USERNAME=<username>
-      - PASSWORD=<password>
-      - DATA_DIR=<directory-path-to-sync>
-      - SEAFILE_UID=<your_uid>
-      - SEAFILE_GID=<your_gid>
-    volumes:
-      - <host-volume-path>:<directory-path-to-sync>

dsc/__init__.py

@@ -0,0 +1 @@
+from .client import SeafileClient, const

dsc/client.py

@@ -0,0 +1,154 @@
+import logging
+import os
+import subprocess
+import time
+from typing import Optional
+from urllib.parse import urlparse
+
+from cached_property import cached_property_with_ttl
+import requests
+
+from dsc import const
+from dsc.misc import create_dir, hide_password
+
+_lg = logging.getLogger(__name__)
+
+
+class SeafileClient:
+    def __init__(self,
+                 host: str,
+                 user: str,
+                 passwd: str,
+                 app_dir: str = const.DEFAULT_APP_DIR):
+        up = urlparse(requests.get(f"http://{host}").url)
+        self.url = f"{up.scheme}://{up.netloc}"
+        self.user = user
+        self.password = passwd
+        self.app_dir = os.path.abspath(app_dir)
+        self.__token = None
+
+    def __str__(self):
+        return f"SeafileClient({self.user}@{self.url})"
+
+    def __gen_cmd(self, cmd: str) -> list:
+        return ["su", "-", const.DEFAULT_USERNAME, "-c", cmd]
+
+    @property
+    def token(self):
+        if self.__token is None:
+            url = f"{self.url}/api2/auth-token/"
+            _lg.info("Fetching token: %s", url)
+            r = requests.post(url, data={"username": self.user, "password": self.password})
+            if r.status_code != 200:
+                raise RuntimeError(f"Can't get token: {r.text}")
+            self.__token = r.json()["token"]
+        return self.__token
+
+    @cached_property_with_ttl(ttl=60)
+    def remote_libraries(self) -> dict:
+        url = f"{self.url}/api2/repos/"
+        _lg.info("Fetching remote libraries: %s", url)
+        auth_header = {"Authorization": f"Token {self.token}"}
+        r = requests.get(url, headers=auth_header)
+        if r.status_code != 200:
+            raise RuntimeError(r.text)
+        r_libs = {lib["id"]: lib["name"] for lib in r.json()}
+        return r_libs
+
+    @property
+    def config_initialized(self) -> bool:
+        return os.path.isdir(os.path.join(self.app_dir, ".ccnet"))
+
+    @property
+    def daemon_ready(self) -> bool:
+        cmd = "seaf-cli status"
+        _lg.info("Checking seafile daemon status: %s", cmd)
+        proc = subprocess.run(
+            self.__gen_cmd(cmd),
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        return proc.returncode == 0
+
+    def init_config(self):
+        if self.config_initialized:
+            return
+        cmd = "seaf-cli init -d %s" % self.app_dir
+        _lg.info("Initializing seafile config: %s", cmd)
+        subprocess.run(self.__gen_cmd(cmd))
+
+    def start_daemon(self):
+        cmd = "seaf-cli start"
+        _lg.info("Starting seafile daemon: %s", cmd)
+        subprocess.run(self.__gen_cmd(cmd))
+        _lg.info("Waiting for seafile daemon to start")
+
+        while True:
+            if self.daemon_ready:
+                break
+            time.sleep(5)
+
+        _lg.info("Seafile daemon is ready")
+
+    def get_library_id(self, library) -> Optional[str]:
+        for lib_id, lib_name in self.remote_libraries.items():
+            if library in (lib_id, lib_name):
+                return lib_id
+        return None
+
+    def sync_lib(self, lib_id: str, parent_dir: str = const.DEFAULT_LIBS_DIR):
+        lib_name = self.remote_libraries[lib_id]
+        lib_dir = os.path.join(parent_dir, lib_name.replace(" ", "_"))
+        create_dir(lib_dir)
+        cmd = [
+            "seaf-cli",
+            "sync",
+            "-l", lib_id,
+            "-s", self.url,
+            "-d", lib_dir,
+            "-u", self.user,
+            "-p", self.password,
+        ]
+        _lg.info(
+            "Syncing library %s: %s", lib_name,
+            " ".join(hide_password(cmd, self.password)),
+        )
+        subprocess.run(self.__gen_cmd(" ".join(cmd)))
+
+    def get_status(self):
+        cmd = "seaf-cli status"
+        _lg.debug("Fetching seafile client status: %s", cmd)
+        out = subprocess.check_output(self.__gen_cmd(cmd))
+        out = out.decode().splitlines()
+
+        statuses = dict()
+        for line in out:
+            if line.startswith("#") or not line.strip():
+                continue
+            lib, status = line.split(sep="\t", maxsplit=1)
+            lib = lib.strip()
+            status = " ".join(status.split())
+            statuses[lib] = status
+        return statuses
+
+    def watch_status(self):
+        prev_status = dict()
+        while True:
+            time.sleep(const.STATUS_POLL_PERIOD)
+            cur_status = self.get_status()
+            for folder, state in cur_status.items():
+                if state != prev_status.get(folder):
+                    logging.info("Library %s:\t%s", folder, state)
+                prev_status[folder] = cur_status[folder]
+
+    def get_local_libraries(self) -> set:
+        cmd = "seaf-cli list"
+        _lg.info("Listing local libraries: %s", cmd)
+        out = subprocess.check_output(self.__gen_cmd(cmd))
+        out = out.decode().splitlines()[1:]  # first line is a header
+
+        local_libs = set()
+        for line in out:
+            lib_name, lib_id, lib_path = line.rsplit(maxsplit=3)
+            local_libs.add(lib_id)
+        return local_libs

dsc/const.py

@@ -0,0 +1,5 @@
+DEFAULT_APP_DIR = "/dsc"
+DEFAULT_LIBS_DIR = "/dsc/seafile"
+DEPRECATED_LIBS_DIR = "/data"
+DEFAULT_USERNAME = "seafile"
+STATUS_POLL_PERIOD = 1

dsc/misc.py

@@ -0,0 +1,48 @@
+import os
+import pwd
+import subprocess
+
+from dsc.const import DEFAULT_USERNAME
+
+
+def setup_uid(uid: int, gid: int):
+    """
+    Set GID and UID of default user so that seafile client creates files with
+    correct permissions.
+    If GID does not match, create a new group with the given GID.
+    Then update UID and GID of default user to match the given ones.
+    """
+    user_pwinfo = pwd.getpwnam(DEFAULT_USERNAME)
+    create_group(gid)
+    if user_pwinfo.pw_uid != uid or user_pwinfo.pw_gid != gid:
+        subprocess.call(["usermod", "-o", "-u", str(uid), "-g", str(gid), DEFAULT_USERNAME])
+
+
+def create_group(gid: int):
+    """Search for a group with the given GID. If not found, create a new one."""
+    if not os.path.exists(f"/etc/group"):
+        raise RuntimeError(f"File /etc/group does not exist")
+    with open("/etc/group", "r") as f:
+        for line in f.readlines():
+            cur_gid = line.split(sep=":", maxsplit=3)[2]
+            if int(cur_gid) == gid:
+                return
+    subprocess.call(["groupadd", "-g", str(gid), DEFAULT_USERNAME + "-data"])
+
+
+def create_dir(dir_path: str):
+    if not os.path.exists(dir_path):
+        os.mkdir(dir_path)
+        user_pwinfo = pwd.getpwnam(DEFAULT_USERNAME)
+        os.chown(dir_path, user_pwinfo.pw_uid, user_pwinfo.pw_gid)
+    else:
+        if not os.path.isdir(dir_path):
+            raise RuntimeError(f"Data dir {dir_path} is not a directory")
+
+
+def hide_password(cmd: list, password: str) -> list:
+    cmd = cmd.copy()
+    for i, arg in enumerate(cmd):
+        if arg == password:
+            cmd[i] = '********'
+    return cmd

requirements.txt

@@ -1 +1,2 @@
-requests
+cached_property==1.5.2
+requests==2.31.0

seafile_client/__init__.py

@@ -1 +0,0 @@
-from .client import SeafileClient, start_seaf_daemon

seafile_client/client.py

@@ -1,87 +0,0 @@
-import logging
-import os
-import subprocess
-import time
-
-import requests
-
-from .consts import DEFAULT_USERNAME
-from .misc import create_dir
-
-
-logging.basicConfig(format="%(asctime)s %(message)s",
-                    level=logging.INFO)
-
-
-class SeafileClient:
-    def __init__(self, host: str, port: int, user: str, passwd: str):
-        self.url = f"https://{host}:{port}"
-        self.user = user
-        self.password = passwd
-        self.__token = None
-
-    def __str__(self):
-        return f"SeafileClient({self.user}@{self.url})"
-
-    @property
-    def token(self):
-        if self.__token is None:
-            url = f"{self.url}/api2/auth-token/"
-            r = requests.post(url, data={"username": self.user,
-                                         "password": self.password})
-            if r.status_code != 200:
-                raise RuntimeError(f"Can't get token: {r.text}")
-            self.__token = r.json()['token']
-        return self.__token
-
-    def get_lib_name(self, lib_id: str) -> str:
-        url = f"{self.url}/api2/repos/{lib_id}"
-        auth_header = {"Authorization": f"Token {self.token}"}
-        r = requests.get(url, headers=auth_header)
-        if r.status_code != 200:
-            raise RuntimeError(r.text)
-        return r.json()['name']
-
-    def sync_lib(self, lib_id: str, data_dir: str):
-        lib_name = self.get_lib_name(lib_id)
-        lib_dir = os.path.join(data_dir, lib_name.replace(' ', '_'))
-        create_dir(lib_dir)
-        cmd = ['seaf-cli', 'sync',
-               '-l', lib_id,
-               '-s', self.url,
-               '-d', lib_dir,
-               '-u', self.user,
-               '-p', self.password]
-        cmd = ' '.join(cmd)
-        subprocess.run(['su', '-', DEFAULT_USERNAME, '-c', cmd])
-
-    def get_status(self):
-        cmd = 'seaf-cli status'
-        out = subprocess.check_output(['su', '-', DEFAULT_USERNAME, '-c', cmd])
-        out = out.decode().splitlines()
-
-        statuses = dict()
-        for line in out:
-            if line.startswith('#') or not line.strip():
-                continue
-            lib, status = line.split(sep='\t', maxsplit=1)
-            lib = lib.strip()
-            status = " ".join(status.split())
-            statuses[lib] = status
-        return statuses
-
-    def watch_status(self):
-        prev_status = dict()
-        while True:
-            time.sleep(5)
-            cur_status = self.get_status()
-            for folder, state in cur_status.items():
-                if state != prev_status.get(folder):
-                    logging.info(f"Library {folder}:\t{state}")
-                prev_status[folder] = cur_status[folder]
-
-
-def start_seaf_daemon():
-    cmd = 'seaf-cli start'
-    subprocess.run(['su', '-', DEFAULT_USERNAME, '-c', cmd])
-    time.sleep(5)

seafile_client/consts.py

@@ -1 +0,0 @@
-DEFAULT_USERNAME = "seafile"

seafile_client/misc.py

@@ -1,25 +0,0 @@
-import os
-import pwd
-import subprocess
-
-from .consts import DEFAULT_USERNAME
-
-
-def setup_uid(uid: int, gid: int):
-    user_pwinfo = pwd.getpwnam(DEFAULT_USERNAME)
-    if user_pwinfo.pw_uid != uid or user_pwinfo.pw_gid != gid:
-        subprocess.call(["usermod", "-o", "-u", str(uid), "-g", str(gid), DEFAULT_USERNAME])
-
-
-def create_dir(dir_path: str):
-    if not os.path.exists(dir_path):
-        os.mkdir(dir_path)
-        user_pwinfo = pwd.getpwnam(DEFAULT_USERNAME)
-        os.chown(dir_path, user_pwinfo.pw_uid, user_pwinfo.pw_gid)
-    else:
-        if not os.path.isdir(dir_path):
-            raise RuntimeError(f"Data dir {dir_path} is not a directory")
-
-
-def tail_f(fpath):
-    os.system(f"tail -f {fpath}")

start.py

@@ -1,32 +1,58 @@
 #!/usr/bin/env python3
 
 import argparse
+import logging
 import os
 import os.path
 import sys
 
-from seafile_client import SeafileClient, start_seaf_daemon
+from dsc import SeafileClient, const
-from seafile_client.misc import setup_uid, create_dir
+from dsc.misc import setup_uid, create_dir
+
+_lg = logging.getLogger('dsc')
 
 
 def main():
+    logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)
+
     parser = argparse.ArgumentParser()
-    parser.add_argument("--uid", default=os.getenv("SEAFILE_UID"), type=int)
+    parser.add_argument("--uid", default=os.getenv("SEAFILE_UID", default=1000), type=int)
-    parser.add_argument("--gid", default=os.getenv("SEAFILE_GID"), type=int)
+    parser.add_argument("--gid", default=os.getenv("SEAFILE_GID", default=1000), type=int)
-    parser.add_argument("--data-dir", default=os.getenv("DATA_DIR"))
     parser.add_argument("--host", default=os.getenv("SERVER_HOST"))
-    parser.add_argument("--port", default=os.getenv("SERVER_PORT"), type=int)
     parser.add_argument("--username", default=os.getenv("USERNAME"))
     parser.add_argument("--password", default=os.getenv("PASSWORD"))
     parser.add_argument("--libs", default=os.getenv("LIBRARY_ID"))
     args = parser.parse_args()
 
     setup_uid(args.uid, args.gid)
-    start_seaf_daemon()
+    create_dir(const.DEFAULT_APP_DIR)
-    create_dir(args.data_dir)
+
-    client = SeafileClient(args.host, args.port, args.username, args.password)
+    client = SeafileClient(args.host, args.username, args.password, const.DEFAULT_APP_DIR)
-    for lib_id in args.libs.split(sep=":"):
+    client.init_config()
-        client.sync_lib(lib_id, args.data_dir)
+    client.start_daemon()
+
+    libs_to_sync = set()
+    for arg_lib in args.libs.split(sep=":"):
+        lib_id = client.get_library_id(arg_lib)
+        if lib_id:
+            libs_to_sync.add(lib_id)
+        else:
+            _lg.warning("Library %s is not found on server %s", arg_lib, args.host)
+
+    # don't start to sync libraries already in sync
+    libs_to_sync -= client.get_local_libraries()
+
+    # check for deprecated /data directory
+    if os.path.isdir(const.DEPRECATED_LIBS_DIR):
+        _lg.warning("*** DEPRECATED DIRECTORY FOUND ***")
+        _lg.warning("Deprecated directory %s is found, please mount your host directory with"
+                    " libraries to %s instead", const.DEPRECATED_LIBS_DIR, const.DEFAULT_LIBS_DIR)
+        libs_dir = const.DEPRECATED_LIBS_DIR
+    else:
+        libs_dir = const.DEFAULT_LIBS_DIR
+
+    for lib_id in libs_to_sync:
+        client.sync_lib(lib_id, libs_dir)
     client.watch_status()
 
     return 0