Bound HTTP request timeouts and retry budget (#28)
* Bound HTTP request timeouts and retry budget With no request timeout, Retry(total=30, backoff_factor=2, backoff_max=60) let a single unreachable server hang the client for around 26 minutes: backoff sleeps alone sum to that, and each of the 30 connection attempts could itself stall indefinitely without a timeout. Measured a black-holed connect attempt taking 75s on this machine with no timeout set, confirming the mechanism. - Add HTTP_TIMEOUT (connect, read) and a tighter retry budget (HTTP_RETRY_TOTAL/BACKOFF_FACTOR/BACKOFF_MAX) in dsc/const.py, bounding the worst case to a few minutes. - Pass timeout=HTTP_TIMEOUT on both SeafileClient HTTP calls (token fetch, remote library listing). - Add NetworkError(DscError) so a network failure is caught by start.py's existing top-level DscError handling instead of producing a bare traceback. - Add tests/test_http.py covering the retry policy values, timeout propagation, and error wrapping. - Check off the corresponding IMPROVEMENTS.md items. * Include URL and status code in NetworkError messages Copilot review on PR #28 flagged that the non-200 path for remote library listing raised NetworkError with only the raw response body, losing which request failed and its status code. The token-fetch error had the same gap. Both now include the URL and HTTP status. --------- Co-authored-by: Maxim Shulga <maxim.shulga@alludo.com>
This commit is contained in:
1 parent
c5b4262a4c
commit
a2887fa6db
5 files changed
+117
-13
No files matched your search
+20
-8
@@ -13,7 +13,7 @@ from urllib3.util.retry import Retry
|
||||
import seafile
|
||||
|
||||
from dsc import const
|
||||
from dsc.errors import ConfigError, DaemonError, DaemonTimeout
|
||||
from dsc.errors import ConfigError, DaemonError, DaemonTimeout, NetworkError
|
||||
from dsc.misc import config_value, create_dir, hide_secrets, user_cmd
|
||||
|
||||
_lg = logging.getLogger(__name__)
|
||||
@@ -48,9 +48,9 @@ class SeafileClient:
|
||||
|
||||
self.session = requests.Session()
|
||||
retry_strategy = Retry(
|
||||
total=30,
|
||||
backoff_factor=2,
|
||||
backoff_max=60,
|
||||
total=const.HTTP_RETRY_TOTAL,
|
||||
backoff_factor=const.HTTP_RETRY_BACKOFF_FACTOR,
|
||||
backoff_max=const.HTTP_RETRY_BACKOFF_MAX,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=["GET", "POST"]
|
||||
)
|
||||
@@ -94,9 +94,18 @@ class SeafileClient:
|
||||
raise ConfigError("No password or token available to authenticate")
|
||||
url = f"{self.url}/api2/auth-token/"
|
||||
_lg.info("Fetching token: %s", url)
|
||||
r = self.session.post(url, data={"username": self.user, "password": self.password})
|
||||
try:
|
||||
r = self.session.post(
|
||||
url,
|
||||
data={"username": self.user, "password": self.password},
|
||||
timeout=const.HTTP_TIMEOUT,
|
||||
)
|
||||
except requests.exceptions.RequestException as err:
|
||||
raise NetworkError(f"Can't reach {url}: {err}") from err
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"Can't get token: {r.text}")
|
||||
raise NetworkError(
|
||||
f"Can't get token from {url}: HTTP {r.status_code}: {r.text}"
|
||||
)
|
||||
self.__token = r.json()["token"]
|
||||
return self.__token
|
||||
|
||||
@@ -105,9 +114,12 @@ class SeafileClient:
|
||||
url = f"{self.url}/api2/repos/"
|
||||
_lg.info("Fetching remote libraries: %s", url)
|
||||
auth_header = {"Authorization": f"Token {self.token}"}
|
||||
r = self.session.get(url, headers=auth_header)
|
||||
try:
|
||||
r = self.session.get(url, headers=auth_header, timeout=const.HTTP_TIMEOUT)
|
||||
except requests.exceptions.RequestException as err:
|
||||
raise NetworkError(f"Can't reach {url}: {err}") from err
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(r.text)
|
||||
raise NetworkError(f"Can't fetch {url}: HTTP {r.status_code}: {r.text}")
|
||||
r_libs = {lib["id"]: lib["name"] for lib in r.json()}
|
||||
return r_libs
|
||||
|
||||
|
||||
Reference in new issue
Block a user