Files
cura-te-ipsum/curateipsum/backup.py
T
a987180653 Replace PID-file process killing with a real fcntl.flock() lock (#6)
* Replace PID-file process killing with a real fcntl.flock() lock

set_backups_lock()/release_backups_lock() now use an exclusive
fcntl.flock() on the lock file's inode instead of trusting the PID
written inside it. The kernel releases the lock automatically on
normal exit, on any signal, and on a crash - no PID liveness check or
staleness heuristic is needed, and there is nothing left in this code
path that sends a signal to another process.

- The open file descriptor is held for the full operation in a
  module-level _held_locks map keyed by the backups directory, and
  release_backups_lock() only ever releases a descriptor this process
  itself acquired, never one it infers from the lock file's content.
- PID and start time are still written into the lock file, but purely
  as diagnostics for whoever reads it (e.g. in a warning log line);
  they play no role in acquisition or release decisions anymore.
- --force no longer sends SIGTERM/SIGKILL to the previous holder. It
  now blocks on the same flock() until that process releases the lock
  on its own (exit, signal, or crash all count).
- release_backups_lock() intentionally stops unlinking the lock file:
  since flock() locks an inode rather than a path, unlinking while
  another process is blocked opening the same path would let that
  process (or a third one) acquire an independent lock on a fresh
  inode at the same path, defeating mutual exclusion.

Rewrites TestBackupLock to match: a real second file descriptor with
its own flock() stands in for "another process" in the fast unit
tests, and a new subprocess-based test proves end-to-end that only one
process can hold the lock and that killing the holder with SIGKILL
frees it for the next acquirer without our code ever calling os.kill.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Address Copilot review feedback on flock-based lock

- Make the fcntl import optional (try/except ImportError -> None), so
  importing curateipsum.backup no longer crashes on Windows before
  cli.py's own "Not supported platform" check gets a chance to run.
- Guard set_backups_lock() against being called twice for a directory
  this process already holds: it now fails fast instead of silently
  overwriting the tracked fd in _held_locks, which would leak the
  original descriptor and leave the lock held even after
  release_backups_lock() closes only the newest one.
- Wrap the post-acquire metadata write (ftruncate/write/fsync) in a
  try/except: a failure there now releases the flock and closes the fd
  before returning False, instead of leaving the lock held with
  no caller aware of it.
- Cap _read_lock_metadata()'s read and strip newlines before logging,
  so a stale or forged lock file can't inject bogus log lines or force
  an unbounded read.

Adds a regression test for the double-acquire guard.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Maxim Shulga <maxim.shulga@alludo.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-03 22:00:21 +00:00

529 lines
20 KiB
Python

"""
Module with backup functions.
"""
import logging
import os
import shutil
import time
from datetime import datetime, timedelta
from typing import Dict, Optional, Iterable, Union
try:
import fcntl
except ImportError:
# Not available on Windows. cli.py's own platform check reports this
# cleanly before any locking function here is ever called; importing
# fcntl lazily/optionally lets that check run instead of crashing at
# `from curateipsum import backup` time.
fcntl = None
from curateipsum import fs
BACKUP_ENT_FMT = "%Y%m%d_%H%M%S"
LOCK_FILE = ".backups_lock"
DELTA_DIR = ".backup_delta"
BACKUP_MARKER = ".backup_finished"
STAGING_PREFIX = ".incomplete-"
_lg = logging.getLogger(__name__)
class BackupFailedError(Exception):
"""Raised when a backup could not be completed successfully.
Any failure while building a snapshot (copy error, parse error,
unsupported entry, nonzero subprocess exit) is fatal: the partially
built snapshot is discarded and this is the only way that failure is
reported to callers.
"""
def _get_backup_marker(
backup_entry: Union[os.DirEntry, fs.PseudoDirEntry]
) -> fs.PseudoDirEntry:
"""Return DirEntry for marker file of given backup."""
marker_name = "%s_%s" % (BACKUP_MARKER, backup_entry.name)
marker_path = os.path.join(backup_entry.path, marker_name)
return fs.PseudoDirEntry(path=marker_path)
def _is_backup(backup_entry: Union[os.DirEntry, fs.PseudoDirEntry]) -> bool:
"""Guess if backup_entry is a real backup."""
backup_marker = _get_backup_marker(backup_entry)
# if there is no marker file in the backup dir, it's not a backup
if not os.path.exists(backup_marker.path):
return False
# if there is only a marker file in the backup dir, it's not a backup
if os.listdir(backup_entry.path) == [backup_marker.name]:
return False
try:
datetime.strptime(backup_entry.name, BACKUP_ENT_FMT)
return True
except ValueError:
return False
def _iterate_backups(backups_dir: str) -> Iterable[os.DirEntry]:
"""Iterate over backups in backups_dir."""
b_iter = os.scandir(backups_dir)
b_ent: os.DirEntry
for b_ent in b_iter:
if not _is_backup(b_ent):
continue
yield b_ent
b_iter.close()
def _get_latest_backup(backups_dir: str) -> Optional[os.DirEntry]:
"""Returns path to latest backup created in backups_dir or None."""
all_backups = sorted(_iterate_backups(backups_dir), key=lambda e: e.name)
if all_backups:
return all_backups[-1]
return None
def _date_from_backup(backup_entry: os.DirEntry) -> datetime:
"""Returns datetime object from backup name."""
return datetime.strptime(backup_entry.name, BACKUP_ENT_FMT)
# Open lock file descriptors currently held by this process, keyed by the
# absolute path of the backups directory they lock. The kernel associates
# an flock() with this exact open file description, so the descriptor must
# stay open for the full duration of the operation: closing it (including
# on process crash or signal) is what releases the lock.
_held_locks: Dict[str, int] = {}
_LOCK_METADATA_MAX_BYTES = 256
def _read_lock_metadata(lock_file_path: str) -> str:
"""
Read lock file content for diagnostic logging only. Never raises, and
never trusted for anything beyond a human-readable log line: the read
is capped and newlines are stripped so a stale or forged lock file
can't inject bogus log lines or force an unbounded read.
"""
try:
with open(lock_file_path, "r") as f:
content = f.read(_LOCK_METADATA_MAX_BYTES)
except OSError:
return "<unreadable>"
return content.replace("\n", " ").replace("\r", " ").strip()
def set_backups_lock(backups_dir: str,
force: bool = False) -> bool:
"""
Acquire an exclusive kernel (flock) lock to prevent multiple backups
running at the same time. The lock file also holds the PID and start
time of the holder, kept for diagnostics only - they play no part in
deciding whether the lock can be acquired.
If another process already holds the lock:
- with force=False, return False immediately.
- with force=True, block until that process releases it (normal exit,
signal, or crash all release the kernel lock), then acquire it.
"""
backups_dir_abs = os.path.abspath(backups_dir)
if backups_dir_abs in _held_locks:
_lg.error("This process already holds the backup lock for %s",
backups_dir)
return False
lock_file_path = os.path.join(backups_dir_abs, LOCK_FILE)
fd = os.open(lock_file_path, os.O_CREAT | os.O_RDWR, 0o644)
flock_flags = fcntl.LOCK_EX if force else fcntl.LOCK_EX | fcntl.LOCK_NB
try:
fcntl.flock(fd, flock_flags)
except BlockingIOError:
held_by = _read_lock_metadata(lock_file_path)
_lg.warning(
"Previous backup is still in progress (%s), exiting", held_by
)
os.close(fd)
return False
except OSError as err:
_lg.error("Failed to acquire backup lock: %s", err)
os.close(fd)
return False
# Lock acquired: replace metadata, it belongs to us now. If writing it
# fails, don't leave the lock held forever - release and report failure.
try:
os.ftruncate(fd, 0)
os.lseek(fd, 0, os.SEEK_SET)
metadata = "%d\n%s\n" % (os.getpid(), datetime.now().isoformat())
os.write(fd, metadata.encode())
os.fsync(fd)
except OSError as err:
_lg.error("Failed to write backup lock metadata: %s", err)
try:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
return False
_held_locks[backups_dir_abs] = fd
return True
def release_backups_lock(backups_dir: str):
"""
Release the kernel lock held by this process for backups_dir, if any.
Only releases a lock this process actually acquired via
set_backups_lock() - verified by looking up our own held descriptor,
never by trusting the lock file's on-disk content.
"""
backups_dir_abs = os.path.abspath(backups_dir)
fd = _held_locks.pop(backups_dir_abs, None)
if fd is None:
_lg.debug("No lock held by this process for %s, nothing to release",
backups_dir)
return
try:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
# Deliberately not unlinking the lock file: flock() locks an inode, not
# a path. Unlinking it here while another process is blocked opening
# the same path (force=True) would let that process (or a third one)
# create a fresh inode at the same path and acquire an independent
# lock, defeating mutual exclusion. The file is small and harmless to
# leave in place; only its content (diagnostics) gets overwritten by
# whoever next holds the lock.
def set_backup_marker(backup_entry: Union[os.DirEntry, fs.PseudoDirEntry]):
"""Create finished backup marker file in backup's directory."""
backup_marker = _get_backup_marker(backup_entry)
if not os.path.exists(backup_marker.path):
open(backup_marker.path, "a").close()
_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0)
def _fsync_dir(path: str):
"""Fsync a directory so its content is durable before it is renamed."""
dir_fd = os.open(path, os.O_RDONLY | _O_DIRECTORY)
try:
os.fsync(dir_fd)
finally:
os.close(dir_fd)
def _reserve_snapshot_name(backups_dir: str) -> str:
"""
Return a snapshot name that doesn't collide with an existing entry or
an in-progress staging directory. Guards against two backups started
within the same second.
"""
def _taken(candidate: str) -> bool:
return (os.path.lexists(os.path.join(backups_dir, candidate))
or os.path.lexists(os.path.join(
backups_dir, f"{STAGING_PREFIX}{candidate}")))
name = datetime.now().strftime(BACKUP_ENT_FMT)
while _taken(name):
time.sleep(1)
name = datetime.now().strftime(BACKUP_ENT_FMT)
return name
def cleanup_stale_staging_dirs(backups_dir: str):
"""
Remove staging directories abandoned by a crashed or interrupted
backup run. Must be called before a new backup starts so a stale
staging directory can never be mistaken for a real snapshot.
"""
with os.scandir(backups_dir) as it:
entry: os.DirEntry
for entry in it:
if entry.name.startswith(STAGING_PREFIX) and entry.is_dir():
_lg.warning("Removing abandoned staging directory: %s",
entry.name)
shutil.rmtree(entry.path, ignore_errors=True)
def cleanup_old_backups(backups_dir: str,
dry_run: bool = False,
keep_all: int = 7,
keep_daily: int = 30,
keep_weekly: int = 52,
keep_monthly: int = 12,
keep_yearly: int = 5):
"""
Delete old backups. Never deletes the only backup.
For keep_* params threshold is inclusive, e.g.:
keep_weekly=1 being run on Thursday will keep one backup from this week and
one from the previous, even if the previous week's backup was created on
Monday.
keep_monthly=3 being run on any day of April will keep one backup from each
of months of January, February and March.
:param backups_dir: full path to backups directory.
:param dry_run: don't do anything.
:param keep_all:
up to this amount of days in the past all backups must be kept.
:param keep_daily:
up to this amount of days in the past one daily backup must be kept.
:param keep_weekly:
up to this amount of weeks in the past one weekly backup must be kept.
:param keep_monthly:
up to this amount of months in the past one monthly backup must be kept.
1 month is considered to be 30 days.
:param keep_yearly:
up to this amount of years in the past one yearly backup must be kept.
1 year is considered to be 365 days.
"""
all_backups = sorted(_iterate_backups(backups_dir),
key=lambda e: e.name, reverse=True)
if dry_run:
_lg.info("Dry-run, no backups will be actually removed")
if not all_backups:
_lg.debug("No backups, exiting")
return
if len(all_backups) == 1:
_lg.debug("Only one backup (%s) exists, will not remove it",
all_backups[0].name)
return
now = datetime.now()
thresholds = {k: now.strftime(BACKUP_ENT_FMT)
for k in ("all", "daily", "weekly", "monthly", "yearly")}
if keep_all is not None:
thresholds["all"] = ((now - timedelta(days=keep_all))
.replace(hour=0, minute=0, second=0)
.strftime(BACKUP_ENT_FMT))
if keep_daily is not None:
thresholds["daily"] = ((now - timedelta(days=keep_daily))
.replace(hour=0, minute=0, second=0)
.strftime(BACKUP_ENT_FMT))
if keep_weekly is not None:
thresholds["weekly"] = (
(now - timedelta(weeks=keep_weekly, days=now.weekday()))
.strftime(BACKUP_ENT_FMT)
)
if keep_monthly is not None:
thresholds["monthly"] = ((now - timedelta(days=30*keep_monthly))
.replace(day=1, hour=0, minute=0, second=0)
.strftime(BACKUP_ENT_FMT))
if keep_yearly is not None:
thresholds["yearly"] = (
(now - timedelta(days=365*keep_yearly))
.replace(month=1, day=1, hour=0, minute=0, second=0)
.strftime(BACKUP_ENT_FMT)
)
prev_backup = all_backups[0]
to_remove = {b: False for b in all_backups}
for backup in all_backups[1:]:
# skip all backups made after threshold
if backup.name > thresholds["all"]:
prev_backup = backup
continue
# leave only one backup per day for backups made after threshold
if backup.name > thresholds["daily"]:
if (_date_from_backup(prev_backup).date()
== _date_from_backup(backup).date()):
to_remove[prev_backup] = True
prev_backup = backup
continue
# leave only one backup per week for backups made after threshold
if backup.name > thresholds["weekly"]:
if (_date_from_backup(prev_backup).isocalendar()[1]
== _date_from_backup(backup).isocalendar()[1]):
to_remove[prev_backup] = True
prev_backup = backup
continue
# leave only one backup per month for backups made after threshold
if backup.name > thresholds["monthly"]:
if (_date_from_backup(prev_backup).date().replace(day=1)
== _date_from_backup(backup).date().replace(day=1)):
to_remove[prev_backup] = True
prev_backup = backup
continue
# leave only one backup per year for backups made after threshold
if backup.name > thresholds["yearly"]:
if (_date_from_backup(prev_backup).date().replace(month=1, day=1)
== _date_from_backup(backup).date().replace(month=1, day=1)):
to_remove[prev_backup] = True
prev_backup = backup
continue
to_remove[backup] = True
for backup, do_delete in to_remove.items():
if do_delete:
if dry_run:
_lg.info("Would remove old backup %s", backup.name)
else:
_lg.info("Removing old backup %s", backup.name)
shutil.rmtree(backup.path)
def process_backed_entry(backup_dir: str,
entry_relpath: str,
action: fs.Actions,
msg: str):
"""
Additional processing of backed up DirEntry (file/dir/symlink).
Actions:
- if DirEntry was not deleted, hardlink it to DELTA_DIR.
"""
_lg.debug("%s %s %s", action, entry_relpath, msg)
if action not in (fs.Actions.ERROR, fs.Actions.DELETE):
fs.nest_hardlink(src_dir=backup_dir, src_relpath=entry_relpath,
dst_dir=os.path.join(backup_dir, DELTA_DIR))
def initiate_backup(sources,
backups_dir: str,
dry_run: bool = False,
external_rsync: bool = False,
external_hardlink: bool = False
) -> Optional[fs.PseudoDirEntry]:
"""
Main backup function.
Builds the new snapshot in a private ".incomplete-<name>" staging
directory, copying data from the latest backup and then syncing data
from sources. Any failure is fatal: the staging directory is discarded
and BackupFailedError is raised, so a failed or interrupted backup can
never be mistaken for a successful one. On success, the staging
directory is fsync'd and atomically renamed into its final snapshot
name.
:param sources: list of directories to backup (relative paths ok)
:param backups_dir: directory where all backups are stored
:param dry_run: if True, no actual changes will be made
:param external_rsync: if True, use external rsync instead of python
:param external_hardlink: if True, use external hardlink instead of python
:return: DirEntry-like object of the created snapshot, or None if no
snapshot was created (dry-run or no changes since last backup).
:raises BackupFailedError: if the backup could not be completed.
"""
cleanup_stale_staging_dirs(backups_dir)
snapshot_name = _reserve_snapshot_name(backups_dir)
staging = fs.PseudoDirEntry(
os.path.join(backups_dir, f"{STAGING_PREFIX}{snapshot_name}")
)
_lg.debug("Staging directory: %s", staging.path)
latest_backup = _get_latest_backup(backups_dir)
if latest_backup is None:
_lg.info("Creating empty staging directory for backup: %s",
snapshot_name)
try:
os.mkdir(staging.path)
except OSError as err:
raise BackupFailedError(
"Failed to create staging directory %s: %s"
% (staging.path, err)
) from err
else:
_lg.info("Copying data from latest backup %s to staging for %s",
latest_backup.name, snapshot_name)
hl_res = fs.hardlink_dir(src_dir=latest_backup.path,
dst_dir=staging.path,
use_external=external_hardlink)
if not hl_res:
_lg.error("Something went wrong during copying data from latest"
" backup, removing staging directory for %s",
snapshot_name)
shutil.rmtree(staging.path, ignore_errors=True)
raise BackupFailedError(
"Failed to copy data from latest backup %s"
% latest_backup.name
)
# remove backup markers from copied backup
for fname in os.listdir(staging.path):
if fname.startswith(BACKUP_MARKER):
os.remove(os.path.join(staging.path, fname))
# clean up delta dir from copied backup
shutil.rmtree(os.path.join(staging.path, DELTA_DIR),
ignore_errors=True)
rsync_func = fs.rsync_ext if external_rsync else fs.rsync
backup_changed = False
try:
for src in sources:
src_abs = os.path.abspath(src)
src_name = os.path.basename(src_abs)
dst_abs = os.path.join(staging.path, src_name)
_lg.info("Backing up directory %s to staging for %s",
src_abs, snapshot_name)
for entry_relpath, action, msg in rsync_func(
src_abs, dst_abs, dry_run=dry_run
):
if action == fs.Actions.ERROR:
raise BackupFailedError(
"Failed to copy %s: %s" % (entry_relpath, msg)
)
# TODO maybe should be run if first backup too?
if latest_backup is not None:
process_backed_entry(
backup_dir=staging.path,
entry_relpath=os.path.join(src_name, entry_relpath),
action=action,
msg=msg,
)
# raise flag if something was changed since last backup
backup_changed = True
except fs.BackupCreationError as err:
_lg.error("Error during backup creation: %s", err)
_lg.error("Failed to create backup %s, removing staging directory",
snapshot_name)
shutil.rmtree(staging.path, ignore_errors=True)
raise BackupFailedError(str(err)) from err
except BackupFailedError:
_lg.error("Failed to create backup %s, removing staging directory",
snapshot_name)
shutil.rmtree(staging.path, ignore_errors=True)
raise
# do not create backup on dry-run
if dry_run:
_lg.info("Dry-run, removing staging directory: %s", snapshot_name)
shutil.rmtree(staging.path, ignore_errors=True)
return None
# do not create backup if no change from previous one
if latest_backup is not None and not backup_changed:
_lg.info("Staged backup is the same as previous one %s, removing",
latest_backup.name)
shutil.rmtree(staging.path, ignore_errors=True)
return None
# write and fsync completion marker only after every source succeeded,
# then atomically rename the staging dir into its final snapshot name
marker_name = "%s_%s" % (BACKUP_MARKER, snapshot_name)
marker_fd = os.open(os.path.join(staging.path, marker_name),
os.O_CREAT | os.O_WRONLY, 0o644)
try:
os.fsync(marker_fd)
finally:
os.close(marker_fd)
_fsync_dir(staging.path)
final_path = os.path.join(backups_dir, snapshot_name)
os.rename(staging.path, final_path)
_fsync_dir(backups_dir)
_lg.info("Backup created: %s", snapshot_name)
return fs.PseudoDirEntry(final_path)