148 lines
4.6 KiB
Python
148 lines
4.6 KiB
Python
"""
|
|
Module with filesystem-related functions.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
from typing import Iterable
|
|
|
|
_lg = logging.getLogger(__name__)
|
|
|
|
|
|
def rsync_ext(src, dst, dry_run=False):
|
|
"""Call external rsync command"""
|
|
rsync_args = ["rsync"]
|
|
if dry_run:
|
|
rsync_args.append("-n")
|
|
rsync_args.append("-a") # archive
|
|
rsync_args.append("-z") # compress
|
|
rsync_args.append("-h") # human-readable
|
|
rsync_args.append("-v") # verbose
|
|
rsync_args.append("-u") # don't touch new files on receiver
|
|
rsync_args.append("--progress")
|
|
rsync_args.append("--del") # delete during
|
|
rsync_args.append(src)
|
|
rsync_args.append(dst)
|
|
res = subprocess.run(rsync_args)
|
|
return res
|
|
|
|
|
|
def scantree(path) -> Iterable[os.DirEntry]:
|
|
"""Recursively yield DirEntry file objects for given directory."""
|
|
entry: os.DirEntry
|
|
for entry in os.scandir(path):
|
|
if entry.is_dir(follow_symlinks=False):
|
|
yield entry
|
|
yield from scantree(entry.path)
|
|
else:
|
|
yield entry
|
|
|
|
|
|
def rsync(src_dir, dst_dir=None):
|
|
"""
|
|
Do sync
|
|
:param src_dir: source dir
|
|
:param dst_dir: dest dir, create if not exists
|
|
:return: nothing
|
|
"""
|
|
|
|
_lg.info(f"Rsync: {src_dir} -> {dst_dir}")
|
|
src_abs = os.path.abspath(src_dir)
|
|
dst_abs = os.path.abspath(dst_dir)
|
|
|
|
if not os.path.isdir(src_abs):
|
|
raise RuntimeError(f"Error during reading source directory: {src_abs}")
|
|
if os.path.exists(dst_abs):
|
|
if not os.path.isdir(dst_abs):
|
|
raise RuntimeError(f"Destination path is not a directory: {dst_abs}")
|
|
else:
|
|
os.mkdir(dst_abs)
|
|
|
|
for src_entry in scantree(src_abs):
|
|
rel_path = src_entry.path[len(src_abs)+1:]
|
|
dst_path = os.path.join(dst_abs, rel_path)
|
|
src_stat = src_entry.stat(follow_symlinks=False)
|
|
|
|
dst_stat = os.lstat(dst_path)
|
|
|
|
if src_entry.is_dir(follow_symlinks=False):
|
|
pass
|
|
|
|
do_update = False
|
|
# check file size
|
|
if src_stat.st_size != dst_stat.st_size:
|
|
do_update = True
|
|
# check modification time (mtime)
|
|
if src_stat.st_mtime > dst_stat.st_mtime:
|
|
do_update = True
|
|
|
|
if do_update:
|
|
_lg.info("Updating %s", src_entry)
|
|
|
|
|
|
def _hardlink_dir_ext(src, dst):
|
|
"""
|
|
Make hardlink for a directory using cp -al. Both src and dst should exist.
|
|
:param src: absolute path to source directory.
|
|
:param dst: absolute path to target directory.
|
|
:return: None
|
|
"""
|
|
res = subprocess.run(["cp", "-v", "-a", "-l", f"{src}/*", dst])
|
|
return res
|
|
|
|
|
|
def _recursive_hardlink(src, dst):
|
|
"""
|
|
Do hardlink directory recursively using python only.
|
|
Both src and dst directories should exist.
|
|
:param src: absolute path to source directory.
|
|
:param dst: absolute path to target directory.
|
|
:return: None
|
|
"""
|
|
with os.scandir(src) as it:
|
|
ent: os.DirEntry
|
|
for ent in it:
|
|
ent_dst_path = os.path.join(dst, ent.name)
|
|
if ent.is_dir(follow_symlinks=False):
|
|
_lg.debug(f"Copying directory: {ent.path} -> {ent_dst_path}")
|
|
os.mkdir(ent_dst_path)
|
|
ent_stat = ent.stat(follow_symlinks=False)
|
|
os.chown(ent_dst_path, ent_stat.st_uid, ent_stat.st_gid)
|
|
os.chmod(ent_dst_path, ent_stat.st_mode)
|
|
|
|
# process directory children
|
|
_recursive_hardlink(ent.path, ent_dst_path)
|
|
continue
|
|
if ent.is_file(follow_symlinks=False) or ent.is_symlink():
|
|
_lg.debug(f"Hardlink file: {ent.path} -> {ent_dst_path}")
|
|
os.link(ent.path, ent_dst_path, follow_symlinks=False)
|
|
continue
|
|
# something that is not a file, symlink or directory
|
|
raise NotImplementedError(ent.path)
|
|
|
|
|
|
def hardlink_dir(src_dir, dst_dir):
|
|
"""
|
|
Make hardlink for a directory with all its content.
|
|
:param src_dir: path to source directory
|
|
:param dst_dir: path to target directory
|
|
:return: None
|
|
"""
|
|
_lg.info(f"Recursive hardlinking: {src_dir} -> {dst_dir}")
|
|
src_abs = os.path.abspath(src_dir)
|
|
dst_abs = os.path.abspath(dst_dir)
|
|
|
|
if not os.path.isdir(src_abs):
|
|
_lg.error(f"Error reading source directory: {src_dir}")
|
|
raise RuntimeError(f"Error reading source directory: {src_dir}")
|
|
|
|
if os.path.exists(dst_abs):
|
|
_lg.error(f"Destination already exists: {dst_dir}")
|
|
raise RuntimeError(f"Destination already exists: {dst_dir}")
|
|
|
|
_lg.debug(f"Creating directory: {dst_abs}")
|
|
os.mkdir(dst_abs)
|
|
_recursive_hardlink(src_abs, dst_abs)
|
|
return
|