snegov/cura-te-ipsum
1
0
Fork 0
Code Issues 22 Releases Wiki Activity

Add running backup lock and marker of finished backup

Browse Source
This commit is contained in:
Maks Snegov
2023-05-06 23:06:15 -07:00
parent 775234a433
commit 62fcfe5881
5 changed files with 297 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
tests/test_backups.py
View File

@@ -18,6 +18,7 @@ class TestBackupCleanup(TestCase):
def _add_backup(self, backup_name: str) -> fs.PseudoDirEntry:
backup = fs.PseudoDirEntry(os.path.join(self.backup_dir.name, backup_name))
os.mkdir(backup.path)
bk.set_backup_marker(backup)
fd, path = tempfile.mkstemp(prefix="backup_file_", dir=backup.path)
with open(fd, "w") as f:
@@ -38,7 +39,7 @@ class TestBackupCleanup(TestCase):
def _run_cleanup(self, **kwargs):
""" Run cleanup_old_backups with null parameters. """
cleanup_kwargs = {
"backup_dir": self.backup_dir.name,
"backups_dir": self.backup_dir.name,
"dry_run": False,
"keep_all": None,
"keep_daily": None,
@@ -58,6 +59,7 @@ class TestBackupCleanup(TestCase):
def test_only_one_backup(self, mock_datetime):
""" Test the only backup will not be removed in any case """
mock_datetime.now.return_value = datetime(2021, 10, 20)
# very old backup
only_backup = self._add_backup("20010101_0000")
self._run_cleanup(keep_all=1)
self._check_backups([only_backup])
@@ -67,8 +69,8 @@ class TestBackupCleanup(TestCase):
""" Test at least one backup should be left """
mock_datetime.now.return_value = datetime(2021, 10, 20)
backups = [
self._add_backup("20211103_0300"),
self._add_backup("20201216_0100"),
self._add_backup("20211103_0300"), # this one is the latest and should be kept
self._add_backup("20201216_0100"), # the rest should be removed
self._add_backup("20200716_0100"),
self._add_backup("20181116_0100"),
]
@@ -81,9 +83,9 @@ class TestBackupCleanup(TestCase):
""" Test threshold for keeping all backups """
mock_datetime.now.return_value = datetime(2021, 10, 20)
backups = [
self._add_backup("20211019_0300"),
self._add_backup("20211017_0100"),
self._add_backup("20211016_2300"),
self._add_backup("20211019_0300"), # keep
self._add_backup("20211017_0100"), # keep
self._add_backup("20211016_2300"), # remove, older than 3 days
]
expected_backups = backups[:2]
self._run_cleanup(keep_all=3)
@@ -94,12 +96,12 @@ class TestBackupCleanup(TestCase):
""" Test threshold for keeping daily backups """
mock_datetime.now.return_value = datetime(2021, 10, 20)
backups = [
self._add_backup("20211019_0300"),
self._add_backup("20211017_2100"),
self._add_backup("20211017_0100"),
self._add_backup("20211017_0030"),
self._add_backup("20211016_2300"),
self._add_backup("20211016_0100"),
self._add_backup("20211019_0300"), # keep, first daily backup at 2021-10-19
self._add_backup("20211017_2100"), # remove, not the first daily backup
self._add_backup("20211017_0100"), # remove, not the first daily backup
self._add_backup("20211017_0030"), # keep, first daily backup at 2021-10-17
self._add_backup("20211016_2300"), # remove, older than 3 days
self._add_backup("20211016_0100"), # remove, older than 3 days
]
expected_backups = [backups[0], backups[3]]
self._run_cleanup(keep_daily=3)
@@ -110,17 +112,17 @@ class TestBackupCleanup(TestCase):
""" Test threshold for keeping all and daily backups """
mock_datetime.now.return_value = datetime(2021, 10, 20)
backups = [
self._add_backup("20211019_0300"),
self._add_backup("20211017_0200"),
self._add_backup("20211017_0100"),
self._add_backup("20211016_2300"),
self._add_backup("20211016_2200"),
self._add_backup("20211015_2200"),
self._add_backup("20211015_1500"),
self._add_backup("20211015_0200"),
self._add_backup("20211014_2200"),
self._add_backup("20211014_2000"),
self._add_backup("20211014_1232"),
self._add_backup("20211019_0300"), # keep, newer than 3 days
self._add_backup("20211017_0200"), # keep, newer than 3 days
self._add_backup("20211017_0100"), # keep, newer than 3 days
self._add_backup("20211016_2300"), # remove, not the first daily backup
self._add_backup("20211016_2200"), # keep, the first daily backup at 2021-10-16
self._add_backup("20211015_2200"), # remove, not the first daily backup
self._add_backup("20211015_1500"), # remove, not the first daily backup
self._add_backup("20211015_0200"), # keep, the first daily backup at 2021-10-15
self._add_backup("20211014_2200"), # remove, older than 5 days
self._add_backup("20211014_2000"), # remove, older than 5 days
self._add_backup("20211014_1232"), # remove, older than 5 days
]
expected_backups = backups[0:3] + [backups[4]] + [backups[7]]
self._run_cleanup(keep_all=3, keep_daily=5)
@@ -131,40 +133,57 @@ class TestBackupCleanup(TestCase):
""" Test threshold for keeping weekly backups """
mock_datetime.now.return_value = datetime(2021, 11, 11)
backups = [
self._add_backup("20211111_0300"),
self._add_backup("20211110_0300"),
self._add_backup("20211108_0100"),
self._add_backup("20211107_2300"),
self._add_backup("20211107_0100"),
self._add_backup("20211031_0100"),
self._add_backup("20211025_0100"),
self._add_backup("20211024_0100"),
self._add_backup("20211023_0100"),
self._add_backup("20211022_0100"),
self._add_backup("20211008_0100"),
self._add_backup("20211007_0100"),
self._add_backup("20211004_0100"),
self._add_backup("20211003_0100"),
self._add_backup("20211111_0300"), # remove, not the first weekly backup (Thursday)
self._add_backup("20211110_0300"), # remove, not the first weekly backup (Wednesday)
self._add_backup("20211108_0100"), # keep, first weekly backup at 2021-11-08 (Monday)
self._add_backup("20211107_2300"), # remove, not the first weekly backup (Sunday)
self._add_backup("20211107_0100"), # keep, first weekly backup at 2021-11-07 (Sunday)
self._add_backup("20211031_0100"), # remove, not the first weekly backup (Sunday)
self._add_backup("20211025_0100"), # keep, first weekly backup at 2021-10-25 (Monday)
self._add_backup("20211024_0100"), # remove, not the first weekly backup (Sunday)
self._add_backup("20211023_0100"), # remove, not the first weekly backup (Saturday)
self._add_backup("20211022_0100"), # keep, first weekly backup at 2021-10-22 (Friday)
self._add_backup("20211008_0100"), # remove, not the first weekly backup (Friday)
self._add_backup("20211007_0100"), # remove, not the first weekly backup (Thursday)
self._add_backup("20211004_0100"), # keep, first weekly backup at 2021-10-04 (Monday)
self._add_backup("20211003_0100"), # remove, older than 5 weeks
self._add_backup("20211002_0100"), # remove, older than 5 weeks
]
expected_backups = [backups[2], backups[4], backups[6],
backups[9], backups[12]]
self._run_cleanup(keep_weekly=5)
self._check_backups(expected_backups)
@mock.patch(f"{bk.__name__}.datetime", wraps=datetime)
def test_keep_weekly_threshold_inclusive(self, mock_datetime):
""" Test threshold for keeping weekly backups """
mock_datetime.now.return_value = datetime(2021, 11, 11)
backups = [
self._add_backup("20211111_0300"), # remove, not the first weekly backup (Thursday)
self._add_backup("20211110_0300"), # keep, first weekly backup (Wednesday)
self._add_backup("20211107_0100"), # remove, not the first weekly backup (Sunday)
self._add_backup("20211102_0100"), # keep, first weekly backup (Tuesday)
]
expected_backups = [backups[1], backups[3]]
self._run_cleanup(keep_weekly=5)
self._check_backups(expected_backups)
@mock.patch(f"{bk.__name__}.datetime", wraps=datetime)
def test_keep_monthly_threshold_only(self, mock_datetime):
""" Test threshold for keeping monthly backups """
mock_datetime.now.return_value = datetime(2021, 11, 11)
backups = [
self._add_backup("20211103_0300"),
self._add_backup("20211019_0300"),
self._add_backup("20211017_2100"),
self._add_backup("20211017_0100"),
self._add_backup("20210916_2300"),
self._add_backup("20210916_0100"),
self._add_backup("20210816_0100"),
self._add_backup("20210810_0000"),
self._add_backup("20210716_0100"),
self._add_backup("20211103_0300"), # keep, first monthly backup at 2021-11
self._add_backup("20211019_0300"), # remove, not the first monthly backup
self._add_backup("20211017_2100"), # remove, not the first monthly backup
self._add_backup("20211017_0100"), # keep, first monthly backup at 2021-10
self._add_backup("20210916_2300"), # remove, not the first monthly backup
self._add_backup("20210916_0100"), # keep, first monthly backup at 2021-09
self._add_backup("20210816_0100"), # remove, not the first monthly backup
self._add_backup("20210810_0000"), # keep, first monthly backup at 2021-08
self._add_backup("20210716_0100"), # remove, older than 3 months
self._add_backup("20210715_0100"), # remove, older than 3 months
]
expected_backups = [backups[0], backups[3], backups[5], backups[7]]
self._run_cleanup(keep_monthly=3)
@@ -175,15 +194,19 @@ class TestBackupCleanup(TestCase):
""" Test threshold for keeping yearly backups """
mock_datetime.now.return_value = datetime(2021, 11, 11)
backups = [
self._add_backup("20211103_0300"),
self._add_backup("20210810_0000"),
self._add_backup("20210716_0100"),
self._add_backup("20201216_0100"),
self._add_backup("20200716_0100"),
self._add_backup("20190316_0100"),
self._add_backup("20181116_0100"),
self._add_backup("20211103_0300"), # remove, not the first yearly backup in 2021
self._add_backup("20210810_0000"), # remove, not the first yearly backup in 2021
self._add_backup("20210716_0100"), # keep, first yearly backup in 2021
self._add_backup("20201216_0100"), # remove, not the first yearly backup in 2020
self._add_backup("20200716_0100"), # keep, first yearly backup in 2020
self._add_backup("20191216_0100"), # remove, not the first yearly backup in 2019
self._add_backup("20190316_0100"), # keep, first yearly backup in 2019
self._add_backup("20181216_0100"), # remove, not the first yearly backup in 2018
self._add_backup("20181116_0100"), # keep, first yearly backup in 2018
self._add_backup("20171116_0100"), # remove, older than 3 years
self._add_backup("20171115_0100"), # remove, older than 3 years
]
expected_backups = [backups[2], backups[4], backups[5], backups[6]]
expected_backups = [backups[2], backups[4], backups[6], backups[8]]
self._run_cleanup(keep_yearly=3)
self._check_backups(expected_backups)
@@ -200,3 +223,7 @@ class TestBackupCleanup(TestCase):
]
self._run_cleanup(keep_all=2, dry_run=True)
self._check_backups(backups)
# TODO add tests for iterating over backups (marker, dirname)
# TODO add tests for backups dir lockfile

3
tests/test_fs.py
View File

@@ -142,7 +142,8 @@ class TestHardlinkDir(CommonFSTestCase):
nfile_relpath = self.relpath(src_nfile_path)
fs.hardlink_dir(self.src_dir, self.dst_dir)
self.check_directory_stats(src_ndir_path, os.path.join(self.dst_dir, ndir_relpath))
self.check_directory_stats(src_ndir_path,
os.path.join(self.dst_dir, ndir_relpath))
# check file in nested directory
src_fstat = os.lstat(src_nfile_path)