Treat a known Message-ID as a duplicate in the mail importer

Outlook for Mac rewrites headers on every export, so a re-exported
message never hashes the same twice. Hash-only matching stored each
re-export as a variant beside the original.
This commit is contained in:
snegov committed 2026-10-07 13:04:52 -07:00
1 parent 011bdc2947
commit 7c8db7999a
2 files changed
+13 -13

No files matched your search

+4 -2
View File
@@ -21,8 +21,10 @@ filename. Unknown or invalid dates are stored under `unknown-date/`.
The importer regenerates `index.csv` from the archive with the full SHA-256, selected normalized
header values, and archive path. Its duplicate rules are:
- An existing SHA-256 is an exact duplicate and is skipped.
- A repeated `Message-ID` with different bytes is preserved and reported as a variant.
- An existing SHA-256 is a duplicate and is skipped.
- An existing `Message-ID` is a duplicate and is skipped. Outlook rewrites headers on every
export, so the same message rarely hashes the same twice.
- A message without a `Message-ID` is checked by SHA-256 only.
- A repeated subject alone is not a duplicate.
Each new message is copied to a temporary file beside its final destination, SHA-256 verified, and
+9 -11
View File
@@ -150,7 +150,7 @@ def main():
parser.error(f"could not scan source directory: {error}")
messages = [path for path in files if path.suffix.lower() == ".eml"]
unsupported = [path for path in files if path.suffix.lower() != ".eml"]
imported = duplicates = variants = failures = 0
imported = duplicates = failures = 0
for path in sorted(messages):
try:
@@ -161,12 +161,13 @@ def main():
continue
headers = read_headers(path)
if headers["message_id"] and headers["message_id"] in message_ids:
print(f"DUPLICATE {path.relative_to(source)}")
duplicates += 1
continue
target = destination(archive, headers, digest)
is_variant = bool(
headers["message_id"] and headers["message_id"] in message_ids
)
label = "VARIANT" if is_variant else "IMPORT"
print(f"{label} {path.relative_to(source)} -> {target.relative_to(archive)}")
print(f"IMPORT {path.relative_to(source)} -> {target.relative_to(archive)}")
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp")
@@ -191,10 +192,7 @@ def main():
hashes.add(digest)
if headers["message_id"]:
message_ids.add(headers["message_id"])
if is_variant:
variants += 1
else:
imported += 1
imported += 1
except (OSError, ValueError) as error:
print(f"FAILED {path.relative_to(source)}: {error}")
failures += 1
@@ -205,7 +203,7 @@ def main():
write_index(archive, rows)
print(
f"{imported} imported, {variants} variants, {duplicates} duplicates, "
f"{imported} imported, {duplicates} duplicates, "
f"{len(unsupported)} unsupported, {failures} failures"
)
return 1 if failures or unsupported else 0