Treat a known Message-ID as a duplicate in the mail importer
Outlook for Mac rewrites headers on every export, so a re-exported message never hashes the same twice. Hash-only matching stored each re-export as a variant beside the original.
This commit is contained in:
1 parent
011bdc2947
commit
7c8db7999a
2 files changed
+12
-12
No files matched your search
@@ -21,8 +21,10 @@ filename. Unknown or invalid dates are stored under `unknown-date/`.
|
|||||||
The importer regenerates `index.csv` from the archive with the full SHA-256, selected normalized
|
The importer regenerates `index.csv` from the archive with the full SHA-256, selected normalized
|
||||||
header values, and archive path. Its duplicate rules are:
|
header values, and archive path. Its duplicate rules are:
|
||||||
|
|
||||||
- An existing SHA-256 is an exact duplicate and is skipped.
|
- An existing SHA-256 is a duplicate and is skipped.
|
||||||
- A repeated `Message-ID` with different bytes is preserved and reported as a variant.
|
- An existing `Message-ID` is a duplicate and is skipped. Outlook rewrites headers on every
|
||||||
|
export, so the same message rarely hashes the same twice.
|
||||||
|
- A message without a `Message-ID` is checked by SHA-256 only.
|
||||||
- A repeated subject alone is not a duplicate.
|
- A repeated subject alone is not a duplicate.
|
||||||
|
|
||||||
Each new message is copied to a temporary file beside its final destination, SHA-256 verified, and
|
Each new message is copied to a temporary file beside its final destination, SHA-256 verified, and
|
||||||
|
|||||||
+8
-10
@@ -150,7 +150,7 @@ def main():
|
|||||||
parser.error(f"could not scan source directory: {error}")
|
parser.error(f"could not scan source directory: {error}")
|
||||||
messages = [path for path in files if path.suffix.lower() == ".eml"]
|
messages = [path for path in files if path.suffix.lower() == ".eml"]
|
||||||
unsupported = [path for path in files if path.suffix.lower() != ".eml"]
|
unsupported = [path for path in files if path.suffix.lower() != ".eml"]
|
||||||
imported = duplicates = variants = failures = 0
|
imported = duplicates = failures = 0
|
||||||
|
|
||||||
for path in sorted(messages):
|
for path in sorted(messages):
|
||||||
try:
|
try:
|
||||||
@@ -161,12 +161,13 @@ def main():
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
headers = read_headers(path)
|
headers = read_headers(path)
|
||||||
|
if headers["message_id"] and headers["message_id"] in message_ids:
|
||||||
|
print(f"DUPLICATE {path.relative_to(source)}")
|
||||||
|
duplicates += 1
|
||||||
|
continue
|
||||||
|
|
||||||
target = destination(archive, headers, digest)
|
target = destination(archive, headers, digest)
|
||||||
is_variant = bool(
|
print(f"IMPORT {path.relative_to(source)} -> {target.relative_to(archive)}")
|
||||||
headers["message_id"] and headers["message_id"] in message_ids
|
|
||||||
)
|
|
||||||
label = "VARIANT" if is_variant else "IMPORT"
|
|
||||||
print(f"{label} {path.relative_to(source)} -> {target.relative_to(archive)}")
|
|
||||||
|
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp")
|
temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp")
|
||||||
@@ -191,9 +192,6 @@ def main():
|
|||||||
hashes.add(digest)
|
hashes.add(digest)
|
||||||
if headers["message_id"]:
|
if headers["message_id"]:
|
||||||
message_ids.add(headers["message_id"])
|
message_ids.add(headers["message_id"])
|
||||||
if is_variant:
|
|
||||||
variants += 1
|
|
||||||
else:
|
|
||||||
imported += 1
|
imported += 1
|
||||||
except (OSError, ValueError) as error:
|
except (OSError, ValueError) as error:
|
||||||
print(f"FAILED {path.relative_to(source)}: {error}")
|
print(f"FAILED {path.relative_to(source)}: {error}")
|
||||||
@@ -205,7 +203,7 @@ def main():
|
|||||||
write_index(archive, rows)
|
write_index(archive, rows)
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f"{imported} imported, {variants} variants, {duplicates} duplicates, "
|
f"{imported} imported, {duplicates} duplicates, "
|
||||||
f"{len(unsupported)} unsupported, {failures} failures"
|
f"{len(unsupported)} unsupported, {failures} failures"
|
||||||
)
|
)
|
||||||
return 1 if failures or unsupported else 0
|
return 1 if failures or unsupported else 0
|
||||||
|
|||||||
Reference in new issue
Block a user