#!/usr/bin/env python3
"""
# Script dedup-readwise
## Metadata
#obsidian/ob-script #python
**Parent**:: [[♯ Readwise]], [[Obsidian Chore Scripts]], [[Executable Markdown File]]
## Synopsis
Repair concatenated Readwise exports, retaining the complete last appended export,
including its frontmatter. Older-only highlights and annotations are not merged.
Single-export files and separately named duplicates are not rewritten.
Both legacy `Full Title` and canonical `full-title` are recognized; an export
containing both names requires review. Retained frontmatter is never reserialized.
Requires PyYAML (`python -m pip install PyYAML`).
- `python shore/bin/dedup-readwise.md -n`: scan the library without writing (also the default).
- `python shore/bin/dedup-readwise.md -n "path/to/note.md"`: inspect one file.
- `python shore/bin/dedup-readwise.md --write --backup-dir "path/to/backups" "path/to/note.md"`: apply to explicit files or directories.
Writes preserve UTF-8/LF bytes and exact modification times. Originals are backed
up by SHA-256 before writing; without `--backup-dir`, backups use the system temp
folder's `readwise-dedup-backups` directory. Keep a persistent backup directory if
needed. Backups must be outside write-target directories. Reports include older-only
lines, removed block IDs, and changed metadata.
Malformed exports, inconsistent titles/URLs, reversed Created dates, non-LF input,
and a newer legacy copy lacking an older copy's frontmatter require review.
Only structurally complete matching exports are selected, not arbitrary covers.
Windows writes deny other writers/deletion while locked; POSIX locks are advisory,
so stop other editors/sync writers before applying. No reset or resync is performed.
## Script
```python
# """
import argparse
from contextlib import contextmanager
from datetime import date
import hashlib
import os
from pathlib import Path
import re
import stat
import sys
import tempfile
from urllib.parse import urlsplit
import yaml
class ReviewRequired(ValueError):
pass
class PropertiesLoader(yaml.SafeLoader):
def construct_mapping(self, node, deep=False):
result = {}
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
if not isinstance(key, str) or key == '<<' or key in result:
raise ReviewRequired('Invalid or duplicate YAML property key')
result[key] = self.construct_object(value_node, deep=deep)
return result
def properties(text):
try:
result = yaml.load(text, Loader=PropertiesLoader)
except yaml.YAMLError as error:
raise ReviewRequired('Malformed YAML') from error
if not isinstance(result, dict):
raise ReviewRequired('Frontmatter must be a mapping')
return result
def visible_lines(lines, masked):
fence = None
comment = None
html = None
visible = set()
for i, line in enumerate(lines):
if i in masked:
continue
if comment:
if comment in line:
comment = None
continue
if html:
if re.search(r'</' + html + r'\s*>', line, re.I):
html = None
continue
marker = re.match(r'^ {0,3}(`{3,}|~{3,})(.*)