Redirect map builder: match old URLs to new ones before a migration

#!/usr/bin/env python3
"""
Redirect map builder
The SEM Dispatch Vault, semdispatch.com/vault

Matches an old URL list to a new one: exact path first, then a fuzzy
fallback with difflib. Writes redirect-map.csv and unmatched.csv.
Reads and writes local files only; it never touches either site.
"""

import csv
import sys
from difflib import SequenceMatcher
from urllib.parse import urlparse

CONFIG = {
    # Input files (one URL per row; a header row is skipped if it is not a URL).
    "OLD_URLS": "old-urls.csv",
    "NEW_URLS": "new-urls.csv",

    # Output files.
    "REDIRECT_MAP": "redirect-map.csv",
    "UNMATCHED": "unmatched.csv",

    # Fuzzy similarity floor, 0.0 to 1.0. Below this, a URL is left unmatched
    # rather than pointed at a weak guess. 0.65 is cautious on purpose.
    "FUZZY_THRESHOLD": 0.65,
}


def normalise(url):
    """Reduce a URL to a comparable path: lowercase, no trailing slash."""
    parsed = urlparse(url.strip())
    path = parsed.path if parsed.scheme else url.strip()
    path = path.split("?", 1)[0].split("#", 1)[0]
    path = path.rstrip("/").lower()
    return path or "/"


def read_urls(filename):
    urls = []
    with open(filename, newline="", encoding="utf-8") as handle:
        for row in csv.reader(handle):
            if not row:
                continue
            value = row[0].strip()
            if not value:
                continue
            # Skip an obvious header cell.
            if value.lower() in ("url", "urls", "address", "old", "new"):
                continue
            urls.append(value)
    return urls


def main():
    old_urls = read_urls(CONFIG["OLD_URLS"])
    new_urls = read_urls(CONFIG["NEW_URLS"])

    if not old_urls or not new_urls:
        sys.exit("Both CSVs need at least one URL. Nothing to do.")

    # Index the new URLs by normalised path for exact matching.
    new_by_path = {}
    for url in new_urls:
        new_by_path.setdefault(normalise(url), url)
    new_paths = list(new_by_path.keys())

    matches = []
    unmatched = []
    exact_count = 0
    fuzzy_count = 0

    for old in old_urls:
        old_path = normalise(old)

        if old_path in new_by_path:
            matches.append((old, new_by_path[old_path], "exact", "1.00"))
            exact_count += 1
            continue

        best_path = None
        best_score = 0.0
        for candidate in new_paths:
            score = SequenceMatcher(None, old_path, candidate).ratio()
            if score > best_score:
                best_score = score
                best_path = candidate

        if best_path is not None and best_score >= CONFIG["FUZZY_THRESHOLD"]:
            matches.append(
                (old, new_by_path[best_path], "fuzzy", "%.2f" % best_score)
            )
            fuzzy_count += 1
        else:
            unmatched.append((old, "%.2f" % best_score))

    with open(CONFIG["REDIRECT_MAP"], "w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(["old_url", "new_url", "match_type", "confidence"])
        writer.writerows(matches)

    with open(CONFIG["UNMATCHED"], "w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(["old_url", "best_score"])
        writer.writerows(unmatched)

    print("Old URLs read:      %d" % len(old_urls))
    print("Exact matches:      %d" % exact_count)
    print("Fuzzy matches:      %d  (review every one before shipping)" % fuzzy_count)
    print("Left unmatched:     %d" % len(unmatched))
    print("Wrote %s and %s" % (CONFIG["REDIRECT_MAP"], CONFIG["UNMATCHED"]))


if __name__ == "__main__":
    main()