marcus-web/scripts/generate_all_nfr_years.py

#!/usr/bin/env python3
"""
Fully automated NFR dictionary generator for all years.

This script uses web search and scraping to find announcement pages
and extract film data for each NFR year from 1989-2023.

Usage:
    python3 scripts/generate_all_nfr_years.py
    python3 scripts/generate_all_nfr_years.py --start 2020 --end 2023
    python3 scripts/generate_all_nfr_years.py --year 2015
"""

import argparse
import json
import re
import time
from pathlib import Path
from urllib.parse import quote_plus

import requests

# Configuration
SCRIPT_DIR = Path(__file__).parent
NFR_DATA_DIR = SCRIPT_DIR / "nfr_data"
FIRST_NFR_YEAR = 1989

# Known URLs to speed things up
KNOWN_ANNOUNCEMENT_URLS = {
    2024: "https://newsroom.loc.gov/news/25-films-named-to-national-film-registry-for-preservation/s/55d5285d-916f-4105-b7d4-7fc3ba8664e3",
    2023: "https://newsroom.loc.gov/news/25-films-selected-for-preservation-in-national-film-registry/s/aa4bef48-95f6-486f-882d-110613633b1e",
    2022: "https://newsroom.loc.gov/news/25-eclectic-films-chosen-for-national-film-registry/s/8c41f7a1-b9d9-4f9e-b252-4795b73a4aaf",
    2021: "https://blogs.loc.gov/now-see-hear/2021/12/librarian-of-congress-adds-25-films-to-the-national-film-registry/",
    2020: "https://blogs.loc.gov/now-see-hear/2020/12/librarian-of-congress-adds-25-films-to-the-national-film-registry/",
    2019: "https://blogs.loc.gov/now-see-hear/2019/12/librarian-of-congress-announces-national-film-registry-selections-for-2019/",
    2018: "https://blogs.loc.gov/now-see-hear/2018/12/librarian-of-congress-announces-national-film-registry-selections-for-2018/",
    2017: "https://blogs.loc.gov/now-see-hear/2017/12/librarian-of-congress-announces-national-film-registry-selections-for-2017/",
    2016: "https://blogs.loc.gov/now-see-hear/2016/12/librarian-of-congress-announces-2016-national-film-registry-selections/",
    2015: "https://blogs.loc.gov/now-see-hear/2015/12/announcing-the-2015-national-film-registry-selections/",
    2014: "https://blogs.loc.gov/now-see-hear/2014/12/announcing-the-2014-national-film-registry-selections/",
    2013: "https://blogs.loc.gov/now-see-hear/2013/12/announcing-the-2013-national-film-registry-selections/",
    2012: "https://blogs.loc.gov/now-see-hear/2012/12/announcing-the-2012-national-film-registry-selections/",
    2011: "https://blogs.loc.gov/now-see-hear/2011/12/announcing-the-2011-national-film-registry-selections/",
    2010: "https://blogs.loc.gov/now-see-hear/2010/12/announcing-the-2010-national-film-registry-selections/",
    2009: "https://blogs.loc.gov/now-see-hear/2009/12/announcing-the-2009-national-film-registry-selections/",
    2008: "https://blogs.loc.gov/now-see-hear/2008/12/announcing-the-2008-national-film-registry-selections/",
    2007: "https://blogs.loc.gov/now-see-hear/2007/12/announcing-the-2007-national-film-registry-selections/",
}


def search_for_announcement_url(year):
    """
    Try to find the announcement URL for a given year using various strategies.
    """
    print(f"  Searching for {year} announcement URL...")

    if year in KNOWN_ANNOUNCEMENT_URLS:
        print(f"  Using known URL for {year}")
        return KNOWN_ANNOUNCEMENT_URLS[year]

    # Try common URL patterns
    # Pattern 1: blogs.loc.gov (most common for older years)
    year_patterns = [
        f"https://blogs.loc.gov/now-see-hear/{year}/12/announcing-the-{year}-national-film-registry-selections/",
        f"https://blogs.loc.gov/now-see-hear/{year}/12/librarian-of-congress-announces-{year}-national-film-registry-selections/",
        f"https://blogs.loc.gov/now-see-hear/{year}/12/announcing-the-{year}-national-film-registry/",
    ]

    for url in year_patterns:
        try:
            resp = requests.head(url, timeout=10, allow_redirects=True)
            if resp.status_code == 200:
                print(f"  Found URL: {url}")
                return url
        except Exception:
            pass

    # If we can't find it automatically, return None
    print(f"  Could not find announcement URL for {year} - will need manual search")
    return None


def fetch_page_content(url):
    """Fetch HTML content from a URL."""
    try:
        resp = requests.get(url, timeout=30)
        resp.raise_for_status()
        return resp.text
    except Exception as e:
        print(f"  Error fetching {url}: {e}")
        return None


def extract_films_basic(html, year):
    """
    Basic extraction of films from HTML content.
    This is a simple heuristic-based approach.
    """
    films = {}

    # Look for film title patterns: "Title" (Year) or Title (Year)
    # This is very basic and may need refinement
    patterns = [
        r'"([^"]+)"\s*\((\d{4})\)',  # "Title" (Year)
        r'<strong>([^<]+)</strong>\s*\((\d{4})\)',  # <strong>Title</strong> (Year)
        r'<b>([^<]+)</b>\s*\((\d{4})\)',  # <b>Title</b> (Year)
    ]

    found_titles = set()

    for pattern in patterns:
        matches = re.findall(pattern, html)
        for title, film_year in matches:
            title = title.strip()
            film_year = int(film_year)

            # Sanity checks
            if (len(title) > 3 and
                1890 <= film_year <= year and
                title not in found_titles):

                films[title] = {
                    "year": film_year,
                    "description": f"[Description needs to be added for this {year} NFR inductee]"
                }
                found_titles.add(title)

    return films


def generate_nfr_dict_file(year, films, source_url=""):
    """Generate a Python file containing the NFR dictionary for a year."""
    if not films:
        print(f"  No films to generate for {year}")
        return None

    output_path = NFR_DATA_DIR / f"nfr_{year}.py"

    # Build the Python code
    code = f'''# {year} National Film Registry inductees
# Source: {source_url if source_url else "[Add source URL]"}
# Generated automatically - descriptions may need review/enhancement

NFR_{year} = {{
'''

    for title, data in films.items():
        # Escape single quotes in strings
        title_escaped = title.replace("'", "\\'").replace('"', '\\"')
        desc_escaped = data["description"].replace("'", "\\'").replace('"', '\\"')

        code += f'''    "{title_escaped}": {{
        "year": {data["year"]},
        "description": "{desc_escaped}"
    }},
'''

    code += "}\n"

    # Save the file
    output_path.write_text(code)
    print(f"  ✓ Saved {len(films)} films to {output_path.relative_to(SCRIPT_DIR)}")

    return output_path


def process_year(year, force=False):
    """Process a single year: find URL, fetch content, extract films, generate file."""
    print(f"\n{'='*60}")
    print(f"Processing NFR {year}")
    print(f"{'='*60}")

    # Check if already exists
    output_file = NFR_DATA_DIR / f"nfr_{year}.py"
    if output_file.exists() and not force:
        print(f"  ⚠️  {output_file.name} already exists (use --force to overwrite)")
        return False

    # Find announcement URL
    url = search_for_announcement_url(year)
    if not url:
        print(f"  ✗ Skipping {year} - no URL found")
        print(f"    You can manually process this year with:")
        print(f"    python3 scripts/setup_nfr.py {year}")
        return False

    # Fetch content
    html = fetch_page_content(url)
    if not html:
        print(f"  ✗ Could not fetch content for {year}")
        return False

    # Extract films
    print(f"  Extracting films...")
    films = extract_films_basic(html, year)

    if not films:
        print(f"  ✗ Could not extract films from {year}")
        print(f"    The page format may require manual processing:")
        print(f"    python3 scripts/setup_nfr.py {year} --url \"{url}\"")
        return False

    print(f"  Found {len(films)} films")

    # Generate file
    output_file = generate_nfr_dict_file(year, films, url)

    if output_file:
        print(f"  ✓ Successfully processed {year}")
        return True

    return False


def main():
    parser = argparse.ArgumentParser(
        description="Generate NFR dictionaries for all years"
    )
    parser.add_argument(
        "--start",
        type=int,
        default=FIRST_NFR_YEAR,
        help=f"Start year (default: {FIRST_NFR_YEAR})"
    )
    parser.add_argument(
        "--end",
        type=int,
        default=2023,
        help="End year (default: 2023)"
    )
    parser.add_argument(
        "--year",
        type=int,
        help="Process a single year"
    )
    parser.add_argument(
        "--force",
        action="store_true",
        help="Overwrite existing files"
    )
    parser.add_argument(
        "--delay",
        type=float,
        default=1.0,
        help="Delay between requests in seconds (default: 1.0)"
    )

    args = parser.parse_args()

    # Create output directory
    NFR_DATA_DIR.mkdir(exist_ok=True)

    # Determine years to process
    if args.year:
        years = [args.year]
    else:
        years = list(range(args.start, args.end + 1))

    print(f"Will process {len(years)} years: {min(years)}-{max(years)}")
    print(f"Output directory: {NFR_DATA_DIR}")
    print(f"Force overwrite: {args.force}")

    # Process years
    successful = []
    failed = []
    skipped = []

    for i, year in enumerate(years):
        # Add delay between requests to be polite
        if i > 0:
            time.sleep(args.delay)

        result = process_year(year, force=args.force)

        if result is True:
            successful.append(year)
        elif result is False:
            # Check if it was skipped vs failed
            output_file = NFR_DATA_DIR / f"nfr_{year}.py"
            if output_file.exists():
                skipped.append(year)
            else:
                failed.append(year)

    # Summary
    print(f"\n{'='*60}")
    print("SUMMARY")
    print(f"{'='*60}")
    print(f"\n✓ Successfully generated: {len(successful)} files")
    if successful:
        print(f"  Years: {successful}")

    if skipped:
        print(f"\n⊘ Skipped (already exist): {len(skipped)}")
        print(f"  Years: {skipped}")

    if failed:
        print(f"\n✗ Failed/Need manual processing: {len(failed)}")
        print(f"  Years: {failed}")
        print(f"\n  Process these manually with:")
        for year in failed:
            print(f"    python3 scripts/setup_nfr.py {year}")

    print(f"\n📁 Generated files: {NFR_DATA_DIR}/")
    print(f"\nNote: Generated files use basic extraction and may need review.")
    print(f"For better results with descriptions, use setup_nfr.py with ollama.")


if __name__ == "__main__":
    main()