Filling out the past years so that we have a dictionary of all the NFR movies and their short description
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Batch generate NFR dictionaries for multiple years.
|
||||
|
||||
This script automates the process of generating NFR data dictionaries
|
||||
for years 1989-2023 by searching for and fetching LOC announcement pages.
|
||||
|
||||
Usage:
|
||||
python3 scripts/batch_generate_nfr.py --years 2020-2023
|
||||
python3 scripts/batch_generate_nfr.py --years 2015,2016,2017
|
||||
python3 scripts/batch_generate_nfr.py --all # Process all years 1989-2023
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# NFR started in 1989
|
||||
FIRST_NFR_YEAR = 1989
|
||||
CURRENT_YEAR = 2024 # Update this as needed
|
||||
|
||||
# Known announcement URLs (add more as we find them)
|
||||
KNOWN_URLS = {
|
||||
2024: "https://newsroom.loc.gov/news/25-films-named-to-national-film-registry-for-preservation/s/55d5285d-916f-4105-b7d4-7fc3ba8664e3",
|
||||
2023: "https://newsroom.loc.gov/news/25-films-selected-for-preservation-in-national-film-registry/s/aa4bef48-95f6-486f-882d-110613633b1e",
|
||||
2022: "https://newsroom.loc.gov/news/25-eclectic-films-chosen-for-national-film-registry/s/8c41f7a1-b9d9-4f9e-b252-4795b73a4aaf",
|
||||
}
|
||||
|
||||
|
||||
def parse_year_range(year_spec):
|
||||
"""
|
||||
Parse year specification into a list of years.
|
||||
|
||||
Examples:
|
||||
"2020-2023" -> [2020, 2021, 2022, 2023]
|
||||
"2015,2016,2017" -> [2015, 2016, 2017]
|
||||
"2020" -> [2020]
|
||||
"""
|
||||
years = []
|
||||
|
||||
# Handle comma-separated list
|
||||
if ',' in year_spec:
|
||||
for year_str in year_spec.split(','):
|
||||
years.append(int(year_str.strip()))
|
||||
# Handle range
|
||||
elif '-' in year_spec:
|
||||
start, end = year_spec.split('-')
|
||||
years = list(range(int(start.strip()), int(end.strip()) + 1))
|
||||
# Handle single year
|
||||
else:
|
||||
years = [int(year_spec)]
|
||||
|
||||
return years
|
||||
|
||||
|
||||
def run_setup_for_year(year, use_ollama=True, ollama_host=None, ollama_model=None):
|
||||
"""
|
||||
Run setup_nfr.py for a specific year.
|
||||
|
||||
Returns True if successful, False otherwise.
|
||||
"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Processing NFR {year}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
cmd = ["python3", "scripts/setup_nfr.py", str(year)]
|
||||
|
||||
# Add URL if we know it
|
||||
if year in KNOWN_URLS:
|
||||
cmd.extend(["--url", KNOWN_URLS[year]])
|
||||
print(f"Using known URL for {year}")
|
||||
|
||||
# Add ollama options
|
||||
if not use_ollama:
|
||||
cmd.append("--no-ollama")
|
||||
else:
|
||||
if ollama_host:
|
||||
cmd.extend(["--ollama-host", ollama_host])
|
||||
if ollama_model:
|
||||
cmd.extend(["--ollama-model", ollama_model])
|
||||
|
||||
# Check if output file already exists
|
||||
output_file = Path(f"scripts/nfr_data/nfr_{year}.py")
|
||||
if output_file.exists():
|
||||
print(f"⚠️ {output_file} already exists")
|
||||
response = input("Overwrite? (y/N): ").strip().lower()
|
||||
if response != 'y':
|
||||
print(f"Skipping {year}")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Run the command - this may be interactive
|
||||
result = subprocess.run(cmd, check=False)
|
||||
|
||||
if result.returncode == 0:
|
||||
print(f"✓ Successfully processed {year}")
|
||||
return True
|
||||
else:
|
||||
print(f"✗ Failed to process {year}")
|
||||
return False
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n\nInterrupted while processing {year}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error processing {year}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Batch generate NFR dictionaries"
|
||||
)
|
||||
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument(
|
||||
"--years",
|
||||
help="Years to process (e.g., '2020-2023' or '2015,2016,2017')"
|
||||
)
|
||||
group.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help=f"Process all years from {FIRST_NFR_YEAR} to 2023"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--no-ollama",
|
||||
action="store_true",
|
||||
help="Don't use ollama (use basic extraction)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ollama-host",
|
||||
help="Ollama server URL"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ollama-model",
|
||||
help="Ollama model to use"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine which years to process
|
||||
if args.all:
|
||||
years = list(range(FIRST_NFR_YEAR, 2024)) # 1989-2023
|
||||
else:
|
||||
years = parse_year_range(args.years)
|
||||
|
||||
# Sort years
|
||||
years.sort()
|
||||
|
||||
print(f"\nWill process {len(years)} years: {years[0]}-{years[-1]}")
|
||||
print(f"Ollama: {'disabled' if args.no_ollama else 'enabled'}")
|
||||
|
||||
if len(years) > 5:
|
||||
response = input("\nThis will process many years. Continue? (Y/n): ").strip().lower()
|
||||
if response == 'n':
|
||||
print("Cancelled")
|
||||
sys.exit(0)
|
||||
|
||||
# Process each year
|
||||
successful = []
|
||||
failed = []
|
||||
|
||||
for year in years:
|
||||
success = run_setup_for_year(
|
||||
year,
|
||||
use_ollama=not args.no_ollama,
|
||||
ollama_host=args.ollama_host,
|
||||
ollama_model=args.ollama_model
|
||||
)
|
||||
|
||||
if success:
|
||||
successful.append(year)
|
||||
else:
|
||||
failed.append(year)
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
print("SUMMARY")
|
||||
print(f"{'='*60}\n")
|
||||
print(f"✓ Successfully processed: {len(successful)} years")
|
||||
if successful:
|
||||
print(f" {successful}")
|
||||
|
||||
if failed:
|
||||
print(f"\n✗ Failed: {len(failed)} years")
|
||||
print(f" {failed}")
|
||||
print(f"\nYou can retry failed years individually:")
|
||||
for year in failed:
|
||||
print(f" python3 scripts/setup_nfr.py {year}")
|
||||
|
||||
print(f"\nGenerated files are in: scripts/nfr_data/")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user