120 lines
3.6 KiB
Python
120 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Extract NFR films by induction year from the LOC complete registry page.
|
|
"""
|
|
|
|
import re
|
|
import os
|
|
from bs4 import BeautifulSoup
|
|
import requests
|
|
|
|
def extract_films_by_year(year):
|
|
"""
|
|
Extract films inducted in a specific year from the LOC registry page.
|
|
"""
|
|
url = "https://www.loc.gov/programs/national-film-preservation-board/film-registry/complete-national-film-registry-listing/"
|
|
|
|
print(f"Fetching {url}...")
|
|
resp = requests.get(url, timeout=30)
|
|
resp.raise_for_status()
|
|
|
|
soup = BeautifulSoup(resp.text, 'html.parser')
|
|
|
|
# Find the table with film listings
|
|
table = soup.find('table', class_='sortable-table')
|
|
if not table:
|
|
print("Could not find table")
|
|
return {}
|
|
|
|
films = {}
|
|
|
|
# Iterate through table rows
|
|
rows = table.find_all('tr')
|
|
for row in rows:
|
|
cells = row.find_all(['th', 'td'])
|
|
if len(cells) >= 3:
|
|
title_cell = cells[0]
|
|
year_cell = cells[1]
|
|
inductee_cell = cells[2]
|
|
|
|
title = title_cell.get_text(strip=True).strip()
|
|
film_year_str = year_cell.get_text(strip=True).strip()
|
|
inductee_year_str = inductee_cell.get_text(strip=True).strip()
|
|
|
|
# Check if this film was inducted in the target year
|
|
try:
|
|
inductee_year = int(inductee_year_str)
|
|
if inductee_year == year:
|
|
# Parse film year (may be range or single year)
|
|
film_year_match = re.search(r'(\d{4})', film_year_str)
|
|
if film_year_match:
|
|
film_year = int(film_year_match.group(1))
|
|
else:
|
|
film_year = year # fallback
|
|
|
|
films[title] = {
|
|
"year": film_year,
|
|
"description": f"[NFR {year} inductee - film released in {film_year}]"
|
|
}
|
|
except ValueError:
|
|
continue
|
|
|
|
print(f"Found {len(films)} films inducted in {year}")
|
|
return films
|
|
|
|
def generate_nfr_file(year, films):
|
|
"""Generate a Python file with NFR data for a given year."""
|
|
if not films:
|
|
print(f"No films found for {year}")
|
|
return
|
|
|
|
filename = f"scripts/nfr_data/nfr_{year}.py"
|
|
|
|
code = f'''# {year} National Film Registry inductees
|
|
# Source: https://www.loc.gov/programs/national-film-preservation-board/film-registry/complete-national-film-registry-listing/
|
|
# Generated by extract_nfr_years.py
|
|
|
|
NFR_{year} = {{
|
|
'''
|
|
|
|
for title, data in sorted(films.items()):
|
|
title_escaped = title.replace("'", "\\'").replace('"', '\\"')
|
|
desc_escaped = data["description"].replace("'", "\\'").replace('"', '\\"')
|
|
|
|
code += f''' "{title_escaped}": {{
|
|
"year": {data["year"]},
|
|
"description": "{desc_escaped}"
|
|
}},
|
|
'''
|
|
|
|
code += "}\n"
|
|
|
|
with open(filename, 'w') as f:
|
|
f.write(code)
|
|
|
|
print(f"✓ Saved {len(films)} films to {filename}")
|
|
|
|
def main():
|
|
import os
|
|
|
|
# Create output directory if needed
|
|
os.makedirs("scripts/nfr_data", exist_ok=True)
|
|
|
|
# Extract films for years 1991-1997
|
|
years_to_extract = [1991, 1992, 1993, 1994, 1995, 1996, 1997]
|
|
|
|
for year in years_to_extract:
|
|
print(f"\n{'='*60}")
|
|
print(f"Extracting films for NFR year {year}")
|
|
print(f"{'='*60}")
|
|
|
|
films = extract_films_by_year(year)
|
|
generate_nfr_file(year, films)
|
|
|
|
print(f"\n{'='*60}")
|
|
print("EXTRACTION COMPLETE")
|
|
print(f"{'='*60}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|