- Fix directories

scripts/extract_wiki_zim.py

@@ -6,7 +6,7 @@ import csv
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 INPUT_TSV = os.path.abspath(os.path.join(BASE_DIR, "../data/raw/imdb_datasets/title.basics.test.tsv"))
-OUTPUT_DIR = os.path.abspath(os.path.join(BASE_DIR, "../data/raw/wikipedia/wikipedia_html"))
+OUTPUT_DIR = os.path.abspath(os.path.join(BASE_DIR, "../data/processed/wikipedia_html"))
 ZIM_PATH = os.path.abspath(os.path.join(BASE_DIR, "../data/raw/wikipedia/wikipedia_en_all_maxi_2025-08.zim"))
 
 os.makedirs(OUTPUT_DIR, exist_ok=True)
@@ -53,16 +53,22 @@ with open(INPUT_TSV, encoding="utf-8") as f:
         tconst = row["tconst"]
         title = row["primaryTitle"]
         year = row["startYear"]
+        titleType = row["titleType"]
+        if year is None or titleType != "movie":
+            print("Skipping from TSV: ", title)
+            continue
         # folder for each movie
         movie_dir = os.path.join(OUTPUT_DIR, tconst)
         os.makedirs(movie_dir, exist_ok=True)
         outfile = os.path.join(movie_dir, f"{tconst}.html")
         if os.path.exists(outfile):
             continue
-        query = f"{title} {year}" if year != "\\N" else title #if year not empty
+        query = f"{title} ({year} film)" if year != "\\N" else title #if year not empty
         print(f"fetching Wikipedia HTML + images for {tconst}: {query}")
         html_with_images = fetch_wikipedia_html_with_images(query, movie_dir)
         if html_with_images:
+            if "Directed by" not in html_with_images:
+                continue
             with open(outfile, "w", encoding="utf-8") as out:
                 out.write(html_with_images)
         else: