"""
Step 1: Filters the raw reviews keeping ONLY those whose ASINs
exist in the metadata catalog (100k products).

Input:  raw/Clothing_Shoes_and_Jewelry.jsonl (26GB)
        artifacts/metadata_cleaned.json (100k products)
Output: artifacts/reviews_cleaned.json (only reviews with ASIN in the catalog)

Estimated time: 5-15 minutes (streaming scan of the 26GB file).
"""
import json
import os
import sys

# Paths relative to the project root (run from ~/Desktop/tesi)
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW_REVIEWS = os.path.join(PROJECT_ROOT, "raw", "Clothing_Shoes_and_Jewelry.jsonl")
METADATA_FILE = os.path.join(PROJECT_ROOT, "artifacts", "metadata_cleaned.json")
OUTPUT_FILE = os.path.join(PROJECT_ROOT, "artifacts", "reviews_cleaned.json")


def main():
    # 1. Load the ASINs from the metadata catalog
    print("--- Loading ASINs from the metadata catalog... ---")
    if not os.path.exists(METADATA_FILE):
        print(f"Error: {METADATA_FILE} not found!")
        sys.exit(1)

    with open(METADATA_FILE, "r", encoding="utf-8") as f:
        products = json.load(f)
    valid_asins = {p["asin"] for p in products}
    print(f"   ASINs in the catalog: {len(valid_asins)}")

    # 2. Scan the raw file and filter
    print(f"--- Scanning raw file: {RAW_REVIEWS} ---")
    if not os.path.exists(RAW_REVIEWS):
        print(f"Error: {RAW_REVIEWS} not found!")
        sys.exit(1)

    count_scanned = 0
    count_saved = 0

    with open(OUTPUT_FILE, "w", encoding="utf-8") as f_out:
        with open(RAW_REVIEWS, "r", encoding="utf-8") as f_in:
            for line in f_in:
                count_scanned += 1

                if count_scanned % 500000 == 0:
                    print(f"   Scanned: {count_scanned:,} | Saved: {count_saved:,}...")

                try:
                    data = json.loads(line)

                    # Filter: rating >= 4 + ASIN present in the catalog
                    rating = data.get("rating") or data.get("overall")
                    if not rating or float(rating) < 4.0:
                        continue

                    user = data.get("reviewerID") or data.get("user_id")
                    item = data.get("asin") or data.get("parent_asin")

                    if not user or not item:
                        continue

                    if item not in valid_asins:
                        continue

                    clean_obj = {
                        "reviewerID": user,
                        "asin": item,
                        "overall": float(rating),
                    }
                    json.dump(clean_obj, f_out)
                    f_out.write("\n")
                    count_saved += 1

                except Exception:
                    continue

    print(f"\n--- SUMMARY ---")
    print(f"Total lines scanned: {count_scanned:,}")
    print(f"Reviews saved (ASIN in catalog, rating >= 4): {count_saved:,}")
    print(f"Output: {OUTPUT_FILE}")


if __name__ == "__main__":
    main()