scraper.py

#!/usr/bin/env python3
"""
Bayut Scraper - Interactive CLI Tool
Powered by RealtyAPI.io | Bayut Scraper API

Scrape Bayut UAE property data - search listings in Dubai, Abu Dhabi,
Sharjah and more. Get property details, agent info, and TruBroker data.

Supports single test mode (preview) and bulk mode (from input.json).

Get your API key at: https://www.realtyapi.io
"""

import csv
import json
import os
import sys
import time

from config import ensure_api_key
from bayut_api import BayutAPI

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
OUTPUT_DIR = os.path.join(SCRIPT_DIR, "output")
INPUT_FILE = os.path.join(SCRIPT_DIR, "input.json")


# ─────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────

def preview(data, max_chars=1000):
    text = json.dumps(data, indent=2)
    if len(text) <= max_chars:
        print(text)
    else:
        print(text[:max_chars])
        print(f"\n... ({len(text) - max_chars} more characters)")


def prompt_choice(options, prompt_text="Select an option"):
    print()
    for i, option in enumerate(options, 1):
        print(f"  {i}. {option}")
    print()
    while True:
        try:
            choice = int(input(f"{prompt_text} (1-{len(options)}): "))
            if 1 <= choice <= len(options):
                return choice
            print(f"Please enter a number between 1 and {len(options)}")
        except ValueError:
            print("Please enter a valid number")


def prompt_output_format():
    options = ["JSON (.json)", "CSV (.csv)", "Both JSON and CSV"]
    return prompt_choice(options, "Save output as")


def flatten_dict(d, parent_key="", sep="_"):
    items = []
    for k, v in d.items():
        new_key = f"{parent_key}{sep}{k}" if parent_key else k
        if isinstance(v, dict):
            items.extend(flatten_dict(v, new_key, sep).items())
        elif isinstance(v, list):
            items.append((new_key, json.dumps(v)))
        else:
            items.append((new_key, v))
    return dict(items)


def save_json(data, filepath):
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w") as f:
        json.dump(data, f, indent=2)


def save_csv(records, filepath):
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    if not records:
        return
    flat = [flatten_dict(r) if isinstance(r, dict) else {"data": json.dumps(r)} for r in records]
    all_keys = []
    seen = set()
    for r in flat:
        for k in r:
            if k not in seen:
                all_keys.append(k)
                seen.add(k)
    with open(filepath, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=all_keys, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(flat)


def save_results(results, base_name, fmt_choice):
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    json_path = os.path.join(OUTPUT_DIR, f"{base_name}.json")
    csv_path = os.path.join(OUTPUT_DIR, f"{base_name}.csv")
    if fmt_choice in (1, 3):
        save_json(results, json_path)
        print(f"  Saved: {json_path}")
    if fmt_choice in (2, 3):
        csv_records = results if isinstance(results, list) else [results]
        save_csv(csv_records, csv_path)
        print(f"  Saved: {csv_path}")


def load_input_file():
    if not os.path.exists(INPUT_FILE):
        print(f"\n  \033[93m⚠  input.json not found at:\033[0m {INPUT_FILE}")
        print(f"  \033[93m   Create it using input_sample.json as a template.\033[0m")
        print(f"  \033[93m   Copy input_sample.json to input.json and fill in your data.\033[0m\n")
        return None
    with open(INPUT_FILE, "r") as f:
        return json.load(f)


def progress_bar(current, total, label=""):
    width = 30
    filled = int(width * current / total)
    bar = "=" * filled + "-" * (width - filled)
    pct = int(100 * current / total)
    print(f"\r  [{bar}] {pct}% ({current}/{total}) {label}", end="", flush=True)
    if current == total:
        print()


def run_bulk(items, label, fetch_fn, base_name):
    total = len(items)
    if total == 0:
        print(f"  No {label} items found in input.json")
        return
    print(f"\n  Found {total} {label} item(s) in input.json")
    fmt_choice = prompt_output_format()
    results, errors = [], []
    print()
    for i, item in enumerate(items, 1):
        item_label = str(item) if not isinstance(item, dict) else json.dumps(item)[:60]
        progress_bar(i, total, item_label)
        try:
            data = fetch_fn(item)
            results.append({"input": item, "result": data})
        except Exception as e:
            errors.append({"input": item, "error": str(e)})
            results.append({"input": item, "result": None, "error": str(e)})
        if i < total:
            time.sleep(0.3)
    print(f"\n  Done: {total - len(errors)} succeeded, {len(errors)} failed")
    save_results(results, base_name, fmt_choice)


def ask_mode():
    options = [
        "Test one (enter input manually, see preview)",
        "Bulk run (read from input.json, save all results)",
    ]
    return prompt_choice(options, "Select mode")


# ─────────────────────────────────────────────
# Property Search
# ─────────────────────────────────────────────

def handle_search(api):
    print("\n--- Search Bayut ---")
    options = ["Property Search", "Autocomplete (get location IDs)"]
    search_choice = prompt_choice(options)
    mode = ask_mode()

    if mode == 1:
        if search_choice == 1:
            data = _property_search_single(api)
        else:
            query = input("\nEnter search term (e.g., dubai, abu dhabi): ").strip()
            if not query:
                return print("Query required.")
            print("\nFetching suggestions...")
            data = api.autocomplete(query)
        if data is None:
            return
        print("\n--- Preview (first 1000 chars) ---")
        preview(data)
        save = input("\nSave full output? (y/n): ").strip().lower()
        if save == "y":
            fmt = prompt_output_format()
            name = "property_search" if search_choice == 1 else "autocomplete"
            save_results(data, name, fmt)
    else:
        input_data = load_input_file()
        if not input_data:
            return
        if search_choice == 1:
            items = input_data.get("property_searches", [])

            def fetch(item):
                if isinstance(item, str):
                    return api.search_property(location_external_id=item)
                return api.search_property(
                    location_external_id=item["location_external_id"],
                    purpose=item.get("purpose", "for-sale"),
                    hits_per_page=item.get("hits_per_page"),
                    page=item.get("page"),
                    category=item.get("category"),
                    rent_frequency=item.get("rent_frequency"),
                    rooms=item.get("rooms"),
                    baths=item.get("baths"),
                    min_price=item.get("min_price"),
                    max_price=item.get("max_price"),
                    min_area=item.get("min_area"),
                    max_area=item.get("max_area"),
                    furnishing_status=item.get("furnishing_status"),
                    category_external_id=item.get("category_external_id"),
                    completion_status=item.get("completion_status"),
                )

            run_bulk(items, "property search", fetch, "property_search")
        else:
            items = input_data.get("autocomplete_queries", [])
            run_bulk(items, "autocomplete", lambda q: api.autocomplete(q), "autocomplete")


def _property_search_single(api):
    loc_id = input("\nEnter location_external_id (use Autocomplete to find it): ").strip()
    if not loc_id:
        print("Location ID required.")
        return None

    purpose_options = ["for-sale", "for-rent"]
    print("\nPurpose:")
    pc = prompt_choice(purpose_options, "Select purpose")
    purpose = purpose_options[pc - 1]

    print("\nOptional filters (press Enter to skip):")
    category = input("Category (residential/commercial): ").strip() or None
    rooms = input("Bedrooms (e.g., 1, 2, 3): ").strip() or None
    baths = input("Bathrooms (e.g., 1, 2): ").strip() or None
    min_price = input("Min price: ").strip() or None
    max_price = input("Max price: ").strip() or None
    furnishing = input("Furnishing (furnished/unfurnished): ").strip() or None
    if purpose == "for-rent":
        rent_freq = input("Rent frequency (monthly/yearly/weekly/daily): ").strip() or None
    else:
        rent_freq = None
    page = input("Page (default 0): ").strip() or None

    print("\nSearching...")
    return api.search_property(
        location_external_id=loc_id, purpose=purpose,
        category=category, rooms=rooms, baths=baths,
        min_price=min_price, max_price=max_price,
        furnishing_status=furnishing, rent_frequency=rent_freq, page=page,
    )


# ─────────────────────────────────────────────
# Property Details
# ─────────────────────────────────────────────

def handle_property_details(api):
    print("\n--- Property Details ---")
    mode = ask_mode()

    if mode == 1:
        pid = input("\nEnter property external ID (from search results): ").strip()
        if not pid:
            return print("Property ID required.")
        print("\nFetching...")
        data = api.get_property_details(pid)
        print("\n--- Preview (first 1000 chars) ---")
        preview(data)
        save = input("\nSave full output? (y/n): ").strip().lower()
        if save == "y":
            fmt = prompt_output_format()
            save_results(data, "property_details", fmt)
    else:
        input_data = load_input_file()
        if not input_data:
            return
        items = input_data.get("property_ids", [])
        run_bulk(items, "property ID", lambda pid: api.get_property_details(pid), "property_details")


# ─────────────────────────────────────────────
# Agency & Agent
# ─────────────────────────────────────────────

def handle_agent(api):
    print("\n--- Agency & Agent ---")
    options = [
        "Agency Details",
        "Agents by Agency",
        "Agent Details (by slug)",
        "Agent Claimed Deals Count",
        "Agent Claimed Deals Total Price",
        "Agent Stories",
        "TruBroker Agent Search",
    ]
    agent_choice = prompt_choice(options)
    mode = ask_mode()

    if mode == 1:
        data = _agent_single(api, agent_choice)
        if data is None:
            return
        print("\n--- Preview (first 1000 chars) ---")
        preview(data)
        save = input("\nSave full output? (y/n): ").strip().lower()
        if save == "y":
            fmt = prompt_output_format()
            name_map = {
                1: "agency_details", 2: "agents_by_agency", 3: "agent_details",
                4: "agent_deals_count", 5: "agent_deals_total", 6: "agent_stories",
                7: "trubroker_agents",
            }
            save_results(data, name_map[agent_choice], fmt)
    else:
        input_data = load_input_file()
        if not input_data:
            return

        bulk_map = {
            1: ("agency_ids", lambda aid: api.get_agency_details(aid), "agency_details"),
            2: ("agency_ids", lambda aid: api.get_agents_by_agency(aid), "agents_by_agency"),
            3: ("agent_slugs", lambda s: api.get_agent_details(s), "agent_details"),
            4: ("agent_ids", lambda aid: api.get_agent_claimed_deals_count(aid), "agent_deals_count"),
            5: ("agent_ids", lambda aid: api.get_agent_claimed_deals_total_price(aid), "agent_deals_total"),
            6: ("agent_ids", lambda aid: api.get_agent_stories(aid), "agent_stories"),
            7: ("trubroker_searches", None, "trubroker_agents"),
        }

        if agent_choice == 7:
            items = input_data.get("trubroker_searches", [])

            def fetch(item):
                if isinstance(item, str):
                    return api.search_trubroker_agents(location_external_id=item)
                return api.search_trubroker_agents(
                    location_external_id=item["location_external_id"],
                    category=item.get("category"),
                    purpose=item.get("purpose"),
                )

            run_bulk(items, "TruBroker search", fetch, "trubroker_agents")
        else:
            key, fetch_fn, base_name = bulk_map[agent_choice]
            items = input_data.get(key, [])
            run_bulk(items, base_name.replace("_", " "), fetch_fn, base_name)


def _agent_single(api, choice):
    if choice == 1:
        aid = input("\nEnter agency external ID: ").strip()
        if not aid: return print("Agency ID required.") or None
        print("\nFetching...")
        return api.get_agency_details(aid)

    elif choice == 2:
        aid = input("\nEnter agency external ID: ").strip()
        if not aid: return print("Agency ID required.") or None
        print("\nFetching...")
        return api.get_agents_by_agency(aid)

    elif choice == 3:
        slug = input("\nEnter agent slug (e.g., salma-mohamed-2521536): ").strip()
        if not slug: return print("Agent slug required.") or None
        print("\nFetching...")
        return api.get_agent_details(slug)

    elif choice == 4:
        aid = input("\nEnter agent external ID: ").strip()
        if not aid: return print("Agent ID required.") or None
        print("\nFetching...")
        return api.get_agent_claimed_deals_count(aid)

    elif choice == 5:
        aid = input("\nEnter agent external ID: ").strip()
        if not aid: return print("Agent ID required.") or None
        print("\nFetching...")
        return api.get_agent_claimed_deals_total_price(aid)

    elif choice == 6:
        aid = input("\nEnter agent external ID: ").strip()
        if not aid: return print("Agent ID required.") or None
        print("\nFetching...")
        return api.get_agent_stories(aid)

    elif choice == 7:
        loc_id = input("\nEnter location_external_id: ").strip()
        if not loc_id: return print("Location ID required.") or None
        category = input("Category (residential/commercial, Enter to skip): ").strip() or None
        purpose = input("Purpose (for-sale/for-rent, Enter to skip): ").strip() or None
        print("\nSearching TruBroker agents...")
        return api.search_trubroker_agents(loc_id, category=category, purpose=purpose)


# ─────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────

def main():
    print("=" * 60)
    print("  Bayut Scraper - Powered by RealtyAPI.io")
    print("  https://www.realtyapi.io")
    print("=" * 60)

    api_key = ensure_api_key()
    if not api_key:
        sys.exit(1)

    api = BayutAPI(api_key)

    while True:
        print("\n" + "-" * 50)
        print("  What would you like to do?")
        print("-" * 50)
        options = [
            "Search Properties / Autocomplete",
            "Property Details",
            "Agency & Agent Data",
            "Exit",
        ]
        choice = prompt_choice(options)

        if choice == 1:
            handle_search(api)
        elif choice == 2:
            handle_property_details(api)
        elif choice == 3:
            handle_agent(api)
        elif choice == 4:
            print("\nGoodbye!")
            break


if __name__ == "__main__":
    main()