scraper.py

#!/usr/bin/env python3
"""
Airbnb Scraper - Interactive CLI Tool
Powered by RealtyAPI.io | Airbnb Scraper API

Scrape Airbnb data - search homes, experiences, services worldwide.
Get details, reviews, availability, and filter reference data.

Supports single test mode (preview) and bulk mode (from input.json).

Get your API key at: https://www.realtyapi.io
"""

import csv
import json
import os
import sys
import time

from config import ensure_api_key
from airbnb_api import AirbnbAPI

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
OUTPUT_DIR = os.path.join(SCRIPT_DIR, "output")
INPUT_FILE = os.path.join(SCRIPT_DIR, "input.json")


# ─────────────────────────────────────────────
# Helpers (same pattern as other scrapers)
# ─────────────────────────────────────────────

def preview(data, max_chars=1000):
    text = json.dumps(data, indent=2)
    if len(text) <= max_chars:
        print(text)
    else:
        print(text[:max_chars])
        print(f"\n... ({len(text) - max_chars} more characters)")


def prompt_choice(options, prompt_text="Select an option"):
    print()
    for i, option in enumerate(options, 1):
        print(f"  {i}. {option}")
    print()
    while True:
        try:
            choice = int(input(f"{prompt_text} (1-{len(options)}): "))
            if 1 <= choice <= len(options):
                return choice
            print(f"Please enter a number between 1 and {len(options)}")
        except ValueError:
            print("Please enter a valid number")


def prompt_output_format():
    options = ["JSON (.json)", "CSV (.csv)", "Both JSON and CSV"]
    return prompt_choice(options, "Save output as")


def flatten_dict(d, parent_key="", sep="_"):
    items = []
    for k, v in d.items():
        new_key = f"{parent_key}{sep}{k}" if parent_key else k
        if isinstance(v, dict):
            items.extend(flatten_dict(v, new_key, sep).items())
        elif isinstance(v, list):
            items.append((new_key, json.dumps(v)))
        else:
            items.append((new_key, v))
    return dict(items)


def save_json(data, filepath):
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w") as f:
        json.dump(data, f, indent=2)


def save_csv(records, filepath):
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    if not records:
        return
    flat = [flatten_dict(r) if isinstance(r, dict) else {"data": json.dumps(r)} for r in records]
    all_keys, seen = [], set()
    for r in flat:
        for k in r:
            if k not in seen:
                all_keys.append(k)
                seen.add(k)
    with open(filepath, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=all_keys, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(flat)


def save_results(results, base_name, fmt_choice):
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    json_path = os.path.join(OUTPUT_DIR, f"{base_name}.json")
    csv_path = os.path.join(OUTPUT_DIR, f"{base_name}.csv")
    if fmt_choice in (1, 3):
        save_json(results, json_path)
        print(f"  Saved: {json_path}")
    if fmt_choice in (2, 3):
        csv_records = results if isinstance(results, list) else [results]
        save_csv(csv_records, csv_path)
        print(f"  Saved: {csv_path}")


def load_input_file():
    if not os.path.exists(INPUT_FILE):
        print(f"\n  \033[93m⚠  input.json not found at:\033[0m {INPUT_FILE}")
        print(f"  \033[93m   Create it using input_sample.json as a template.\033[0m")
        print(f"  \033[93m   Copy input_sample.json to input.json and fill in your data.\033[0m\n")
        return None
    with open(INPUT_FILE, "r") as f:
        return json.load(f)


def progress_bar(current, total, label=""):
    width = 30
    filled = int(width * current / total)
    bar = "=" * filled + "-" * (width - filled)
    pct = int(100 * current / total)
    print(f"\r  [{bar}] {pct}% ({current}/{total}) {label}", end="", flush=True)
    if current == total:
        print()


def run_bulk(items, label, fetch_fn, base_name):
    total = len(items)
    if total == 0:
        print(f"  No {label} items found in input.json")
        return
    print(f"\n  Found {total} {label} item(s) in input.json")
    fmt_choice = prompt_output_format()
    results, errors = [], []
    print()
    for i, item in enumerate(items, 1):
        item_label = str(item) if not isinstance(item, dict) else json.dumps(item)[:60]
        progress_bar(i, total, item_label)
        try:
            data = fetch_fn(item)
            results.append({"input": item, "result": data})
        except Exception as e:
            errors.append({"input": item, "error": str(e)})
            results.append({"input": item, "result": None, "error": str(e)})
        if i < total:
            time.sleep(0.3)
    print(f"\n  Done: {total - len(errors)} succeeded, {len(errors)} failed")
    save_results(results, base_name, fmt_choice)


def ask_mode():
    options = [
        "Test one (enter input manually, see preview)",
        "Bulk run (read from input.json, save all results)",
    ]
    return prompt_choice(options, "Select mode")


def single_then_save(data, base_name):
    """Show preview and optionally save - used by all single-test flows."""
    if data is None:
        return
    print("\n--- Preview (first 1000 chars) ---")
    preview(data)
    save = input("\nSave full output? (y/n): ").strip().lower()
    if save == "y":
        fmt = prompt_output_format()
        save_results(data, base_name, fmt)


# ─────────────────────────────────────────────
# Search Homes
# ─────────────────────────────────────────────

def handle_search_homes(api):
    print("\n--- Search Airbnb Homes ---")
    options = [
        "Search by Destination",
        "Search by Coordinates (bounding box)",
        "Search by Place ID",
        "Search by Category",
        "Autocomplete (location suggestions)",
    ]
    choice = prompt_choice(options)
    mode = ask_mode()

    if mode == 1:
        data = _homes_single(api, choice)
        name_map = {1: "homes_destination", 2: "homes_coordinates",
                    3: "homes_placeid", 4: "homes_category", 5: "autocomplete"}
        single_then_save(data, name_map[choice])
    else:
        input_data = load_input_file()
        if not input_data:
            return
        key_map = {
            1: "search_homes_destinations",
            2: "search_homes_coordinates",
            3: "search_homes_place_ids",
            4: "search_homes_categories",
            5: "autocomplete_queries",
        }
        items = input_data.get(key_map[choice], [])

        fn_map = {
            1: lambda item: api.search_homes_by_destination(
                destination=item.get("destination", item) if isinstance(item, dict) else item,
                check_in=item.get("check_in") if isinstance(item, dict) else None,
                check_out=item.get("check_out") if isinstance(item, dict) else None,
                adults=item.get("adults") if isinstance(item, dict) else None,
                type_of_place=item.get("type_of_place") if isinstance(item, dict) else None,
                price_min=item.get("price_min") if isinstance(item, dict) else None,
                price_max=item.get("price_max") if isinstance(item, dict) else None,
                bedrooms=item.get("bedrooms") if isinstance(item, dict) else None,
                property_type=item.get("property_type") if isinstance(item, dict) else None,
            ),
            2: lambda item: api.search_homes_by_coordinates(
                ne_lat=item["ne_lat"], ne_lng=item["ne_lng"],
                sw_lat=item["sw_lat"], sw_lng=item["sw_lng"],
                check_in=item.get("check_in"), check_out=item.get("check_out"),
            ),
            3: lambda item: api.search_homes_by_place_id(
                place_id=item.get("place_id", item) if isinstance(item, dict) else item,
                check_in=item.get("check_in") if isinstance(item, dict) else None,
                check_out=item.get("check_out") if isinstance(item, dict) else None,
            ),
            4: lambda item: api.search_homes_by_category(
                category_tag=item.get("category_tag", item) if isinstance(item, dict) else item,
            ),
            5: lambda q: api.autocomplete(q),
        }
        label_map = {1: "destination", 2: "coordinates", 3: "place ID",
                     4: "category", 5: "autocomplete"}
        run_bulk(items, label_map[choice], fn_map[choice],
                 f"homes_{label_map[choice].replace(' ', '_')}")


def _homes_single(api, choice):
    if choice == 1:
        dest = input("\nEnter destination (e.g., toronto, canada): ").strip()
        if not dest: return print("Destination required.") or None
        check_in = input("Check-in date YYYY-MM-DD (Enter to skip): ").strip() or None
        check_out = input("Check-out date YYYY-MM-DD (Enter to skip): ").strip() or None
        print("\nOptional filters (press Enter to skip):")
        adults = input("Adults: ").strip() or None
        adults = int(adults) if adults else None
        bedrooms = input("Bedrooms: ").strip() or None
        bedrooms = int(bedrooms) if bedrooms else None
        price_min = input("Min price/night: ").strip() or None
        price_min = int(price_min) if price_min else None
        price_max = input("Max price/night: ").strip() or None
        price_max = int(price_max) if price_max else None
        type_of_place = input("Type (Any_type/Room/Entire_home): ").strip() or None
        property_type = input("Property (House/Flat/Guest house/Hotel): ").strip() or None
        print("\nSearching...")
        return api.search_homes_by_destination(
            dest, check_in=check_in, check_out=check_out,
            adults=adults, bedrooms=bedrooms, price_min=price_min,
            price_max=price_max, type_of_place=type_of_place,
            property_type=property_type,
        )
    elif choice == 2:
        ne_lat = input("\nNE latitude (top-right): ").strip()
        ne_lng = input("NE longitude: ").strip()
        sw_lat = input("SW latitude (bottom-left): ").strip()
        sw_lng = input("SW longitude: ").strip()
        if not all([ne_lat, ne_lng, sw_lat, sw_lng]):
            return print("All 4 coordinates required.") or None
        check_in = input("Check-in YYYY-MM-DD (Enter to skip): ").strip() or None
        check_out = input("Check-out YYYY-MM-DD (Enter to skip): ").strip() or None
        print("\nSearching...")
        return api.search_homes_by_coordinates(
            ne_lat, ne_lng, sw_lat, sw_lng,
            check_in=check_in, check_out=check_out,
        )
    elif choice == 3:
        pid = input("\nEnter Google Place ID: ").strip()
        if not pid: return print("Place ID required.") or None
        print("\nSearching...")
        return api.search_homes_by_place_id(pid)
    elif choice == 4:
        tag = input("\nEnter category tag (e.g., Tag:4104): ").strip()
        if not tag: return print("Category tag required.") or None
        print("\nSearching...")
        return api.search_homes_by_category(tag)
    elif choice == 5:
        kw = input("\nEnter keyword (e.g., paris): ").strip()
        if not kw: return print("Keyword required.") or None
        print("\nFetching suggestions...")
        return api.autocomplete(kw)


# ─────────────────────────────────────────────
# Search Experiences & Services
# ─────────────────────────────────────────────

def handle_search_exp_svc(api):
    print("\n--- Search Experiences & Services ---")
    options = [
        "Experiences by Destination",
        "Experiences by Place ID",
        "Services by Destination",
    ]
    choice = prompt_choice(options)
    mode = ask_mode()

    if mode == 1:
        if choice == 1:
            dest = input("\nEnter destination: ").strip()
            if not dest: return print("Destination required.")
            exp_type = input("Experience type (art,cooking,outdoor, etc.): ").strip() or None
            print("\nSearching...")
            data = api.search_experiences_by_destination(dest, experience_type=exp_type)
        elif choice == 2:
            pid = input("\nEnter Google Place ID: ").strip()
            if not pid: return print("Place ID required.")
            print("\nSearching...")
            data = api.search_experiences_by_place_id(pid)
        else:
            dest = input("\nEnter destination: ").strip()
            if not dest: return print("Destination required.")
            check_in = input("Check-in YYYY-MM-DD: ").strip()
            check_out = input("Check-out YYYY-MM-DD: ").strip()
            if not check_in or not check_out:
                return print("Check-in and check-out required for services.")
            svc_opts = ["Photography", "Prepared_Meals", "Training", "Makeup",
                        "Hair", "Spa_Treatment", "Catering", "Nails", "Chefs"]
            print("\nService type:")
            sc = prompt_choice(svc_opts, "Select service")
            print("\nSearching...")
            data = api.search_services_by_destination(
                dest, check_in, check_out, svc_opts[sc - 1])

        name_map = {1: "exp_destination", 2: "exp_placeid", 3: "services"}
        single_then_save(data, name_map[choice])
    else:
        input_data = load_input_file()
        if not input_data:
            return
        key_map = {1: "search_exp_destinations", 2: "search_exp_place_ids",
                   3: "search_services"}
        items = input_data.get(key_map[choice], [])
        fn_map = {
            1: lambda item: api.search_experiences_by_destination(
                destination=item.get("destination", item) if isinstance(item, dict) else item,
                experience_type=item.get("experience_type") if isinstance(item, dict) else None,
            ),
            2: lambda item: api.search_experiences_by_place_id(
                place_id=item.get("place_id", item) if isinstance(item, dict) else item,
            ),
            3: lambda item: api.search_services_by_destination(
                destination=item["destination"], check_in=item["check_in"],
                check_out=item["check_out"], type_of_service=item["type_of_service"],
            ),
        }
        label_map = {1: "experience", 2: "experience place ID", 3: "service"}
        run_bulk(items, label_map[choice], fn_map[choice],
                 f"search_{label_map[choice].replace(' ', '_')}")


# ─────────────────────────────────────────────
# Details & Reviews
# ─────────────────────────────────────────────

DETAIL_OPTIONS = [
    ("Home Details",             "get_home_details",             "home_details",       "home"),
    ("Home Reviews",             "get_home_reviews",             "home_reviews",       "home_review"),
    ("Home Availability",        "get_home_availability",        "home_availability",  "home_avail"),
    ("Experience Details",       "get_experience_details",       "exp_details",        "exp"),
    ("Experience Reviews",       "get_experience_reviews",       "exp_reviews",        "exp_review"),
    ("Experience Availability",  "get_experience_availability",  "exp_availability",   "exp_avail"),
    ("Service Details",          "get_service_details",          "svc_details",        "svc"),
    ("Service Reviews",          "get_service_reviews",          "svc_reviews",        "svc_review"),
]


def handle_details(api):
    print("\n--- Details & Reviews ---")
    labels = [opt[0] for opt in DETAIL_OPTIONS]
    choice = prompt_choice(labels)
    _, method_name, base_name, input_type = DETAIL_OPTIONS[choice - 1]
    method = getattr(api, method_name)
    mode = ask_mode()

    if mode == 1:
        data = _detail_single(method, input_type)
        single_then_save(data, base_name)
    else:
        input_data = load_input_file()
        if not input_data:
            return
        bulk_key_map = {
            "home": "home_listing_ids",
            "home_review": "home_listing_ids",
            "home_avail": "home_product_ids",
            "exp": "experience_listing_ids",
            "exp_review": "experience_listing_ids",
            "exp_avail": "experience_availability",
            "svc": "service_listing_ids",
            "svc_review": "service_listing_ids",
        }
        items = input_data.get(bulk_key_map[input_type], [])

        fetch_map = {
            "home": lambda item: method(item["stay_listing_id"], item["check_in"], item["check_out"]),
            "home_review": lambda item: method(item if isinstance(item, str) else item["stay_listing_id"]),
            "home_avail": lambda lid: method(lid),
            "exp": lambda lid: method(lid),
            "exp_review": lambda lid: method(lid),
            "exp_avail": lambda item: method(item["experience_listing_id"], item["start_date"], item["end_date"]),
            "svc": lambda lid: method(lid),
            "svc_review": lambda lid: method(lid),
        }
        run_bulk(items, base_name.replace("_", " "), fetch_map[input_type], base_name)


def _detail_single(method, input_type):
    if input_type == "home":
        lid = input("\nEnter stayListingId (from search results): ").strip()
        ci = input("Check-in YYYY-MM-DD: ").strip()
        co = input("Check-out YYYY-MM-DD: ").strip()
        if not all([lid, ci, co]):
            return print("All fields required.") or None
        print("\nFetching...")
        return method(lid, ci, co)
    elif input_type == "home_review":
        lid = input("\nEnter stayListingId: ").strip()
        if not lid: return print("ID required.") or None
        print("\nFetching...")
        return method(lid)
    elif input_type == "home_avail":
        lid = input("\nEnter listingId (productId from home details): ").strip()
        if not lid: return print("ID required.") or None
        print("\nFetching...")
        return method(lid)
    elif input_type in ("exp", "exp_review"):
        lid = input("\nEnter experienceListingId: ").strip()
        if not lid: return print("ID required.") or None
        print("\nFetching...")
        return method(lid)
    elif input_type == "exp_avail":
        lid = input("\nEnter experienceListingId: ").strip()
        sd = input("Start date YYYY-MM-DD: ").strip()
        ed = input("End date YYYY-MM-DD: ").strip()
        if not all([lid, sd, ed]):
            return print("All fields required.") or None
        print("\nFetching...")
        return method(lid, sd, ed)
    elif input_type in ("svc", "svc_review"):
        lid = input("\nEnter serviceListingId: ").strip()
        if not lid: return print("ID required.") or None
        print("\nFetching...")
        return method(lid)


# ─────────────────────────────────────────────
# Filters & Reference
# ─────────────────────────────────────────────

def handle_filters(api):
    print("\n--- Filters & Reference Data ---")
    options = [
        "Categories (Countryside, Beachfront, etc.)",
        "Amenities Filter (IDs for search)",
        "Property Types Filter",
        "Accessibility Filter",
        "Host Languages Filter",
        "Currencies",
        "Language Codes",
    ]
    choice = prompt_choice(options)
    print("\nFetching...")
    method_map = {
        1: api.get_categories, 2: api.get_amenities_filter,
        3: api.get_property_types_filter, 4: api.get_accessibility_filter,
        5: api.get_host_language_filter, 6: api.get_currencies,
        7: api.get_language_codes,
    }
    name_map = {
        1: "categories", 2: "amenities_filter", 3: "property_types_filter",
        4: "accessibility_filter", 5: "host_language_filter",
        6: "currencies", 7: "language_codes",
    }
    data = method_map[choice]()
    single_then_save(data, name_map[choice])


# ─────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────

def main():
    print("=" * 60)
    print("  Airbnb Scraper - Powered by RealtyAPI.io")
    print("  https://www.realtyapi.io")
    print("=" * 60)

    api_key = ensure_api_key()
    if not api_key:
        sys.exit(1)

    api = AirbnbAPI(api_key)

    while True:
        print("\n" + "-" * 50)
        print("  What would you like to do?")
        print("-" * 50)
        options = [
            "Search Homes (by destination, coordinates, place ID, category)",
            "Search Experiences & Services",
            "Details & Reviews (homes, experiences, services)",
            "Filters & Reference Data",
            "Exit",
        ]
        choice = prompt_choice(options)

        if choice == 1:
            handle_search_homes(api)
        elif choice == 2:
            handle_search_exp_svc(api)
        elif choice == 3:
            handle_details(api)
        elif choice == 4:
            handle_filters(api)
        elif choice == 5:
            print("\nGoodbye!")
            break


if __name__ == "__main__":
    main()