"""CT Secretary of State integration — business filings data.

Fetches CT SoS filings via Socrata Open Data API with pagination,
checkpointing for resumable historical sweeps, and proper field mapping.

References:
- Old gardener-fork/scripts/harvest.py for daily harvest
- Old gardener-fork/scripts/historical-sweep.py for pagination/checkpointing
"""

import requests
import json
from datetime import datetime, date
from pathlib import Path
from typing import Iterator, Optional

CT_API_BASE = "https://data.ct.gov/resource/n7gp-d28j.json"
AGENTS_API_BASE = "https://data.ct.gov/resource/qh2m-n44y.json"
PRINCIPALS_API_BASE = "https://data.ct.gov/resource/ka36-64k6.json"
FILING_HISTORY_API_BASE = "https://data.ct.gov/resource/ah3s-bes7.json"
DEFAULT_APP_TOKEN = "DEMO_KEY"  # Public placeholder

BULK_BATCH_SIZE = 100

FILING_SERVICE_DOMAINS = [
    "incfile.com",
    "legalzoom.com",
    "northwestregisteredagent.com",
    "nwregisteredagent.com",
    "registeredagents.com",
    "incorporate.com",
    "bizfilings.com",
    "corpnet.com",
    "zenbusiness.com",
    "swyftfilings.com",
    "mycompanyworks.com",
    "harborcompliance.com",
    "wolterskluwer.com",
    "cogencyglobal.com",
    "ctagent.com",
    "rlsagent.com",
    "ctcorporation.com",
    "paychex.com",
    "incorp.com",
    "sundocfilings.com",
]


def fetch_filings_since(
    since_date: date,
    app_token: str = DEFAULT_APP_TOKEN,
    status_filter: str = "Active"
) -> list[dict]:
    """Fetch all CT SoS filings registered on or after since_date.
    
    Args:
        since_date: Start date for fetching (inclusive)
        app_token: Socrata app token (default: DEMO_KEY)
        status_filter: Status to filter by (default: "Active")
    
    Returns:
        List of raw Socrata records
    """
    date_str = since_date.strftime("%Y-%m-%d")
    
    # Build $where clause
    where_parts = [f"status='{status_filter}'"]
    where_parts.append(f"date_registration >= '{date_str}'")
    where_clause = " AND ".join(where_parts)
    
    params = {
        "$where": where_clause,
        "$order": "date_registration DESC",
        "$limit": 5000,  # Max per page
        "$offset": 0
    }
    
    headers = {"X-App-Token": app_token} if app_token != DEFAULT_APP_TOKEN else {}
    
    all_records = []
    while True:
        try:
            resp = requests.get(CT_API_BASE, params=params, headers=headers, timeout=60)
            resp.raise_for_status()
            records = resp.json()
            
            if not records:
                break
            
            all_records.extend(records)
            
            # Check if we got a full page (might be more)
            if len(records) < 5000:
                break
            
            params["$offset"] += 5000
            
        except Exception as e:
            print(f"[CTSOS] Error fetching filings since {date_str}: {e}")
            break
    
    return all_records


def fetch_filings_in_range(
    start_date: date,
    end_date: date,
    app_token: str = DEFAULT_APP_TOKEN,
    status_filter: str = "Active",
    checkpoint_path: Optional[Path] = None,
    resume: bool = False
) -> Iterator[dict]:
    """Fetch CT SoS filings in a date range with pagination and optional checkpointing.
    
    Yields raw records as they're fetched. Supports resuming from checkpoint.
    
    Args:
        start_date: Start date (inclusive)
        end_date: End date (exclusive)
        app_token: Socrata app token
        status_filter: Status to filter by
        checkpoint_path: Path to checkpoint file for resumable sweeps
        resume: If True, resume from checkpoint; if False, start fresh
    
    Yields:
        Raw Socrata record dicts
    """
    start_str = start_date.strftime("%Y-%m-%d")
    end_str = end_date.strftime("%Y-%m-%d")
    
    # Build $where clause
    where_parts = [f"status='{status_filter}'"]
    where_parts.append(f"date_registration >= '{start_str}'")
    where_parts.append(f"date_registration < '{end_str}'")
    where_clause = " AND ".join(where_parts)
    
    # Checkpoint handling
    checkpoint = {"offset": 0, "page": 0}
    if resume and checkpoint_path and checkpoint_path.exists():
        try:
            with open(checkpoint_path) as f:
                checkpoint = json.load(f)
            print(f"[CTSOS] Resuming from page {checkpoint['page']}, offset {checkpoint['offset']}")
        except Exception as e:
            print(f"[CTSOS] Checkpoint load failed: {e}, starting fresh")
            checkpoint = {"offset": 0, "page": 0}
    
    params = {
        "$where": where_clause,
        "$order": "date_registration DESC",
        "$limit": 5000,
        "$offset": checkpoint["offset"]
    }
    
    headers = {"X-App-Token": app_token} if app_token != DEFAULT_APP_TOKEN else {}
    
    page = checkpoint["page"]
    
    while True:
        try:
            resp = requests.get(CT_API_BASE, params=params, headers=headers, timeout=60)
            resp.raise_for_status()
            records = resp.json()
            
            if not records:
                break
            
            # Yield each record
            for record in records:
                yield record
            
            # Save checkpoint
            if checkpoint_path:
                checkpoint["page"] = page
                checkpoint["offset"] = params["$offset"] + 5000
                with open(checkpoint_path, 'w') as f:
                    json.dump(checkpoint, f, indent=2)
            
            # Check if we got a full page
            if len(records) < 5000:
                break
            
            params["$offset"] += 5000
            page += 1
            
        except Exception as e:
            print(f"[CTSOS] Error fetching at page {page}, offset {params['$offset']}: {e}")
            break


def map_socrata_to_lead(record: dict, harvested_at: datetime) -> dict:
    """Transform a raw Socrata record into a lead dict matching the new schema.
    
    Pure field mapping + light cleanup. Does NOT compute temporal classification
    or milestones — that happens in the harvest capability.
    
    Args:
        record: Raw Socrata API response
        harvested_at: Timestamp when this lead was harvested
    
    Returns:
        Lead dict with mapped fields
    """
    from capabilities.patterns import proper_case_name
    
    # Extract and map fields according to field mapping table
    # Identity and entity fields
    name = record.get("business_name") or record.get("name") or ""
    name = proper_case_name(name.strip()) if name else ""
    
    entity_type = record.get("business_type") or record.get("entity_type") or ""
    citizenship = record.get("citizenship") or ""
    naics = record.get("naics_code") or record.get("naics") or ""
    filing_date = record.get("date_registration") or ""
    agent_name = record.get("agent_name") or ""
    agent_address = record.get("agent_address") or ""
    
    # Location fields — use first non-empty
    city = (
        record.get("city") or 
        record.get("billingcity") or 
        record.get("principal_office_city") or 
        ""
    )
    city = _normalize_city(city)
    
    state = (
        record.get("state") or 
        record.get("billingstate") or 
        record.get("principal_office_state") or 
        "CT"  # Default to CT
    )
    
    zip_code = (
        record.get("zip") or 
        record.get("billingpostalcode") or 
        record.get("principal_office_zip") or 
        ""
    )
    # Preserve as string (don't cast to int — leading zeros)
    zip_code = str(zip_code) if zip_code else ""
    
    street_address = record.get("billingstreet") or record.get("principal_office_address") or ""
    
    # Contact fields (often missing — that's expected)
    email = record.get("business_email_address") or ""
    phone = record.get("business_phone_number") or ""

    # Detect filing-service emails
    email_is_filing_service = False
    if email and "@" in email:
        email_domain = email.split("@")[-1].lower().strip()
        if email_domain in FILING_SERVICE_DOMAINS:
            email_is_filing_service = True

    # Build lead dict
    lead = {
        # Identity (always required)
        "business_id": record.get("id", ""),
        "name": name,
        "entity_type": entity_type,
        "citizenship": citizenship,
        "naics": naics,
        "filing_date": filing_date,  # REQUIRED
        "agent_name": agent_name,
        "agent_address": agent_address,

        # Location
        "city": city,
        "state": state,
        "zip": zip_code,
        "street_address": street_address,

        # Contact (often empty)
        "email": email,
        "phone": phone,

        # Filing-service detection
        "email_is_filing_service": email_is_filing_service,

        # Status (new leads start active)
        "lead_status": "active",

        # Metadata (computed at harvest)
        "harvested_at": harvested_at.isoformat(),
        "harvest_source": "ct_sos",
    }
    
    return lead


def _normalize_city(city: str) -> str:
    """Normalize city name to proper case."""
    if not city:
        return city
    
    s = city.strip()
    if s.upper() == s and any(c.isalpha() for c in s):
        return s.title()
    return s


def _socrata_request(url: str, params: dict, app_token: str = DEFAULT_APP_TOKEN) -> list:
    """Make a Socrata API request with error handling."""
    headers = {"X-App-Token": app_token} if app_token != DEFAULT_APP_TOKEN else {}
    try:
        resp = requests.get(url, params=params, headers=headers, timeout=60)
        resp.raise_for_status()
        return resp.json()
    except Exception as e:
        print(f"[CTSOS] API request failed: {e}")
        return []


def classify_agent_type(agent_record: dict) -> str:
    """Classify agent type from Agents dataset record.
    
    Returns:
        'individual', 'business', or 'secretary_of_state'
    """
    agent_type = (agent_record.get("type") or "").lower().strip()
    name = (agent_record.get("name__c") or "").lower()
    
    if "secretary" in name and "state" in name:
        return "secretary_of_state"
    
    if agent_type in ("individual", "person", "natural person"):
        return "individual"
    
    if agent_type in ("business", "commercial", "entity", "corporation", "llc", "company"):
        return "business"
    
    name_lower = name.lower()
    company_keywords = [
        "llc", "inc", "corp", "corporation", "company", "registered agent",
        "associates", "group", "services", "agency", "trust",
    ]
    if any(kw in name_lower for kw in company_keywords):
        return "business"
    
    firstname = agent_record.get("firstname", "")
    lastname = agent_record.get("lastname", "")
    if firstname or lastname:
        return "individual"
    
    return "business"


def fetch_agent_for_business(business_id: str, app_token: str = DEFAULT_APP_TOKEN) -> dict | None:
    """Fetch the registered agent record for a business from the Agents dataset.
    
    Args:
        business_id: Business Master ID (join key)
        app_token: Socrata app token
    
    Returns:
        Dict with agent_name, agent_type, agent_address fields, or None.
    """
    params = {
        "$where": f"business_key='{business_id}'",
        "$limit": 1,
    }
    
    records = _socrata_request(AGENTS_API_BASE, params, app_token)
    
    if not records:
        return None
    
    record = records[0]
    
    name = record.get("name__c", "")
    if not name:
        first = record.get("firstname", "")
        last = record.get("lastname", "")
        if first or last:
            name = f"{first} {last}".strip()
    
    address_parts = []
    street = record.get("business_street_address_1", "")
    if street:
        address_parts.append(street)
    city = record.get("business_city", "")
    state = record.get("business_state", "")
    zip_code = record.get("business_zip_code", "")
    if city or state or zip_code:
        location = " ".join(p for p in [city, state, zip_code] if p)
        if address_parts:
            address_parts.append(location)
        else:
            address_parts.append(location)
    
    return {
        "name": name,
        "type": classify_agent_type(record),
        "address": ", ".join(address_parts) if address_parts else "",
    }


def fetch_principals_for_business(business_id: str, app_token: str = DEFAULT_APP_TOKEN) -> list[dict]:
    """Fetch all principal records for a business from the Principals dataset.
    
    Args:
        business_id: Business Master ID (join key)
        app_token: Socrata app token
    
    Returns:
        List of dicts with: name, first_name, last_name, designation, type.
        Sorted by designation priority: President > CEO > Managing Member >
        Member > Director > Officer > other.
    """
    params = {
        "$where": f"business_id='{business_id}'",
        "$limit": 50,
    }
    
    records = _socrata_request(PRINCIPALS_API_BASE, params, app_token)
    
    if not records:
        return []
    
    principals = []
    for record in records:
        name = record.get("name__c", "")
        first_name = record.get("first_name", "")
        last_name = record.get("last_name", "")
        designation = record.get("designation", "")
        principal_type = record.get("type", "")
        
        principals.append({
            "name": name,
            "first_name": first_name,
            "last_name": last_name,
            "designation": designation,
            "type": principal_type,
        })
    
    principals.sort(key=lambda p: _designation_priority(p.get("designation", "")))
    
    return principals


def _designation_priority(designation: str) -> int:
    """Sort designation by priority. Lower number = higher priority."""
    priority_map = {
        "president": 1,
        "ceo": 2,
        "managing member": 3,
        "member": 4,
        "director": 5,
        "officer": 6,
        "manager": 7,
        "partner": 8,
    }
    return priority_map.get(designation.lower().strip(), 99)


def fetch_agents_bulk(business_ids: list[str], app_token: str = DEFAULT_APP_TOKEN, batch_size: int = BULK_BATCH_SIZE) -> dict[str, dict]:
    """Bulk fetch agents for multiple businesses.
    
    Uses $where=business_key IN ('id1','id2',...) for efficiency.
    
    Args:
        business_ids: List of Business Master IDs
        app_token: Socrata app token
        batch_size: Number of IDs per request (default 200)
    
    Returns:
        Dict mapping business_id -> agent_record (first agent per business).
    """
    result = {}
    
    for i in range(0, len(business_ids), batch_size):
        batch = business_ids[i:i + batch_size]
        escaped_ids = [bid.replace("'", "''") for bid in batch if bid]
        
        if not escaped_ids:
            continue
        
        in_clause = ",".join(f"'{eid}'" for eid in escaped_ids)
        params = {
            "$where": f"business_key IN ({in_clause})",
            "$limit": len(batch) * 2,
        }
        
        records = _socrata_request(AGENTS_API_BASE, params, app_token)
        
        for record in records:
            biz_key = record.get("business_key", "")
            if biz_key and biz_key not in result:
                name = record.get("name__c", "")
                if not name:
                    first = record.get("firstname", "")
                    last = record.get("lastname", "")
                    if first or last:
                        name = f"{first} {last}".strip()
                
                address_parts = []
                street = record.get("business_street_address_1", "")
                if street:
                    address_parts.append(street)
                city = record.get("business_city", "")
                state = record.get("business_state", "")
                zip_code = record.get("business_zip_code", "")
                if city or state or zip_code:
                    location = " ".join(p for p in [city, state, zip_code] if p)
                    if address_parts:
                        address_parts.append(location)
                    else:
                        address_parts.append(location)
                
                result[biz_key] = {
                    "name": name,
                    "type": classify_agent_type(record),
                    "address": ", ".join(address_parts) if address_parts else "",
                }
    
    return result


def fetch_principals_bulk(business_ids: list[str], app_token: str = DEFAULT_APP_TOKEN, batch_size: int = BULK_BATCH_SIZE) -> dict[str, list[dict]]:
    """Bulk fetch principals for multiple businesses.
    
    Uses $where=business_id IN ('id1','id2',...) for efficiency.
    
    Args:
        business_ids: List of Business Master IDs
        app_token: Socrata app token
        batch_size: Number of IDs per request (default 100)
    
    Returns:
        Dict mapping business_id -> list of principal records.
    """
    result = {}
    
    for i in range(0, len(business_ids), batch_size):
        batch = business_ids[i:i + batch_size]
        escaped_ids = [bid.replace("'", "''") for bid in batch if bid]
        
        if not escaped_ids:
            continue
        
        in_clause = ",".join(f"'{eid}'" for eid in escaped_ids)
        params = {
            "$where": f"business_id IN ({in_clause})",
            "$limit": len(batch) * 10,
        }
        
        records = _socrata_request(PRINCIPALS_API_BASE, params, app_token)
        
        for record in records:
            biz_id = record.get("business_id", "")
            if not biz_id:
                continue
            
            if biz_id not in result:
                result[biz_id] = []
            
            name = record.get("name__c", "")
            first_name = record.get("first_name", "")
            last_name = record.get("last_name", "")
            designation = record.get("designation", "")
            principal_type = record.get("type", "")
            
            result[biz_id].append({
                "name": name,
                "first_name": first_name,
                "last_name": last_name,
                "designation": designation,
                "type": principal_type,
            })
    
    # Sort each list by designation priority
    for biz_id in result:
        result[biz_id].sort(key=lambda p: _designation_priority(p.get("designation", "")))
    
    return result


def fetch_filing_history(business_id: str, app_token: str = DEFAULT_APP_TOKEN) -> list[dict]:
    """Fetch filing history from Filing History dataset.
    
    Args:
        business_id: Business Master ID (join key)
        app_token: Socrata app token
    
    Returns:
        List of dicts with: filing_date, filing_type, filing_category,
        report_year, physical_filing, organizer_name.
        Sorted by filing_date descending (most recent first).
    """
    params = {
        "$where": f"account='{business_id}'",
        "$order": "filing_date DESC",
        "$limit": 100,
    }
    
    records = _socrata_request(FILING_HISTORY_API_BASE, params, app_token)
    
    if not records:
        return []
    
    history = []
    for record in records:
        history.append({
            "filing_date": record.get("filing_date", ""),
            "filing_type": record.get("filing_type", ""),
            "filing_category": record.get("filing_category", ""),
            "report_year": record.get("report_year", ""),
            "physical_filing": record.get("physical_filing", False),
            "organizer_name": record.get("organizer_name", ""),
        })
    
    return history


def fetch_filing_history_bulk(business_ids: list[str], app_token: str = DEFAULT_APP_TOKEN, batch_size: int = BULK_BATCH_SIZE) -> dict[str, list[dict]]:
    """Bulk fetch filing histories for multiple businesses.
    
    Uses $where=business_id IN ('id1','id2',...) for efficiency.
    
    Args:
        business_ids: List of Business Master IDs
        app_token: Socrata app token
        batch_size: Number of IDs per request (default 100)
    
    Returns:
        Dict mapping business_id -> list of filing history records.
    """
    result = {}
    
    for i in range(0, len(business_ids), batch_size):
        batch = business_ids[i:i + batch_size]
        escaped_ids = [bid.replace("'", "''") for bid in batch if bid]
        
        if not escaped_ids:
            continue
        
        in_clause = ",".join(f"'{eid}'" for eid in escaped_ids)
        params = {
            "$where": f"account IN ({in_clause})",
            "$order": "account, filing_date DESC",
            "$limit": len(batch) * 20,
        }
        
        records = _socrata_request(FILING_HISTORY_API_BASE, params, app_token)
        
        for record in records:
            biz_id = record.get("account", "")
            if not biz_id:
                continue
            
            if biz_id not in result:
                result[biz_id] = []
            
            result[biz_id].append({
                "filing_date": record.get("filing_date", ""),
                "filing_type": record.get("filing_type", ""),
                "filing_category": record.get("filing_category", ""),
                "report_year": record.get("report_year", ""),
                "physical_filing": record.get("physical_filing", False),
                "organizer_name": record.get("organizer_name", ""),
            })
    
    return result


def search_filings_by_address(
    street: str, city: str, app_token: str = DEFAULT_APP_TOKEN
) -> list[dict]:
    """Search CT SoS filings by billingstreet + billingcity.

    Args:
        street: Street address to match (will be lowercased for matching)
        city: City to match (will be lowercased for matching)
        app_token: Socrata app token

    Returns:
        List of matching Socrata records with id, name, billingstreet, billingcity, etc.
    """
    if not street or not city:
        return []

    street_clean = street.strip().lower()
    city_clean = city.strip().lower()
    if not street_clean or not city_clean:
        return []

    street_escaped = street_clean.replace("'", "''")
    city_escaped = city_clean.replace("'", "''")

    params = {
        "$where": f"lower(billingstreet)='{street_escaped}' AND lower(billingcity)='{city_escaped}'",
        "$limit": 50,
    }

    return _socrata_request(CT_API_BASE, params, app_token)


def search_filings_by_name(
    name: str, city: str = "", app_token: str = DEFAULT_APP_TOKEN
) -> list[dict]:
    """Search CT SoS filings by business name (contains) with optional city filter.

    Uses Socrata contains() for case-insensitive partial matching.

    Args:
        name: Business name to search for
        city: Optional city filter to narrow results
        app_token: Socrata app token

    Returns:
        List of matching Socrata records.
    """
    if not name:
        return []

    name_clean = name.strip().lower()
    if not name_clean:
        return []

    name_escaped = name_clean.replace("'", "''")

    if city:
        city_clean = city.strip().lower()
        city_escaped = city_clean.replace("'", "''") if city_clean else ""
        if city_escaped:
            where = f"contains(lower(name),'{name_escaped}') AND lower(billingcity)='{city_escaped}' AND status='Active'"
        else:
            where = f"contains(lower(name),'{name_escaped}') AND status='Active'"
    else:
        where = f"contains(lower(name),'{name_escaped}') AND status='Active'"

    params = {
        "$where": where,
        "$limit": 50,
    }

    return _socrata_request(CT_API_BASE, params, app_token)