"""Harvest capability — CT SoS filings with temporal classification and milestone metadata.

Coordinates integrations/ctsos.py with state/backlog.py to:
- Fetch raw CT SoS filings
- Map to new schema
- Apply filters (status, citizenship, entity type, shell names)
- Compute temporal classification (fresh/recent/historical)
- Compute milestone metadata for historical leads
- Deduplicate against existing backlog
- Write validated leads to state

References old gardener-fork/scripts/harvest.py and historical-sweep.py logic.
"""

import uuid
from datetime import datetime, date, timedelta
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

from integrations.ctsos import fetch_filings_since, fetch_filings_in_range, map_socrata_to_lead
from capabilities.patterns import proper_case_name, is_shell_lead
from state.backlog import load_backlog, save_backlog, get_lead
from state.schema import validate_lead


# Constants from assessment — do not tune
FRESH_DAYS = 14  # 0-14 days = fresh
RECENT_DAYS = 89  # 15-89 days = recent, 90+ = historical
MILESTONE_YEARS = [2, 3, 5, 7, 10]
MILESTONE_WINDOW_DAYS = 90

# Default shell patterns (should come from config in real impl)
DEFAULT_SHELL_PATTERNS = [
    {"pattern": r"\bholdings\b", "flags": "i"},
    {"pattern": r"\binvestments?\b", "flags": "i"},
    {"pattern": r"\bcapital\b", "flags": "i"},
    {"pattern": r"\bventures?\b", "flags": "i"},
    {"pattern": r"\bproperties\b", "flags": "i"},
    {"pattern": r"\bgroup\b", "flags": "i"},
    {"pattern": r"\bmanagement\b", "flags": "i"},
]


@dataclass
class HarvestResult:
    """Result of a harvest operation."""
    total_fetched: int = 0
    total_kept: int = 0
    total_new: int = 0
    total_skipped_duplicate: int = 0
    total_zoho_duplicate: int = 0
    filter_breakdown: dict = field(default_factory=dict)
    by_temporal_category: dict = field(default_factory=dict)
    by_milestone: dict = field(default_factory=dict)
    errors: list = field(default_factory=list)
    
    def __str__(self) -> str:
        """Format harvest result for CLI output."""
        skip_parts = [f"Duplicates: {self.total_skipped_duplicate}", f"Zoho exists: {self.total_zoho_duplicate}"]
        skip_line = f"  Skipped: {', '.join(skip_parts)}"

        lines = [
            f"Harvest Complete",
            f"  Total fetched: {self.total_fetched}",
            f"  Total kept: {self.total_kept}",
            f"  New leads: {self.total_new}",
            skip_line,
            f"",
            f"Filter breakdown:",
        ]
        for reason, count in self.filter_breakdown.items():
            lines.append(f"  {reason}: {count}")
        
        lines.append(f"")
        lines.append(f"Temporal categories:")
        for cat, count in self.by_temporal_category.items():
            lines.append(f"  {cat}: {count}")
        
        lines.append(f"")
        lines.append(f"Milestones (historical only):")
        for ms, count in self.by_milestone.items():
            lines.append(f"  {ms}: {count}")
        
        if self.errors:
            lines.append(f"")
            lines.append(f"Errors ({len(self.errors)}):")
            for err in self.errors[:5]:  # Show first 5
                lines.append(f"  {err}")
        
        return "\n".join(lines)


def harvest_recent(days: int = 1, skip_zoho_check: bool = False) -> HarvestResult:
    """Harvest filings from the last N days.
    
    Args:
        days: Number of days back to harvest (default: 1)
        skip_zoho_check: Skip Zoho dedup check
    
    Returns:
        HarvestResult with counts and breakdowns
    """
    since_date = date.today() - timedelta(days=days)
    harvested_at = datetime.utcnow()
    
    return _harvest_batch(since_date, date.today(), harvested_at, resume=False, skip_zoho_check=skip_zoho_check)


def harvest_historical_range(
    start_date: date,
    end_date: date,
    resume: bool = False,
    checkpoint_path: Optional[Path] = None,
    skip_zoho_check: bool = False,
) -> HarvestResult:
    """Harvest filings from a historical date range.
    
    Args:
        start_date: Start date (inclusive)
        end_date: End date (exclusive)
        resume: If True, resume from checkpoint
        checkpoint_path: Path to checkpoint file
        skip_zoho_check: Skip Zoho dedup check
    
    Returns:
        HarvestResult with counts and breakdowns
    """
    harvested_at = datetime.utcnow()
    return _harvest_batch(start_date, end_date, harvested_at, resume, checkpoint_path, skip_zoho_check=skip_zoho_check)


def harvest_for_milestone(
    target_milestone_years: int,
    window_days: int = MILESTONE_WINDOW_DAYS,
    resume: bool = False,
    checkpoint_path: Optional[Path] = None,
    skip_zoho_check: bool = False,
) -> HarvestResult:
    """Harvest filings whose age puts them within window_days of a target milestone.
    
    For target=5, window=90, fetches filings from approximately 5 years ago ± 90 days.
    
    Args:
        target_milestone_years: Target milestone (3, 5, 7, or 10)
        window_days: Days around milestone (default: 90)
        resume: If True, resume from checkpoint
        checkpoint_path: Path to checkpoint file
    
    Returns:
        HarvestResult with counts and breakdowns
    """
    if target_milestone_years not in MILESTONE_YEARS:
        raise ValueError(f"Invalid milestone year: {target_milestone_years}. Must be one of {MILESTONE_YEARS}")
    
    # Calculate date range: target_milestone_years ago ± window_days
    today = date.today()
    target_years_ago = today - timedelta(days=target_milestone_years * 365.25)
    start_date = target_years_ago - timedelta(days=window_days)
    end_date = target_years_ago + timedelta(days=window_days)
    
    harvested_at = datetime.utcnow()
    return _harvest_batch(start_date, end_date, harvested_at, resume, checkpoint_path, skip_zoho_check=skip_zoho_check)


def _harvest_batch(
    start_date: date,
    end_date: date,
    harvested_at: datetime,
    resume: bool = False,
    checkpoint_path: Optional[Path] = None,
    skip_zoho_check: bool = False,
) -> HarvestResult:
    """Internal harvest implementation.
    
    Args:
        start_date: Start date (inclusive)
        end_date: End date (exclusive)
        harvested_at: Timestamp for all leads in this batch
        resume: If True, resume from checkpoint
        checkpoint_path: Path to checkpoint file
        skip_zoho_check: Skip Zoho dedup check (faster for fresh leads)
    
    Returns:
        HarvestResult with counts and breakdowns
    """
    result = HarvestResult()
    
    # Load existing backlog for deduplication
    existing_backlog = load_backlog()
    existing_keys = set()
    for lead in existing_backlog:
        # Dedup key: name + filing_date
        key = f"{lead.get('name', '').lower()}|{lead.get('filing_date', '')}"
        existing_keys.add(key)
    
    # Pre-fetch Zoho email sets for batch dedup (one API call, not per-lead)
    zoho_lead_emails = None
    zoho_contact_emails = None
    if not skip_zoho_check:
        from state.zoho_dedup import build_zoho_email_sets
        try:
            zoho_lead_emails, zoho_contact_emails = build_zoho_email_sets()
        except Exception:
            pass  # Fail open — proceed without Zoho dedup
    
    # Fetch and process
    leads_to_save = []
    
    # Use generator for memory efficiency
    raw_gen = fetch_filings_in_range(
        start_date, end_date,
        checkpoint_path=checkpoint_path,
        resume=resume
    )
    
    for raw_record in raw_gen:
        result.total_fetched += 1
        
        # Map to lead schema
        try:
            lead = map_socrata_to_lead(raw_record, harvested_at)
        except Exception as e:
            result.errors.append(f"Mapping error: {e}")
            continue
        
        # Apply filters
        filter_reason = _apply_filters(lead)
        if filter_reason:
            result.filter_breakdown[filter_reason] = result.filter_breakdown.get(filter_reason, 0) + 1
            continue
        
        # Compute temporal classification
        temporal_cat = _compute_temporal_category(lead["filing_date"], harvested_at.date())
        lead["temporal_category"] = temporal_cat
        
        # Compute milestone metadata (only for historical)
        if temporal_cat == "historical":
            milestone_meta = _compute_milestone_metadata(lead["filing_date"], harvested_at.date())
            lead.update(milestone_meta)
        else:
            lead["milestone_year"] = None
            lead["milestone_window_hit"] = False
            lead["milestone_proximity_days"] = None
        
        # Deduplication
        dedup_key = f"{lead['name'].lower()}|{lead['filing_date']}"
        if dedup_key in existing_keys:
            result.total_skipped_duplicate += 1
            continue

        # Suppression check
        from state.suppressions import is_suppressed
        suppressed, reason = is_suppressed(lead.get("email", ""))
        if suppressed:
            result.filter_breakdown["suppressed"] = result.filter_breakdown.get("suppressed", 0) + 1
            continue

        # Zoho dedup check (batch mode)
        if zoho_lead_emails is not None and zoho_contact_emails is not None:
            from state.zoho_dedup import check_email_in_sets
            in_zoho, _ = check_email_in_sets(lead.get("email", ""), zoho_lead_emails, zoho_contact_emails)
            if in_zoho:
                result.total_zoho_duplicate += 1
                continue

        # Generate ID
        lead["id"] = str(uuid.uuid4())
        
        # Validate
        is_valid, errors = validate_lead(lead)
        if not is_valid:
            result.errors.append(f"Validation failed for {lead['name']}: {errors}")
            continue
        
        leads_to_save.append(lead)
        result.total_kept += 1
        result.total_new += 1
        
        # Track temporal category
        result.by_temporal_category[temporal_cat] = result.by_temporal_category.get(temporal_cat, 0) + 1
        
        # Track milestone (historical only)
        if temporal_cat == "historical":
            ms_key = f"{lead.get('milestone_year', 'none')}yr" if lead.get("milestone_window_hit") else "none"
            result.by_milestone[ms_key] = result.by_milestone.get(ms_key, 0) + 1
    
    # Save to backlog
    if leads_to_save:
        existing_backlog.extend(leads_to_save)
        save_backlog(existing_backlog)
    
    return result


def _apply_filters(lead: dict) -> Optional[str]:
    """Apply harvest-time filters. Returns filter reason if filtered, None if kept.
    
    Filters:
    1. Empty name
    2. Status not Active (already filtered at fetch, but double-check)
    3. Citizenship not Domestic (configurable, default: Domestic only)
    4. Entity type exclusion (drop sole proprietorships)
    5. Shell name detection
    
    Args:
        lead: Lead dict
    
    Returns:
        Filter reason string if filtered, None if kept
    """
    # Empty name
    if not lead.get("name") or len(lead["name"]) < 3:
        return "empty_name"
    
    # Citizenship filter (default: Domestic only)
    citizenship = lead.get("citizenship", "")
    if citizenship and citizenship != "Domestic":
        return "citizenship_foreign"
    
    # Entity type filter (drop non-corporate)
    entity_type = lead.get("entity_type", "").upper()
    excluded_types = ["SOLE PROPRIETORSHIP", "DBA", "FICTITIOUS NAME"]
    for excluded in excluded_types:
        if excluded in entity_type:
            return "entity_type"
    
    # Shell name filter
    name = lead.get("name", "")
    if is_shell_lead(name, DEFAULT_SHELL_PATTERNS):
        return "shell_name"
    
    return None


def _compute_temporal_category(filing_date_str: str, today: date) -> str:
    """Compute temporal category from filing date.
    
    Categories:
    - fresh: 0-14 days
    - recent: 15-89 days
    - historical: 90+ days
    
    Args:
        filing_date_str: ISO date string
        today: Current date
    
    Returns:
        "fresh", "recent", or "historical"
    """
    try:
        filing_date = datetime.fromisoformat(filing_date_str.replace('Z', '+00:00')).date()
    except (ValueError, TypeError):
        filing_date = datetime.strptime(filing_date_str[:10], "%Y-%m-%d").date()
    
    age_days = (today - filing_date).days
    
    if age_days <= FRESH_DAYS:
        return "fresh"
    elif age_days <= RECENT_DAYS:
        return "recent"
    else:
        return "historical"


def _compute_milestone_metadata(filing_date_str: str, today: date) -> dict:
    """Compute milestone metadata for a historical lead.
    
    Args:
        filing_date_str: ISO date string
        today: Current date
    
    Returns:
        Dict with milestone_year, milestone_window_hit, milestone_proximity_days
    """
    try:
        filing_date = datetime.fromisoformat(filing_date_str.replace('Z', '+00:00')).date()
    except (ValueError, TypeError):
        filing_date = datetime.strptime(filing_date_str[:10], "%Y-%m-%d").date()
    
    age_days = (today - filing_date).days
    age_years_float = age_days / 365.25
    
    # Find nearest milestone
    nearest_milestone = min(MILESTONE_YEARS, key=lambda y: abs(y - age_years_float))
    
    # Calculate target anniversary date (handle leap year edge case)
    try:
        target_anniversary_date = filing_date.replace(year=filing_date.year + nearest_milestone)
    except ValueError:
        # Feb 29 filing — fall back to Feb 28
        target_anniversary_date = filing_date.replace(year=filing_date.year + nearest_milestone, day=28)
    
    # Days until/since milestone
    proximity_days = (target_anniversary_date - today).days
    
    # Is within window?
    in_window = abs(proximity_days) <= MILESTONE_WINDOW_DAYS
    
    return {
        "milestone_year": nearest_milestone if in_window else None,
        "milestone_window_hit": in_window,
        "milestone_proximity_days": proximity_days,
    }