"""
SEAO Web Scraper using Playwright.
Parses the SEAO list page, follows detail links, and extracts contract information.
"""

import asyncio
import logging
import re
from urllib.parse import urlparse, parse_qs, urljoin

from playwright.async_api import async_playwright, Page, Browser

import config

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# URL helpers
# ---------------------------------------------------------------------------

def extract_item_id_from_url(url: str) -> str:
    """Extract the ItemId GUID from a SEAO detail page URL."""
    parsed = urlparse(url)
    params = parse_qs(parsed.query)
    return params.get("ItemId", [""])[0]


def build_detail_url(item_id: str) -> str:
    """Build a detail page URL from an ItemId."""
    return (
        f"{config.SEAO_BASE_URL}/avis-resultat-recherche/consulter"
        f"?ItemId={item_id}"
        f"&prov=AccueilResultatOuverture"
        f"&search=statIds=9%2C10%2C11%2C12%2C7"
        f"&tpIds=2%2C3%2C5%2C6%2C7%2C8%2C9%2C10%2C13%2C14%2C15%2C16%2C17%2C18%2C19"
        f"&isArchive=true&catIds=53&prov=AccueilResultatOuverture&addendaPublieDerniereVisite=false"
    )


# ---------------------------------------------------------------------------
# List page parsing
# ---------------------------------------------------------------------------

async def parse_list_page(page: Page) -> list[dict]:
    """
    Parse all contract rows from the current list page using browser JS.
    DOM structure (real HTML, from inspection):
      <td>
        <span>Contrat conclu</span>
        <div class="row"><label class="code-secret">Du jour</label></div>
      </td>
      <td>
        <a href="...?ItemId=..."><span>P1458 /2026-067</span></a>
        <span>/20149087</span>
        <div class="row"><span class="res-avis">DESCRIPTION</span></div>
        <div class="row"><span class="res-avis">TYPE - CATEGORY</span></div>
        <div class="row"><span class="res-avis-do">VILLE</span></div>
      </td>
      <td class="table-date-column"><span>2026-05-28 10:52</span></td>
      <td class="table-date-column"></td>
    """
    await page.wait_for_selector("table", timeout=15000)

    contracts = await page.evaluate("""
        () => {
            const results = [];
            const tbody = document.querySelector('table tbody');
            if (!tbody) return results;

            const rows = tbody.querySelectorAll('tr');
            for (const row of rows) {
                const cells = row.querySelectorAll('td');
                if (cells.length < 3) continue;

                // ---- Cell 0: Status ----
                // Status is in the first <span> child (before any "Du jour" label)
                const statusCell = cells[0];
                const statusSpan = statusCell.querySelector(':scope > span');
                const statusText = statusSpan ? statusSpan.textContent.trim() : '';

                // ---- Cell 1: Avis (composite) ----
                const avisCell = cells[1];

                // Contract number: inside <a><span>
                const link = avisCell.querySelector('a');
                const contractNumber = link ? link.textContent.trim() : '';
                const detailUrl = link ? link.getAttribute('href') || '' : '';

                // ItemId from URL
                let itemId = '';
                try {
                    const url = new URL(detailUrl, 'https://seao.gouv.qc.ca');
                    itemId = url.searchParams.get('ItemId') || '';
                } catch(e) {}

                // Reference number: the <span> directly after the <a> (not inside a div.row)
                // It's a direct child <span> that starts with /
                let refNumber = '';
                const directSpans = avisCell.querySelectorAll(':scope > span');
                for (const sp of directSpans) {
                    const txt = sp.textContent.trim();
                    if (txt.startsWith('/')) {
                        refNumber = txt.replace(/^\\//, '');
                        break;
                    }
                }

                // Description: first span.res-avis
                const descEl = avisCell.querySelector('span.res-avis');
                const description = descEl ? descEl.textContent.trim() : '';

                // Type + Category: second span.res-avis
                const resAvisSpans = avisCell.querySelectorAll('span.res-avis');
                const typeCategory = resAvisSpans.length > 1 ? resAvisSpans[1].textContent.trim() : '';

                // City: span.res-avis-do
                const cityEl = avisCell.querySelector('span.res-avis-do');
                const city = cityEl ? cityEl.textContent.trim() : '';

                // ---- Cell 2: Publication date ----
                const pubDate = cells[2] ? cells[2].textContent.trim() : '';

                // ---- Cell 3: Closing date ----
                const closingDate = cells[3] ? cells[3].textContent.trim() : '';

                results.push({
                    item_id: itemId,
                    contract_number: contractNumber,
                    reference_number: refNumber,
                    description: description,
                    type_category: typeCategory,
                    city: city,
                    status: statusText,
                    publication_date: pubDate,
                    closing_date: closingDate,
                });
            }
            return results;
        }
    """)

    logger.info(f"Found {len(contracts)} contracts on current page")
    return contracts


# ---------------------------------------------------------------------------
# Detail page parsing
# ---------------------------------------------------------------------------

async def _click_tab(page: Page, tab_name: str) -> bool:
    """Click a tab by its text content (case-insensitive partial match)."""
    try:
        tab = page.locator('[role="tab"]').filter(has_text=tab_name).first
        if await tab.count() > 0:
            await tab.click()
            await page.wait_for_timeout(1500)
            logger.debug(f"Tab matching '{tab_name}' clicked successfully")
            return True
        else:
            logger.debug(f"Tab matching '{tab_name}' not found")
    except Exception as exc:
        logger.debug(f"Tab matching '{tab_name}' error: {exc}")
    return False


async def _parse_ouverture_tab(page: Page) -> list[dict]:
    """
    Parse the 'Résultats d'ouverture' tab.
    Extracts ALL bidders: supplier_name, supplier_address, neq, contact, bid_amount.

    DOM structure (nested table per bidder):
      <generic>
        <generic>COMPANY NAME</generic>
        <table>
          <tr><th>Soumissionnaire</th><th>NEQ</th><th>Contact</th><th>Montant soumis</th></tr>
          <tr><td>NAME + ADDRESS</td><td>NEQ</td><td></td><td>AMOUNT</td></tr>
        </table>
      </generic>
    """
    submissions = []

    try:
        await page.wait_for_selector('[role="tabpanel"][aria-hidden="false"] table', timeout=10000)
    except Exception:
        logger.debug("No table found in Résultats d'ouverture tab")
        return submissions

    # SEAO nests one table per bidder.  The old parser inspected every <th>
    # below the outer wrapper and consequently addressed the wrong columns.
    extracted = await page.evaluate("""
        () => {
            const text = node => (node?.innerText || node?.textContent || '')
                .replace(/\\u00a0/g, ' ').trim();
            const panel = Array.from(document.querySelectorAll('[role="tabpanel"]'))
                .find(node => node.getAttribute('aria-hidden') === 'false');
            if (!panel) return [];

            const results = [];
            for (const table of panel.querySelectorAll('table')) {
                const headers = Array.from(table.querySelectorAll(':scope > thead th, :scope > tbody > tr:first-child th'))
                    .map(header => text(header).toLowerCase());
                const supplierCol = headers.findIndex(header => header.includes('soumissionnaire') || header.includes('fournisseur'));
                const neqCol = headers.findIndex(header => header === 'neq');
                const contactCol = headers.findIndex(header => header === 'contact');
                const amountCol = headers.findIndex(header => header.includes('montant') || header.includes('prix'));

                // A bidder table has the four direct columns below.  This excludes
                // the outer wrapper, which contains nested bidder tables.
                if (headers.length !== 4 || supplierCol < 0 || amountCol < 0) continue;

                for (const row of table.querySelectorAll(':scope > tbody > tr')) {
                    const cells = row.querySelectorAll(':scope > td');
                    if (cells.length !== headers.length) continue;

                    const supplierCell = cells[supplierCol];
                    const parts = text(supplierCell).split(/\\n+/).map(value => value.trim()).filter(Boolean);
                    const supplierName = parts.shift() || '';
                    if (!supplierName) continue;

                    results.push({
                        supplier_name: supplierName,
                        supplier_address: parts.join(', '),
                        neq: neqCol >= 0 ? text(cells[neqCol]) : '',
                        contact: contactCol >= 0 ? text(cells[contactCol]) : '',
                        bid_amount: text(cells[amountCol]),
                        is_winner: false,
                        contract_date: '',
                        contract_publication_date: '',
                        contract_amount: '',
                        final_publication_date: '',
                        final_end_date: '',
                        renewal_options_exercised: 0,
                        additional_acquisition_options: 0,
                        total_paid_amount: '',
                        supplementary_info: '',
                        source_tab: 'ouverture',
                    });
                }
            }
            return results;
        }
    """)

    submissions.extend(extracted)
    logger.debug(f"Extracted {len(submissions)} bidders from Résultats d'ouverture")
    return submissions


async def _parse_conclusion_tab(page: Page, submissions: list[dict]) -> dict:
    """
    Parse the 'Conclusion de contrat' tab.
    Enriches existing submissions with winner flag, contract dates and contract amount.

    DOM structure (flat table):
      <table>
        <tr><th>Soumissionaire retenu</th><th>Fournisseur</th><th>NEQ</th><th>Contact</th>
            <th>Montant soumis</th><th>Date conclusion</th><th>Date publication</th><th>Montant contrat</th></tr>
        <tr><td><svg/>|</td><td>name</td><td>NEQ</td><td></td><td>amount</td><td>date</td><td>date</td><td>amount</td></tr>
      </table>

    Returns dict with winning_bidder and contract_price for backward compatibility.
    """
    result = {"winning_bidder": "", "contract_price": ""}

    try:
        await page.wait_for_selector('[role="tabpanel"][aria-hidden="false"] table', timeout=10000)
    except Exception:
        logger.debug("No table found in Conclusion tab")
        return result

    enriched = await page.evaluate("""
        (submissions) => {
            const output = {
                winning_bidder: '',
                contract_price: '',
                enriched: JSON.parse(JSON.stringify(submissions)),
            };

            const text = node => (node?.innerText || node?.textContent || '')
                .replace(/\\u00a0/g, ' ').trim();
            const panel = Array.from(document.querySelectorAll('[role="tabpanel"]'))
                .find(node => node.getAttribute('aria-hidden') === 'false');
            if (!panel) return output;

            for (const table of panel.querySelectorAll('table')) {
                const ths = Array.from(table.querySelectorAll(':scope > thead th, :scope > tbody > tr:first-child th'));
                let winnerCol = -1, supplierCol = -1, neqCol = -1, contactCol = -1, bidCol = -1,
                    dateConclusionCol = -1, datePubCol = -1, contractAmountCol = -1;

                for (let i = 0; i < ths.length; i++) {
                    const t = ths[i].textContent.trim().toLowerCase();
                    if (t.includes('soumissionaire retenu') || t.includes('retenu')) winnerCol = i;
                    if (t.includes('soumissionnaire') || t.includes('fournisseur')) supplierCol = i;
                    if (t === 'neq') neqCol = i;
                    if (t === 'contact') contactCol = i;
                    if (t.includes('montant soumis') || t.includes('prix soumis')) bidCol = i;
                    if (t.includes('date de conclusion')) dateConclusionCol = i;
                    if (t.includes('date de publication')) datePubCol = i;
                    if (t.includes('montant du contrat') || t.includes('prix du contrat')) contractAmountCol = i;
                }

                if (supplierCol < 0 || winnerCol < 0 || contractAmountCol < 0) continue;

                const tbody = table.querySelector('tbody');
                if (!tbody) continue;

                const rows = tbody.querySelectorAll(':scope > tr');
                for (const row of rows) {
                    const cells = row.querySelectorAll(':scope > td');
                    if (cells.length !== ths.length) continue;

                    const hasSvg = winnerCol >= 0 && winnerCol < cells.length
                        && cells[winnerCol].querySelector('svg, img') !== null;
                    const isWinner = hasSvg;

                    const supplierCell = supplierCol < cells.length ? cells[supplierCol] : null;
                    if (!supplierCell) continue;

                    const supplierName = text(supplierCell).split(/\\n+/)[0].trim();
                    if (!supplierName) continue;

                    const neq = neqCol >= 0 && neqCol < cells.length
                        ? text(cells[neqCol]) : '';

                    // A conclusion may be published without an opening-result tab.
                    // Preserve that supplier instead of dropping the award data.
                    let submission = output.enriched.find(sub => sub.neq === neq && sub.supplier_name === supplierName);
                    if (!submission) {
                        submission = {
                            supplier_name: supplierName,
                            supplier_address: '',
                            neq,
                            contact: contactCol >= 0 ? text(cells[contactCol]) : '',
                            bid_amount: bidCol >= 0 ? text(cells[bidCol]) : '',
                            is_winner: false,
                            contract_date: '',
                            contract_publication_date: '',
                            contract_amount: '',
                            final_publication_date: '',
                            final_end_date: '',
                            renewal_options_exercised: 0,
                            additional_acquisition_options: 0,
                            total_paid_amount: '',
                            supplementary_info: '',
                            source_tab: 'conclusion',
                        };
                        output.enriched.push(submission);
                    }

                    submission.is_winner = isWinner;
                    if (isWinner) {
                        if (dateConclusionCol >= 0) submission.contract_date = text(cells[dateConclusionCol]);
                        if (datePubCol >= 0) submission.contract_publication_date = text(cells[datePubCol]);
                        submission.contract_amount = text(cells[contractAmountCol]);
                        output.contract_price = submission.contract_amount;
                        output.winning_bidder = supplierName;
                    }
                }
            }
            return output;
        }
    """, submissions)

    # Update submissions in place
    submissions.clear()
    submissions.extend(enriched.get("enriched", []))

    result["winning_bidder"] = enriched.get("winning_bidder", "")
    result["contract_price"] = enriched.get("contract_price", "")
    logger.debug(
        f"Conclusion: winner='{result['winning_bidder']}', price='{result['contract_price']}'"
    )
    return result


async def _parse_finale_tab(page: Page, submissions: list[dict]) -> None:
    """
    Parse the 'Information finale' tab (Terminé contracts only).
    Enriches the winner submission with final dates, options, total paid.
    """
    try:
        await page.wait_for_selector('[role="tabpanel"][aria-hidden="false"] table', timeout=10000)
    except Exception:
        logger.debug("No table found in Information finale tab")
        return

    final_data = await page.evaluate("""
        () => {
            const result = {};

            const text = node => (node?.innerText || node?.textContent || '')
                .replace(/\\u00a0/g, ' ').trim();
            const panel = Array.from(document.querySelectorAll('[role="tabpanel"]'))
                .find(node => node.getAttribute('aria-hidden') === 'false');
            if (!panel) return result;

            for (const table of panel.querySelectorAll('table')) {
                const ths = Array.from(table.querySelectorAll(':scope > thead th, :scope > tbody > tr:first-child th'));
                let finPubCol = -1, finEndCol = -1, renewalCol = -1, addAcqCol = -1, totalPaidCol = -1;
                for (let i = 0; i < ths.length; i++) {
                    const t = ths[i].textContent.trim().toLowerCase();
                    if (t.includes('date de publication') && t.includes('fin')) finPubCol = i;
                    if (t.includes('date de fin')) finEndCol = i;
                    if (t.includes('renouvellement')) renewalCol = i;
                    if (t.includes('acquisition')) addAcqCol = i;
                    if (t.includes('montant total')) totalPaidCol = i;
                }

                if (finEndCol < 0) continue;

                const tbody = table.querySelector('tbody');
                if (!tbody) continue;

                const rows = tbody.querySelectorAll(':scope > tr');
                for (const row of rows) {
                    const cells = row.querySelectorAll(':scope > td');
                    if (cells.length !== ths.length) continue;

                    if (finPubCol >= 0 && finPubCol < cells.length) result.final_publication_date =
                        text(cells[finPubCol]);
                    if (finEndCol >= 0 && finEndCol < cells.length) result.final_end_date =
                        text(cells[finEndCol]);
                    if (renewalCol >= 0 && renewalCol < cells.length) result.renewal_options_exercised =
                        parseInt(text(cells[renewalCol])) || 0;
                    if (addAcqCol >= 0 && addAcqCol < cells.length) result.additional_acquisition_options =
                        parseInt(text(cells[addAcqCol])) || 0;
                    if (totalPaidCol >= 0 && totalPaidCol < cells.length) result.total_paid_amount =
                        text(cells[totalPaidCol]);
                    break;
                }
                break;
            }

            // Also try to capture supplementary_info text
            const h3s = document.querySelectorAll('h3');
            for (const h3 of h3s) {
                if (h3.textContent.trim().toLowerCase().includes('information supplémentaire')) {
                    const nextEl = h3.nextElementSibling;
                    if (nextEl && nextEl.tagName !== 'H3' && nextEl.tagName !== 'TABLE') {
                        result.supplementary_info = nextEl.textContent.trim();
                    }
                    break;
                }
            }

            return result;
        }
    """)

    if not final_data:
        return

    # Enrich the winner submission
    for sub in submissions:
        if sub.get("is_winner"):
            sub["final_publication_date"] = final_data.get("final_publication_date", "")
            sub["final_end_date"] = final_data.get("final_end_date", "")
            sub["renewal_options_exercised"] = final_data.get("renewal_options_exercised", 0)
            sub["additional_acquisition_options"] = final_data.get("additional_acquisition_options", 0)
            sub["total_paid_amount"] = final_data.get("total_paid_amount", "")
            sub["supplementary_info"] = final_data.get("supplementary_info", "")
            sub["source_tab"] = "finale"
            break

    logger.debug(f"Info finale enriched: {final_data}")


async def parse_detail_page(page: Page, list_status: str = "") -> dict:
    """
    Parse detail page tabs and extract:
    - All submissions (from Résultats d'ouverture, always available)
    - Winning bidder + contract price (from Conclusion de contrat, if status is Contrat conclu/Terminé)
    - Final info (from Information finale, if status is Terminé)

    Returns dict with: winning_bidder, contract_price, submissions
    """
    result = {
        "winning_bidder": "",
        "contract_price": "",
        "submissions": [],
    }

    await page.wait_for_selector("main", timeout=15000)
    status_lower = list_status.lower()

    # Step 1: Always parse "Résultats d'ouverture" - contains all bidders
    if await _click_tab(page, "Résultats d"):
        result["submissions"] = await _parse_ouverture_tab(page)

    # Step 2: If contrat conclu or terminé, parse "Conclusion de contrat" for winner info
    if "contrat conclu" in status_lower or "terminé" in status_lower:
        if await _click_tab(page, "Conclusion de contrat"):
            conclusion_result = await _parse_conclusion_tab(page, result["submissions"])
            result["winning_bidder"] = conclusion_result["winning_bidder"]
            result["contract_price"] = conclusion_result["contract_price"]

    # Step 3: If terminé, parse "Information finale"
    if "terminé" in status_lower:
        if await _click_tab(page, "Information finale"):
            await _parse_finale_tab(page, result["submissions"])

    return result


# ---------------------------------------------------------------------------
# Pagination
# ---------------------------------------------------------------------------


async def go_to_next_page(page: Page) -> bool:
    """
    Click the "Page suivante" button. Returns True if successful, False if no more pages.
    """
    try:
        next_btn = page.get_by_role("button", name="Page suivante")
        if await next_btn.count() > 0 and await next_btn.is_enabled():
            previous_row = await page.locator("table tbody tr").first.text_content()
            await next_btn.click()
            await page.wait_for_function(
                "previous => document.querySelector('table tbody tr')?.textContent !== previous",
                previous_row,
                timeout=15000,
            )
            return True
    except Exception as exc:
        logger.debug(f"No next page: {exc}")
    return False


# ---------------------------------------------------------------------------
# Main scraping orchestration
# ---------------------------------------------------------------------------

async def refresh_existing_details(browser: Browser, contracts: list[dict]) -> list[dict]:
    """Refresh detail tabs for contracts already stored in SQLite."""
    context = await browser.new_context(
        user_agent=(
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/125.0.0.0 Safari/537.36"
        ),
        locale="fr-CA",
    )
    detail_page = await context.new_page()
    enriched_count = 0

    try:
        for index, contract in enumerate(contracts):
            item_id = contract.get("item_id")
            if not item_id:
                continue

            contract["source_url"] = build_detail_url(item_id)
            try:
                await detail_page.goto(contract["source_url"], wait_until="domcontentloaded", timeout=45000)
                await detail_page.wait_for_timeout(1500)
                contract.update(await parse_detail_page(detail_page, contract.get("status", "")))

                from datetime import datetime, timezone
                contract["detail_scraped_at"] = datetime.now(timezone.utc).isoformat()
                enriched_count += 1
            except Exception as exc:
                logger.warning(f"Failed to refresh detail for {item_id}: {exc}")

            if (index + 1) % 10 == 0:
                logger.info(f"Refreshed {index + 1}/{len(contracts)} existing detail pages...")
            await asyncio.sleep(config.REQUEST_DELAY)
    finally:
        await detail_page.close()
        await context.close()

    logger.info(f"Refreshed {enriched_count}/{len(contracts)} existing contracts with detail data")
    return contracts


async def scrape_all(browser: Browser) -> list[dict]:
    """
    Scrape all contracts from the SEAO list page and their detail pages.

    Args:
        browser: A Playwright Browser instance.

    Returns:
        List of complete contract dictionaries.
    """
    all_contracts = []
    context = await browser.new_context(
        user_agent=(
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/125.0.0.0 Safari/537.36"
        ),
        locale="fr-CA",
    )
    list_page = await context.new_page()
    detail_page = await context.new_page()

    try:
        # --- Step 1: Parse list pages ---
        list_url = config.build_list_url()
        logger.info(f"Navigating to list page: {list_url}")
        await list_page.goto(list_url, wait_until="networkidle", timeout=60000)
        await list_page.wait_for_timeout(2000)

        page_num = 1
        while True:
            if config.MAX_PAGES > 0 and page_num > config.MAX_PAGES:
                logger.info(f"Reached MAX_PAGES limit ({config.MAX_PAGES})")
                break

            logger.info(f"--- Parsing list page {page_num} ---")
            contracts = await parse_list_page(list_page)
            all_contracts.extend(contracts)
            logger.info(f"Extracted {len(contracts)} contracts from page {page_num}")

            # Stop if we've reached MAX_ITEMS limit
            if config.MAX_ITEMS > 0 and len(all_contracts) >= config.MAX_ITEMS:
                logger.info(f"Reached MAX_ITEMS limit ({config.MAX_ITEMS})")
                all_contracts = all_contracts[: config.MAX_ITEMS]
                break

            # Try next page
            if not await go_to_next_page(list_page):
                logger.info("No more list pages.")
                break

            page_num += 1
            await asyncio.sleep(config.REQUEST_DELAY)

        logger.info(f"Total contracts found across {page_num} pages: {len(all_contracts)}")

        # --- Step 2: Visit detail pages ---
        enriched_count = 0
        for i, contract in enumerate(all_contracts):
            item_id = contract.get("item_id")
            if not item_id:
                continue

            detail_url = build_detail_url(item_id)
            contract["source_url"] = detail_url
            try:
                await detail_page.goto(detail_url, wait_until="domcontentloaded", timeout=45000)
                await detail_page.wait_for_timeout(1500)

                detail_data = await parse_detail_page(
                    detail_page,
                    list_status=contract.get("status", ""),
                )
                contract.update(detail_data)
                from datetime import datetime, timezone
                contract["detail_scraped_at"] = datetime.now(timezone.utc).isoformat()
                enriched_count += 1

                if (i + 1) % 10 == 0:
                    logger.info(f"Enriched {i + 1}/{len(all_contracts)} detail pages...")

            except Exception as exc:
                logger.warning(f"Failed to parse detail for {item_id}: {exc}")

            await asyncio.sleep(config.REQUEST_DELAY)

        logger.info(f"Enriched {enriched_count}/{len(all_contracts)} contracts with detail data")

    finally:
        await list_page.close()
        await detail_page.close()
        await context.close()

    return all_contracts