from flask import Flask, render_template, request, redirect, url_for, flash, jsonify, send_from_directory, g, session, abort, make_response
from flask_login import LoginManager, login_user, logout_user, login_required, current_user
from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.utils import secure_filename
from flask_wtf.csrf import CSRFProtect, CSRFError
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
from config import Config
from models import db, User, Post, Page, MenuItem, Image, File, Settings, Redirect, AuditLog, LoginAttempt, Form, FormSubmission, Newsletter, TwoFactorRecoveryCode
from models import AVAILABLE_LANGUAGES, LANG_CODE_TO_NAME, LANG_NAME_TO_CODE
from models import init_default_settings, create_default_pages, create_default_menu_items
import os
import re
import secrets
from datetime import datetime, timedelta, timezone
def utc_now():
    """Naive UTC datetime — drop-in replacement for the deprecated stdlib utcnow()."""
    return datetime.now(timezone.utc).replace(tzinfo=None)

from slugify import slugify
import json
import markdown
from markupsafe import escape as html_escape
from PIL import Image as PILImage
import io
from functools import wraps
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError
import socket
import hashlib
import hmac as hmac_module
# Cross-platform file locking (fcntl on Linux, msvcrt on Windows)
try:
    import fcntl
    def _lock(f):
        fcntl.flock(f, fcntl.LOCK_EX)
    def _unlock(f):
        fcntl.flock(f, fcntl.LOCK_UN)
except ImportError:
    try:
        import msvcrt
        def _lock(f):
            msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1)
        def _unlock(f):
            try:
                msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
            except OSError:
                pass
    except ImportError:
        # Fallback: no locking (single-process dev mode)
        def _lock(f):
            pass
        def _unlock(f):
            pass
from i18n import init_i18n, t, t_for, set_admin_language, pt_for
from block_renderer import init_block_renderer
import schema_org
from updater import (
    LicenseValidator, UpdateChecker, UpdatePerformer,
    get_current_version, get_license_status, is_licensed,
    has_plan, should_revalidate
)

# ── Startup checks (terminal/log only — before app is created) ───────────────
# These catch fatal misconfigurations early with clear messages.

def _startup_checks():
    """Verify critical dependencies and configuration at startup."""
    errors = []
    
    # cryptography is imported lazily inside encrypt/decrypt functions
    # Without it, license keys, SMTP passwords, and API keys can't be stored securely
    try:
        from cryptography.fernet import Fernet  # noqa: F401
    except ImportError:
        errors.append(
            "CRITICAL: 'cryptography' package is not installed.\n"
            "  License keys, SMTP passwords, and API keys cannot be encrypted without it.\n"
            "  Fix: pip install cryptography  (or: venv/bin/pip install cryptography)"
        )
    
    # SECRET_KEY must exist — without it, sessions and encryption break
    from config import Config as _Cfg
    if not _Cfg.SECRET_KEY or len(_Cfg.SECRET_KEY) < 16:
        errors.append(
            "CRITICAL: SECRET_KEY is missing or too short in .env\n"
            "  Sessions will reset on every restart. Encryption will be insecure.\n"
            "  Fix: Add to .env: SECRET_KEY=<random 64-char hex string>\n"
            "  Generate: python3 -c \"import secrets; print(secrets.token_hex(32))\""
        )
    
    if errors:
        print("\n" + "=" * 60)
        print("  AliothPress — Startup Check Failed")
        print("=" * 60)
        for err in errors:
            print(f"\n  ⚠️  {err}")
        print("\n" + "=" * 60 + "\n")
        # Don't exit — let the app try to start so the setup wizard works
        # But log the warning clearly

_startup_checks()

# Initialize Flask app
app = Flask(__name__)

# Production runs behind nginx proxying to a unix socket, where WSGI has no
# REMOTE_ADDR. Trust exactly one hop of X-Forwarded-For/-Proto set by our own
# nginx (the socket is unreachable from outside), so request.remote_addr,
# brute-force counters, rate limits, and the audit log see the real client IP.
from werkzeug.middleware.proxy_fix import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
app.config.from_object(Config)
Config.init_app(app)

# Initialize extensions
db.init_app(app)

# CSRF Protection — all POST/PUT/DELETE forms require a token
csrf = CSRFProtect(app)

# Rate Limiting — protects against brute force, spam, and API abuse
limiter = Limiter(
    app=app,
    key_func=get_remote_address,
    default_limits=["200 per minute"],       # Global: 200 req/min per IP
    storage_uri="memory://",                  # In-memory (no Redis needed)
)

login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'admin_login'

# Initialize i18n (makes t() available in all templates)
init_i18n(app)

# Initialize page builder block renderer (makes render_blocks filter available in templates)
init_block_renderer(app)

# Register absolute_url as a Jinja filter so templates can normalize any
# stored URL (relative or absolute) to absolute form. Used by og:image,
# twitter:image, canonical, and anywhere else a URL must be absolute.
app.jinja_env.filters['absolute_url'] = schema_org.absolute_url


# ============================================================================
# SECURITY HELPERS
# ============================================================================

def validate_password(password):
    """Validate password strength based on Config policy.
    Returns (is_valid: bool, error_message: str).
    """
    if len(password) < app.config.get('PASSWORD_MIN_LENGTH', 8):
        return False, t('flash.password_min_length')
    if app.config.get('PASSWORD_REQUIRE_UPPERCASE', True) and not re.search(r'[A-Z]', password):
        return False, t('flash.password_needs_uppercase')
    if app.config.get('PASSWORD_REQUIRE_LOWERCASE', True) and not re.search(r'[a-z]', password):
        return False, t('flash.password_needs_lowercase')
    if app.config.get('PASSWORD_REQUIRE_DIGIT', True) and not re.search(r'[0-9]', password):
        return False, t('flash.password_needs_digit')
    return True, ''


def escape_like(value):
    """Escape SQL LIKE wildcards (% and _) in user input."""
    return value.replace('\\', '\\\\').replace('%', r'\%').replace('_', r'\_')


ALLOWED_PER_PAGE = (20, 50, 100)

def get_per_page(default=20):
    """Get validated per_page from request args."""
    per_page = request.args.get('per_page', default, type=int)
    return per_page if per_page in ALLOWED_PER_PAGE else default


# Python ints are unbounded, so `?page=99999999999999999999` used to flow
# straight into OFFSET and crash PostgreSQL with `bigint out of range` → 500,
# reachable by any anonymous visitor. Clamp to a sane window instead.
MAX_PAGE = 1_000_000

def get_page(default=1):
    """Get validated ?page= from request args, clamped to [1, MAX_PAGE]."""
    page = request.args.get('page', default, type=int)
    if page is None:
        page = default
    return min(max(1, page), MAX_PAGE)


# ── Site timezone ───────────────────────────────────────────────────────────
# All datetimes are STORED as naive UTC (utc_now()); nothing about storage
# changes here. The setting `site_timezone` (IANA name, e.g. 'Europe/Berlin')
# only affects DISPLAY: the i18n date filters, admin JSON list endpoints,
# CSV exports and public post dates all convert through to_site_time().

from zoneinfo import ZoneInfo, available_timezones


def get_site_timezone():
    """tzinfo for the configured site timezone, cached per request so a page
    that formats 50 audit rows doesn't hit the settings table 50 times."""
    try:
        cached = getattr(g, '_site_tzinfo', None)
        if cached is not None:
            return cached
    except RuntimeError:  # outside request context (CLI, tests)
        cached = None
    name = Settings.get('site_timezone', 'UTC') or 'UTC'
    if name == 'UTC':
        tz = timezone.utc
    else:
        try:
            tz = ZoneInfo(name)
        except Exception:
            tz = timezone.utc
    try:
        g._site_tzinfo = tz
    except RuntimeError:
        pass
    return tz


def to_site_time(dt):
    """Convert a stored naive-UTC datetime to the site timezone.
    Returns a naive datetime (site wall-clock) for display; None passes
    through. Already-aware datetimes are converted from their own zone."""
    if dt is None:
        return None
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(get_site_timezone()).replace(tzinfo=None)


def timezone_choices():
    """Sorted IANA Region/City names for the settings dropdown, with UTC on
    top. Aliases like Etc/*, US/* and posix/right variants are filtered out
    to keep the list clean."""
    zones = [z for z in available_timezones()
             if '/' in z and not z.startswith(('Etc/', 'US/', 'SystemV/',
                                              'posix/', 'right/'))]
    return ['UTC'] + sorted(zones)


# Let the i18n date filters (format_date / format_datetime / format_time)
# convert through the site timezone. Registered as a callback to avoid an
# import cycle between i18n.py and models.py.
import i18n as _i18n_module
_i18n_module.set_timezone_converter(to_site_time)


# ── Upload filename handling ────────────────────────────────────────────────
# The old one-liner `secure_filename(file.filename)` had three problems:
#   1. Non-Latin names lost their extension: secure_filename() drops every
#      non-ASCII character, so 'тест.png' / '写真.jpg' collapsed to just
#      'png' / 'jpg' — splitext() then saw no extension at all and the file
#      landed on disk as 'png_20260726_154949' (extension-less).
#   2. Length was never trimmed. The filesystem limit is 255 BYTES, not
#      characters, so a long Cyrillic name (2 bytes/char) or CJK name
#      (3 bytes/char) hit OSError far earlier than a Latin one — and the
#      timestamp suffix added 16 more characters on top.
#   3. All non-Latin names of the same type collapsed to the same base
#      ('png', 'pdf'), so two admins uploading within the same second would
#      silently overwrite each other's file.
# Fix: split the extension off the ORIGINAL name first, sanitize the parts
# separately (with slugify-transliteration as fallback, same approach as the
# public download_slug), cap the length with headroom, and check the target
# directory for collisions before saving.

UPLOAD_BASENAME_MAX = 180  # ASCII after sanitizing → chars == bytes; leaves
                           # room for '_YYYYMMDD_HHMMSS', '-NN' collision
                           # counters and variant suffixes under 255 bytes.

def make_upload_filename(original_filename):
    """Build a safe, extension-preserving on-disk filename (without
    timestamp). Always returns ASCII of bounded length."""
    # Split on the LAST dot, mirroring allowed_file()/allowed_document() —
    # os.path.splitext treats leading dots differently ('....png' → no ext).
    original_filename = original_filename or ''
    if '.' in original_filename:
        base, _, ext = original_filename.rpartition('.')
        ext = '.' + ext
    else:
        base, ext = original_filename, ''

    # Sanitize the extension on its own: keep alphanumerics only.
    ext = re.sub(r'[^A-Za-z0-9]', '', ext).lower()
    ext = f'.{ext[:10]}' if ext else ''

    # Sanitize the base name; fall back to transliteration for non-ASCII
    # ('Закат над морем' → 'zakat-nad-morem'), then to a generic stub.
    name = secure_filename(base)
    if not name:
        name = slugify(base)
    if not name:
        name = 'file'

    return f'{name[:UPLOAD_BASENAME_MAX]}{ext}'


def unique_upload_path(directory, filename):
    """Return a filename that does not yet exist in `directory`, appending
    '-2', '-3', … before the extension when needed. Protects against two
    uploads producing the same sanitized name in the same second."""
    name, ext = os.path.splitext(filename)
    candidate = filename
    counter = 2
    while os.path.exists(os.path.join(directory, candidate)):
        candidate = f'{name}-{counter}{ext}'
        counter += 1
    return candidate


def validate_next_url(next_url):
    """Validate redirect URL to prevent open redirect attacks.
    Only allows relative URLs starting with '/'.
    Returns safe URL or None.
    """
    if not next_url:
        return None
    parsed = urlparse(next_url)
    # Block absolute URLs (with scheme or netloc) — prevents redirecting to external sites
    if parsed.netloc or parsed.scheme:
        return None
    # Must start with /
    if not next_url.startswith('/'):
        return None
    return next_url


def _is_editor_path(path):
    """True if a same-origin admin path points at an editor screen: a
    new/edit/translate form or the visual Page Builder.

    Cancel and post-save redirects must never land on these. A stale
    referrer would otherwise turn Cancel into a confusing jump into another
    editor: e.g. a post created via /admin/posts/new redirects to its edit
    screen whose HTTP referrer is still /admin/posts/new, so a
    referrer-derived Cancel would reopen the empty "new post" form —
    looking like an "undo" to the user.
    """
    p = (path or '').split('?', 1)[0].rstrip('/')
    return p.endswith(('/new', '/edit', '/translate', '/builder'))


def _clean_return_to(raw):
    """Normalise a return_to/referrer candidate to a safe same-origin admin path.

    Returns '' when the candidate is off-site, not an admin page, an editor
    screen (new/edit/translate/builder), or contains header-injection
    characters. Editor screens are excluded on purpose: leaving an edit
    screen should land on the list (with its filters), never bounce the user
    into another editor.
    """
    target = (raw or '').strip()
    if not target:
        return ''
    parsed = urlparse(target)
    if parsed.netloc and parsed.netloc != request.host:
        return ''
    path = parsed.path + (('?' + parsed.query) if parsed.query else '')
    if not path.startswith('/admin/'):
        return ''
    if '\n' in path or '\r' in path:
        return ''
    if _is_editor_path(path):
        return ''
    return path


def _safe_return_to(fallback_endpoint, **fallback_kwargs):
    """Redirect to user-supplied return_to from form, restricted to same-origin admin pages."""
    target = _clean_return_to(request.form.get('return_to'))
    if target:
        return redirect(target)
    return redirect(url_for(fallback_endpoint, **fallback_kwargs))


def _is_inline_save():
    """True when an edit form was submitted by the shared inline AJAX saver."""
    return request.headers.get('X-Requested-With') == 'XMLHttpRequest'


def _inline_saved(message, fallback_endpoint, **fallback_kwargs):
    """Finish a successful inline edit save.

    - Inline AJAX submit: stay on the edit screen and return JSON so the page
      shows the standard admin success toast in place — no redirect, so the
      editor never gets bounced into the Page Builder or back to the list.
    - Plain POST (no JS / progressive enhancement fallback): keep the previous
      behaviour of flashing the message and redirecting to a safe return_to.
    """
    if _is_inline_save():
        return jsonify(success=True, message=message)
    flash(message, 'success')
    return _safe_return_to(fallback_endpoint, **fallback_kwargs)


def _is_agent_json():
    """True for WebMCP / agent callers that asked for the JSON contract.

    Human forms never send _format=json — only the agent tool layer does — so
    this flag reliably separates the two callers on a shared route.
    """
    return request.form.get('_format') == 'json' or request.args.get('format') == 'json'


def _agent_safe_status(status):
    """Agents may save drafts but never publish.

    Server-side enforcement of the WebMCP contract: when the caller is an
    agent (_format=json), a requested 'published' status is silently demoted
    to 'draft'. Human form posts pass through untouched. Publishing stays a
    human-only action in the admin UI.
    """
    if _is_agent_json() and status == 'published':
        return 'draft'
    return status


def _after_create(edit_endpoint, message, **edit_kwargs):
    """Finish a successful create / translate.

    Keeps "Save" meaning save: instead of dropping the user back on the list,
    land them in the editor of the freshly created item. The record now exists
    (so re-saving updates it in place via the inline saver instead of creating
    a duplicate, and the Page Builder becomes reachable). The success message is
    flashed and shows as the usual admin toast on the editor. The original
    return_to is threaded through, so a later Cancel still returns to the list
    the user came from — filters intact.
    """
    flash(message, 'success')
    return_to = _clean_return_to(request.form.get('return_to'))
    # Agent / WebMCP callers ask for JSON and get a clean contract instead of a
    # redirect they would have to scrape an id out of. Same create path, no dupe.
    if request.form.get('_format') == 'json' or request.args.get('format') == 'json':
        new_id = next((v for v in edit_kwargs.values() if isinstance(v, int)), None)
        return jsonify({
            'success': True,
            'id': new_id,
            'edit_url': url_for(edit_endpoint, **edit_kwargs),
        })
    return redirect(url_for(edit_endpoint, return_to=return_to or None, **edit_kwargs))


def sanitize_html(html_content):
    """Sanitize HTML content from editor before saving to DB.
    Removes script tags, event handlers, and dangerous elements
    while preserving rich formatting from TinyMCE/editor.
    """
    if not html_content:
        return html_content
    try:
        import bleach
        ALLOWED_TAGS = [
            'p', 'br', 'strong', 'b', 'em', 'i', 'u', 's', 'a', 'ul', 'ol', 'li',
            'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'code', 'pre',
            'img', 'table', 'thead', 'tbody', 'tfoot', 'tr', 'th', 'td', 'hr',
            'figure', 'figcaption', 'sup', 'sub', 'abbr', 'mark', 'del', 'ins',
            'div', 'span', 'section', 'article', 'aside', 'header', 'footer',
            'details', 'summary', 'picture', 'source', 'video', 'audio',
            'iframe',  # Needed for YouTube/Vimeo embeds
        ]
        # NOTE on `style`: intentionally NOT allowed anywhere. Content styling
        # must come from the active theme, never from inline styles pasted in
        # from Word / Docs / chats. Allowing 'style' without a css_sanitizer
        # also made bleach 6 empty the value but keep the attribute, leaving
        # ugly `<p style="">` artifacts in the DB. With 'style' out of the
        # whitelist bleach drops the whole attribute -> clean `<p>`, while
        # `id`/`class` (anchors like <p id="multilingual-system">, Prism
        # language-* classes, ap-* block classes) survive untouched.
        # Old content with `style=""` self-heals on its next save.
        ALLOWED_ATTRS = {
            '*': ['class', 'id'],
            'a': ['href', 'title', 'target', 'rel'],
            'img': ['src', 'alt', 'title', 'width', 'height', 'loading'],
            'iframe': ['src', 'width', 'height', 'frameborder', 'allowfullscreen',
                        'allow', 'loading', 'title'],
            'video': ['src', 'controls', 'width', 'height', 'poster', 'preload'],
            'audio': ['src', 'controls', 'preload'],
            'source': ['src', 'type', 'srcset', 'sizes', 'media'],
            'td': ['colspan', 'rowspan'],
            'th': ['colspan', 'rowspan', 'scope'],
            'abbr': ['title'],
            'div': ['class', 'id', 'data-form-id', 'data-block-type'],
            'span': ['class', 'id'],
        }
        return bleach.clean(
            html_content,
            tags=ALLOWED_TAGS,
            attributes=ALLOWED_ATTRS,
            strip=True,
            protocols=['http', 'https', 'mailto']
        )
    except ImportError:
        # bleach not installed — strip <script> tags as basic fallback
        import re as _re
        html_content = _re.sub(r'<script[^>]*>.*?</script>', '', html_content, flags=_re.DOTALL | _re.IGNORECASE)


def sanitize_svg(filepath):
    """Sanitize an SVG file by removing dangerous elements.
    
    SVG files can contain <script>, onclick, onload, and other executable content.
    This strips all dangerous elements while preserving the visual SVG content.
    
    Uses defusedxml if available (safer XML parsing), falls back to regex.
    """
    try:
        with open(filepath, 'r', encoding='utf-8') as f:
            content = f.read()
    except (UnicodeDecodeError, IOError):
        return False  # Not a valid text SVG
    
    # Dangerous elements and attributes to remove
    import re as _re
    
    # Remove <script> tags and content
    content = _re.sub(r'<script[^>]*>.*?</script>', '', content, flags=_re.DOTALL | _re.IGNORECASE)
    content = _re.sub(r'<script[^>]*/>', '', content, flags=_re.IGNORECASE)
    
    # Remove event handler attributes (onclick, onload, onmouseover, etc.)
    content = _re.sub(r'\s+on\w+\s*=\s*["\'][^"\']*["\']', '', content, flags=_re.IGNORECASE)
    content = _re.sub(r"\s+on\w+\s*=\s*['][^']*[']", '', content, flags=_re.IGNORECASE)
    
    # Remove javascript: URLs in href/xlink:href
    content = _re.sub(r'(href\s*=\s*["\'])javascript:[^"\']*(["\'])', r'\1#\2', content, flags=_re.IGNORECASE)
    content = _re.sub(r'(xlink:href\s*=\s*["\'])javascript:[^"\']*(["\'])', r'\1#\2', content, flags=_re.IGNORECASE)
    
    # Remove data: URLs (can contain encoded scripts)
    content = _re.sub(r'(href\s*=\s*["\'])data:[^"\']*(["\'])', r'\1#\2', content, flags=_re.IGNORECASE)
    
    # Remove <foreignObject> (can embed HTML with scripts)
    content = _re.sub(r'<foreignObject[^>]*>.*?</foreignObject>', '', content, flags=_re.DOTALL | _re.IGNORECASE)
    
    # Remove <use> with external references (can load remote SVGs)
    content = _re.sub(r'<use[^>]*href\s*=\s*["\']https?://[^"\']*["\'][^>]*/>', '', content, flags=_re.IGNORECASE)
    
    # Remove <set> and <animate> with event attributes
    content = _re.sub(r'<set[^>]*on\w+[^>]*/>', '', content, flags=_re.IGNORECASE)
    
    # Write sanitized SVG back
    try:
        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(content)
        return True
    except IOError:
        return False
        html_content = _re.sub(r'\s+on\w+\s*=\s*["\'][^"\']*["\']', '', html_content, flags=_re.IGNORECASE)
        return html_content


def _extract_svg_dimensions(filepath):
    """Extract intrinsic pixel dimensions from an SVG file.

    Pillow cannot open SVGs (they are vector XML, not raster), so we parse
    the file as XML and read the root <svg> element's width/height/viewBox
    attributes. Used by process_uploaded_image() and the one-time backfill.

    Returns (width, height) as ints, or (None, None) if dimensions cannot
    be determined. Never raises — failures degrade silently and the caller
    keeps the existing NULL state, which is safe.

    Strategy:
      1. Parse the file as XML, find the root <svg> element.
      2. Try root width/height attributes if they're in pixels (or unitless).
      3. Otherwise fall back to viewBox (last two values).
      4. Anything weird (percentages, em, no viewBox) → (None, None).
    """
    try:
        # Stdlib ET. We only read attributes of the root element — we don't
        # expand entities, follow includes, or evaluate anything. Files in
        # /static/uploads have already passed sanitize_svg(), so they're
        # known-safe text that we wrote ourselves.
        import xml.etree.ElementTree as ET
        tree = ET.parse(filepath)
        root = tree.getroot()
    except Exception:
        return (None, None)

    # SVG root tag may be namespaced as '{http://www.w3.org/2000/svg}svg'
    tag = root.tag.split('}', 1)[-1].lower() if '}' in root.tag else root.tag.lower()
    if tag != 'svg':
        return (None, None)

    def _parse_length(value):
        """Parse SVG length like '100', '100px', '100.5px'.
        Returns positive int pixels, or None for non-pixel units (%, em,
        ex, pt, …) or unparseable values.
        """
        if not value:
            return None
        v = value.strip()
        if not v:
            return None
        m = re.match(r'^([+-]?\d+(?:\.\d+)?)(\s*px)?\s*$', v, re.IGNORECASE)
        if not m:
            return None
        try:
            num = float(m.group(1))
            return int(round(num)) if num > 0 else None
        except (ValueError, TypeError):
            return None

    # Try explicit width/height first
    w = _parse_length(root.get('width'))
    h = _parse_length(root.get('height'))
    if w and h:
        return (w, h)

    # Fall back to viewBox: "min-x min-y width height"
    # Separators may be spaces, commas, or both.
    viewbox = root.get('viewBox') or root.get('viewbox')
    if viewbox:
        parts = re.split(r'[\s,]+', viewbox.strip())
        if len(parts) == 4:
            try:
                vb_w = float(parts[2])
                vb_h = float(parts[3])
                if vb_w > 0 and vb_h > 0:
                    return (int(round(vb_w)), int(round(vb_h)))
            except (ValueError, TypeError):
                pass

    return (None, None)



def validate_pg_identifier(name):
    """Validate PostgreSQL identifier (database name, user) to prevent SQL injection."""
    if not name or not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]{0,62}$', name):
        raise ValueError(f'Invalid database identifier: {name}')
    return name

# Add markdown filter to Jinja2
@app.template_filter('markdown')
def markdown_filter(text):
    """Convert markdown to HTML (with sanitization against XSS)"""
    if not text:
        return ''
    html = markdown.markdown(text, extensions=['extra', 'nl2br', 'sane_lists'])
    # Sanitize output to prevent XSS from user-supplied markdown
    return sanitize_html(html)


def _ad(key, **params):
    """Audit detail: store translatable message as JSON.
    
    Usage: _ad('created_post', title=post.title, lang=post.language)
    Returns: '{"_t": "created_post", "title": "My Post", "lang": "en"}'
    """
    return json.dumps({"_t": key, **params}, ensure_ascii=False)


# ── Custom CSS: validation + render-time sanitization ──
# Block any url(...) or @import in user-supplied CSS. Custom CSS is meant
# for styling tweaks (spacing, colors, typography, borders, hover, animations)
# on classes that blocks expose via Page Builder's "CSS class" field. It is
# not a mechanism for loading external resources — that stays with the
# theme/Media Library/Page Builder block settings.
_CUSTOM_CSS_URL_RE = re.compile(
    r"(?:url\s*\(|@import\b)",
    re.IGNORECASE
)


def _validate_custom_css(text):
    """Validate Custom CSS — rejects url() and @import entirely.
    
    Returns (is_valid: bool, error_translation_key: str|None).
    Empty input is valid (clears the setting).
    """
    if not text:
        return True, None
    if _CUSTOM_CSS_URL_RE.search(text):
        return False, 'flash.custom_css_url_not_supported'
    return True, None


@app.template_filter('sanitize_css')
def sanitize_css_filter(text):
    """Neutralize </style breakout in user-supplied CSS before injecting
    into a <style> tag.
    
    HTML5 parsers close a <style> element on the literal sequence </style
    (case-insensitive). Replacing </style with <\\/style breaks the literal
    match — the backslash is not interpreted as an escape in HTML, so the
    parser does not recognize it as a closing tag. The sequence </style has
    no meaningful use in real CSS, so this transformation is side-effect-free
    for legitimate stylesheets.
    
    This is defense-in-depth: external URLs are already blocked at save time
    by _validate_custom_css(), but render-time neutralization protects against
    any value that might reach the template through other paths.
    """
    if not text:
        return ''
    return re.sub(r'</style', r'<\\/style', text, flags=re.IGNORECASE)


def _custom_css_value_and_etag():
    """Return (css_text, etag_string) for the current site_custom_css setting.
    ETag is the MD5 of the CSS content, enabling conditional GET (304 Not Modified)
    and URL-based cache-busting (?v=<etag>).
    """
    css = Settings.get('site_custom_css', '') or ''
    etag = hashlib.md5(css.encode('utf-8')).hexdigest()
    return css, etag


@app.context_processor
def _inject_custom_css_version():
    """Expose custom_css_version() to all templates for cache-busting the
    public /custom.css endpoint in <link> tags."""
    def custom_css_version():
        _, etag = _custom_css_value_and_etag()
        return etag[:8]
    return {'custom_css_version': custom_css_version}


# In-memory cache for static asset versions: {filename: (mtime, hash)}.
# Invalidates automatically when the file changes on disk (mtime check),
# so updates ship instantly without restarting the app.
_static_version_cache = {}


@app.context_processor
def _inject_static_version_helper():
    """Expose static_v() to all templates for cache-busting static CSS/JS.

    static_v('css/core.css') -> /static/css/core.css?v=<8-char content hash>

    The version is derived from file content, so the URL only changes when
    the file actually changes. Combined with the long-lived
    'Cache-Control: public, max-age=31536000, immutable' header on /static,
    this gives returning visitors instant updates AND a perfect
    "efficient cache policy" Lighthouse audit.
    """
    def static_v(filename):
        try:
            path = os.path.normpath(os.path.join(app.static_folder, filename))
            # Safety: never resolve outside the static folder
            if not path.startswith(os.path.abspath(app.static_folder)):
                return url_for('static', filename=filename)
            mtime = os.path.getmtime(path)
            cached = _static_version_cache.get(filename)
            if cached and cached[0] == mtime:
                version = cached[1]
            else:
                with open(path, 'rb') as f:
                    version = hashlib.md5(f.read()).hexdigest()[:8]
                _static_version_cache[filename] = (mtime, version)
            return url_for('static', filename=filename) + '?v=' + version
        except OSError:
            # Missing file: fall back to a plain URL, let the 404 surface
            return url_for('static', filename=filename)
    return {'static_v': static_v}


@app.context_processor
def _inject_admin_back_helpers():
    """Expose a single, shared "safe back target" to admin templates.

    Both helpers reuse _clean_return_to(), so the Page Builder is always
    excluded and same-origin admin filters (e.g. ?status=draft) are preserved.
    Templates no longer hand-roll `request.args.get('return_to') or
    request.referrer or url_for(...)`, which is what previously let the builder
    URL leak into the Cancel button and the hidden return_to field.

      admin_back_url('admin_pages_list')  -> href for Cancel/Back links
      admin_back_value()                  -> value for hidden <input name=return_to>
    """
    def admin_back_url(fallback_endpoint, **kwargs):
        candidate = request.args.get('return_to') or request.referrer
        return _clean_return_to(candidate) or url_for(fallback_endpoint, **kwargs)

    def admin_back_value():
        candidate = request.args.get('return_to') or request.referrer
        return _clean_return_to(candidate)

    return {'admin_back_url': admin_back_url, 'admin_back_value': admin_back_value}


@app.route('/custom.css')
def public_custom_css():
    """Serve user-supplied Custom CSS as an external stylesheet.
    
    Benefits over inlining <style>{{ ... }}</style>:
      - Clean HTML: no template expressions inside <style> (no dev-tool warnings)
      - Browser caching across pages via ETag + long max-age
      - Cache-busting automatic: URL includes ?v=<hash>, URL changes when CSS changes
    
    Security notes:
      - External URLs are blocked at save time by _validate_custom_css() per
        the privacy policy (no third-party resource loading).
      - Served as text/css — no HTML parsing context, so </style breakout is
        not applicable here (unlike the historical <style> injection path).
    """
    css, etag = _custom_css_value_and_etag()
    quoted_etag = f'"{etag}"'
    
    # Conditional GET: if the client already has this version, reply 304
    if request.headers.get('If-None-Match') == quoted_etag:
        response = make_response('', 304)
        response.headers['ETag'] = quoted_etag
        return response
    
    response = make_response(css)
    response.headers['Content-Type'] = 'text/css; charset=utf-8'
    response.headers['ETag'] = quoted_etag
    # max-age=0 + must-revalidate forces browsers to revalidate with ETag on
    # every navigation. The ?v=<hash> query param in the <link> URL also
    # ensures a fresh URL whenever the content changes, so CDN/proxy caches
    # bust automatically.
    response.headers['Cache-Control'] = 'public, max-age=0, must-revalidate'
    return response


@app.template_filter('translate_detail')
def translate_detail_filter(detail_str):
    """Translate audit log detail string.
    
    New entries (JSON with _t key) → translated via i18n.
    Old entries (plain strings) → shown as-is (backward compatible).
    """
    if not detail_str:
        return ''
    try:
        data = json.loads(detail_str)
        if isinstance(data, dict) and '_t' in data:
            key = 'audit_log.detail_' + data.pop('_t')
            template = t(key)
            if template != key:  # translation found
                return template.format(**data)
            # Fallback: return key name + params for debugging
            return template
    except (json.JSONDecodeError, TypeError, ValueError, KeyError):
        pass
    return detail_str

@login_manager.user_loader
def load_user(user_id):
    user = User.query.get(int(user_id))
    if not user:
        return None
    # Deactivated users lose their session immediately
    if not user.is_active_account:
        return None
    # Session invalidation: if password was changed, force re-login
    stored_version = session.get('_session_version')
    if stored_version is not None and stored_version != user.session_version:
        return None
    return user

# Context processor - make settings available in all templates
@app.context_processor
def inject_settings():
    """Make settings dict available in all templates"""
    settings_dict = {}
    try:
        for s in Settings.query.all():
            settings_dict[s.key] = s.value
    except Exception:
        pass  # Database not initialized yet
    # Demo mode: a guest's session theme choice overrides the site default
    # for their own requests only (public pages read the theme from here)
    if app.config.get('DEMO_MODE'):
        _cand = session.get('demo_active_theme')
        if _cand and _theme_exists(_cand):
            settings_dict['active_theme'] = _cand
    # Whether the AI Assistant has a working provider key. The WebMCP layer
    # uses this to register the ai_* tools only when they can actually run,
    # so every tool an agent sees is a tool that works.
    ai_ready = False
    try:
        _prov, _key, _err = get_ai_config()
        ai_ready = _err is None
    except Exception:
        pass
    return {'settings': settings_dict, 'ai_provider_ready': ai_ready}


# ── License integrity & revalidation ─────────────────────────────────────────

_LICENSE_INTEGRITY_SALT = b'AP_LIC_integrity_v1_9c4e'

def _compute_license_signature(encrypted_key=None, last_checked=None):
    """Compute HMAC signature to detect tampering with license_valid in database.
    
    Uses SECRET_KEY + hardcoded salt + encrypted key + last_checked timestamp.
    If someone manually edits license_valid to 'true' in SQLite,
    the signature won't match and is_licensed will return False.

    Values can be passed in explicitly so the signature is computed over the exact
    data being written in the same transaction (avoids re-reading mid-update).
    """
    if encrypted_key is None:
        encrypted_key = Settings.get('license_key_encrypted', '')
    if last_checked is None:
        last_checked = Settings.get('license_last_checked', '')
    if not encrypted_key:
        return ''
    msg = f"{encrypted_key}|{last_checked}|valid".encode()
    return hmac_module.new(
        app.config['SECRET_KEY'].encode() + _LICENSE_INTEGRITY_SALT,
        msg,
        hashlib.sha256
    ).hexdigest()


def _verify_license_integrity():
    """Verify that license data hasn't been tampered with in the database.
    
    Returns True only if:
    - license_valid is 'true'
    - license_key_encrypted exists
    - stored signature matches computed signature
    """
    if Settings.get('license_valid', 'false') != 'true':
        return False
    
    encrypted_key = Settings.get('license_key_encrypted', '')
    if not encrypted_key:
        return False
    
    stored_sig = Settings.get('_license_sig', '')
    if not stored_sig:
        return False
    
    expected_sig = _compute_license_signature()
    if not hmac_module.compare_digest(stored_sig, expected_sig):
        return False
    
    return True


def _save_license_result(result, commit=True):
    """Centralized: save license validation result with integrity signature.
    
    Called from wizard, settings page, and revalidation hook.
    Stores the validation result AND a tamper-evident HMAC signature.

    An 'inconclusive' result means we couldn't get an authoritative answer
    (server unreachable, timeout, non-200, malformed). In that case we keep the
    current stored state — a paying user must never be downgraded because of a
    transient outage; a later check resolves it.
    """
    if result.get('inconclusive'):
        return

    valid = result.get('valid', False)
    last_checked = to_site_time(utc_now()).strftime('%Y-%m-%d %H:%M')
    encrypted_key = Settings.get('license_key_encrypted', '')

    # All fields + signature written in ONE transaction. Committing the timestamp
    # and the signature separately would leave a window where the signature is
    # computed over a different timestamp than the one stored — a concurrent
    # reader would then see a valid license as "tampered" and show the footer link.
    Settings.set_many({
        'license_valid': 'true' if valid else 'false',
        'license_platform': result.get('platform', ''),
        'license_plan': result.get('plan', 'standard'),
        'license_last_checked': last_checked,
        '_license_sig': _compute_license_signature(encrypted_key, last_checked) if valid else '',
    }, commit=commit)


def _save_license_invalid(commit=True):
    """Mark license as invalid and clear signature."""
    Settings.set_many({
        'license_valid': 'false',
        '_license_sig': '',
    }, commit=commit)


@app.before_request
def license_revalidation_hook():
    """Periodically re-validate the license against the update server.
    
    Runs on admin pages only. If the stored signature doesn't match,
    re-validate immediately rather than just marking invalid — the
    signature might be stale from a previous bug or server migration.
    Every 7 days the key is also routinely re-checked.
    """
    if not request.path.startswith('/admin'):
        return
    
    # Skip static files, API calls that need to be fast, and setup
    if request.path.startswith('/admin/static') or request.path == '/setup':
        return
    
    try:
        encrypted_key = Settings.get('license_key_encrypted', '')
        if not encrypted_key:
            return  # No license key installed
        
        claimed_valid = Settings.get('license_valid', 'false') == 'true'
        
        # If signature doesn't match OR periodic check is due → re-validate
        needs_check = False
        if claimed_valid and not _verify_license_integrity():
            needs_check = True  # Signature mismatch — re-validate instead of killing
        elif should_revalidate(Settings.get, days=7):
            needs_check = True  # Periodic revalidation
        
        if not needs_check:
            return
        
        # Decrypt the key and revalidate with the update server
        raw_key = decrypt_smtp_password(encrypted_key)
        if not raw_key:
            _save_license_invalid(commit=True)
            return
        
        validator = LicenseValidator(app.config.get('UPDATE_SERVER_URL', ''))
        site_url = request.url_root.rstrip('/')
        result = validator.validate(raw_key, site_url)
        _save_license_result(result, commit=True)
    except Exception:
        db.session.rollback()  # Clean up dirty session to prevent PendingRollbackError


@app.context_processor
def inject_license_status():
    """Make license status available in all templates (for 'Powered by' link)"""
    try:
        licensed = _verify_license_integrity()
        needs_revalidation = licensed and should_revalidate(Settings.get, days=7)
    except Exception:
        licensed = False
        needs_revalidation = False
    return {
        'is_licensed': licensed,
        'cms_version': get_current_version(),
        'needs_license_revalidation': needs_revalidation,
    }


# Auto cache-busting for static files (CSS, JS, images)
# Appends ?v=<modification_time> so browsers always load fresh files after changes
@app.url_defaults
def static_cache_buster(endpoint, values):
    """Automatically add file modification timestamp to static file URLs."""
    if endpoint == 'static':
        filename = values.get('filename')
        if filename:
            filepath = os.path.join(app.static_folder, filename)
            try:
                values['v'] = int(os.path.getmtime(filepath))
            except OSError:
                pass


# Set admin language for each request (for i18n)
@app.before_request
def set_request_language():
    """Set admin interface language based on user preference or default."""
    lang = 'en'
    if current_user.is_authenticated and current_user.admin_language:
        lang = current_user.admin_language
    # Demo mode: the guest's language choice lives in their own browser
    # session (see demo_mode_guard), so it overrides the shared demo
    # account's DB value and never leaks to the next visitor.
    if app.config.get('DEMO_MODE'):
        lang = session.get('demo_admin_language') or lang
    set_admin_language(lang)


# Make sessions permanent (uses PERMANENT_SESSION_LIFETIME from config)
@app.before_request
def make_session_permanent():
    session.permanent = True


# CSRF error handler — shows a friendly message instead of generic 400
@app.errorhandler(CSRFError)
def handle_csrf_error(e):
    """Handle CSRF token errors with a user-friendly message."""
    if (request.headers.get('X-Requested-With') == 'XMLHttpRequest'
            or request.is_json or request.path.startswith('/api/')):
        return jsonify(success=False, error=t('flash.session_expired')), 400
    flash(t('flash.session_expired'), 'error')
    return redirect(request.referrer or url_for('admin_dashboard'))


@app.errorhandler(429)
def handle_rate_limit_error(e):
    """Handle rate limit exceeded — too many requests."""
    if request.is_json or request.path.startswith('/api/'):
        return jsonify({'error': t('api.too_many_requests')}), 429
    flash(t('flash.too_many_requests'), 'error')
    return redirect(request.referrer or url_for('admin_dashboard'))


# Periodic cleanup of old login attempts and audit logs
@app.before_request
def periodic_cleanup():
    """Run cleanup tasks occasionally (~1 in 500 requests)."""
    import random
    if random.random() < 0.002:
        try:
            LoginAttempt.clear_old(days=7)
        except Exception:
            db.session.rollback()


# ============================================================================
# DEMO MODE
# ============================================================================
# Read-only guest mode for public demo installations (DEMO_MODE=true in .env).
#
#   • Blocks EVERY state-changing request (POST/PUT/PATCH/DELETE) app-wide —
#     nothing can be saved, published, uploaded, sent, installed or updated.
#     Because this runs as a global before_request it also covers plugin
#     routes, WebMCP write endpoints and public form/newsletter submissions.
#   • Login/logout (incl. the 2FA step) remain functional so guests can sign
#     in with the prefilled demo credentials.
#   • Password recovery and invitation/registration pages are switched off.
#   • The demo account is (re)created automatically at startup (see
#     _ensure_demo_user) with role 'admin': guests see Settings, Users,
#     Plugins, Tools, AI, Newsletter etc., while owner-only areas (license,
#     granting the admin role) stay hidden — and the write barrier keeps
#     absolutely everything read-only anyway.

DEMO_WRITE_ALLOWED_PATHS = {
    '/admin/login',            # guests must be able to sign in…
    '/admin/login/2fa',        # …including a (theoretical) second factor…
    '/admin/login/2fa/email',
    '/admin/logout',           # …and sign out again
}

DEMO_DISABLED_PAGE_PREFIXES = (
    '/admin/forgot-password',  # no password recovery on a demo
    '/admin/reset-password',
    '/invite/',                # no account registration on a demo
)


def _theme_exists(name):
    """True if `name` is a safe slug naming an installed theme."""
    if not name or not re.fullmatch(r'[A-Za-z0-9_-]+', name):
        return False
    return os.path.isfile(os.path.join(
        app.static_folder, 'css', 'themes', name, 'theme.css'))


def effective_active_theme():
    """Site theme for the current request.
    
    Normally this is exactly Settings.get('active_theme') — byte-for-byte
    the behaviour the call sites had before this helper existed. In demo
    mode only, a guest's own session choice (stored by demo_mode_guard when
    they press Activate on the themes page) overrides it for that guest,
    so every visitor previews their own pick without affecting the site
    default in the DB or what other concurrent visitors see.
    """
    theme = Settings.get('active_theme', 'stellar-origin')
    if app.config.get('DEMO_MODE'):
        cand = session.get('demo_active_theme')
        if cand and _theme_exists(cand):
            theme = cand
    return theme


@app.before_request
def demo_mode_guard():
    """Global write barrier for demo installations. See block comment above."""
    if not app.config.get('DEMO_MODE'):
        return
    
    path = request.path
    
    # Recovery & registration pages are disabled entirely (any HTTP method)
    if path.startswith(DEMO_DISABLED_PAGE_PREFIXES):
        flash(t('demo.feature_disabled'), 'info')
        return redirect(url_for('admin_login'))
    
    # Special case: guests may switch the admin UI language and theme — but
    # only for their own browser session, never on the shared demo account
    # in the database. Otherwise one visitor picking Japanese would flip the
    # interface for every subsequent guest. The rest of the profile form
    # (email, password, display name…) stays blocked like everything else.
    if (request.method == 'POST' and path == '/admin/profile'
            and current_user.is_authenticated):
        new_lang = request.form.get('admin_language', '').strip()
        if new_lang in LANG_CODE_TO_NAME:
            session['demo_admin_language'] = new_lang
            set_admin_language(new_lang)  # flash below already in the new language
        new_theme = request.form.get('admin_theme', '').strip()
        if new_theme in ('light', 'dark', 'system'):
            session['demo_admin_theme'] = new_theme
        flash(t('demo.pref_saved'), 'info')
        return redirect(url_for('admin_profile'))
    
    # Special case: guests may "activate" any site theme — the choice is
    # stored in their own browser session (see effective_active_theme), so
    # each visitor previews their own pick while the site default in the DB
    # stays untouched for everyone else. The custom-CSS form posts to the
    # same endpoint but injects site-wide code, so it stays blocked, as do
    # mode-lock / default-mode changes (also site-wide).
    if (request.method == 'POST' and path == '/admin/themes'
            and current_user.is_authenticated
            and 'custom_css' not in request.form):
        _cand = request.form.get('active_theme', '').strip()
        if _cand and _theme_exists(_cand):
            session['demo_active_theme'] = _cand
            flash(t('demo.theme_preview'), 'info')
            return redirect(url_for('admin_themes'))
        # no valid theme in the form → fall through to the generic block
    
    # Everything is read-only: block all state-changing methods
    if (request.method in ('POST', 'PUT', 'PATCH', 'DELETE')
            and path not in DEMO_WRITE_ALLOWED_PATHS):
        wants_json = (request.is_json
                      or request.headers.get('X-Requested-With') == 'XMLHttpRequest'
                      or path.startswith('/api/')
                      or path.startswith('/admin/api/'))
        if wants_json:
            return jsonify(success=False, error=t('demo.read_only'), demo=True), 403
        flash(t('demo.read_only'), 'info')
        # Bounce back to the page the form was submitted from (same-host only,
        # to avoid an open redirect via a forged Referer header).
        ref = urlparse(request.referrer or '')
        safe_ref = None
        if ref.netloc == request.host and ref.path.startswith('/'):
            safe_ref = ref.path + (f'?{ref.query}' if ref.query else '')
        fallback = url_for('admin_dashboard') if path.startswith('/admin') else url_for('index')
        return redirect(safe_ref or fallback)


@app.context_processor
def inject_demo_mode():
    """Expose the demo flag (+ prefill credentials & session prefs) to templates."""
    if not app.config.get('DEMO_MODE'):
        return {'demo_mode': False, 'demo_pref_language': None, 'demo_pref_theme': None}
    return {
        'demo_mode': True,
        'demo_login_username': app.config.get('DEMO_USERNAME', 'demo'),
        'demo_login_password': app.config.get('DEMO_PASSWORD', 'demo'),
        # Guest's per-session UI preferences (see demo_mode_guard)
        'demo_pref_language': session.get('demo_admin_language'),
        'demo_pref_theme': session.get('demo_admin_theme'),
    }


# ============================================================================
# SECURITY HEADERS
# ============================================================================

@app.after_request
def set_security_headers(response):
    """Set security headers on every response."""
    # API responses must never be indexed: robots.txt now ALLOWS /api/public/
    # (it is an agent-facing surface advertised in /.well-known/webmcp), so
    # crawlers may fetch it — noindex keeps JSON out of search results and
    # turns the old GSC "Blocked due to 4xx" entry into an intentional,
    # clean "excluded by noindex" state.
    if request.path.startswith('/api/'):
        response.headers.setdefault('X-Robots-Tag', 'noindex')
    # Prevent clickjacking
    response.headers['X-Frame-Options'] = 'SAMEORIGIN'
    # Prevent MIME-type sniffing
    response.headers['X-Content-Type-Options'] = 'nosniff'
    # XSS protection for older browsers
    response.headers['X-XSS-Protection'] = '1; mode=block'
    # Referrer policy
    response.headers['Referrer-Policy'] = 'strict-origin-when-cross-origin'
    # Permissions policy
    response.headers['Permissions-Policy'] = 'camera=(), microphone=(), geolocation=()'
    # HSTS (only over HTTPS)
    if request.is_secure or request.headers.get('X-Forwarded-Proto') == 'https':
        response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
    return response


@app.teardown_appcontext
def shutdown_session(exception=None):
    """Clean up database session after each request.
    
    If a transaction failed (e.g. during license validation or settings save),
    the session is left in a 'dirty' state with a pending rollback.
    Without this cleanup, ALL subsequent requests on the same gunicorn worker
    will fail with PendingRollbackError until the worker is restarted.
    
    db.session.remove() rolls back any pending transaction and returns the
    connection to the pool in a clean state.
    """
    if exception:
        db.session.rollback()
    db.session.remove()


# ============================================================================
# HTTP CACHE HEADERS — Improves PageSpeed, reduces server load
# ============================================================================

@app.after_request
def set_cache_headers(response):
    """Set intelligent cache headers based on content type and route.
    
    Strategy:
    - Static files (CSS/JS/images): long cache with immutable hint
    - Public pages/posts: short cache, must revalidate (content can change)
    - Admin routes: no cache (dynamic, authenticated)
    - Sitemap/robots: medium cache
    - API endpoints: no cache
    """
    path = request.path
    
    # Never cache admin pages or API endpoints
    if path.startswith('/admin') or path.startswith('/api'):
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, max-age=0'
        return response
    
    # Static files: aggressive caching (served by Flask in dev, nginx/CDN in prod)
    if path.startswith('/static'):
        response.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
        return response
    
    # Sitemap and robots.txt: cache for 1 hour
    if path in ('/sitemap.xml', '/robots.txt'):
        response.headers['Cache-Control'] = 'public, max-age=3600'
        return response
    
    # RSS/Atom feeds: cache for 15 minutes
    if path.endswith('/feed') or path.endswith('.xml'):
        response.headers['Cache-Control'] = 'public, max-age=900'
        return response
    
    # Public pages and posts: short cache, allow CDN caching
    # s-maxage is for shared caches (CDN), max-age is for browsers
    if response.status_code == 200 and response.content_type and 'text/html' in response.content_type:
        response.headers['Cache-Control'] = 'public, max-age=60, s-maxage=300, must-revalidate'
        # Add Vary header for proper multilingual caching
        response.headers['Vary'] = 'Accept-Language, Cookie'
        return response
    
    return response

@app.before_request
def check_setup_required():
    """If no users exist, redirect everything to /setup.
    This means the user just opens their domain and sees the Wizard."""
    # Allow static files and the setup page itself
    if request.path.startswith('/static') or request.path == '/setup':
        return None
    
    # Allow setup API endpoints (DB test, auto-install, finalize, health)
    if request.path.startswith('/setup/'):
        return None
    
    # Increase upload limit for backup restore (500 MB instead of default 16 MB)
    if request.path == '/admin/backup/restore':
        request.max_content_length = 500 * 1024 * 1024
    
    try:
        if User.query.count() == 0:
            return redirect(url_for('setup'))
    except Exception:
        # Database tables don't exist yet — definitely needs setup
        return redirect(url_for('setup'))
    
    return None


# Query parameters preserved as part of canonical content identity.
# Anything else (utm_*, ref, fbclid, gclid, mc_cid, etc.) is treated
# as tracking metadata and stripped via redirect before the page renders.
#   - 'lang' : selects language version
#   - 'page' : paginates content
_CANONICAL_QUERY_WHITELIST = frozenset({'lang', 'page'})

# Paths exempt from stripping. Admin/setup use legitimate query params
# (search, filters, OAuth); static and api routes don't benefit from
# stripping. /unsubscribe and /confirm-email read ?email= and ?token=
# from the query string and 404 on missing/invalid tokens, so passing
# tracking params there is safe — the endpoint ignores them.
_STRIP_EXEMPT_PREFIXES = ('/admin/', '/static/', '/api/', '/setup')
_STRIP_EXEMPT_PATHS = frozenset({'/sitemap.xml', '/robots.txt', '/custom.css',
                                  '/admin', '/setup',
                                  '/unsubscribe', '/confirm-email'})


@app.before_request
def strip_tracking_params():
    """Redirect visitors arriving with tracking/affiliate query parameters to
    a clean URL before the page renders.

    This runs as a before_request hook because the canonical <link> only
    affects search-engine indexing — the visitor's address bar still holds
    the dirty URL, and any outbound click carries it as Referer. Stripping
    via HTTP redirect makes the dirty URL genuinely disappear from the
    browser.

    Constraints:
      - No DB lookups. The whitelist is a static frozenset; the 'lang'
        parameter is preserved as-is and validated by per-route handlers.
      - No validation of language codes. A request with ?lang=<deactivated>
        must reach its route and 404 normally, not be silently rewritten.
      - No interaction with the /?lang=<default> → / redirect in index().
    """
    # Fast bailouts first — keep this hook cheap and unable to fail.
    if request.method != 'GET':
        return None
    path = request.path
    if path.startswith(_STRIP_EXEMPT_PREFIXES):
        return None
    if path in _STRIP_EXEMPT_PATHS:
        return None
    if not request.args:
        return None
    
    # ?page=1 is the same document as the clean URL — a duplicate that
    # Google demotes with "Duplicate, Google chose different canonical"
    # (seen live on <slug>?page=1). Stripping it needs no DB lookup, so it
    # belongs here with the tracking params. ?page=0 and non-numeric junk
    # keep reaching the route to 404/normalize as before.
    needs_page1_strip = request.args.get('page') == '1'
    
    # Are all current parameters already in the whitelist? Use a generator
    # so we exit on the first non-whitelisted key, and avoid building a dict
    # (which would also silently lose multi-value parameters like ?tag=a&tag=b).
    if not needs_page1_strip and all(k in _CANONICAL_QUERY_WHITELIST for k in request.args.keys()):
        return None
    
    # Something needs stripping. Preserve order and multi-values via items(multi=True).
    from urllib.parse import urlencode
    kept = [(k, v) for k, v in request.args.items(multi=True)
            if k in _CANONICAL_QUERY_WHITELIST and not (k == 'page' and v == '1')]
    qs = ('?' + urlencode(kept)) if kept else ''
    return redirect(path + qs, code=301)


@app.context_processor
def inject_canonical_url():
    """Provide canonical_url() to every public template.
    
    Returns the current page's canonical URL with all non-whitelisted query
    parameters stripped. The strip_tracking_params before_request hook above
    will redirect visitors away from dirty URLs before they reach a template,
    so in normal traffic this function returns request.url unchanged. The
    defensive strip here covers two corner cases:
      - exempt paths (/admin/, /static/, /api/, /setup, /sitemap.xml, etc.)
        that bypass the redirect hook but might still render templates that
        extend public/base.html (e.g. an admin preview of a public page);
      - any future code path that renders a public template via test_client
        or internal request without going through the full before_request
        chain.
    
    Default-language 'lang' parameter is dropped because index() already
    301-redirects /?lang=<default> to /, making /?lang=<default> a URL that
    should never appear as canonical anywhere.
    
    Wrapped in try/except so a malformed request or settings issue cannot
    take down the page render — falls back to request.url.
    """
    def canonical_url():
        try:
            from urllib.parse import urlencode, quote
            
            # Read default language once. Settings.get is safe inside a
            # template render context; the except below catches any
            # unexpected failure.
            default_lang = Settings.get('default_language', 'en')
            
            kept = []
            for k, v in request.args.items(multi=True):
                if k not in _CANONICAL_QUERY_WHITELIST:
                    continue
                # Drop ?lang=<default> — that URL 301-redirects to /, so it
                # must never appear as canonical (would contradict the redirect).
                if k == 'lang' and v == default_lang:
                    continue
                # Drop ?page=1 for the same reason — the strip hook above
                # 301-redirects it to the clean URL.
                if k == 'page' and v == '1':
                    continue
                kept.append((k, v))
            
            qs = ('?' + urlencode(kept)) if kept else ''
            base = request.url_root.rstrip('/')
            # request.path is DECODED by Flask — tag/category names with
            # spaces or non-ASCII would emit an invalid canonical like
            # "/tag/CMS features" (exactly the raw-space URLs GSC reports).
            # Re-encode; '/' stays a separator.
            return f'{base}{quote(request.path, safe="/")}{qs}'
        except Exception:
            # Never let canonical generation break a page render.
            # Fall back to request.url unmodified.
            return request.url
    
    return dict(canonical_url=canonical_url)


# ============================================================================
# DATABASE INITIALIZATION
# ============================================================================

def _finalize_pending_setup():
    """Phase 2 of two-phase setup: read .setup-pending.json, create admin & settings.
    Called on startup after a restart triggered by PostgreSQL setup."""
    pending_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.setup-pending.json')
    if not os.path.exists(pending_path):
        return False
    
    try:
        with open(pending_path, 'r') as f:
            data = json.load(f)
        
        # Create all tables in the (now correct) database
        db.create_all()
        
        # Create admin user (password was already hashed before saving)
        admin = User(
            username=data['username'],
            email=data['email'],
            password=data['password_hash'],
            is_admin=True,
            role='owner',
            display_name=data['username'],
            is_active_account=True
        )
        admin.admin_language = data.get('default_language', 'en')
        db.session.add(admin)
        db.session.commit()
        
        # Initialize default settings
        init_default_settings()
        
        # Save language settings
        default_language = data.get('default_language', 'en')
        if default_language:
            Settings.set('default_language', default_language)
            Settings.set('active_languages', default_language)
        
        # Create default pages & menus
        create_default_pages(admin.id, language=default_language)
        create_default_menu_items()
        
        # Save site info
        if data.get('site_name'):
            Settings.set('site_name', data['site_name'])
        if data.get('site_tagline'):
            Settings.set('site_tagline', data['site_tagline'])
        
        # Process license key
        license_key = data.get('license_key', '').strip()
        if license_key and not Settings.get('license_key_encrypted', ''):
            try:
                encrypted = encrypt_smtp_password(license_key)
                Settings.set('license_key_encrypted', encrypted)
                validator = LicenseValidator(app.config.get('UPDATE_SERVER_URL', ''))
                # Use saved request_url_root or fallback
                site_url = data.get('request_url_root', '').rstrip('/')
                if site_url:
                    result = validator.validate(license_key, site_url)
                    _save_license_result(result)
            except Exception:
                _save_license_invalid()
        
        # Ensure SECRET_KEY is in .env
        env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.env')
        if os.path.exists(env_path):
            with open(env_path, 'r') as f:
                env_content = f.read()
        else:
            env_content = ''
        if 'SECRET_KEY=' not in env_content:
            generated_key = secrets.token_hex(32)
            with open(env_path, 'a') as f:
                f.write(f"\nSECRET_KEY={generated_key}\n")
            app.config['SECRET_KEY'] = generated_key
        
        # Clean up SQLite file left from initial startup (before wizard chose PostgreSQL)
        app_dir = os.path.dirname(os.path.abspath(__file__))
        sqlite_instance = os.path.join(app_dir, 'instance')
        if os.path.isdir(sqlite_instance):
            import shutil
            shutil.rmtree(sqlite_instance, ignore_errors=True)
        
        # Remove pending file — setup complete
        os.remove(pending_path)
        return True
        
    except Exception as e:
        # Log but don't crash — let the user re-attempt setup
        print(f"[AliothPress] Error finalizing pending setup: {e}")
        try:
            os.remove(pending_path)
        except OSError:
            pass
        return False


def _ensure_demo_user():
    """Create or refresh the read-only guest account for DEMO_MODE.
    
    Runs at startup from init_database(). Idempotent: resets the password,
    role and 2FA state on every start, so the publicly advertised demo
    credentials always work even if a previous session or deploy touched
    the account. Role is 'admin' (not 'owner'): guests see every admin
    section, while owner-only areas (license, granting the admin role)
    stay hidden. Actual write protection is enforced globally by
    demo_mode_guard — this account has no special powers beyond viewing.
    """
    username = app.config.get('DEMO_USERNAME', 'demo')
    email = app.config.get('DEMO_EMAIL', 'demo@example.com')
    password = app.config.get('DEMO_PASSWORD', 'demo')
    
    user = User.query.filter(
        (User.username == username) |
        (db.func.lower(User.email) == email.lower())
    ).first()
    if not user:
        user = User(username=username, email=email)
        db.session.add(user)
    
    user.username = username
    user.email = email
    user.password = generate_password_hash(password)
    user.role = 'admin'
    user.is_active_account = True
    if not user.display_name:
        user.display_name = 'Demo'
    # Never let 2FA or stale tokens lock guests out of the demo account
    user.twofa_enabled = False
    user.twofa_secret = ''
    user.invite_token = None
    user.reset_token = None
    db.session.commit()


def init_database():
    """Initialize database with tables and default data.
    
    Uses a file lock so that when Gunicorn starts multiple workers,
    only one worker initializes the DB at a time — the others wait.
    Without this, concurrent db.create_all() on SQLite causes
    'table already exists' errors and 502 Bad Gateway.
    """
    lock_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.db-init.lock')
    lock_file = open(lock_path, 'w')
    try:
        _lock(lock_file)
        
        with app.app_context():
            db.create_all()
            
            # Phase 2: finalize two-phase setup if pending (PostgreSQL restart flow)
            _finalize_pending_setup()
            
            # Check if admin user exists
            if User.query.count() == 0:
                return False  # Redirect to setup
            
            # Migration: ensure first user has 'owner' role (for existing databases)
            first_user = User.query.order_by(User.created_at.asc()).first()
            if first_user and (not first_user.role or first_user.role != 'owner'):
                first_user.role = 'owner'
                first_user.is_active_account = True
                if not first_user.display_name:
                    first_user.display_name = first_user.username
                db.session.commit()
            
            # Initialize default settings
            init_default_settings()
            
            # Create default pages if they don't exist
            admin = User.query.first()
            if admin:
                # Use the site's default language (set during installation)
                default_lang = Settings.get('default_language', 'en')
                create_default_pages(admin.id, language=default_lang)
                create_default_menu_items()
            
            # Demo mode: make sure the public guest account exists and its
            # advertised credentials work (only reachable once installed —
            # the User.query.count() == 0 early-return above guarantees it)
            if app.config.get('DEMO_MODE'):
                _ensure_demo_user()
            
            return True
    finally:
        _unlock(lock_file)
        lock_file.close()


# ============================================================================
# TEMPLATE CONTEXT PROCESSORS
# ============================================================================

def _get_active_languages():
    """Get list of active languages from Settings, filtered from AVAILABLE_LANGUAGES.
    Returns list of dicts [{code, name}, ...] preserving AVAILABLE_LANGUAGES order."""
    active_raw = Settings.get('active_languages', 'en')
    if isinstance(active_raw, str):
        active_codes = {l.strip() for l in active_raw.split(',') if l.strip()}
    else:
        active_codes = {'en'}
    return [lang for lang in AVAILABLE_LANGUAGES if lang['code'] in active_codes]


def _get_active_language_codes():
    """Get set of active language codes."""
    active_raw = Settings.get('active_languages', 'en')
    if isinstance(active_raw, str):
        return {l.strip() for l in active_raw.split(',') if l.strip()}
    return {'en'}


def _is_language_active(lang_code):
    """Check if a given language code is currently active in site settings."""
    if not lang_code:
        return False
    return lang_code in _get_active_language_codes()


def _filter_missing_by_active(missing_languages):
    """Filter missing_languages list to only include active languages."""
    active_codes = _get_active_language_codes()
    return [lang for lang in missing_languages if lang['code'] in active_codes]


def _redirect_hub_default_lang():
    """301-redirect ?lang=<default> on hub/tag/category URLs to the clean URL.

    Same duplicate-content rule as index(): the default language lives on the
    clean URL (no ?lang= tail), every other language keeps its tail. Languages
    stay equal in weight — this only removes the ONE redundant spelling of the
    default-language URL. Preserves pagination (?page=N, N>1; page=1 is
    already stripped by strip_tracking_params before routes run)."""
    if request.args.get('lang') and request.args.get('lang') == Settings.get('default_language', 'en'):
        from urllib.parse import urlencode
        kept = [(k, v) for k, v in request.args.items(multi=True)
                if k != 'lang' and not (k == 'page' and v == '1')]
        qs = ('?' + urlencode(kept)) if kept else ''
        return redirect(request.path + qs, code=301)
    return None


def _build_hub_lang_versions(current_path):
    """Build virtual language versions for hub/aggregate pages (tags, categories).

    These pages don't have content translations, but the language switcher
    should still be visible so visitors can navigate to other languages.
    Links point to the same page with ?lang= parameter.
    """
    active_langs = Settings.get('active_languages', 'en')
    if isinstance(active_langs, str):
        active_langs = [l.strip() for l in active_langs.split(',') if l.strip()]
    else:
        active_langs = ['en']

    class HubLangVersion:
        def __init__(self, lang, path):
            self.language = lang
            self.slug = f'{path.lstrip("/")}?lang={lang}'

    return [HubLangVersion(lang, current_path) for lang in active_langs]


def _build_content_404_versions(slug):
    """Build language switcher entries for a 404 on a content slug (page/post).

    Page and post slugs are unique per language (no ?lang= parameter), so the
    hub-style switcher (which appends ?lang=) produces broken URLs like
    /ru-test?lang=de. Instead, find any Page/Post with this slug — including
    drafts and content in deactivated languages — and return real published
    translations from its translation group, with their real slugs.

    Returns an empty list for a true 404 (slug belongs to nothing in the
    database) — no translation group means no other-language equivalents
    exist, so the switcher should not appear.
    """
    obj = (Page.query.filter_by(slug=slug).first()
           or Post.query.filter_by(slug=slug).first())
    if obj and obj.translation_group:
        active_codes = _get_active_language_codes()
        return obj.get_all_language_versions(active_codes=active_codes)
    return []


def _build_system_lang_versions():
    """Build language versions for system pages (confirm-email, unsubscribe, etc).

    Unlike hub pages, these pages have essential query parameters (token, email)
    that must be preserved. Only the lang parameter is replaced.
    """
    active_langs = Settings.get('active_languages', 'en')
    if isinstance(active_langs, str):
        active_langs = [l.strip() for l in active_langs.split(',') if l.strip()]
    else:
        active_langs = ['en']

    # Keep all current query params except 'lang'
    base_params = [(k, v) for k, v in request.args.items() if k != 'lang']

    class SystemLangVersion:
        def __init__(self, lang):
            self.language = lang
            params = base_params + [('lang', lang)]
            self.slug = f'{request.path.lstrip("/")}?{urlencode(params)}'

    return [SystemLangVersion(lang) for lang in active_langs]


class _ResolvedItem:
    """Lightweight resolved menu item used in get_menu_for_language."""
    def __init__(self, label, url, target='_self', css_class=''):
        self.label = label
        self.url = url
        self.target = target or '_self'
        self.css_class = css_class or ''
        self.children = []   # populated after construction if needed


# Module-level cache for the site logo Image record.
def _resolve_image_by_public_url(url):
    """Resolve a stored image URL to its Image record.

    Handles the clean short public URL the media picker stores
    (e.g. '/og-cloud-init.png' → download_slug 'og-cloud-init'),
    absolute URLs (https://host/slug.png), and the legacy internal
    '/static/uploads/...' path. Returns the Image or None.

    This mirrors build_images_map()'s matching so single-value lookups
    (site OG image, logo, social meta) resolve the same way featured
    images do — by public URL, not the internal Image.url column.
    """
    if not url or not isinstance(url, str):
        return None
    path = url
    if '://' in path:
        try:
            path = '/' + path.split('://', 1)[1].split('/', 1)[1]
        except Exception:
            path = url
    # Legacy long path stored directly in Image.url
    if path.startswith('/static/uploads/'):
        try:
            return Image.query.filter_by(url=path).first()
        except Exception:
            return None
    # Clean short public URL → download_slug
    try:
        parsed = _parse_image_short_url(path.lstrip('/'))
    except Exception:
        parsed = None
    if parsed and parsed[0]:
        try:
            img = Image.query.filter_by(download_slug=parsed[0]).first()
            if img:
                return img
        except Exception:
            return None
    # Last resort: exact match on the stored URL column
    try:
        return Image.query.filter_by(url=url).first()
    except Exception:
        return None


# The logo URL changes only when admin uploads a new logo, so we can
# cache the lookup process-wide and invalidate automatically whenever
# the URL changes. This keeps inject_site_settings() free of an extra
# DB query on every request.
_SITE_LOGO_IMAGE_CACHE = {'url': None, 'image': None}


def _get_site_logo_image(logo_url):
    """Look up the Image record for the site logo, with process-level
    cache. Returns the Image object (with .get_variants(), .get_srcset(),
    .width, .height) or None if the logo isn't in the media library
    (e.g. referenced by external URL, or uploaded before variants
    pipeline existed).

    Safe to call on every request — does a DB query only when the logo
    URL changes in Settings.
    """
    if not logo_url or not logo_url.startswith('/'):
        return None
    if _SITE_LOGO_IMAGE_CACHE['url'] == logo_url:
        return _SITE_LOGO_IMAGE_CACHE['image']
    try:
        img = _resolve_image_by_public_url(logo_url)
    except Exception:
        img = None
    _SITE_LOGO_IMAGE_CACHE['url'] = logo_url
    _SITE_LOGO_IMAGE_CACHE['image'] = img
    return img


# Process-level cache for the site OG image lookup. Same pattern as the
# logo cache above. Stores a (url, mime_type) tuple so templates can set
# both <meta og:image> and <meta og:image:type> consistently — important
# when the OG source is a transparent PNG (we drop the JPG variant for
# RGBA originals, so the best available is WebP).
_SITE_OG_META_CACHE = {'url': None, 'meta': None}


def _get_site_og_meta(og_url):
    """Resolve the site-wide OG image URL into a (url, mime_type) tuple.
    
    Looks up the Image record by URL; if found and it has an OG variant,
    returns Image.get_og_meta(). Otherwise returns the raw URL with an
    'image/jpeg' fallback type.
    
    Returns None if og_url is empty (templates check for None).
    """
    if not og_url:
        return None
    if _SITE_OG_META_CACHE['url'] == og_url:
        return _SITE_OG_META_CACHE['meta']
    
    meta = (og_url, 'image/jpeg')  # safe default
    if og_url.startswith('/'):
        try:
            img = _resolve_image_by_public_url(og_url)
            if img and hasattr(img, 'get_og_meta'):
                resolved = img.get_og_meta()
                if resolved:
                    meta = resolved
        except Exception:
            pass
    
    _SITE_OG_META_CACHE['url'] = og_url
    _SITE_OG_META_CACHE['meta'] = meta
    return meta


# Same process-level cache pattern for the site-wide Twitter image. The
# Twitter card image should also use the 1200×630 social variant rather
# than the full-size original.
_SITE_TWITTER_META_CACHE = {'url': None, 'meta': None}


def _get_site_twitter_meta(tw_url):
    """Resolve the site-wide Twitter image URL into a (url, mime_type)
    tuple, preferring the social (1200×630) variant. tw_url is the
    configured twitter_image, or the og_image when no separate Twitter
    image is set. Returns None when neither is configured."""
    if not tw_url:
        return None
    if _SITE_TWITTER_META_CACHE['url'] == tw_url:
        return _SITE_TWITTER_META_CACHE['meta']
    meta = (tw_url, 'image/jpeg')
    if tw_url.startswith('/'):
        try:
            img = _resolve_image_by_public_url(tw_url)
            if img and hasattr(img, 'get_og_meta'):
                resolved = img.get_og_meta()
                if resolved:
                    meta = resolved
        except Exception:
            pass
    _SITE_TWITTER_META_CACHE['url'] = tw_url
    _SITE_TWITTER_META_CACHE['meta'] = meta
    return meta


def get_menu_for_language(location, lang):
    """Get menu items resolved for visitor's language, with submenu support.

    Returns a list of resolved items. Each item may have a `.children` attribute
    (list of resolved child items) when submenus are present.

    Resolution rules (same as before, now applied recursively):
    - Page/post links: resolve to visitor's language translation if available
    - Custom/external links: always show as-is
    - Submenus: children are resolved with the same logic
    """
    import json as _json

    def get_custom_label(item, language):
        if item.labels_json:
            try:
                labels = _json.loads(item.labels_json)
                return labels.get(language, '')
            except (ValueError, TypeError):
                pass
        return ''

    def resolve_single(item):
        """Resolve one MenuItem to a display object. Returns None to hide."""
        # Custom / external / no content link
        if item.page_type in ('custom', 'external') or (not item.page_id and not item.post_id):
            # Per-language URL and label (set via Custom Links tab in
            # Menu Builder). Hide the item entirely if neither this
            # language nor a legacy fallback have data — symmetric with
            # how page/post items are hidden when no published translation
            # exists for the visitor's language.
            lang_url = ''
            lang_label = ''
            if item.urls_json:
                try:
                    urls = _json.loads(item.urls_json)
                    lang_url = (urls.get(lang) or '').strip()
                except (ValueError, TypeError):
                    pass
            if item.labels_json:
                try:
                    labels = _json.loads(item.labels_json)
                    lang_label = (labels.get(lang) or '').strip()
                except (ValueError, TypeError):
                    pass
            # Fallback to legacy single url/label (pre-2.0.1 custom links
            # that predate urls_json). New per-language items without data
            # for this language hit `return None` below.
            if not lang_url and not item.urls_json:
                lang_url = item.url or ''
            if not lang_label and not item.labels_json:
                lang_label = item.label or ''
            if not lang_url or not lang_label:
                return None
            return _ResolvedItem(
                lang_label,
                lang_url,
                item.target,
                item.css_class,
            )

        custom_label = get_custom_label(item, lang)

        # Helper: wrap a MenuItem into a _ResolvedItem with fallback label/url.
        # We always return _ResolvedItem (never the raw ORM object) because
        # later code assigns .children to the result — and SQLAlchemy would
        # reject a list of _ResolvedItem on an ORM relationship.
        def _wrap_item(lbl, url):
            return _ResolvedItem(lbl, url, item.target, item.css_class)

        if item.page_id and item.page:
            page = item.page
            # Show link only to published content in the visitor's language.
            # No fallback to a different language: a menu link must lead to
            # content the visitor can read. Symmetric with view_content,
            # which 404s drafts and content in deactivated languages.
            if page.status == 'published' and page.language == lang:
                return _wrap_item(custom_label or page.title, f'/{page.slug}')
            translation = page.get_translation(lang)
            if translation:
                return _wrap_item(
                    custom_label or translation.title,
                    f'/{translation.slug}',
                )
            return None

        if item.post_id and item.post:
            post = item.post
            # Show link only to published content in the visitor's language.
            # No fallback to a different language: a menu link must lead to
            # content the visitor can read. Symmetric with view_content,
            # which 404s drafts and content in deactivated languages.
            if post.status == 'published' and post.language == lang:
                return _wrap_item(custom_label or post.title, f'/{post.slug}')
            translation = post.get_translation(lang)
            if translation:
                return _wrap_item(
                    custom_label or translation.title,
                    f'/{translation.slug}',
                )
            return None

        # Plain custom-URL item (no page/post link)
        return _wrap_item(custom_label or item.label or '', item.url or '')

    # Fetch ALL active items for this location in ONE query and group by
    # parent — recursion then walks the in-memory tree (no N+1 queries).
    all_items = MenuItem.query.filter_by(
        menu_location=location, is_active=True
    ).order_by(MenuItem.order).all()
    by_parent = {}
    for it in all_items:
        by_parent.setdefault(it.parent_id, []).append(it)

    max_depth = app.config.get('MENU_MAX_DEPTH', 3)

    def _build_level(parent_db_id, depth):
        """Recursively resolve one sibling level.

        The battle-tested per-item rules in resolve_single apply unchanged at
        every depth. A hidden item (draft / missing translation / deactivated
        content) hides its whole branch — same semantics the one-level menu
        always had, applied recursively.
        """
        if depth >= max_depth:
            return []
        level = []
        for item in by_parent.get(parent_db_id, []):
            ri = resolve_single(item)
            if ri is None:
                continue
            ri.children = _build_level(item.id, depth + 1)
            level.append(ri)
        return level

    return _build_level(None, 0)


@app.context_processor
def inject_site_settings():
    """Make site settings available in all templates"""
    
    # Determine current visitor language from context
    # Priority: ?lang= parameter > slug-based detection > fallback
    current_language = request.args.get('lang', '')
    
    # Validate ?lang= against active languages. Garbage like ?lang=кг must not
    # leak into UI elements (logo URL, page title, header). Falls through to
    # slug-based detection or default below.
    if current_language:
        active_codes = _get_active_language_codes()
        if current_language not in active_codes:
            current_language = ''
    
    if not current_language:
        # Try to detect from current page/post slug
        slug = request.view_args.get('slug', '') if request.view_args else ''
        if slug:
            page = Page.query.filter_by(slug=slug).first()
            if page:
                current_language = page.language
            else:
                post = Post.query.filter_by(slug=slug).first()
                if post:
                    current_language = post.language
    
    # Authoritative language signal set by page/post/preview routes (the
    # language the current document is actually rendered in). Needed for admin
    # preview URLs like /admin/posts/<id>/preview, which carry no slug and no
    # ?lang=, so without this the menu falls back to the default language and
    # its links drop the visitor's language. Only consulted when ?lang= and slug
    # detection found nothing, so public pages are unaffected.
    if not current_language:
        _page_lang = getattr(g, 'current_page_language', None)
        if _page_lang:
            current_language = _page_lang
    
    if not current_language:
        current_language = Settings.get('default_language', 'en')
    
    # Store in g so pt() (public translate) can access visitor's language
    g.current_language = current_language
    
    logo_url = Settings.get('site_logo', '')

    # Per-language URL map of custom menu links, used client-side by the
    # language switcher to follow anchor links across languages (each entry
    # is a {lang_code: url} dict, e.g. {'en': '/docs#x', 'de': '/dok#y'}).
    # Anchors may differ per language; missing languages fall back gracefully.
    def _custom_link_lang_map():
        import json as _json
        out = []
        items = MenuItem.query.filter(
            MenuItem.page_type == 'custom',
            MenuItem.urls_json.isnot(None),
        ).all()
        for it in items:
            try:
                urls = _json.loads(it.urls_json)
            except (ValueError, TypeError):
                continue
            if isinstance(urls, dict) and urls:
                out.append(urls)
        return out

    return {
        'site_name': Settings.get('site_name', 'AliothPress'),
        'site_tagline': Settings.get('site_tagline', 'Create once, publish everywhere'),
        'site_logo': logo_url,
        'site_logo_image': _get_site_logo_image(logo_url),
        'site_og_meta': _get_site_og_meta(Settings.get('og_image', '')),
        'site_twitter_meta': _get_site_twitter_meta(Settings.get('twitter_image', '') or Settings.get('og_image', '')),
        'header_menu': get_menu_for_language('header', current_language),
        'footer_menu': get_menu_for_language('footer', current_language),
        'footer_copyright': Settings.get('footer_copyright', 'AliothPress'),
        'current_year': datetime.now().year,
        'available_languages': AVAILABLE_LANGUAGES,
        'active_languages': _get_active_languages(),
        'lang_code_to_name': LANG_CODE_TO_NAME,
        'current_language': current_language,
        'default_language': Settings.get('default_language', 'en'),
        'custom_link_lang_map': _custom_link_lang_map(),
    }


def ensure_unique_slug(slug, model_class, exclude_id=None):
    """Ensure slug is unique across Post, Page, AND File models.
    If slug already exists in any model, appends -2, -3, etc.
    
    Args:
        slug: the desired slug (already sanitized via slugify)
        model_class: Post or Page (used with exclude_id to skip self)
        exclude_id: ID to exclude from model_class (for editing existing content)
    Returns:
        unique slug string
    """
    if not slug:
        return slug
    
    other_model = Page if model_class == Post else Post
    
    base_slug = slug
    counter = 2
    while True:
        # Check the primary model (with exclude_id support)
        query = model_class.query.filter_by(slug=slug)
        if exclude_id:
            query = query.filter(model_class.id != int(exclude_id))
        taken_in_own = query.first()
        
        # Check the other model (no exclude — slug must be globally unique)
        taken_in_other = other_model.query.filter_by(slug=slug).first()
        
        # Check file download slugs (no post/page slug should collide with a file)
        taken_in_files = bool(File.query.filter_by(download_slug=slug).first())
        
        # Check image base slugs — a post slug 'sunset' and an image base slug
        # 'sunset' produce different URLs (/sunset vs /sunset.jpg) but we keep
        # the whole /<slug> namespace distinct to avoid admin confusion.
        taken_in_images = bool(Image.query.filter_by(download_slug=slug).first())
        
        if not taken_in_own and not taken_in_other and not taken_in_files and not taken_in_images:
            return slug
        slug = f"{base_slug}-{counter}"
        counter += 1


# ============================================================================
# PUBLIC ROUTES
# ============================================================================

def build_images_map(*image_urls):
    """Build a dict mapping image URLs (as stored in content) to Image objects.
    
    In the short-URL world, content stores URLs like '/sunset.jpg' or
    '/sunset-og.jpg'. Each URL is parsed into a base `download_slug` and the
    Image is fetched by that slug — one SQL query per batch regardless of how
    many variant-shaped URLs we were given.
    
    Returns a dict keyed by the ORIGINAL URL (as passed in), so callers can
    do `images_map.get(post.featured_image)` without caring about variant
    shape. Also stores the result in Flask g._ap_images_map so that the
    `render_blocks` Jinja filter can pass it to block_renderer.
    """
    urls = set()
    for arg in image_urls:
        if isinstance(arg, (list, tuple, set)):
            urls.update(u for u in arg if u)
        elif arg:
            urls.add(arg)
    
    if not urls:
        g._ap_images_map = {}
        return {}
    
    # Parse each URL into its base slug and remember which original URL
    # produced which slug — so we can key the result by the original URL.
    slug_to_urls = {}  # {base_slug: [original_url, ...]}
    legacy_long_urls = set()
    for url in urls:
        if not isinstance(url, str):
            continue
        path = url.lstrip('/')
        # Strip an absolute domain prefix if present (covers hard-coded
        # https://mysite.com/sunset.jpg that may exist in pasted content).
        if '://' in path:
            try:
                path = path.split('://', 1)[1].split('/', 1)[1]
            except IndexError:
                continue
        parsed = _parse_image_short_url(path)
        if parsed is not None:
            base_slug = parsed[0]
            slug_to_urls.setdefault(base_slug, []).append(url)
        elif url.startswith('/static/uploads/'):
            # Legacy long path — shouldn't happen in clean install, but a
            # safety net so nothing blows up if one slips through.
            legacy_long_urls.add(url)
    
    result = {}
    
    if slug_to_urls:
        images = Image.query.filter(Image.download_slug.in_(slug_to_urls.keys())).all()
        for img in images:
            for original_url in slug_to_urls.get(img.download_slug, []):
                result[original_url] = img
    
    if legacy_long_urls:
        legacy_images = Image.query.filter(Image.url.in_(legacy_long_urls)).all()
        for img in legacy_images:
            result[img.url] = img
    
    g._ap_images_map = result
    return result


def _collect_block_image_urls(page):
    """Extract all image URLs from page builder blocks for images_map."""
    if not page.use_builder or not page.blocks:
        return []
    
    try:
        data = json.loads(page.blocks)
    except (ValueError, TypeError):
        return []
    
    urls = []
    blocks = data.get('blocks', []) if isinstance(data, dict) else data
    _extract_urls_recursive(blocks, urls)
    return urls


def _extract_urls_recursive(blocks, urls):
    """Recursively extract image URLs from nested blocks."""
    for block in blocks:
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        if block_type == 'image' and data.get('url'):
            urls.append(data['url'])
        elif block_type == 'gallery':
            for img in data.get('images', []):
                if img.get('url'):
                    urls.append(img['url'])
        elif block_type == 'ticker':
            for img in data.get('images', []):
                if img.get('url'):
                    urls.append(img['url'])
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('image'):
                    urls.append(slide['image'])
                # Recurse into hero-mode nested blocks (heading, image, button, etc.)
                if slide.get('blocks'):
                    _extract_urls_recursive(slide['blocks'], urls)
        elif block_type == 'columns':
            for col in data.get('columns', []):
                _extract_urls_recursive(col.get('blocks', []), urls)
        elif block_type == 'section':
            # Collect background image URLs (light mode + dark mode), so
            # the renderer can find their Image objects in images_map and
            # build image-set() with WebP/AVIF variants. Without this, the
            # section background would always be the original PNG/JPG.
            if data.get('background_type') == 'image':
                light_bg = data.get('background_image') or data.get('background_value')
                if light_bg:
                    urls.append(light_bg)
            if data.get('background_type_dark') == 'image':
                dark_bg = data.get('background_value_dark')
                if dark_bg:
                    urls.append(dark_bg)
            _extract_urls_recursive(data.get('blocks', []), urls)

# Posts per page on GRID listings: the homepage blog view and pages with the
# "Show latest posts on this page" checkbox (is_posts_page). The card grid is
# repeat(auto-fill, minmax(350px, 1fr)): 3 columns on a wide desktop, 2 on
# tablet, 1 on mobile. 12 is the only small count that fills BOTH layouts
# with complete rows — 4×3 on desktop and 6×2 on tablet — so the last row
# never dangles a lone card (10 did).
# Tag/category archives are vertical LISTS, not grids — they keep their own
# per_page and are unaffected. Admin pagination is separate (get_per_page).
POSTS_GRID_PER_PAGE = 12


def _switch_page_numbers(lang_counts, per_page, current_page):
    """Which page number each language's switcher link should point at.

    The language switcher keeps the visitor's place in a paginated listing.
    A language only gets ?page=N when it genuinely has an Nth page; when it
    doesn't, its link is left alone and the visitor lands on page 1, exactly
    as today. Nothing is ever rounded down to a "nearest" page — the visitor
    is either in the same place or at the start, with no third case.

    That is also what keeps this safe: a link to a page a language does not
    have is never generated, so paginate() never sees an out-of-range page
    and its 404 behaviour stays untouched.

    lang_counts is an iterable of (language, published_post_count) rows.
    Returns {language: page_number}, containing only languages that really
    have that page. Page 1 is never listed — those links need no ?page=.
    """
    numbers = {}
    if current_page <= 1:
        return numbers
    for lang, total in lang_counts:
        if not total:
            continue
        last_page = -(-total // per_page)  # ceil division
        if current_page <= last_page:
            numbers[lang] = current_page
    return numbers


def _build_page_posts_context(page_obj, images_map):
    """Extra template context for pages flagged as a posts page.

    Returns {} for normal pages. For posts pages: a paginated query of the
    latest published posts in the page's language (?page=N), their
    featured-image variants merged into images_map, and an ItemList schema.
    """
    if not getattr(page_obj, 'is_posts_page', False):
        return {}
    page_num = get_page()
    posts = Post.query.filter_by(
        status='published', language=page_obj.language
    ).order_by(Post.published_at.desc().nullslast(), Post.created_at.desc()).paginate(page=page_num, per_page=POSTS_GRID_PER_PAGE)

    # Merge post-card featured images so the cards get responsive <picture>
    card_urls = [p.featured_image for p in posts.items if p.featured_image]
    if card_urls:
        images_map.update(build_images_map(*card_urls))

    # Page numbers for the language switcher. Only needed past page 1, so
    # page-1 views — the overwhelming majority — run no extra query.
    switch_page_numbers = {}
    if page_num > 1:
        switch_page_numbers = _switch_page_numbers(
            db.session.query(Post.language, db.func.count(Post.id))
            .filter(Post.status == 'published')
            .group_by(Post.language).all(),
            POSTS_GRID_PER_PAGE, page_num)

    return {
        'posts': posts,
        'switch_page_numbers': switch_page_numbers,
        'posts_schema': (schema_org.build_itemlist(
            posts.items,
            lambda p: url_for('view_content', slug=p.slug, _external=True),
        ) if posts.items else None),
    }


@app.route('/')
def index():
    """Homepage — shows static page if configured, otherwise posts list"""
    
    active_codes = _get_active_language_codes()
    
    # Prevent duplicate content: /?lang=en and / are the same page when
    # 'en' is the default language. 301-redirect to clean URL.
    default_lang = Settings.get('default_language', 'en')
    if request.args.get('lang') == default_lang:
        return redirect('/', code=301)
    
    # If a language is requested via ?lang= but it's deactivated — 404
    requested_lang = request.args.get('lang')
    if requested_lang and requested_lang not in active_codes:
        return render_template('public/404.html', all_versions=[]), 404
    
    # Check if a static homepage is configured (by translation_group)
    homepage_group = Settings.get('homepage_group', '')
    
    if homepage_group:
        # Each language has its own URL with its own content. If the visitor
        # explicitly requests a language and no published page exists in that
        # language — return 404, same as view_content does for /<slug>. No
        # silent fallback to another language: that creates duplicate content
        # for SEO and confuses visitors who asked for a specific language.
        if requested_lang:
            # Explicit ?lang=X request. Deactivated language already returned
            # 404 above; ?lang=<default> already 301-redirected to /.
            homepage = Page.query.filter_by(
                translation_group=homepage_group,
                language=requested_lang,
                status='published'
            ).first()
            if not homepage:
                return render_template('public/404.html',
                                       all_versions=_build_hub_lang_versions(request.path)), 404
        else:
            # No ?lang= — try the default language. If missing, fall through
            # to the posts-list fallback below (same as when homepage_group
            # is not configured at all).
            homepage = Page.query.filter_by(
                translation_group=homepage_group,
                language=default_lang,
                status='published'
            ).first()
        
        if homepage:
            # Skip homepage whose language got deactivated
            if homepage.language not in active_codes:
                return render_template('public/404.html',
                                       all_versions=_build_hub_lang_versions(request.path)), 404
            all_versions = homepage.get_all_language_versions(active_codes=active_codes)
            homepage_image_urls = _collect_block_image_urls(homepage)
            g.current_page_language = homepage.language
            images_map = build_images_map(homepage.featured_image, homepage.og_image, homepage.twitter_image, homepage_image_urls)
            posts_ctx = _build_page_posts_context(homepage, images_map)
            return render_template('public/page.html', 
                                 page=homepage,
                                 current_language=homepage.language,
                                 all_versions=all_versions,
                                 is_homepage=True,
                                 images_map=images_map,
                                 page_schema=schema_org.build_page_graph(
                                     homepage,
                                     site_name=Settings.get('site_name', 'AliothPress'),
                                     settings=Settings,
                                     org_sameas=_build_org_sameas(),
                                     author_sameas=_build_author_sameas(homepage.author),
                                     images_map=images_map,
                                 ),
                                 **posts_ctx)
    
    # Fallback: show posts list (blog-style)
    page = get_page()
    current_language = request.args.get('lang', Settings.get('default_language', 'en'))
    
    posts = Post.query.filter_by(
        status='published', language=current_language
    ).order_by(Post.published_at.desc().nullslast(), Post.created_at.desc()).paginate(page=page, per_page=POSTS_GRID_PER_PAGE)
    
    if posts.total == 0:
        posts = Post.query.filter(
            Post.status == 'published',
            Post.language.in_(list(active_codes))
        ).order_by(Post.published_at.desc().nullslast(), Post.created_at.desc()).paginate(page=page, per_page=POSTS_GRID_PER_PAGE)
    
    # Build virtual "versions" for language switcher on homepage.
    # Each active language (active_codes was loaded at top of this function).
    class HomepageLangVersion:
        def __init__(self, lang):
            self.language = lang
            self.slug = ''  # not used — is_homepage triggers /?lang= URL
    
    # Preserve AVAILABLE_LANGUAGES order
    ordered_codes = [l['code'] for l in AVAILABLE_LANGUAGES if l['code'] in active_codes]
    all_versions = [HomepageLangVersion(lang) for lang in ordered_codes]
    
    # Featured-image variants for the post cards, fetched in one batched
    # query, so the listing can emit responsive <picture> with explicit
    # width/height (avoids CLS and oversized downloads on the homepage).
    images_map = build_images_map(*[p.featured_image for p in posts.items if p.featured_image])
    
    return render_template('public/index.html', posts=posts, 
                         current_language=current_language,
                         images_map=images_map,
                         all_versions=all_versions,
                         is_homepage=True,
                         page_schema=schema_org.build_website_graph(
                             site_name=Settings.get('site_name', 'AliothPress'),
                             site_tagline=Settings.get('site_tagline', 'Create once, publish everywhere'),
                             settings=Settings,
                             available_languages=AVAILABLE_LANGUAGES,
                             org_sameas=_build_org_sameas(),
                         ),
                         posts_schema=(schema_org.build_itemlist(
                             posts.items,
                             lambda p: url_for('view_content', slug=p.slug, _external=True),
                         ) if posts.items else None))


@app.route('/<slug>')
def view_content(slug):
    """View post or page by slug — all languages are equal, no prefix needed"""
    active_codes = _get_active_language_codes()
    # Try to find post
    post = Post.query.filter_by(slug=slug, status='published').first()
    if post:
        # Content in deactivated languages is hidden from public (404)
        if post.language not in active_codes:
            return render_template('public/404.html',
                                   all_versions=_build_content_404_versions(slug)), 404
        # Get all language versions for hreflang tags (published + active only)
        all_versions = post.get_all_language_versions(active_codes=active_codes)
        images_map = build_images_map(post.featured_image, post.og_image, post.twitter_image)
        return render_template('public/post.html', 
                             post=post, 
                             current_language=post.language,
                             all_versions=all_versions,
                             images_map=images_map,
                             page_schema=schema_org.build_post_graph(
                                 post,
                                 site_name=Settings.get('site_name', 'AliothPress'),
                                 settings=Settings,
                                 org_sameas=_build_org_sameas(),
                                 author_sameas=_build_author_sameas(post.author),
                                 images_map=images_map,
                             ))
    
    # Try to find page
    page = Page.query.filter_by(slug=slug, status='published').first()
    if page:
        # Content in deactivated languages is hidden from public (404)
        if page.language not in active_codes:
            return render_template('public/404.html',
                                   all_versions=_build_content_404_versions(slug)), 404
        # If this page is the homepage, redirect to / to avoid duplicate content
        homepage_group = Settings.get('homepage_group', '')
        if homepage_group and page.translation_group == homepage_group:
            return redirect('/?lang=' + page.language if page.language != Settings.get('default_language', 'en') else '/', code=301)
        all_versions = page.get_all_language_versions(active_codes=active_codes)
        # Collect all image URLs from builder blocks + featured image
        page_image_urls = _collect_block_image_urls(page)
        g.current_page_language = page.language
        images_map = build_images_map(page.featured_image, page.og_image, page.twitter_image, page_image_urls)
        posts_ctx = _build_page_posts_context(page, images_map)
        return render_template('public/page.html', 
                             page=page, 
                             current_language=page.language,
                             all_versions=all_versions,
                             images_map=images_map,
                             page_schema=schema_org.build_page_graph(
                                 page,
                                 site_name=Settings.get('site_name', 'AliothPress'),
                                 settings=Settings,
                                 org_sameas=_build_org_sameas(),
                                 author_sameas=_build_author_sameas(page.author),
                                 images_map=images_map,
                             ),
                             **posts_ctx)
    
    # Check for 301 redirect (slug was changed)
    redirect_entry = Redirect.query.filter_by(old_path=slug).first()
    if redirect_entry:
        return redirect('/' + redirect_entry.new_path, code=301)
    
    # Try to find a downloadable file (e.g. report.pdf)
    doc = File.query.filter_by(download_slug=slug).first()
    if doc:
        files_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'files')
        
        # Files the browser can display — open inline (PDF, TXT, CSV)
        # Everything else (DOC, XLS, ZIP, RAR) — force download
        viewable_extensions = {'pdf', 'txt', 'csv'}
        ext = doc.get_extension()
        force_download = ext not in viewable_extensions
        
        return send_from_directory(
            files_dir,
            doc.filename,
            as_attachment=force_download,
            download_name=doc.original_filename or doc.filename
        )
    
    # Try to serve an image variant by short URL: /sunset.jpg, /sunset-800w.webp,
    # /sunset-og.jpg, etc. Images are tried last because they're the only
    # category whose URLs must contain a dot, so this lookup can't shadow
    # post/page/file slugs above.
    image_response = _serve_image_short_url(slug)
    if image_response is not None:
        return image_response
    
    # 404 if not found
    return render_template('public/404.html',
                           all_versions=_build_content_404_versions(slug)), 404


@app.route('/admin/posts/<int:post_id>/preview')
@login_required
def admin_posts_preview(post_id):
    """Preview post (works for drafts too)"""
    post = Post.query.get_or_404(post_id)
    all_versions = post.get_all_language_versions(include_drafts=True)
    images_map = build_images_map(post.featured_image, post.og_image, post.twitter_image)
    g.current_page_language = post.language
    return render_template('public/post.html', post=post, current_language=post.language, 
                         all_versions=all_versions, preview_mode=True,
                         images_map=images_map,
                         page_schema=schema_org.build_post_graph(
                             post,
                             site_name=Settings.get('site_name', 'AliothPress'),
                             settings=Settings,
                             org_sameas=_build_org_sameas(),
                             author_sameas=_build_author_sameas(post.author),
                             images_map=images_map,
                         ))


@app.route('/admin/pages/<int:page_id>/preview')
@login_required
def admin_pages_preview(page_id):
    """Preview page (works for drafts too)"""
    page = Page.query.get_or_404(page_id)
    all_versions = page.get_all_language_versions(include_drafts=True)
    page_image_urls = _collect_block_image_urls(page)
    g.current_page_language = page.language
    images_map = build_images_map(page.featured_image, page.og_image, page.twitter_image, page_image_urls)
    posts_ctx = _build_page_posts_context(page, images_map)
    return render_template('public/page.html', page=page, current_language=page.language,
                         all_versions=all_versions, preview_mode=True,
                         images_map=images_map,
                         page_schema=schema_org.build_page_graph(
                             page,
                             site_name=Settings.get('site_name', 'AliothPress'),
                             settings=Settings,
                             org_sameas=_build_org_sameas(),
                             author_sameas=_build_author_sameas(page.author),
                             images_map=images_map,
                         ),
                         **posts_ctx)


# ============================================================================
# ERROR HANDLERS
# ============================================================================

@app.errorhandler(404)
def page_not_found(e):
    """Custom 404 — try the themed template, fallback to simple HTML"""
    try:
        current_language = getattr(g, 'current_language', None) or request.args.get('lang', Settings.get('default_language', 'en'))
        return render_template('public/404.html', current_language=current_language,
                               all_versions=_build_hub_lang_versions(request.path)), 404
    except Exception:
        return '''<!DOCTYPE html><html><head><title>404</title>
        <style>body{font-family:system-ui;display:flex;align-items:center;justify-content:center;min-height:100vh;margin:0;background:#1a1a2e;color:#e2e8f0}
        .box{text-align:center}.box h1{font-size:72px;margin:0;color:#667eea}.box p{margin:12px 0;color:#a0aec0}
        a{color:#667eea;text-decoration:none}a:hover{text-decoration:underline}</style></head>
        <body><div class="box"><h1>404</h1><p>Page not found</p>
        <a href="/">← Home</a></div></body></html>''', 404


@app.errorhandler(500)
def internal_server_error(e):
    """Custom 500 — roll back the failed transaction and show a themed page
    instead of Flask's bare white error page. The session rollback matters:
    without it the connection can stay in a failed-transaction state and
    poison subsequent requests on the same worker."""
    try:
        db.session.rollback()
    except Exception:
        pass
    try:
        current_language = getattr(g, 'current_language', None) or request.args.get('lang', Settings.get('default_language', 'en'))
        return render_template('public/500.html', current_language=current_language), 500
    except Exception:
        return '''<!DOCTYPE html><html><head><title>500</title>
        <style>body{font-family:system-ui;display:flex;align-items:center;justify-content:center;min-height:100vh;margin:0;background:#1a1a2e;color:#e2e8f0}
        .box{text-align:center}.box h1{font-size:72px;margin:0;color:#667eea}.box p{margin:12px 0;color:#a0aec0}
        a{color:#667eea;text-decoration:none}a:hover{text-decoration:underline}</style></head>
        <body><div class="box"><h1>500</h1><p>Something went wrong on our side</p>
        <a href="/">← Home</a></div></body></html>''', 500


@app.route('/tag/')
def view_all_tags():
    """Hub page — list all tags used across published posts and pages"""
    active_codes = _get_active_language_codes()
    # ?lang=<default> duplicates the clean URL — 301 away (same as index()).
    _r = _redirect_hub_default_lang()
    if _r:
        return _r
    current_language = request.args.get('lang', Settings.get('default_language', 'en'))
    
    # If visitor asked for a deactivated language — 404
    if request.args.get('lang') and request.args.get('lang') not in active_codes:
        return render_template('public/404.html',
                               all_versions=_build_hub_lang_versions(request.path)), 404
    
    # Collect all unique tags from published posts and pages in active languages
    # and in the currently viewed language (so a 'de' tag hub lists only 'de' tags).
    tags_count = {}
    
    for post in Post.query.filter(
        Post.status == 'published',
        Post.language == current_language,
        Post.language.in_(list(active_codes))
    ).all():
        for tag in (post.get_tags_list() or []):
            tags_count[tag] = tags_count.get(tag, 0) + 1
    
    for page in Page.query.filter(
        Page.status == 'published',
        Page.language == current_language,
        Page.language.in_(list(active_codes))
    ).all():
        for tag in (page.get_tags_list() or []):
            tags_count[tag] = tags_count.get(tag, 0) + 1
    
    # Sort alphabetically
    tags_sorted = sorted(tags_count.items(), key=lambda x: x[0].lower())
    
    return render_template('public/tags_hub.html',
                         tags=tags_sorted,
                         current_language=current_language,
                         all_versions=_build_hub_lang_versions(request.path),
                         page_schema=schema_org.build_collection_page(
                             name=pt_for('all_tags', current_language),
                             description=pt_for('all_tags_desc', current_language),
                             url=request.url,
                         ))


@app.route('/category/')
def view_all_categories():
    """Hub page — list all categories used across published posts"""
    active_codes = _get_active_language_codes()
    # ?lang=<default> duplicates the clean URL — 301 away (same as index()).
    _r = _redirect_hub_default_lang()
    if _r:
        return _r
    current_language = request.args.get('lang', Settings.get('default_language', 'en'))
    
    # If visitor asked for a deactivated language — 404
    if request.args.get('lang') and request.args.get('lang') not in active_codes:
        return render_template('public/404.html',
                               all_versions=_build_hub_lang_versions(request.path)), 404
    
    # Collect all unique categories with post count, filtered by current_language
    # and active languages only
    categories_count = {}
    
    for post in Post.query.filter(
        Post.status == 'published',
        Post.language == current_language,
        Post.language.in_(list(active_codes)),
        Post.category.isnot(None), Post.category != ''
    ).all():
        cat = post.category
        categories_count[cat] = categories_count.get(cat, 0) + 1
    
    for page in Page.query.filter(
        Page.status == 'published',
        Page.language == current_language,
        Page.language.in_(list(active_codes)),
        Page.category.isnot(None), Page.category != ''
    ).all():
        cat = page.category
        categories_count[cat] = categories_count.get(cat, 0) + 1
    
    # Sort alphabetically
    categories_sorted = sorted(categories_count.items(), key=lambda x: x[0].lower())
    
    return render_template('public/categories_hub.html',
                         categories=categories_sorted,
                         current_language=current_language,
                         all_versions=_build_hub_lang_versions(request.path),
                         page_schema=schema_org.build_collection_page(
                             name=pt_for('all_categories', current_language),
                             description=pt_for('all_categories_desc', current_language),
                             url=request.url,
                         ))


@app.route('/tag/<tag_name>')
def view_tag(tag_name):
    """Show all posts and pages with a given tag"""
    active_codes = _get_active_language_codes()
    page_num = get_page()
    # ?lang=<default> duplicates the clean URL — 301 away (same as index()).
    _r = _redirect_hub_default_lang()
    if _r:
        return _r
    current_language = request.args.get('lang', Settings.get('default_language', 'en'))
    
    # If visitor asked for a deactivated language — 404
    if request.args.get('lang') and request.args.get('lang') not in active_codes:
        return render_template('public/404.html',
                               all_versions=_build_hub_lang_versions(request.path)), 404
    
    # Search posts with this tag (tags stored as JSON array), filter by language
    safe_tag = escape_like(tag_name)
    posts = Post.query.filter(
        Post.status == 'published',
        Post.language == current_language,
        Post.language.in_(list(active_codes)),
        Post.tags.ilike(f'%{safe_tag}%', escape='\\')
    ).order_by(Post.published_at.desc().nullslast(), Post.created_at.desc()).paginate(page=page_num, per_page=10)
    
    # Search pages with this tag
    pages = Page.query.filter(
        Page.status == 'published',
        Page.language == current_language,
        Page.language.in_(list(active_codes)),
        Page.tags.ilike(f'%{safe_tag}%', escape='\\')
    ).order_by(Page.updated_at.desc()).all()
    
    # Build ItemList combining posts and pages
    item_list = schema_org.build_itemlist(
        list(posts.items) + list(pages),
        lambda item: url_for('view_content', slug=item.slug, _external=True),
    )
    # Page numbers for the language switcher (see _switch_page_numbers).
    # Skipped on page 1, so the common case costs no extra query.
    switch_page_numbers = {}
    if page_num > 1:
        switch_page_numbers = _switch_page_numbers(
            db.session.query(Post.language, db.func.count(Post.id)).filter(
                Post.status == 'published',
                Post.tags.ilike(f'%{safe_tag}%', escape='\\'),
            ).group_by(Post.language).all(),
            10, page_num)

    return render_template('public/tag.html',
                         tag_name=tag_name,
                         posts=posts,
                         tagged_pages=pages,
                         switch_page_numbers=switch_page_numbers,
                         current_language=current_language,
                         all_versions=_build_hub_lang_versions(request.path),
                         page_schema=schema_org.build_collection_page(
                             name=tag_name,
                             description=pt_for('tagged_with', current_language) + ': ' + tag_name,
                             url=request.url,
                             item_list=item_list,
                         ))


@app.route('/category/<category_name>')
def view_category(category_name):
    """Show all posts in a given category"""
    active_codes = _get_active_language_codes()
    page_num = get_page()
    # ?lang=<default> duplicates the clean URL — 301 away (same as index()).
    _r = _redirect_hub_default_lang()
    if _r:
        return _r
    current_language = request.args.get('lang', Settings.get('default_language', 'en'))
    
    # If visitor asked for a deactivated language — 404
    if request.args.get('lang') and request.args.get('lang') not in active_codes:
        return render_template('public/404.html',
                               all_versions=_build_hub_lang_versions(request.path)), 404
    
    posts = Post.query.filter(
        Post.status == 'published',
        Post.language == current_language,
        Post.language.in_(list(active_codes)),
        Post.category == category_name
    ).order_by(Post.published_at.desc().nullslast(), Post.created_at.desc()).paginate(page=page_num, per_page=10)
    
    item_list = schema_org.build_itemlist(
        list(posts.items),
        lambda item: url_for('view_content', slug=item.slug, _external=True),
    )
    # Page numbers for the language switcher (see _switch_page_numbers).
    switch_page_numbers = {}
    if page_num > 1:
        switch_page_numbers = _switch_page_numbers(
            db.session.query(Post.language, db.func.count(Post.id)).filter(
                Post.status == 'published',
                Post.category == category_name,
            ).group_by(Post.language).all(),
            10, page_num)

    return render_template('public/tag.html',
                         tag_name=category_name,
                         posts=posts,
                         tagged_pages=[],
                         switch_page_numbers=switch_page_numbers,
                         current_language=current_language,
                         is_category=True,
                         category_name=category_name,
                         all_versions=_build_hub_lang_versions(request.path),
                         page_schema=schema_org.build_collection_page(
                             name=category_name,
                             description=pt_for('in_category', current_language) + ': ' + category_name,
                             url=request.url,
                             item_list=item_list,
                         ))


# ============================================================================
# AUTHENTICATION ROUTES
# ============================================================================

@app.route('/setup', methods=['GET', 'POST'])
def setup():
    """Initial setup - Installation Wizard"""
    # Check if users already exist
    if User.query.count() > 0:
        flash(t('flash.setup_already_done'), 'info')
        return redirect(url_for('admin_login'))
    
    if request.method == 'POST':
        username = request.form.get('username')
        email = request.form.get('email')
        password = request.form.get('password')
        confirm_password = request.form.get('confirm_password')
        
        # Validation
        if not all([username, email, password, confirm_password]):
            flash(t('flash.fields_required'), 'error')
            return render_template('setup.html')
        
        if password != confirm_password:
            flash(t('flash.passwords_mismatch'), 'error')
            return render_template('setup.html')
        
        is_valid, pwd_error = validate_password(password)
        if not is_valid:
            flash(pwd_error, 'error')
            return render_template('setup.html')
        
        # --- Handle database configuration ---
        db_type = request.form.get('db_type', 'sqlite')
        
        if db_type in ('postgresql', 'postgresql_auto'):
            # ── Two-phase setup (same logic as /setup/finalize AJAX endpoint) ──
            # SQLAlchemy cannot switch its engine mid-request, so we:
            #   1. Save form data to .setup-pending.json
            #   2. Update .env with the PostgreSQL URI
            #   3. Restart the service
            #   4. On restart, _finalize_pending_setup() completes the setup
            db_host = request.form.get('db_host', 'localhost').strip()
            db_port = request.form.get('db_port', '5432').strip()
            db_name = request.form.get('db_name', '').strip()
            db_user = request.form.get('db_user', '').strip()
            db_password = request.form.get('db_password', '')
            
            if not all([db_host, db_name, db_user]):
                flash(t('flash.fields_required'), 'error')
                return render_template('setup.html')
            
            db_uri = f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}"
            
            # Save pending setup data
            hashed_password = generate_password_hash(password)
            wizard_language = request.form.get('default_language', 'en')
            
            pending_data = {
                'username': username,
                'email': email,
                'password_hash': hashed_password,
                'db_type': db_type,
                'default_language': wizard_language,
                'site_name': request.form.get('site_name', '').strip(),
                'site_tagline': request.form.get('site_tagline', '').strip(),
                'license_key': request.form.get('license_key', '').strip(),
                'request_url_root': request.url_root,
            }
            
            app_dir = os.path.dirname(os.path.abspath(__file__))
            pending_path = os.path.join(app_dir, '.setup-pending.json')
            
            try:
                with open(pending_path, 'w') as f:
                    json.dump(pending_data, f)
            except Exception as e:
                flash(t('flash.db_connection_error', error=str(e)), 'error')
                return render_template('setup.html')
            
            # Update .env with PostgreSQL URI
            try:
                env_path = os.path.join(app_dir, '.env')
                env_lines = []
                if os.path.exists(env_path):
                    with open(env_path, 'r') as f:
                        env_lines = [line for line in f.readlines() if not line.startswith('DATABASE_URI=')]
                env_lines.append(f"DATABASE_URI={db_uri}\n")
                with open(env_path, 'w') as f:
                    f.writelines(env_lines)
            except Exception as e:
                try:
                    os.remove(pending_path)
                except OSError:
                    pass
                flash(t('flash.db_connection_error', error=str(e)), 'error')
                return render_template('setup.html')
            
            # Trigger service restart
            service_name = os.getenv('SERVICE_NAME', '')
            if service_name:
                try:
                    import subprocess as sp
                    sp.Popen(
                        ['sudo', 'systemctl', 'restart', service_name],
                        stdout=sp.DEVNULL, stderr=sp.DEVNULL
                    )
                except Exception:
                    pass
            
            # Return a simple page that waits for restart and redirects
            return '''<!DOCTYPE html>
<html><head><meta charset="UTF-8">
<title>Setting up...</title>
<style>body{font-family:sans-serif;display:flex;align-items:center;justify-content:center;
min-height:100vh;margin:0;background:linear-gradient(135deg,#667eea,#764ba2);color:white;
text-align:center}.box{max-width:400px}h2{margin-bottom:8px}
.spinner{width:40px;height:40px;border:4px solid rgba(255,255,255,.3);
border-top-color:white;border-radius:50%;animation:s .8s linear infinite;margin:0 auto 20px}
@keyframes s{to{transform:rotate(360deg)}}</style>
</head><body><div class="box"><div class="spinner"></div>
<h2>Setting up your CMS...</h2>
<p>This takes a few seconds. Please don't close the browser.</p>
</div><script>
function poll(n){if(n>30){document.querySelector("h2").textContent="Please refresh the page.";return}
fetch("/setup/health",{cache:"no-store"}).then(function(r){return r.json()})
.then(function(d){if(d.ready&&d.has_users){window.location.href="/admin"}
else{setTimeout(function(){poll(n+1)},2000)}})
.catch(function(){setTimeout(function(){poll(n+1)},2000)})}
setTimeout(function(){poll(0)},3000);
</script></body></html>'''
        
        # ── SQLite path — direct setup (no restart needed) ──────────────
        hashed_password = generate_password_hash(password)
        
        # Ensure SECRET_KEY exists in .env (generate if missing)
        env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.env')
        if os.path.exists(env_path):
            with open(env_path, 'r') as f:
                env_content = f.read()
        else:
            env_content = ''
        if 'SECRET_KEY=' not in env_content:
            generated_key = secrets.token_hex(32)
            with open(env_path, 'a') as f:
                f.write(f"\nSECRET_KEY={generated_key}\n")
            app.config['SECRET_KEY'] = generated_key
        
        admin = User(
            username=username,
            email=email,
            password=hashed_password,
            is_admin=True,
            role='owner',
            display_name=username,
            is_active_account=True
        )
        
        db.session.add(admin)
        
        # Set admin panel language from wizard selection
        wizard_language = request.form.get('default_language', 'en')
        admin.admin_language = wizard_language
        
        db.session.commit()
        
        # Initialize default settings and pages
        init_default_settings()
        
        # Save language settings BEFORE creating default pages
        # so pages are created in the correct language
        default_language = request.form.get('default_language', 'en')
        if default_language:
            Settings.set('default_language', default_language)
            Settings.set('active_languages', default_language)
        
        create_default_pages(admin.id, language=wizard_language)
        create_default_menu_items()
        
        # --- Save site info from wizard ---
        site_name = request.form.get('site_name', '').strip()
        site_tagline = request.form.get('site_tagline', '').strip()
        
        if site_name:
            Settings.set('site_name', site_name)
        if site_tagline:
            Settings.set('site_tagline', site_tagline)
        
        # Process license key from wizard form
        license_key = request.form.get('license_key', '').strip()
        
        if license_key and not Settings.get('license_key_encrypted', ''):
            try:
                encrypted = encrypt_smtp_password(license_key)
                Settings.set('license_key_encrypted', encrypted)
                # Validate against the update server
                validator = LicenseValidator(app.config.get('UPDATE_SERVER_URL', ''))
                site_url = request.url_root.rstrip('/')
                result = validator.validate(license_key, site_url)
                _save_license_result(result)
            except Exception:
                # If update server unreachable, store key anyway — will validate later
                _save_license_invalid()
                flash(t('flash.license_validation_failed'), 'warning')
        
        flash(t('flash.setup_completed'), 'success')
        login_user(admin)
        return redirect(url_for('admin_dashboard'))
    
    return render_template('setup.html')


@app.route('/setup/test-db', methods=['POST'])
@csrf.exempt  # Setup endpoint — no session/CSRF available yet
def setup_test_db():
    """AJAX endpoint — test PostgreSQL connection from the Wizard."""
    if User.query.count() > 0:
        return jsonify({'ok': False, 'error': t('api.setup_already_completed')}), 403
    
    data = request.get_json(silent=True) or {}
    db_host = data.get('host', 'localhost').strip()
    db_port = data.get('port', '5432').strip()
    db_name = data.get('name', '').strip()
    db_user = data.get('user', '').strip()
    db_pass = data.get('password', '')
    
    if not all([db_host, db_name, db_user]):
        return jsonify({'ok': False, 'error': t('api.setup_db_fields_required')})
    
    try:
        import psycopg2
        conn = psycopg2.connect(
            host=db_host, port=int(db_port),
            dbname=db_name, user=db_user, password=db_pass,
            connect_timeout=5
        )
        # Quick check — can we read?
        cur = conn.cursor()
        cur.execute('SELECT 1')
        cur.close()
        conn.close()
        return jsonify({'ok': True, 'message': t('api.setup_connected', db_name=db_name, host=db_host, port=db_port)})
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e)})


@app.route('/setup/install-postgres', methods=['POST'])
@csrf.exempt  # Setup endpoint — no session/CSRF available yet
def setup_install_postgres():
    """AJAX endpoint — auto-install PostgreSQL + create database on a fresh VPS.
    
    This runs system commands, so it only works on a Linux VPS with root/sudo.
    Designed for a first-time Hetzner Cloud / DigitalOcean / Linode setup.
    """
    import subprocess
    import secrets
    
    if User.query.count() > 0:
        return jsonify({'ok': False, 'error': t('api.setup_already_completed')}), 403
    
    data = request.get_json(silent=True) or {}
    raw_db_name = data.get('name', 'aliothpress').strip() or 'aliothpress'
    raw_db_user = data.get('user', 'aliothpress').strip() or 'aliothpress'
    
    # Validate identifiers to prevent SQL injection
    try:
        db_name = validate_pg_identifier(raw_db_name)
        db_user = validate_pg_identifier(raw_db_user)
    except ValueError as e:
        return jsonify({'ok': False, 'error': str(e)}), 400
    
    # Generate a strong random password
    db_pass = secrets.token_urlsafe(20)
    
    steps = []
    
    try:
        # Step 1 — Check if PostgreSQL is already installed
        pg_check = subprocess.run(['which', 'psql'], capture_output=True, text=True)
        pg_installed = pg_check.returncode == 0
        
        if not pg_installed:
            # Step 2 — Install PostgreSQL
            steps.append('Installing PostgreSQL...')
            
            # Update package list
            r = subprocess.run(['sudo', 'apt-get', 'update', '-qq'],
                               capture_output=True, text=True, timeout=120)
            if r.returncode != 0:
                return jsonify({'ok': False, 'error': t('api.apt_update_failed', error=r.stderr.strip()),
                                'steps': steps})
            
            # Install PostgreSQL
            r = subprocess.run(
                ['sudo', 'apt-get', 'install', '-y', '-qq', 'postgresql', 'postgresql-contrib'],
                capture_output=True, text=True, timeout=300
            )
            if r.returncode != 0:
                return jsonify({'ok': False, 'error': t('api.postgres_install_failed', error=r.stderr.strip()),
                                'steps': steps})
            steps.append('PostgreSQL installed')
            
            # Ensure the service is running
            subprocess.run(['sudo', 'systemctl', 'start', 'postgresql'],
                           capture_output=True, text=True, timeout=30)
            subprocess.run(['sudo', 'systemctl', 'enable', 'postgresql'],
                           capture_output=True, text=True, timeout=30)
            steps.append('PostgreSQL service started')
        else:
            steps.append('PostgreSQL already installed')
        
        # Step 3 — Create database user and database
        # Run commands as postgres system user
        def pg_exec(sql):
            """Execute SQL as the postgres superuser."""
            r = subprocess.run(
                ['sudo', '-u', 'postgres', 'psql', '-c', sql],
                capture_output=True, text=True, timeout=15
            )
            return r
        
        # Create user (ignore error if already exists)
        pg_exec(f"CREATE USER {db_user} WITH PASSWORD '{db_pass}';")
        pg_exec(f"ALTER USER {db_user} WITH PASSWORD '{db_pass}';")
        steps.append(f'Database user "{db_user}" ready')
        
        # Create database (ignore error if already exists)
        pg_exec(f"CREATE DATABASE {db_name} OWNER {db_user};")
        steps.append(f'Database "{db_name}" ready')
        
        # Grant privileges
        pg_exec(f"GRANT ALL PRIVILEGES ON DATABASE {db_name} TO {db_user};")
        steps.append('Privileges granted')
        
        # Step 4 — Verify connection
        import psycopg2
        conn = psycopg2.connect(
            host='127.0.0.1', port=5432,
            dbname=db_name, user=db_user, password=db_pass,
            connect_timeout=5
        )
        conn.close()
        steps.append('Connection verified')
        
        return jsonify({
            'ok': True,
            'steps': steps,
            'credentials': {
                'host': '127.0.0.1',
                'port': '5432',
                'name': db_name,
                'user': db_user,
                'password': db_pass
            },
            'message': t('api.pg_ready', db_name=db_name)
        })
        
    except subprocess.TimeoutExpired:
        steps.append('Command timed out')
        return jsonify({'ok': False, 'error': t('api.setup_timeout'),
                        'steps': steps})
    except Exception as e:
        return jsonify({'ok': False, 'error': str(e), 'steps': steps})


@app.route('/setup/health', methods=['GET'])
def setup_health():
    """Health check endpoint — the wizard polls this during restart."""
    try:
        user_count = User.query.count()
        return jsonify({'ready': True, 'has_users': user_count > 0})
    except Exception:
        return jsonify({'ready': False}), 503


@app.route('/setup/finalize', methods=['POST'])
@csrf.exempt  # Setup endpoint — no session/CSRF available yet
def setup_finalize():
    """Two-phase setup: save wizard data to .setup-pending.json, update .env, restart.
    
    This is the AJAX endpoint called by the wizard when PostgreSQL is selected.
    Instead of trying to switch SQLAlchemy's engine mid-request, we:
      1. Save all form data (with hashed password) to a JSON file
      2. Update .env with the new DATABASE_URI
      3. Trigger a service restart
      4. On restart, init_database() picks up the pending file and finalizes setup
    """
    if User.query.count() > 0:
        return jsonify({'ok': False, 'error': 'Setup already completed.'}), 403
    
    data = request.get_json(silent=True) or {}
    
    # ── Validate required fields ─────────────────────────────────────
    username = (data.get('username') or '').strip()
    email = (data.get('email') or '').strip()
    password = data.get('password', '')
    confirm = data.get('confirm_password', '')
    db_type = data.get('db_type', 'sqlite')
    
    if not all([username, email, password]):
        return jsonify({'ok': False, 'error': 'Missing required fields.'}), 400
    
    if password != confirm:
        return jsonify({'ok': False, 'error': 'Passwords do not match.'}), 400
    
    is_valid, pwd_error = validate_password(password)
    if not is_valid:
        return jsonify({'ok': False, 'error': pwd_error}), 400
    
    # ── Build the new DATABASE_URI ───────────────────────────────────
    if db_type in ('postgresql', 'postgresql_auto'):
        db_host = (data.get('db_host') or 'localhost').strip()
        db_port = (data.get('db_port') or '5432').strip()
        db_name = (data.get('db_name') or '').strip()
        db_user = (data.get('db_user') or '').strip()
        db_password = data.get('db_password', '')
        
        if not all([db_host, db_name, db_user]):
            return jsonify({'ok': False, 'error': 'Missing database fields.'}), 400
        
        db_uri = f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}"
    else:
        # SQLite — shouldn't normally hit this path, but handle gracefully
        db_uri = 'sqlite:///aliothpress.db'
    
    # ── Save pending setup data ──────────────────────────────────────
    hashed_password = generate_password_hash(password)
    
    pending_data = {
        'username': username,
        'email': email,
        'password_hash': hashed_password,
        'db_type': db_type,
        'default_language': data.get('default_language', 'en'),
        'site_name': (data.get('site_name') or '').strip(),
        'site_tagline': (data.get('site_tagline') or '').strip(),
        'license_key': (data.get('license_key') or '').strip(),
        'request_url_root': request.url_root,
    }
    
    app_dir = os.path.dirname(os.path.abspath(__file__))
    pending_path = os.path.join(app_dir, '.setup-pending.json')
    
    try:
        with open(pending_path, 'w') as f:
            json.dump(pending_data, f)
    except Exception as e:
        return jsonify({'ok': False, 'error': f'Could not save setup data: {e}'}), 500
    
    # ── Update .env with new DATABASE_URI ────────────────────────────
    env_path = os.path.join(app_dir, '.env')
    try:
        env_lines = []
        if os.path.exists(env_path):
            with open(env_path, 'r') as f:
                env_lines = [line for line in f.readlines() if not line.startswith('DATABASE_URI=')]
        env_lines.append(f"DATABASE_URI={db_uri}\n")
        with open(env_path, 'w') as f:
            f.writelines(env_lines)
    except Exception as e:
        # Clean up pending file
        try:
            os.remove(pending_path)
        except OSError:
            pass
        return jsonify({'ok': False, 'error': f'Could not update .env: {e}'}), 500
    
    # ── Trigger service restart ──────────────────────────────────────
    service_name = os.getenv('SERVICE_NAME', '')
    restart_triggered = False
    
    if service_name:
        try:
            import subprocess as sp
            sp.Popen(
                ['sudo', 'systemctl', 'restart', service_name],
                stdout=sp.DEVNULL, stderr=sp.DEVNULL
            )
            restart_triggered = True
        except Exception:
            restart_triggered = False
    
    return jsonify({
        'ok': True,
        'restart': restart_triggered,
        'message': 'Setup data saved. Restarting...' if restart_triggered else 'Setup data saved. Please restart the service manually.'
    })


@app.route('/admin/login', methods=['GET', 'POST'])
@limiter.limit("10 per minute")
def admin_login():
    """Admin login page with brute-force protection"""
    if current_user.is_authenticated:
        return redirect(url_for('admin_dashboard'))
    
    ip = request.remote_addr
    
    if request.method == 'POST':
        username = request.form.get('username')
        password = request.form.get('password')
        
        if LoginAttempt.is_blocked(ip, username=username):
            flash(t('flash.too_many_attempts'), 'error')
            AuditLog.log('login_blocked', details=_ad('login_blocked', ip=ip), ip=ip)
            db.session.commit()
            return render_template('admin/login.html')
        
        login_id = (username or '').strip()
        user = User.query.filter(
            (User.username == login_id) |
            (db.func.lower(User.email) == login_id.lower())
        ).first()
        
        if user and check_password_hash(user.password, password):
            # Check if account is active
            if not user.is_active_account:
                flash(t('flash.invalid_credentials'), 'error')
                return render_template('admin/login.html')
            
            # Optional second factor (see twofa.py): returns a redirect
            # to the verification step when required, or None to proceed
            # with the normal single-factor login below.
            import twofa as _twofa
            second = _twofa.intercept_login(user, ip,
                                            request.args.get('next'))
            if second is not None:
                return second
            login_user(user)
            # Store session version for password-change invalidation
            session['_session_version'] = user.session_version or 0
            LoginAttempt.record(ip, username, success=True)
            AuditLog.log('login', details=_ad('login_success'), user=user, ip=ip)
            db.session.commit()
            # Validate next_page to prevent open redirect attacks
            next_page = validate_next_url(request.args.get('next'))
            return redirect(next_page or url_for('admin_dashboard'))
        else:
            LoginAttempt.record(ip, username, success=False)
            AuditLog.log('login_failed', details=_ad('login_failed', name=username), ip=ip)
            db.session.commit()
            flash(t('flash.invalid_credentials'), 'error')
    
    return render_template('admin/login.html')


@app.route('/admin/logout', methods=['POST'])
@login_required
def admin_logout():
    """Logout"""
    AuditLog.log('logout', details=_ad('logout'), user=current_user, ip=request.remote_addr)
    db.session.commit()
    logout_user()
    flash(t('flash.logged_out'), 'info')
    return redirect(url_for('index'))


@app.route('/admin/forgot-password', methods=['GET', 'POST'])
def admin_forgot_password():
    """Request a password reset link via email."""
    if current_user.is_authenticated:
        return redirect(url_for('admin_dashboard'))
    
    if request.method == 'POST':
        email = request.form.get('email', '').strip()
        if not email:
            flash(t('flash.email_enter_address'), 'error')
            return render_template('admin/forgot_password.html')
        
        user = User.query.filter_by(email=email, is_active_account=True).first()
        
        if user:
            # Generate secure token (valid 1 hour)
            token = secrets.token_urlsafe(32)
            user.reset_token = token
            user.reset_token_expires = utc_now() + timedelta(hours=1)
            db.session.commit()
            
            # Build reset URL
            reset_url = url_for('admin_reset_password', token=token, _external=True)
            
            # Send email (in the user's preferred language, not the requester's)
            settings = Settings.query.all()
            site_name = Settings.get('site_name', 'AliothPress')
            lang = user.admin_language or 'en'
            powered_by = '' if _verify_license_integrity() else ' &bull; Powered by <a href="https://aliothpress.com" style="color: #a0aec0; text-decoration: underline;">AliothPress</a>'
            
            html_body = f'''
            <div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 600px; margin: 0 auto; padding: 20px;">
                <h2 style="color: #2d3748;">🔑 {t_for('email.reset_title', lang)}</h2>
                <p>{t_for('email.reset_intro', lang, site_name=site_name)}</p>
                <p style="font-size: 14px; color: #4a5568; padding: 12px 16px; background: #f7fafc; border-radius: 6px;">{t_for('email.reset_your_username', lang)}: <strong>{user.username}</strong></p>
                <p style="margin: 24px 0;">
                    <a href="{reset_url}" style="background: #667eea; color: white; padding: 12px 24px; border-radius: 8px; text-decoration: none; font-weight: 600;">
                        {t_for('email.reset_button', lang)}
                    </a>
                </p>
                <p style="font-size: 13px; color: #718096;">{t_for('email.reset_expires', lang)}</p>
                <p style="font-size: 13px; color: #718096;">{t_for('email.reset_ignore', lang)}</p>
                <hr style="border: none; border-top: 1px solid #e2e8f0; margin: 24px 0;">
                <p style="font-size: 12px; color: #a0aec0;">{t_for('email.sent_by', lang, site_name=site_name)}{powered_by}</p>
            </div>
            '''
            
            success, msg = send_email(email, f'{site_name} — {t_for("email.reset_subject", lang)}', html_body)
            
            if not success:
                AuditLog.log('password_reset_email_failed', details=_ad('password_reset_email_failed', email=email, error=msg), ip=request.remote_addr)
                db.session.commit()
        
        # Always show the same message (prevents email enumeration)
        flash(t('flash.password_reset_sent'), 'info')
        AuditLog.log('password_reset_request', details=_ad('password_reset_request', email=email), ip=request.remote_addr)
        db.session.commit()
        return render_template('admin/forgot_password.html')
    
    return render_template('admin/forgot_password.html')


@app.route('/admin/reset-password/<token>', methods=['GET', 'POST'])
def admin_reset_password(token):
    """Reset password using a valid token."""
    if current_user.is_authenticated:
        return redirect(url_for('admin_dashboard'))
    
    user = User.query.filter_by(reset_token=token, is_active_account=True).first()
    
    if not user or not user.reset_token_expires or user.reset_token_expires < utc_now():
        flash(t('flash.password_reset_invalid'), 'error')
        return redirect(url_for('admin_login'))
    
    if request.method == 'POST':
        password = request.form.get('password', '')
        confirm = request.form.get('confirm_password', '')
        
        if password != confirm:
            flash(t('flash.passwords_mismatch'), 'error')
            return render_template('admin/reset_password.html', token=token)
        
        is_valid, pwd_error = validate_password(password)
        if not is_valid:
            flash(pwd_error, 'error')
            return render_template('admin/reset_password.html', token=token)
        
        # Update password and clear token
        user.password = generate_password_hash(password)
        user.reset_token = None
        user.reset_token_expires = None
        user.session_version = (user.session_version or 0) + 1  # Invalidate all sessions
        db.session.commit()
        
        AuditLog.log('password_reset', details=_ad('password_reset', email=user.email), user=user, ip=request.remote_addr)
        db.session.commit()
        
        flash(t('flash.password_reset_done'), 'success')
        return redirect(url_for('admin_login'))
    
    return render_template('admin/reset_password.html', token=token)


# ============================================================================
# ADMIN DASHBOARD
# ============================================================================

@app.route('/admin')
@login_required
def admin_dashboard():
    """Admin dashboard - overview"""
    stats = {
        'total_posts': Post.query.count(),
        'published_posts': Post.query.filter_by(status='published').count(),
        'draft_posts': Post.query.filter_by(status='draft').count(),
        'total_pages': Page.query.count(),
        'total_images': Image.query.count(),
        'total_users': User.query.count(),
        'total_forms': Form.query.count(),
        'new_submissions': FormSubmission.query.filter_by(status='new').count(),
    }
    
    recent_posts = Post.query.order_by(Post.updated_at.desc()).limit(5).all()
    recent_pages = Page.query.order_by(Page.updated_at.desc()).limit(5).all()
    recent_activity = AuditLog.query.order_by(AuditLog.created_at.desc()).limit(8).all()
    recent_submissions = FormSubmission.query.order_by(FormSubmission.created_at.desc()).limit(5).all()
    
    return render_template('admin/dashboard.html', 
                         stats=stats, 
                         recent_posts=recent_posts,
                         recent_pages=recent_pages,
                         recent_activity=recent_activity,
                         recent_submissions=recent_submissions,
                         license_info=get_license_status(Settings.get),
                         cms_version=get_current_version())


# ============================================================================
# POSTS MANAGEMENT
# ============================================================================

@app.route('/admin/posts')
@login_required
def admin_posts_list():
    """List all posts"""
    page = get_page()
    per_page = get_per_page()
    status_filter = request.args.get('status', 'all')
    lang_filter = request.args.get('lang', 'all')
    sort_by = request.args.get('sort', 'updated-desc')
    search_query = request.args.get('q', '').strip()
    
    query = Post.query
    if status_filter != 'all':
        query = query.filter_by(status=status_filter)
    if lang_filter != 'all':
        query = query.filter_by(language=lang_filter)
    if search_query:
        query = query.filter(Post.title.ilike(f'%{escape_like(search_query)}%', escape='\\'))
    
    # Sorting
    if sort_by == 'title-asc':
        query = query.order_by(Post.title.asc())
    elif sort_by == 'title-desc':
        query = query.order_by(Post.title.desc())
    elif sort_by == 'published-desc':
        query = query.order_by(Post.published_at.desc().nullslast())
    elif sort_by == 'published-asc':
        query = query.order_by(Post.published_at.asc().nullslast())
    else:
        query = query.order_by(Post.updated_at.desc())
    
    posts = query.paginate(
        page=page, per_page=per_page, error_out=False
    )
    
    # Compute which translation_groups have at least one MenuItem.
    # A post is shown as "in menu" if any translation in its group is pinned to a menu.
    in_menu_groups = {
        tg for (tg,) in db.session.query(Post.translation_group)
            .join(MenuItem, MenuItem.post_id == Post.id)
            .filter(Post.translation_group.isnot(None))
            .distinct()
    }
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': p.id,
                'title': p.title,
                'slug': p.slug,
                'status': p.status,
                'language': p.language,
                'is_in_menu': p.translation_group in in_menu_groups,
                'author': p.author.username if p.author else '—',
                'published_at': to_site_time(p.published_at).isoformat() if p.published_at else None,
                'updated_at': to_site_time(p.updated_at).isoformat() if p.updated_at else None,
            } for p in posts.items],
            'page': posts.page,
            'pages': posts.pages,
            'total': posts.total,
            'has_prev': posts.has_prev,
            'has_next': posts.has_next,
            'prev_num': posts.prev_num,
            'next_num': posts.next_num,
            'per_page': per_page,
        })
    
    return render_template('admin/posts/list.html', posts=posts, 
                         status_filter=status_filter, lang_filter=lang_filter,
                         sort_by=sort_by, search_query=search_query,
                         per_page=per_page,
                         in_menu_groups=in_menu_groups,
                         available_languages=_get_active_languages())


@app.route('/admin/posts/new', methods=['GET', 'POST'])
@login_required
def admin_posts_new():
    """Create new post"""
    if request.method == 'POST':
        # Get form data
        title = request.form.get('title')
        content = sanitize_html(request.form.get('content'))
        excerpt = request.form.get('excerpt')
        language = request.form.get('language') or 'en'
        status = _agent_safe_status(request.form.get('status', 'draft'))
        
        # Translation group (from hidden field if creating as translation, otherwise auto-generated)
        translation_group = request.form.get('translation_group', '')
        
        # SEO fields
        meta_title = request.form.get('meta_title')
        meta_description = request.form.get('meta_description')
        meta_keywords = request.form.get('meta_keywords')
        
        # Open Graph
        og_title = request.form.get('og_title')
        og_description = request.form.get('og_description')
        og_image = request.form.get('og_image')
        
        # Twitter
        twitter_title = request.form.get('twitter_title')
        twitter_description = request.form.get('twitter_description')
        twitter_image = request.form.get('twitter_image')
        
        # Featured image
        featured_image = request.form.get('featured_image')
        featured_image_alt = request.form.get('featured_image_alt')
        featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags (JSON array)
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        tags_json = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        category = request.form.get('category')
        
        # Schema.org structured data
        schema_type = request.form.get('schema_type') or 'BlogPosting'
        schema_author = request.form.get('schema_author') or None
        schema_publisher = request.form.get('schema_publisher') or None
        
        # Slug (auto-generate if not provided)
        slug = request.form.get('slug')
        if not slug:
            slug = slugify(title)
            slug = ensure_unique_slug(slug, Post)
        else:
            slug = slugify(slug)  # Sanitize: lowercase, no spaces/special chars
            slug = ensure_unique_slug(slug, Post)
        
        # Create post
        post = Post(
            title=title,
            slug=slug,
            content=content,
            excerpt=excerpt,
            language=language,
            status=status,
            user_id=current_user.id,
            meta_title=meta_title,
            meta_description=meta_description,
            meta_keywords=meta_keywords,
            og_title=og_title,
            og_description=og_description,
            og_image=og_image,
            twitter_title=twitter_title,
            twitter_description=twitter_description,
            twitter_image=twitter_image,
            featured_image=featured_image,
            featured_image_alt=featured_image_alt,
            featured_image_caption=featured_image_caption,
            tags=tags_json,
            category=category,
            schema_type=schema_type,
            schema_author=schema_author,
            schema_publisher=schema_publisher,
            show_author=request.form.get('show_author') == 'on',
            show_date=request.form.get('show_date') == 'on',
            faq_json=request.form.get('faq_json', ''),
            translation_group=translation_group if translation_group else None,  # None = auto-generate
        )
        
        # Set published_at if publishing
        if status == 'published':
            post.published_at = utc_now()
        
        db.session.add(post)
        db.session.commit()
        
        AuditLog.log('create_post', 'post', post.id,
                     _ad('created_post', title=post.title, lang=post.language),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return _after_create('admin_posts_edit', t('flash.post_created'), post_id=post.id)
    
    # GET request — check if creating a translation of an existing post
    source_id = request.args.get('translate_from')
    target_lang = request.args.get('target_lang')
    source_post = None
    
    if source_id:
        source_post = Post.query.get(source_id)
    
    return render_template('admin/posts/new.html', 
                         source_post=source_post,
                         target_lang=target_lang,
                         available_languages=AVAILABLE_LANGUAGES)


@app.route('/admin/posts/<int:post_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_posts_edit(post_id):
    """Edit existing post"""
    post = Post.query.get_or_404(post_id)
    
    if request.method == 'POST':
        # Update basic fields
        post.title = request.form.get('title')
        post.content = sanitize_html(request.form.get('content'))
        post.excerpt = request.form.get('excerpt')
        post.language = request.form.get('language') or 'en'
        
        # Update status
        old_status = post.status
        post.status = request.form.get('status', 'draft')
        
        # Set published_at when publishing for first time
        if post.status == 'published' and old_status != 'published':
            post.published_at = utc_now()
        
        # SEO fields
        post.meta_title = request.form.get('meta_title')
        post.meta_description = request.form.get('meta_description')
        post.meta_keywords = request.form.get('meta_keywords')
        
        # Schema.org structured data
        post.schema_type = request.form.get('schema_type') or 'BlogPosting'
        post.schema_author = request.form.get('schema_author') or None
        post.schema_publisher = request.form.get('schema_publisher') or None
        
        # Open Graph
        post.og_title = request.form.get('og_title')
        post.og_description = request.form.get('og_description')
        post.og_image = request.form.get('og_image')
        
        # Twitter
        post.twitter_title = request.form.get('twitter_title')
        post.twitter_description = request.form.get('twitter_description')
        post.twitter_image = request.form.get('twitter_image')
        
        # Featured image
        post.featured_image = request.form.get('featured_image')
        post.featured_image_alt = request.form.get('featured_image_alt')
        post.featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        post.tags = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        post.category = request.form.get('category')
        
        # Display options
        post.show_author = request.form.get('show_author') == 'on'
        post.show_date = request.form.get('show_date') == 'on'
        
        # FAQ (AEO)
        post.faq_json = request.form.get('faq_json', '')
        
        # Slug
        new_slug = request.form.get('slug')
        if new_slug:
            new_slug = slugify(new_slug)  # Sanitize user input
        if new_slug and new_slug != post.slug:
            # Check uniqueness across both Post and Page
            existing_post = Post.query.filter_by(slug=new_slug).first()
            existing_page = Page.query.filter_by(slug=new_slug).first()
            if (existing_post and existing_post.id != post.id) or existing_page:
                flash(t('flash.slug_exists'), 'warning')
                old_slug = post.slug
                post.slug = ensure_unique_slug(slugify(post.title), Post, exclude_id=post.id)
                Redirect.create_redirect(old_slug, post.slug, 'post', post.id)
            else:
                old_slug = post.slug
                post.slug = new_slug
                Redirect.create_redirect(old_slug, new_slug, 'post', post.id)
        
        post.updated_at = utc_now()
        
        db.session.commit()
        
        AuditLog.log('edit_post', 'post', post.id,
                     _ad('edited_post', title=post.title, lang=post.language),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return _inline_saved(t('flash.post_updated'), 'admin_posts_list')
    
    # Get tags as comma-separated string
    tags_str = ', '.join(post.get_tags_list())
    
    # Get translations info
    translations = post.get_translations()
    missing_languages = _filter_missing_by_active(post.get_missing_languages())
    
    return render_template('admin/posts/edit.html', post=post, tags_str=tags_str,
                         translations=translations, missing_languages=missing_languages,
                         available_languages=AVAILABLE_LANGUAGES)


@app.route('/admin/posts/<int:post_id>/delete', methods=['POST'])
@login_required
def admin_posts_delete(post_id):
    """Delete post, cleaning up internal links site-wide."""
    post = Post.query.get_or_404(post_id)
    post_title = post.title
    
    # Clean up all internal links pointing to this post
    _cleanup_content_references(post.slug, 'post', post.id)
    
    AuditLog.log('delete_post', 'post', post_id,
                 _ad('deleted_post', title=post_title),
                 user=current_user, ip=request.remote_addr)
    db.session.delete(post)
    db.session.commit()
    flash(t('flash.post_deleted'), 'success')
    return _safe_return_to('admin_posts_list')


@app.route('/admin/posts/<int:post_id>/translate', methods=['GET', 'POST'])
@login_required
def admin_posts_translate(post_id):
    """Translate post — create a new post in another language"""
    source_post = Post.query.get_or_404(post_id)
    
    if request.method == 'POST':
        # This creates a new post linked via translation_group
        title = request.form.get('title')
        content = sanitize_html(request.form.get('content'))
        excerpt = request.form.get('excerpt')
        language = request.form.get('language') or 'en'
        status = _agent_safe_status(request.form.get('status', 'draft'))
        
        # Check if translation in this language already exists
        existing = Post.query.filter_by(
            translation_group=source_post.translation_group, language=language
        ).first()
        if existing:
            # Agent / WebMCP callers asked for JSON — give them a machine-readable
            # conflict instead of an HTML redirect they cannot interpret.
            if request.form.get('_format') == 'json' or request.args.get('format') == 'json':
                return jsonify({
                    'success': False,
                    'error': 'translation_exists',
                    'language': language,
                    'existing_id': existing.id,
                    'existing_slug': existing.slug,
                    'edit_url': url_for('admin_posts_edit', post_id=existing.id),
                }), 409
            flash(t('flash.translation_exists', language=LANG_CODE_TO_NAME.get(language, language)), 'warning')
            return _safe_return_to('admin_posts_list')
        
        # SEO fields
        meta_title = request.form.get('meta_title')
        meta_description = request.form.get('meta_description')
        meta_keywords = request.form.get('meta_keywords')
        
        # Open Graph
        og_title = request.form.get('og_title')
        og_description = request.form.get('og_description')
        og_image = request.form.get('og_image')
        
        # Twitter
        twitter_title = request.form.get('twitter_title')
        twitter_description = request.form.get('twitter_description')
        twitter_image = request.form.get('twitter_image')
        
        # Featured image
        featured_image = request.form.get('featured_image')
        featured_image_alt = request.form.get('featured_image_alt')
        featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        tags_json = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        category = request.form.get('category')
        
        # Slug
        slug = request.form.get('slug')
        if not slug:
            slug = slugify(title)
            slug = ensure_unique_slug(slug, Post)
        else:
            slug = slugify(slug)  # Sanitize: lowercase, no spaces/special chars
            slug = ensure_unique_slug(slug, Post)
        
        new_post = Post(
            title=title,
            slug=slug,
            content=content,
            excerpt=excerpt,
            language=language,
            status=status,
            translation_group=source_post.translation_group,
            user_id=current_user.id,
            meta_title=meta_title,
            meta_description=meta_description,
            meta_keywords=meta_keywords,
            og_title=og_title,
            og_description=og_description,
            og_image=og_image,
            twitter_title=twitter_title,
            twitter_description=twitter_description,
            twitter_image=twitter_image,
            featured_image=featured_image,
            featured_image_alt=featured_image_alt,
            featured_image_caption=featured_image_caption,
            tags=tags_json,
            category=category,
            schema_type=request.form.get('schema_type') or source_post.schema_type,
            schema_author=request.form.get('schema_author', source_post.schema_author),
            schema_publisher=request.form.get('schema_publisher', source_post.schema_publisher),
            show_author=request.form.get('show_author') == 'on',
            show_date=request.form.get('show_date') == 'on',
            faq_json=request.form.get('faq_json', ''),
        )
        
        if status == 'published':
            new_post.published_at = utc_now()
        
        db.session.add(new_post)
        db.session.commit()
        
        return _after_create('admin_posts_edit', t('flash.translation_created', status=status), post_id=new_post.id)
    
    # GET — show translate form
    translations = source_post.get_translations()
    missing_languages = _filter_missing_by_active(source_post.get_missing_languages())
    tags_str = ', '.join(source_post.get_tags_list())
    
    return render_template('admin/posts/translate.html',
                         source=source_post,
                         translations=translations,
                         missing_languages=missing_languages,
                         tags_str=tags_str,
                         available_languages=AVAILABLE_LANGUAGES)


# ============================================================================
# PAGES MANAGEMENT
# ============================================================================

@app.route('/admin/pages')
@login_required
def admin_pages_list():
    """List all pages"""
    page = get_page()
    per_page = get_per_page()
    status_filter = request.args.get('status', 'all')
    lang_filter = request.args.get('lang', 'all')
    sort_by = request.args.get('sort', 'updated-desc')
    search_query = request.args.get('q', '').strip()
    
    query = Page.query
    if status_filter != 'all':
        query = query.filter_by(status=status_filter)
    if lang_filter != 'all':
        query = query.filter_by(language=lang_filter)
    if search_query:
        query = query.filter(Page.title.ilike(f'%{escape_like(search_query)}%', escape='\\'))
    
    # Sorting
    if sort_by == 'title-asc':
        query = query.order_by(Page.title.asc())
    elif sort_by == 'title-desc':
        query = query.order_by(Page.title.desc())
    elif sort_by == 'published-desc':
        query = query.order_by(Page.published_at.desc().nullslast())
    elif sort_by == 'published-asc':
        query = query.order_by(Page.published_at.asc().nullslast())
    else:
        query = query.order_by(Page.updated_at.desc())
    
    pages = query.paginate(
        page=page, per_page=per_page, error_out=False
    )
    
    # Compute which translation_groups have at least one MenuItem.
    # A page is shown as "in menu" if any translation in its group is pinned to a menu.
    in_menu_groups = {
        tg for (tg,) in db.session.query(Page.translation_group)
            .join(MenuItem, MenuItem.page_id == Page.id)
            .filter(Page.translation_group.isnot(None))
            .distinct()
    }
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': p.id,
                'title': p.title,
                'slug': p.slug,
                'status': p.status,
                'language': p.language,
                'is_in_menu': p.translation_group in in_menu_groups,
                'author': p.author.username if p.author else '—',
                'published_at': to_site_time(p.published_at).isoformat() if p.published_at else None,
                'updated_at': to_site_time(p.updated_at).isoformat() if p.updated_at else None,
            } for p in pages.items],
            'page': pages.page,
            'pages': pages.pages,
            'total': pages.total,
            'has_prev': pages.has_prev,
            'has_next': pages.has_next,
            'prev_num': pages.prev_num,
            'next_num': pages.next_num,
            'per_page': per_page,
        })
    
    return render_template('admin/pages/list.html', pages=pages,
                         status_filter=status_filter, lang_filter=lang_filter,
                         sort_by=sort_by, search_query=search_query,
                         per_page=per_page,
                         in_menu_groups=in_menu_groups,
                         available_languages=_get_active_languages())


@app.route('/admin/pages/new', methods=['GET', 'POST'])
@login_required
def admin_pages_new():
    """Create new page"""
    if request.method == 'POST':
        # Get form data
        title = request.form.get('title')
        content = sanitize_html(request.form.get('content'))
        excerpt = request.form.get('excerpt')
        language = request.form.get('language') or 'en'
        status = _agent_safe_status(request.form.get('status', 'published'))
        template = request.form.get('template', 'default')
        
        # Translation group
        translation_group = request.form.get('translation_group', '')
        
        # SEO fields
        meta_title = request.form.get('meta_title')
        meta_description = request.form.get('meta_description')
        meta_keywords = request.form.get('meta_keywords')
        
        # Open Graph
        og_title = request.form.get('og_title')
        og_description = request.form.get('og_description')
        og_image = request.form.get('og_image')
        
        # Twitter
        twitter_title = request.form.get('twitter_title')
        twitter_description = request.form.get('twitter_description')
        twitter_image = request.form.get('twitter_image')
        
        # Featured image
        featured_image = request.form.get('featured_image')
        featured_image_alt = request.form.get('featured_image_alt')
        featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags (JSON array)
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        tags_json = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        category = request.form.get('category')
        
        # Schema.org structured data
        schema_type = request.form.get('schema_type') or 'WebPage'
        schema_author = request.form.get('schema_author') or None
        schema_publisher = request.form.get('schema_publisher') or None
        
        # Slug (auto-generate if not provided)
        slug = request.form.get('slug')
        if not slug:
            slug = slugify(title)
            slug = ensure_unique_slug(slug, Page)
        else:
            slug = slugify(slug)  # Sanitize: lowercase, no spaces/special chars
            slug = ensure_unique_slug(slug, Page)
        
        # Create page
        page = Page(
            title=title,
            slug=slug,
            content=content,
            excerpt=excerpt,
            language=language,
            status=status,
            template=template,
            user_id=current_user.id,
            meta_title=meta_title,
            meta_description=meta_description,
            meta_keywords=meta_keywords,
            og_title=og_title,
            og_description=og_description,
            og_image=og_image,
            twitter_title=twitter_title,
            twitter_description=twitter_description,
            twitter_image=twitter_image,
            featured_image=featured_image,
            featured_image_alt=featured_image_alt,
            featured_image_caption=featured_image_caption,
            tags=tags_json,
            category=category,
            schema_type=schema_type,
            schema_author=schema_author,
            schema_publisher=schema_publisher,
            show_author=request.form.get('show_author') == 'on',
            show_date=request.form.get('show_date') == 'on',
            faq_json=request.form.get('faq_json', ''),
            translation_group=translation_group if translation_group else None,
            use_builder=request.form.get('use_builder') == 'on',
            is_posts_page=request.form.get('is_posts_page') == 'on',
        )
        
        # Initialize empty builder blocks if using page builder
        if page.use_builder:
            page.blocks = '{"blocks": [], "version": 1}'
        
        # Set published_at if publishing
        if status == 'published':
            page.published_at = utc_now()
        
        db.session.add(page)
        db.session.commit()
        
        # Posts-page flag consistency: on CREATE the group wins. If this
        # page joins an existing translation group, inherit the group's
        # flag — a new translation can never silently flip the originals.
        # (Changing the flag for the whole group is done via EDIT, where
        # the user's explicit choice is synced across the group.)
        if page.translation_group:
            sibling = Page.query.filter(
                Page.translation_group == page.translation_group,
                Page.id != page.id
            ).first()
            if sibling is not None and page.is_posts_page != sibling.is_posts_page:
                page.is_posts_page = sibling.is_posts_page
                db.session.commit()
        
        AuditLog.log('create_page', 'page', page.id,
                     _ad('created_page', title=page.title, lang=page.language),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return _after_create('admin_pages_edit', t('flash.page_created'), page_id=page.id)
    
    # GET request — check if creating a translation
    source_id = request.args.get('translate_from')
    target_lang = request.args.get('target_lang')
    source_page = None
    
    if source_id:
        source_page = Page.query.get(source_id)
    
    return render_template('admin/pages/new.html',
                         source_page=source_page,
                         target_lang=target_lang,
                         available_languages=AVAILABLE_LANGUAGES)


@app.route('/admin/pages/<int:page_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_pages_edit(page_id):
    """Edit existing page"""
    page = Page.query.get_or_404(page_id)
    
    if request.method == 'POST':
        # Update basic fields
        page.title = request.form.get('title')
        # Builder pages: page.content holds the RENDERED block HTML produced
        # by block_renderer (regenerated on every builder save). It is shown
        # read-only in the editor and posted back unchanged — re-sanitizing
        # it here would strip block markup (<button>, <col>) and the
        # renderer's own inline styles (spacer heights, overlays, column
        # widths), silently degrading the preview/SEO fallback. The builder
        # save endpoint is the sole writer of builder content.
        if not (page.use_builder and request.form.get('use_builder') == 'on'):
            page.content = sanitize_html(request.form.get('content'))
        page.excerpt = request.form.get('excerpt')
        page.language = request.form.get('language') or 'en'
        page.template = request.form.get('template', 'default')
        
        # Update status
        old_status = page.status
        page.status = request.form.get('status', 'published')
        
        # Set published_at when publishing for first time
        if page.status == 'published' and old_status != 'published':
            page.published_at = utc_now()
        
        # SEO fields
        page.meta_title = request.form.get('meta_title')
        page.meta_description = request.form.get('meta_description')
        page.meta_keywords = request.form.get('meta_keywords')
        
        # Schema.org structured data
        page.schema_type = request.form.get('schema_type') or 'WebPage'
        page.schema_author = request.form.get('schema_author') or None
        page.schema_publisher = request.form.get('schema_publisher') or None
        
        # Open Graph
        page.og_title = request.form.get('og_title')
        page.og_description = request.form.get('og_description')
        page.og_image = request.form.get('og_image')
        
        # Twitter
        page.twitter_title = request.form.get('twitter_title')
        page.twitter_description = request.form.get('twitter_description')
        page.twitter_image = request.form.get('twitter_image')
        
        # Featured image
        page.featured_image = request.form.get('featured_image')
        page.featured_image_alt = request.form.get('featured_image_alt')
        page.featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        page.tags = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        page.category = request.form.get('category')
        
        # Display options
        page.show_author = request.form.get('show_author') == 'on'
        page.show_date = request.form.get('show_date') == 'on'
        
        # FAQ (AEO)
        page.faq_json = request.form.get('faq_json', '')
        
        # Page Builder mode
        page.use_builder = request.form.get('use_builder') == 'on'
        if page.use_builder and not page.blocks:
            page.blocks = '{"blocks": [], "version": 1}'
        
        # Posts page flag — sync across the whole translation group so the
        # language switcher lands on the same kind of page in every language.
        page.is_posts_page = request.form.get('is_posts_page') == 'on'
        if page.translation_group:
            Page.query.filter(
                Page.translation_group == page.translation_group,
                Page.id != page.id
            ).update({'is_posts_page': page.is_posts_page},
                     synchronize_session=False)
        
        # Slug
        new_slug = request.form.get('slug')
        if new_slug:
            new_slug = slugify(new_slug)  # Sanitize user input
        if new_slug and new_slug != page.slug:
            # Check uniqueness across both Post and Page
            existing_page = Page.query.filter_by(slug=new_slug).first()
            existing_post = Post.query.filter_by(slug=new_slug).first()
            if (existing_page and existing_page.id != page.id) or existing_post:
                flash(t('flash.slug_exists'), 'warning')
                old_slug = page.slug
                page.slug = ensure_unique_slug(slugify(page.title), Page, exclude_id=page.id)
                Redirect.create_redirect(old_slug, page.slug, 'page', page.id)
            else:
                old_slug = page.slug
                page.slug = new_slug
                Redirect.create_redirect(old_slug, new_slug, 'page', page.id)
        
        page.updated_at = utc_now()
        
        db.session.commit()
        
        AuditLog.log('edit_page', 'page', page.id,
                     _ad('edited_page', title=page.title, lang=page.language),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return _inline_saved(t('flash.page_updated'), 'admin_pages_list')
    
    # Get tags as comma-separated string
    tags_str = ', '.join(page.get_tags_list())
    
    # Get translations info
    translations = page.get_translations()
    missing_languages = _filter_missing_by_active(page.get_missing_languages())
    
    return render_template('admin/pages/edit.html', page=page, tags_str=tags_str,
                         translations=translations, missing_languages=missing_languages,
                         available_languages=AVAILABLE_LANGUAGES)


@app.route('/admin/pages/<int:page_id>/delete', methods=['POST'])
@login_required
def admin_pages_delete(page_id):
    """Delete page, cleaning up internal links site-wide."""
    page = Page.query.get_or_404(page_id)
    page_title = page.title
    
    # Clean up all internal links pointing to this page
    _cleanup_content_references(page.slug, 'page', page.id)
    
    AuditLog.log('delete_page', 'page', page_id,
                 _ad('deleted_page', title=page_title),
                 user=current_user, ip=request.remote_addr)
    db.session.delete(page)
    db.session.commit()
    flash(t('flash.page_deleted'), 'success')
    return _safe_return_to('admin_pages_list')


@app.route('/admin/pages/<int:page_id>/translate', methods=['GET', 'POST'])
@login_required
def admin_pages_translate(page_id):
    """Translate page — create a new page in another language"""
    source_page = Page.query.get_or_404(page_id)
    
    if request.method == 'POST':
        title = request.form.get('title')
        content = sanitize_html(request.form.get('content'))
        excerpt = request.form.get('excerpt')
        language = request.form.get('language') or 'en'
        status = _agent_safe_status(request.form.get('status', 'draft'))
        template = request.form.get('template', 'default')
        
        # Check if translation already exists
        existing = Page.query.filter_by(
            translation_group=source_page.translation_group, language=language
        ).first()
        if existing:
            # Agent / WebMCP callers asked for JSON — machine-readable conflict
            # instead of an HTML redirect (mirrors admin_posts_translate).
            if request.form.get('_format') == 'json' or request.args.get('format') == 'json':
                return jsonify({
                    'success': False,
                    'error': 'translation_exists',
                    'language': language,
                    'existing_id': existing.id,
                    'existing_slug': existing.slug,
                    'edit_url': url_for('admin_pages_edit', page_id=existing.id),
                }), 409
            flash(t('flash.translation_exists', language=LANG_CODE_TO_NAME.get(language, language)), 'warning')
            return _safe_return_to('admin_pages_list')
        
        # SEO fields
        meta_title = request.form.get('meta_title')
        meta_description = request.form.get('meta_description')
        meta_keywords = request.form.get('meta_keywords')
        
        # Open Graph
        og_title = request.form.get('og_title')
        og_description = request.form.get('og_description')
        og_image = request.form.get('og_image')
        
        # Twitter
        twitter_title = request.form.get('twitter_title')
        twitter_description = request.form.get('twitter_description')
        twitter_image = request.form.get('twitter_image')
        
        # Featured image
        featured_image = request.form.get('featured_image')
        featured_image_alt = request.form.get('featured_image_alt')
        featured_image_caption = request.form.get('featured_image_caption')
        
        # Tags
        tags_input = request.form.get('tags', '')
        tags_list = [tag.strip() for tag in tags_input.split(',') if tag.strip()]
        tags_json = json.dumps(tags_list, ensure_ascii=False)
        
        # Category
        category = request.form.get('category')
        
        # Slug
        slug = request.form.get('slug')
        if not slug:
            slug = slugify(title)
            slug = ensure_unique_slug(slug, Page)
        else:
            slug = slugify(slug)  # Sanitize: lowercase, no spaces/special chars
            slug = ensure_unique_slug(slug, Page)
        
        new_page = Page(
            title=title,
            slug=slug,
            content=content,
            excerpt=excerpt,
            language=language,
            status=status,
            template=template,
            translation_group=source_page.translation_group,
            is_posts_page=source_page.is_posts_page,
            user_id=current_user.id,
            meta_title=meta_title,
            meta_description=meta_description,
            meta_keywords=meta_keywords,
            og_title=og_title,
            og_description=og_description,
            og_image=og_image,
            twitter_title=twitter_title,
            twitter_description=twitter_description,
            twitter_image=twitter_image,
            featured_image=featured_image,
            featured_image_alt=featured_image_alt,
            featured_image_caption=featured_image_caption,
            tags=tags_json,
            category=category,
            schema_type=request.form.get('schema_type') or source_page.schema_type,
            schema_author=request.form.get('schema_author', source_page.schema_author),
            schema_publisher=request.form.get('schema_publisher', source_page.schema_publisher),
            show_author=request.form.get('show_author') == 'on',
            show_date=request.form.get('show_date') == 'on',
            faq_json=request.form.get('faq_json', ''),
        )
        
        if status == 'published':
            new_page.published_at = utc_now()
        
        # Copy Page Builder data from source
        if getattr(source_page, 'use_builder', False) and source_page.blocks:
            new_page.blocks = source_page.blocks
            new_page.use_builder = True
            # Regenerate the flat HTML from the copied blocks instead of
            # keeping the form-posted (re-sanitized, degraded) copy — same
            # content the builder save endpoint would produce.
            try:
                from block_renderer import render_blocks_to_html
                new_page.content = render_blocks_to_html(source_page.blocks)
            except Exception:
                pass  # keep the form copy; builder save will regenerate it
        
        db.session.add(new_page)
        db.session.commit()
        
        return _after_create('admin_pages_edit', t('flash.translation_created', status=status), page_id=new_page.id)
    
    # GET — show translate form
    translations = source_page.get_translations()
    missing_languages = _filter_missing_by_active(source_page.get_missing_languages())
    tags_str = ', '.join(source_page.get_tags_list())
    
    return render_template('admin/pages/translate.html',
                         source=source_page,
                         translations=translations,
                         missing_languages=missing_languages,
                         tags_str=tags_str,
                         available_languages=AVAILABLE_LANGUAGES)


# ============================================================================
# FORMS — Smart Form Builder Routes
# ============================================================================


# --- Forms List ---
@app.route('/admin/forms')
@login_required
def admin_forms_list():
    """List all forms with filters, search, sorting, and pagination"""
    page = get_page()
    per_page = get_per_page()
    status_filter = request.args.get('status', 'all')
    type_filter = request.args.get('type', 'all')
    lang_filter = request.args.get('lang', 'all')
    sort_by = request.args.get('sort', 'updated-desc')
    search_query = request.args.get('q', '').strip()

    query = Form.query

    # Status filter (active / inactive)
    if status_filter == 'active':
        query = query.filter_by(is_active=True)
    elif status_filter == 'inactive':
        query = query.filter_by(is_active=False)

    # Type filter
    if type_filter != 'all':
        query = query.filter_by(form_type=type_filter)

    # Language filter
    if lang_filter != 'all':
        query = query.filter_by(language=lang_filter)

    # Search by title
    if search_query:
        query = query.filter(Form.title.ilike(f'%{escape_like(search_query)}%', escape='\\'))

    # Sorting
    if sort_by == 'title-asc':
        query = query.order_by(Form.title.asc())
    elif sort_by == 'title-desc':
        query = query.order_by(Form.title.desc())
    elif sort_by == 'updated-asc':
        query = query.order_by(Form.updated_at.asc())
    else:
        query = query.order_by(Form.updated_at.desc())

    forms = query.paginate(page=page, per_page=per_page, error_out=False)

    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': f.id,
                'title': f.title,
                'slug': f.slug,
                'form_type': f.form_type,
                'is_active': f.is_active,
                'language': f.language,
                'submissions_count': f.get_submission_count(),
                'new_submissions': f.get_new_submission_count(),
                'fields_count': len(f.get_fields()),
                'updated_at': to_site_time(f.updated_at).isoformat() if f.updated_at else None,
            } for f in forms.items],
            'page': forms.page,
            'pages': forms.pages,
            'total': forms.total,
            'has_prev': forms.has_prev,
            'has_next': forms.has_next,
            'prev_num': forms.prev_num,
            'next_num': forms.next_num,
            'per_page': per_page,
        })

    return render_template('admin/forms/list.html', forms=forms,
                           status_filter=status_filter, type_filter=type_filter,
                           lang_filter=lang_filter, sort_by=sort_by,
                           search_query=search_query,
                           per_page=per_page,
                           available_languages=_get_active_languages())


# --- Create New Form ---
@app.route('/admin/forms/new', methods=['GET', 'POST'])
@login_required
def admin_forms_new():
    if request.method == 'POST':
        title = request.form.get('title', '').strip()
        form_type = request.form.get('form_type', 'custom')
        language = request.form.get('language') or 'en'
        description = request.form.get('description', '').strip()
        
        if not title:
            flash(t('flash.fields_required'), 'error')
            return redirect(url_for('admin_forms_new'))
        
        form = Form(
            title=title,
            form_type=form_type,
            language=language,
            description=description,
            user_id=current_user.id
        )
        
        # Use fields from the constructor (sent as JSON from the client)
        fields_json = request.form.get('fields_json', '[]')
        try:
            fields = json.loads(fields_json)
            form.set_fields(fields)
        except json.JSONDecodeError:
            # Fallback to server-side template if JSON is invalid
            template_fields = get_form_template(form_type)
            form.set_fields(template_fields['fields'])
        
        # Build settings from form inputs
        settings = {
            'submit_label': request.form.get('submit_label', t('forms.default_submit')).strip(),
            'success_message': request.form.get('success_message', '').strip(),
            'redirect_url': request.form.get('redirect_url', '').strip(),
            'notify_email': request.form.get('notify_email', '').strip(),
            'notify_on_submit': request.form.get('notify_on_submit') == 'on',
            'store_submissions': request.form.get('store_submissions') == 'on',
            'anti_spam': request.form.get('anti_spam', 'honeypot'),
            'double_opt_in': request.form.get('double_opt_in') == 'on',
        }
        settings['confirmation_email'] = settings['double_opt_in'] or request.form.get('confirmation_email') == 'on'
        form.set_settings(settings)
        
        db.session.add(form)
        db.session.commit()
        
        AuditLog.log('create_form', 'form', form.id, _ad('created_form', title=title), current_user)
        db.session.commit()
        
        return _after_create('admin_forms_edit', t('flash.form_created'), form_id=form.id)
    
    return render_template('admin/forms/new.html', available_languages=AVAILABLE_LANGUAGES)


# --- Edit Form (Constructor) ---
@app.route('/admin/forms/<int:form_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_forms_edit(form_id):
    form = Form.query.get_or_404(form_id)
    
    if request.method == 'POST':
        form.title = request.form.get('title', form.title).strip()
        form.description = request.form.get('description', '').strip()
        form.is_active = request.form.get('is_active') == 'on'
        
        # Update language
        new_language = request.form.get('language', form.language)
        if new_language and new_language != form.language:
            # Check no other translation in this group uses this language
            existing = Form.query.filter(
                Form.translation_group == form.translation_group,
                Form.language == new_language,
                Form.id != form.id
            ).first()
            if not existing:
                form.language = new_language
        
        # Update slug if title changed
        new_slug = request.form.get('slug', '').strip()
        if new_slug and new_slug != form.slug:
            existing = Form.query.filter(Form.slug == new_slug, Form.id != form.id).first()
            if not existing:
                form.slug = new_slug
        
        # Update fields from JSON
        fields_json = request.form.get('fields_json', '[]')
        try:
            fields = json.loads(fields_json)
            form.set_fields(fields)
        except json.JSONDecodeError:
            flash(t('flash.form_fields_invalid'), 'error')
        
        # Update settings
        settings = form.get_settings()
        settings['submit_label'] = request.form.get('submit_label', t('forms.default_submit')).strip()
        settings['success_message'] = request.form.get('success_message', '').strip()
        settings['redirect_url'] = request.form.get('redirect_url', '').strip()
        settings['notify_email'] = request.form.get('notify_email', '').strip()
        settings['notify_on_submit'] = request.form.get('notify_on_submit') == 'on'
        settings['store_submissions'] = request.form.get('store_submissions') == 'on'
        settings['anti_spam'] = request.form.get('anti_spam', 'honeypot')
        settings['double_opt_in'] = request.form.get('double_opt_in') == 'on'
        # Double opt-in implies confirmation email (disabled checkbox won't be sent)
        settings['confirmation_email'] = settings['double_opt_in'] or request.form.get('confirmation_email') == 'on'
        form.set_settings(settings)
        
        db.session.commit()
        
        AuditLog.log('edit_form', 'form', form.id, _ad('updated_form', title=form.title), current_user)
        db.session.commit()
        
        return _inline_saved(t('flash.form_updated'), 'admin_forms_list')
    
    translations = form.get_translations()
    missing_languages = _filter_missing_by_active(form.get_missing_languages())
    
    return render_template('admin/forms/edit.html', form=form,
                         translations=translations,
                         missing_languages=missing_languages,
                         available_languages=AVAILABLE_LANGUAGES)


# --- Translate Form ---
@app.route('/admin/forms/<int:form_id>/translate', methods=['GET', 'POST'])
@login_required
def admin_forms_translate(form_id):
    """Translate form — create a new form in another language with same structure"""
    source_form = Form.query.get_or_404(form_id)
    
    if request.method == 'POST':
        title = request.form.get('title', '').strip()
        language = request.form.get('language') or 'en'
        
        if not title:
            flash(t('flash.fields_required'), 'error')
            return redirect(url_for('admin_forms_translate', form_id=form_id))
        
        # Check if translation already exists
        existing = Form.query.filter_by(
            translation_group=source_form.translation_group, language=language
        ).first()
        if existing:
            flash(t('flash.translation_exists', language=LANG_CODE_TO_NAME.get(language, language)), 'warning')
            return _safe_return_to('admin_forms_list')
        
        # Create translated form — copy fields and settings structure
        fields_json = request.form.get('fields_json', source_form.fields)
        settings_json = request.form.get('settings_json', source_form.settings)
        
        new_form = Form(
            title=title,
            description=request.form.get('description', source_form.description or ''),
            form_type=source_form.form_type,
            language=language,
            translation_group=source_form.translation_group,
            is_active=source_form.is_active,
            user_id=current_user.id
        )
        new_form.fields = fields_json
        new_form.settings = settings_json
        
        db.session.add(new_form)
        db.session.commit()
        
        AuditLog.log('translate_form', 'form', new_form.id, _ad('translated_form', title=title, lang=language), current_user)
        db.session.commit()
        
        return _after_create('admin_forms_edit', t('flash.translation_created', status='draft'), form_id=new_form.id)
    
    # GET — show translate form
    translations = source_form.get_translations()
    missing_languages = _filter_missing_by_active(source_form.get_missing_languages())
    
    return render_template('admin/forms/translate.html',
                         source=source_form,
                         translations=translations,
                         missing_languages=missing_languages,
                         available_languages=AVAILABLE_LANGUAGES)


# --- Delete Form ---
@app.route('/admin/forms/<int:form_id>/delete', methods=['POST'])
@login_required
def admin_forms_delete(form_id):
    """Delete form and clean up page builder references."""
    form = Form.query.get_or_404(form_id)
    title = form.title
    
    # Clean up builder blocks that reference this form
    _cleanup_form_references(form_id)
    
    db.session.delete(form)
    
    AuditLog.log('delete_form', 'form', form_id, _ad('deleted_form', title=title), current_user)
    db.session.commit()
    
    flash(t('flash.form_deleted'), 'success')
    return _safe_return_to('admin_forms_list')


# --- View Submissions ---
@app.route('/admin/forms/<int:form_id>/submissions')
@login_required
def admin_forms_submissions(form_id):
    form = Form.query.get_or_404(form_id)
    
    # Filter by status
    status_filter = request.args.get('status', '')
    search_query = request.args.get('q', '').strip()
    query = form.submissions
    if status_filter:
        query = query.filter_by(status=status_filter)
    
    # Search across submission data (JSON), IP address
    if search_query:
        pattern = f'%{escape_like(search_query)}%'
        query = query.filter(
            db.or_(
                FormSubmission.data.ilike(pattern, escape='\\'),
                FormSubmission.ip_address.ilike(pattern, escape='\\')
            )
        )
    
    # Pagination
    page_num = get_page()
    per_page = get_per_page()
    pagination = query.order_by(FormSubmission.created_at.desc()).paginate(
        page=page_num, per_page=per_page, error_out=False
    )
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        fields = form.get_fields()
        form_settings = form.get_settings()
        return jsonify({
            'items': [{
                'id': sub.id,
                'created_at': to_site_time(sub.created_at).isoformat() if sub.created_at else None,
                'status': sub.status,
                'email_confirmed': sub.email_confirmed if hasattr(sub, 'email_confirmed') else None,
                'data': sub.get_data(),
            } for sub in pagination.items],
            'fields': [{'name': f.get('name', '') if isinstance(f, dict) else f.name, 'label': f.get('label', '') if isinstance(f, dict) else f.label} for f in fields[:5]],
            'double_opt_in': bool(form_settings.get('double_opt_in', False) if isinstance(form_settings, dict) else getattr(form_settings, 'double_opt_in', False)),
            'page': pagination.page,
            'pages': pagination.pages,
            'total': pagination.total,
            'has_prev': pagination.has_prev,
            'has_next': pagination.has_next,
            'prev_num': pagination.prev_num,
            'next_num': pagination.next_num,
            'per_page': per_page,
        })
    
    # Mark new submissions as read when viewing (only unfiltered / non-search)
    if not search_query:
        new_subs = form.submissions.filter_by(status='new').all()
        for sub in new_subs:
            sub.status = 'read'
        if new_subs:
            db.session.commit()
    
    return render_template('admin/forms/submissions.html',
                         form=form,
                         submissions=pagination.items,
                         pagination=pagination,
                         status_filter=status_filter,
                         search_query=search_query,
                         per_page=per_page)


# --- Bulk Delete Submissions ---
@app.route('/admin/forms/<int:form_id>/submissions/bulk-delete', methods=['POST'])
@login_required
def admin_forms_submissions_bulk_delete(form_id):
    form = Form.query.get_or_404(form_id)
    selected_ids = request.form.getlist('selected_ids', type=int)
    status_filter = request.form.get('status', '')
    search_query = request.form.get('q', '')
    
    if selected_ids:
        deleted = FormSubmission.query.filter(
            FormSubmission.id.in_(selected_ids),
            FormSubmission.form_id == form_id
        ).delete(synchronize_session=False)
        db.session.commit()
        
        AuditLog.log('bulk_delete_submissions', 'form', form_id,
                     _ad('bulk_deleted_submissions', count=str(deleted), title=form.title),
                     current_user)
        db.session.commit()
        
        flash(t('flash.submissions_bulk_deleted', count=deleted), 'success')
    
    return redirect(url_for('admin_forms_submissions', form_id=form_id,
                           status=status_filter, q=search_query))


# --- Single Submission Detail ---
@app.route('/admin/forms/<int:form_id>/submissions/<int:sub_id>')
@login_required
def admin_forms_submission_detail(form_id, sub_id):
    form = Form.query.get_or_404(form_id)
    submission = FormSubmission.query.filter_by(id=sub_id, form_id=form_id).first_or_404()
    
    if submission.status == 'new':
        submission.status = 'read'
        db.session.commit()
    
    return render_template('admin/forms/submission_detail.html', form=form, submission=submission)


# --- Update Submission Status ---
@app.route('/admin/forms/submissions/<int:sub_id>/status', methods=['POST'])
@login_required
def admin_forms_submission_status(sub_id):
    submission = FormSubmission.query.get_or_404(sub_id)
    new_status = request.form.get('status', 'read')
    if new_status in ('new', 'read', 'archived', 'spam'):
        submission.status = new_status
        db.session.commit()
    return _inline_saved(t('builder.saved'), 'admin_forms_submissions', form_id=submission.form_id)


# --- Delete Submission ---
@app.route('/admin/forms/submissions/<int:sub_id>/delete', methods=['POST'])
@login_required
def admin_forms_submission_delete(sub_id):
    submission = FormSubmission.query.get_or_404(sub_id)
    form_id = submission.form_id
    db.session.delete(submission)
    db.session.commit()
    flash(t('flash.submission_deleted'), 'success')
    return _safe_return_to('admin_forms_submissions', form_id=form_id)


# --- Export Submissions (CSV/Excel) ---
@app.route('/admin/forms/<int:form_id>/export')
@login_required
def admin_forms_export(form_id):
    form = Form.query.get_or_404(form_id)
    export_format = request.args.get('format', 'csv')
    
    submissions = form.submissions.order_by(FormSubmission.created_at.desc()).all()
    fields = form.get_fields()
    field_names = [f['name'] for f in fields]
    field_labels = {f['name']: f['label'] for f in fields}
    
    if export_format == 'csv':
        import csv
        output = io.StringIO()
        writer = csv.writer(output)
        
        # Header row
        headers = ['#', 'Date', 'Status', 'IP'] + [field_labels.get(n, n) for n in field_names]
        writer.writerow(headers)
        
        # Data rows
        for i, sub in enumerate(submissions, 1):
            data = sub.get_data()
            row = [
                i,
                to_site_time(sub.created_at).strftime('%Y-%m-%d %H:%M'),
                sub.status,
                sub.ip_address or ''
            ]
            for name in field_names:
                val = data.get(name, '')
                if isinstance(val, list):
                    val = ', '.join(str(v) for v in val)
                row.append(str(val))
            writer.writerow(row)
        
        csv_data = output.getvalue()
        output.close()
        
        from flask import Response
        return Response(
            csv_data,
            mimetype='text/csv',
            headers={'Content-Disposition': f'attachment; filename={form.slug}_submissions.csv'}
        )
    
    elif export_format == 'excel':
        try:
            import openpyxl
            wb = openpyxl.Workbook()
            ws = wb.active
            ws.title = t('submissions.title')
            
            # Header
            headers = ['#', 'Date', 'Status', 'IP'] + [field_labels.get(n, n) for n in field_names]
            ws.append(headers)
            
            # Style header
            from openpyxl.styles import Font
            for cell in ws[1]:
                cell.font = Font(bold=True)
            
            # Data rows
            for i, sub in enumerate(submissions, 1):
                data = sub.get_data()
                row = [
                    i,
                    to_site_time(sub.created_at).strftime('%Y-%m-%d %H:%M'),
                    sub.status,
                    sub.ip_address or ''
                ]
                for name in field_names:
                    val = data.get(name, '')
                    if isinstance(val, list):
                        val = ', '.join(str(v) for v in val)
                    row.append(str(val))
                ws.append(row)
            
            # Auto-width columns
            for col in ws.columns:
                max_len = max(len(str(cell.value or '')) for cell in col)
                ws.column_dimensions[col[0].column_letter].width = min(max_len + 2, 50)
            
            excel_output = io.BytesIO()
            wb.save(excel_output)
            excel_output.seek(0)
            
            from flask import Response
            return Response(
                excel_output.getvalue(),
                mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
                headers={'Content-Disposition': f'attachment; filename={form.slug}_submissions.xlsx'}
            )
        except ImportError:
            flash(t('flash.openpyxl_required'), 'error')
            return _safe_return_to('admin_forms_submissions', form_id=form_id)
    
    return _safe_return_to('admin_forms_submissions', form_id=form_id)


# ============================================================================
# NEWSLETTER — Email campaigns
# ============================================================================

@app.route('/admin/newsletter')
@login_required
def admin_newsletter_list():
    """List all newsletter campaigns with filters, search, pagination"""
    page = get_page()
    per_page = get_per_page()
    status_filter = request.args.get('status', 'all')
    lang_filter = request.args.get('lang', 'all')
    sort_by = request.args.get('sort', 'updated-desc')
    search_query = request.args.get('q', '').strip()
    
    query = Newsletter.query
    if status_filter == 'draft':
        query = query.filter_by(status='draft')
    elif status_filter == 'sent':
        query = query.filter_by(status='sent')
    if lang_filter != 'all':
        query = query.filter_by(language=lang_filter)
    if search_query:
        query = query.filter(Newsletter.subject.ilike(f'%{escape_like(search_query)}%', escape='\\'))
    
    # Sorting
    if sort_by == 'subject-asc':
        query = query.order_by(Newsletter.subject.asc())
    elif sort_by == 'subject-desc':
        query = query.order_by(Newsletter.subject.desc())
    elif sort_by == 'sent-desc':
        query = query.order_by(Newsletter.sent_at.desc().nullslast())
    elif sort_by == 'sent-asc':
        query = query.order_by(Newsletter.sent_at.asc().nullslast())
    else:
        query = query.order_by(Newsletter.updated_at.desc())
    
    newsletters = query.paginate(page=page, per_page=per_page, error_out=False)
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': nl.id,
                'subject': nl.subject,
                'status': nl.status,
                'language': nl.language,
                'recipients_count': nl.recipients_count if nl.status == 'sent' else None,
                'failed_count': nl.failed_count if nl.status == 'sent' else 0,
                'sent_at': to_site_time(nl.sent_at).isoformat() if nl.sent_at else None,
                'updated_at': to_site_time(nl.updated_at).isoformat() if nl.updated_at else None,
            } for nl in newsletters.items],
            'page': newsletters.page,
            'pages': newsletters.pages,
            'total': newsletters.total,
            'has_prev': newsletters.has_prev,
            'has_next': newsletters.has_next,
            'prev_num': newsletters.prev_num,
            'next_num': newsletters.next_num,
            'per_page': per_page,
        })
    
    return render_template('admin/newsletter/list.html',
                         newsletters=newsletters,
                         status_filter=status_filter,
                         lang_filter=lang_filter,
                         sort_by=sort_by,
                         search_query=search_query,
                         per_page=per_page,
                         available_languages=_get_active_languages())


@app.route('/admin/newsletter/new', methods=['GET', 'POST'])
@login_required
def admin_newsletter_new():
    """Create a new newsletter campaign"""
    if request.method == 'POST':
        subject = request.form.get('subject', '').strip()
        content = request.form.get('content', '').strip()
        target_languages = request.form.getlist('target_languages')
        target_form_ids = request.form.getlist('target_form_ids')
        excluded_emails = request.form.getlist('excluded_emails')
        imported_json = request.form.get('imported_emails', '[]')

        if not subject:
            flash(t('flash.fields_required'), 'error')
            return redirect(request.url)

        try:
            imported_emails = json.loads(imported_json) if imported_json else []
        except (json.JSONDecodeError, TypeError):
            imported_emails = []

        newsletter = Newsletter(
            subject=subject,
            content=content,
            user_id=current_user.id,
            language=request.form.get('language') or 'en'
        )
        newsletter.set_target_languages(target_languages)
        newsletter.set_target_form_ids([int(fid) for fid in target_form_ids if fid])
        newsletter.set_excluded_emails(excluded_emails)
        newsletter.set_imported_emails(imported_emails)
        newsletter.confirmed_only = request.form.get('confirmed_only') == 'on'
        db.session.add(newsletter)
        db.session.commit()

        AuditLog.log('create_newsletter', 'newsletter', newsletter.id,
                     _ad('created_newsletter', title=subject), current_user, request.remote_addr)
        db.session.commit()

        return _after_create('admin_newsletter_edit', t('flash.newsletter_saved'), newsletter_id=newsletter.id)

    # GET — prepare data for the form
    all_forms = Form.query.filter_by(is_active=True).order_by(Form.title).all()
    email_forms = []
    for f in all_forms:
        fields = f.get_fields()
        has_email = any(field.get('type') == 'email' for field in fields)
        if has_email:
            email_forms.append(f)

    recipient_stats, form_stats = _get_recipient_stats(email_forms)

    return render_template('admin/newsletter/new.html',
                         email_forms=email_forms,
                         recipient_stats=recipient_stats,
                         form_stats=form_stats)


@app.route('/admin/newsletter/<int:newsletter_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_newsletter_edit(newsletter_id):
    """Edit an existing newsletter campaign"""
    newsletter = Newsletter.query.get_or_404(newsletter_id)

    if request.method == 'POST':
        if newsletter.status == 'sent':
            flash(t('flash.newsletter_already_sent'), 'error')
            return redirect(url_for('admin_newsletter_edit', newsletter_id=newsletter.id))

        subject = request.form.get('subject', '').strip()
        content = request.form.get('content', '').strip()
        target_languages = request.form.getlist('target_languages')
        target_form_ids = request.form.getlist('target_form_ids')
        excluded_emails = request.form.getlist('excluded_emails')
        imported_json = request.form.get('imported_emails', '[]')

        if not subject:
            flash(t('flash.fields_required'), 'error')
            return redirect(request.url)

        try:
            imported_emails = json.loads(imported_json) if imported_json else []
        except (json.JSONDecodeError, TypeError):
            imported_emails = []

        newsletter.subject = subject
        newsletter.content = content
        newsletter.language = request.form.get('language') or 'en'
        newsletter.set_target_languages(target_languages)
        newsletter.set_target_form_ids([int(fid) for fid in target_form_ids if fid])
        newsletter.set_excluded_emails(excluded_emails)
        newsletter.set_imported_emails(imported_emails)
        newsletter.confirmed_only = request.form.get('confirmed_only') == 'on'
        db.session.commit()

        AuditLog.log('save_newsletter', 'newsletter', newsletter.id,
                     _ad('saved_newsletter', title=subject), current_user, request.remote_addr)
        db.session.commit()

        return _inline_saved(t('flash.newsletter_saved'), 'admin_newsletter_list')

    # GET — prepare data for the form
    all_forms = Form.query.filter_by(is_active=True).order_by(Form.title).all()
    email_forms = []
    for f in all_forms:
        fields = f.get_fields()
        has_email = any(field.get('type') == 'email' for field in fields)
        if has_email:
            email_forms.append(f)

    recipient_stats, form_stats = _get_recipient_stats(email_forms)
    translations = newsletter.get_translations()
    missing_languages = _filter_missing_by_active(newsletter.get_missing_languages())

    return render_template('admin/newsletter/edit.html',
                         newsletter=newsletter,
                         email_forms=email_forms,
                         recipient_stats=recipient_stats,
                         form_stats=form_stats,
                         translations=translations,
                         missing_languages=missing_languages)


@app.route('/admin/newsletter/<int:newsletter_id>/translate', methods=['GET', 'POST'])
@login_required
def admin_newsletter_translate(newsletter_id):
    """Translate newsletter — create a new newsletter in another language"""
    source = Newsletter.query.get_or_404(newsletter_id)

    if request.method == 'POST':
        subject = request.form.get('subject', '').strip()
        content = request.form.get('content', '').strip()
        language = request.form.get('language') or 'en'

        if not subject:
            flash(t('flash.fields_required'), 'error')
            return redirect(request.url)

        # Check if translation already exists
        existing = Newsletter.query.filter_by(
            translation_group=source.translation_group, language=language
        ).first()
        if existing:
            flash(t('flash.translation_exists', language=LANG_CODE_TO_NAME.get(language, language)), 'warning')
            return _safe_return_to('admin_newsletter_list')

        new_newsletter = Newsletter(
            subject=subject,
            content=content,
            user_id=current_user.id,
            language=language,
            translation_group=source.translation_group
        )
        # Copy audience settings from source
        new_newsletter.set_target_languages(source.get_target_languages())
        new_newsletter.set_target_form_ids(source.get_target_form_ids())
        new_newsletter.confirmed_only = source.confirmed_only
        db.session.add(new_newsletter)
        db.session.commit()

        AuditLog.log('translate_newsletter', 'newsletter', new_newsletter.id,
                     _ad('translated_newsletter', title=subject, lang=language),
                     current_user, request.remote_addr)
        db.session.commit()

        return _after_create('admin_newsletter_edit', t('flash.translation_created', status='draft'), newsletter_id=new_newsletter.id)

    # GET — show translate form
    translations = source.get_translations()
    missing_languages = _filter_missing_by_active(source.get_missing_languages())

    return render_template('admin/newsletter/translate.html',
                         source=source,
                         translations=translations,
                         missing_languages=missing_languages)


@app.route('/admin/newsletter/<int:newsletter_id>/delete', methods=['POST'])
@login_required
def admin_newsletter_delete(newsletter_id):
    """Delete a newsletter"""
    newsletter = Newsletter.query.get_or_404(newsletter_id)
    db.session.delete(newsletter)
    db.session.commit()
    
    AuditLog.log('delete_newsletter', 'newsletter', newsletter_id,
                 _ad('deleted_newsletter'), current_user, request.remote_addr)
    db.session.commit()
    
    flash(t('flash.newsletter_deleted'), 'success')
    return _safe_return_to('admin_newsletter_list')


@app.route('/admin/newsletter/<int:newsletter_id>/test', methods=['POST'])
@login_required
def admin_newsletter_test(newsletter_id):
    """Send a test email to the current admin"""
    newsletter = Newsletter.query.get_or_404(newsletter_id)
    test_email = request.form.get('test_email', current_user.email)
    
    if not test_email:
        flash(t('flash.email_address_required'), 'error')
        return redirect(url_for('admin_newsletter_edit', newsletter_id=newsletter_id))
    
    site_name = Settings.get('site_name', 'AliothPress')
    
    # Build email with unsubscribe placeholder
    html_body = _build_newsletter_html(newsletter.content, site_name, test_email, is_test=True)
    
    success, message = send_email(test_email, f"[TEST] {newsletter.subject}", html_body)
    
    if success:
        flash(t('flash.newsletter_test_sent').replace('{email}', test_email), 'success')
    else:
        flash(t('flash.newsletter_test_failed').replace('{message}', message), 'error')
    
    return redirect(url_for('admin_newsletter_edit', newsletter_id=newsletter_id))


@app.route('/admin/newsletter/<int:newsletter_id>/send', methods=['POST'])
@login_required
def admin_newsletter_send(newsletter_id):
    """Send the newsletter to all matching recipients"""
    newsletter = Newsletter.query.get_or_404(newsletter_id)
    
    if newsletter.status == 'sent':
        flash(t('flash.newsletter_already_sent'), 'warning')
        return _safe_return_to('admin_newsletter_list')
    
    site_name = Settings.get('site_name', 'AliothPress')
    target_langs = newsletter.get_target_languages()
    target_form_ids = newsletter.get_target_form_ids()
    excluded = newsletter.get_excluded_emails()
    imported = newsletter.get_imported_emails()
    
    # Collect recipient emails
    recipients = _collect_newsletter_recipients(target_langs, target_form_ids, excluded, imported, confirmed_only=newsletter.confirmed_only)
    
    if not recipients:
        flash(t('flash.newsletter_no_recipients'), 'warning')
        return redirect(url_for('admin_newsletter_edit', newsletter_id=newsletter_id))
    
    # Send emails
    newsletter.status = 'sending'
    db.session.commit()
    
    sent_count = 0
    fail_count = 0
    
    for email_addr in recipients:
        html_body = _build_newsletter_html(newsletter.content, site_name, email_addr, is_test=False)
        success, msg = send_email(email_addr, newsletter.subject, html_body)
        if success:
            sent_count += 1
        else:
            fail_count += 1
    
    # Update newsletter status
    newsletter.status = 'sent'
    newsletter.recipients_count = sent_count
    newsletter.failed_count = fail_count
    newsletter.sent_at = utc_now()
    db.session.commit()
    
    AuditLog.log('send_newsletter', 'newsletter', newsletter.id,
                 _ad('sent_newsletter', count=str(sent_count), failed=str(fail_count)),
                 current_user, request.remote_addr)
    db.session.commit()
    
    flash(t('flash.newsletter_sent').replace('{count}', str(sent_count)).replace('{failed}', str(fail_count)), 'success')
    return _safe_return_to('admin_newsletter_list')


@app.route('/admin/newsletter/recipients')
@login_required
def admin_newsletter_recipients_api():
    """Paginated JSON endpoint — recipient list for newsletter editor.

    Query params:
        langs[]        — language codes to filter forms
        form_ids[]     — specific form IDs
        excluded[]     — emails the user has unchecked
        imported       — JSON array of {email, name} from CSV
        search         — filter by email/name/source (server-side)
        sort           — field to sort by: email (default), name, source
        sort_dir       — asc (default) or desc
        page           — 1-based page number
        per_page       — items per page (default 20, max 100)
        emails_only    — if 'true', return only {emails: [...]} for bulk select/deselect
    """
    target_langs = request.args.getlist('langs')
    target_form_ids = [int(x) for x in request.args.getlist('form_ids') if x.isdigit()]
    excluded = request.args.getlist('excluded')
    imported_raw = request.args.get('imported', '[]')

    try:
        imported = json.loads(imported_raw) if imported_raw else []
    except (json.JSONDecodeError, TypeError):
        imported = []

    # ── Collect full recipient list ──
    recipients = _collect_newsletter_recipients_detailed(target_langs, target_form_ids)

    # Add imported emails
    existing_emails = {r['email'] for r in recipients}
    for imp in imported:
        email = imp.get('email', '').strip().lower()
        if email and '@' in email and email not in existing_emails:
            recipients.append({
                'email': email,
                'name': imp.get('name', ''),
                'source': t('newsletter.imported_source'),
                'form_id': None,
                'email_confirmed': None
            })
            existing_emails.add(email)

    # ── Mark excluded ──
    excluded_set = {e.strip().lower() for e in excluded if e.strip()}
    for r in recipients:
        r['excluded'] = r['email'] in excluded_set

    # Global counts (before search filtering — reflects true audience size)
    active_count = sum(1 for r in recipients if not r['excluded'])
    total_all = len(recipients)

    # ── emails_only mode — for bulk Select All / Deselect All ──
    if request.args.get('emails_only') == 'true':
        return jsonify({
            'emails': [r['email'] for r in recipients],
            'count': active_count,
            'total': total_all
        })

    # ── Server-side search ──
    search = request.args.get('search', '').strip().lower()
    if search:
        recipients = [r for r in recipients if
                      search in r['email'] or
                      search in (r.get('name') or '').lower() or
                      search in (r.get('source') or '').lower()]

    # ── Server-side sort ──
    sort_field = request.args.get('sort', 'email')
    if sort_field not in ('email', 'name', 'source'):
        sort_field = 'email'
    sort_desc = request.args.get('sort_dir', 'asc') == 'desc'
    recipients.sort(key=lambda r: (r.get(sort_field) or '').lower(), reverse=sort_desc)

    # ── Pagination ──
    page = get_page()
    per_page = min(max(1, request.args.get('per_page', 20, type=int)), 100)
    total_filtered = len(recipients)
    total_pages = max(1, (total_filtered + per_page - 1) // per_page)
    page = min(page, total_pages)
    start = (page - 1) * per_page
    page_items = recipients[start:start + per_page]

    return jsonify({
        'items': page_items,
        'page': page,
        'per_page': per_page,
        'total': total_filtered,
        'total_pages': total_pages,
        'count': active_count,
        'total_all': total_all
    })


@app.route('/admin/newsletter/import-csv', methods=['POST'])
@login_required
def admin_newsletter_import_csv():
    """AJAX endpoint — parse CSV file and return email list"""
    import csv
    import io
    
    file = request.files.get('csv_file')
    if not file:
        return jsonify({'error': t('api.no_file_uploaded')}), 400
    
    try:
        content = file.read().decode('utf-8-sig')  # handle BOM
        reader = csv.reader(io.StringIO(content))
        
        rows = list(reader)
        if not rows:
            return jsonify({'error': t('api.empty_file')}), 400
        
        # Try to detect header
        header = [h.strip().lower() for h in rows[0]]
        email_col = None
        name_col = None
        
        for i, h in enumerate(header):
            if h in ('email', 'e-mail', 'email address', 'emailaddress', 'mail'):
                email_col = i
            if h in ('name', 'full name', 'fullname', 'first name', 'firstname', 'имя', 'nom'):
                name_col = i
        
        # If no header detected, check if first row looks like data
        start_row = 0
        if email_col is not None:
            start_row = 1
        else:
            # No header — try to find email column by content
            for i, val in enumerate(rows[0]):
                if '@' in val:
                    email_col = i
                    break
            if email_col is None:
                email_col = 0  # fallback: first column
        
        imported = []
        seen = set()
        for row in rows[start_row:]:
            if len(row) <= email_col:
                continue
            email = row[email_col].strip().lower()
            if not email or '@' not in email or email in seen:
                continue
            seen.add(email)
            name = ''
            if name_col is not None and len(row) > name_col:
                name = row[name_col].strip()
            imported.append({'email': email, 'name': name})
        
        return jsonify({'imported': imported, 'count': len(imported)})
    
    except Exception as e:
        return jsonify({'error': str(e)}), 400


# --- Public: Unsubscribe ---

@app.route('/unsubscribe')
def newsletter_unsubscribe():
    """Public unsubscribe page — marks email as unsubscribed in settings"""
    email = request.args.get('email', '')
    token = request.args.get('token', '')
    
    if not email or not token:
        return render_template('public/404.html',
                               all_versions=_build_system_lang_versions()), 404
    
    # Verify token: HMAC-SHA256 with constant-time comparison.
    import hmac as _hmac
    if not _hmac.compare_digest(token, _unsubscribe_token(email)):
        return render_template('public/404.html',
                               all_versions=_build_system_lang_versions()), 404
    
    # Store unsubscribed email
    unsubscribed = Settings.get('newsletter_unsubscribed', '')
    unsub_list = [e.strip() for e in unsubscribed.split(',') if e.strip()] if unsubscribed else []
    
    if email not in unsub_list:
        unsub_list.append(email)
        Settings.set('newsletter_unsubscribed', ','.join(unsub_list))
    
    site_name = Settings.get('site_name', 'AliothPress')
    lang = request.args.get('lang', Settings.get('default_language', 'en'))
    title = pt_for('unsubscribed_title', lang)
    body = pt_for('unsubscribed_body', lang, site_name=site_name)
    return render_template('public/message.html',
                           message_title=title, message_body=body,
                           current_language=lang,
                           all_versions=_build_system_lang_versions())


# --- Newsletter Helper functions ---

def _unsubscribe_token(email):
    """Unsubscribe token — HMAC-SHA256, truncated to 16 hex chars (64 bits).

    Truncating HMAC output is explicitly allowed by RFC 2104 and keeps the
    URL short. 64 bits is far beyond brute-force reach for this purpose.
    """
    import hmac as _hmac
    import hashlib
    return _hmac.new(
        app.config['SECRET_KEY'].encode(),
        email.encode(),
        hashlib.sha256,
    ).hexdigest()[:16]


def _get_unsubscribe_link(email, language=None):
    """Generate unsubscribe URL for an email address"""
    token = _unsubscribe_token(email)
    lang = language or Settings.get('default_language', 'en')
    return url_for('newsletter_unsubscribe', email=email, token=token, lang=lang, _external=True)


# --- Public: Confirm Email (Double Opt-In) ---

@app.route('/confirm-email')
def confirm_email():
    """Public page — visitor clicks link in email to confirm their address."""
    token = request.args.get('token', '')

    if not token:
        return render_template('public/404.html',
                               all_versions=_build_system_lang_versions()), 404

    submission = FormSubmission.query.filter_by(confirmation_token=token).first()

    if not submission:
        return render_template('public/404.html',
                               all_versions=_build_system_lang_versions()), 404

    # Already confirmed? Show success anyway
    if submission.email_confirmed:
        already = True
    else:
        submission.confirm_email()
        
        # Retroactively confirm all other pending submissions with the same email
        visitor_email = None
        sub_data = submission.get_data()
        for key, val in sub_data.items():
            if isinstance(val, str) and '@' in val and not key.startswith('_'):
                visitor_email = val.strip()
                break
        
        if visitor_email:
            pending = FormSubmission.query.filter(
                FormSubmission.id != submission.id,
                FormSubmission.email_confirmed == False,
                FormSubmission.data.like(f'%{visitor_email}%')
            ).all()
            for ps in pending:
                ps.confirm_email()
        
        db.session.commit()
        already = False

    # Determine language: prefer ?lang= from URL (language switcher), fallback to form language
    form = Form.query.get(submission.form_id)
    form_lang = form.language if form else Settings.get('default_language', 'en')
    lang = request.args.get('lang', form_lang)
    site_name = Settings.get('site_name', 'AliothPress')

    if already:
        title = pt_for('confirm_already_title', lang)
        body = pt_for('confirm_already_body', lang, site_name=site_name)
    else:
        title = pt_for('confirm_success_title', lang)
        body = pt_for('confirm_success_body', lang, site_name=site_name)

    return render_template('public/message.html',
                           message_title=title, message_body=body,
                           current_language=lang,
                           all_versions=_build_system_lang_versions())


# --- Helper: Send confirmation / double opt-in email to visitor ---

def _send_confirmation_email(form, submission, visitor_email, double_opt_in=False):
    """Send confirmation email to visitor after form submission.

    Args:
        form: Form object (used for language and title)
        submission: FormSubmission object (has token if double opt-in)
        visitor_email: visitor's email address
        double_opt_in: if True, include confirmation link with token
    """
    lang = form.language
    site_name = Settings.get('site_name', 'AliothPress')
    form_title = form.title
    powered_by = '' if _verify_license_integrity() else ' &bull; Powered by <a href="https://aliothpress.com" style="color: #64748b; text-decoration: underline;">AliothPress</a>'

    if double_opt_in and submission.confirmation_token:
        # Double opt-in: email with confirmation link
        confirm_url = url_for('confirm_email', token=submission.confirmation_token, lang=lang, _external=True)
        subject = t_for('email.confirm_subject', lang, site_name=site_name)
        html_body = f'''<!DOCTYPE html>
<html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"></head>
<body style="margin:0;padding:0;background-color:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellpadding="0" cellspacing="0" style="background-color:#f1f5f9;padding:32px 16px;">
<tr><td align="center">
<table width="600" cellpadding="0" cellspacing="0" style="background:#ffffff;border-radius:12px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,0.1);">
    <tr><td style="background:linear-gradient(135deg,#5b5bd6,#4f46e5);padding:28px 32px;text-align:center;">
        <h1 style="margin:0;color:#ffffff;font-size:22px;font-weight:700;">{site_name}</h1>
    </td></tr>
    <tr><td style="padding:32px;">
        <h2 style="color:#1e293b;margin:0 0 16px;font-size:20px;">{t_for('email.confirm_title', lang)}</h2>
        <p style="color:#475569;line-height:1.6;margin:0 0 24px;">{t_for('email.confirm_intro', lang, form_title=html_escape(form_title))}</p>
        <div style="text-align:center;margin:24px 0;">
            <a href="{confirm_url}" style="display:inline-block;background:#4f46e5;color:#ffffff;text-decoration:none;padding:14px 32px;border-radius:8px;font-weight:600;font-size:16px;">
                {t_for('email.confirm_button', lang)}
            </a>
        </div>
        <p style="color:#94a3b8;font-size:13px;margin:16px 0 0;">{t_for('email.confirm_ignore', lang)}</p>
    </td></tr>
    <tr><td style="padding:20px 32px;background:#f8fafc;border-top:1px solid #e2e8f0;text-align:center;font-size:12px;color:#64748b;">
        <p style="margin:0;">{t_for('email.sent_by', lang, site_name=site_name)}{powered_by}</p>
    </td></tr>
</table>
</td></tr></table>
</body></html>'''
    else:
        # Simple confirmation: thank-you email, no link
        subject = t_for('email.confirmation_subject', lang, site_name=site_name)
        html_body = f'''<!DOCTYPE html>
<html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"></head>
<body style="margin:0;padding:0;background-color:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
<table width="100%" cellpadding="0" cellspacing="0" style="background-color:#f1f5f9;padding:32px 16px;">
<tr><td align="center">
<table width="600" cellpadding="0" cellspacing="0" style="background:#ffffff;border-radius:12px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,0.1);">
    <tr><td style="background:linear-gradient(135deg,#5b5bd6,#4f46e5);padding:28px 32px;text-align:center;">
        <h1 style="margin:0;color:#ffffff;font-size:22px;font-weight:700;">{site_name}</h1>
    </td></tr>
    <tr><td style="padding:32px;">
        <h2 style="color:#1e293b;margin:0 0 16px;font-size:20px;">{t_for('email.confirmation_title', lang)}</h2>
        <p style="color:#475569;line-height:1.6;margin:0 0 8px;">{t_for('email.confirmation_body', lang, form_title=html_escape(form_title), site_name=site_name)}</p>
    </td></tr>
    <tr><td style="padding:20px 32px;background:#f8fafc;border-top:1px solid #e2e8f0;text-align:center;font-size:12px;color:#64748b;">
        <p style="margin:0;">{t_for('email.sent_by', lang, site_name=site_name)}{powered_by}</p>
    </td></tr>
</table>
</td></tr></table>
</body></html>'''

    send_email(visitor_email, subject, html_body)


def _build_newsletter_html(content, site_name, recipient_email, is_test=False, language=None):
    """Wrap newsletter content in a styled email template with unsubscribe link"""
    lang = language or Settings.get('default_language', 'en')
    unsubscribe_url = _get_unsubscribe_link(recipient_email, lang)
    test_banner_text = t_for('newsletter.test_banner', lang)
    test_banner = f'<div style="background:#fef3c7;color:#92400e;padding:10px;text-align:center;font-size:13px;font-weight:600;">⚠️ {test_banner_text}</div>' if is_test else ''
    unsub_text = t_for('newsletter.unsubscribe_link', lang)
    powered_by = '' if _verify_license_integrity() else '<br>Powered by <a href="https://aliothpress.com" style="color:#64748b;text-decoration:underline;">AliothPress</a>'
    
    return f'''<!DOCTYPE html>
<html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"></head>
<body style="margin:0;padding:0;background-color:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;">
{test_banner}
<table width="100%" cellpadding="0" cellspacing="0" style="background-color:#f1f5f9;padding:32px 16px;">
<tr><td align="center">
<table width="600" cellpadding="0" cellspacing="0" style="background:#ffffff;border-radius:12px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,0.1);">
    <tr><td style="background:linear-gradient(135deg,#5b5bd6,#4f46e5);padding:28px 32px;text-align:center;">
        <h1 style="margin:0;color:#ffffff;font-size:22px;font-weight:700;">{site_name}</h1>
    </td></tr>
    <tr><td style="padding:32px;">
        {content}
    </td></tr>
    <tr><td style="padding:20px 32px;background:#f8fafc;border-top:1px solid #e2e8f0;text-align:center;font-size:12px;color:#64748b;">
        <p style="margin:0 0 8px;">{t_for('email.sent_by', lang, site_name=site_name)}{powered_by}</p>
        <a href="{unsubscribe_url}" style="color:#64748b;text-decoration:underline;">{unsub_text}</a>
    </td></tr>
</table>
</td></tr></table>
</body></html>'''


def _collect_newsletter_recipients(target_langs, target_form_ids, excluded_emails=None, imported_emails=None, confirmed_only=False):
    """Collect unique email addresses from form submissions matching filters.
    
    Excludes unsubscribed and manually excluded emails.
    Includes imported CSV emails.
    
    If confirmed_only=True, excludes recipients with email_confirmed=False (pending).
    Recipients with email_confirmed=None (no opt-in) and imported emails are always included.
    
    Returns:
        set of email addresses
    """
    detailed = _collect_newsletter_recipients_detailed(target_langs, target_form_ids)
    
    # Apply confirmed_only filter: exclude pending (False), keep confirmed (True) and no-opt-in (None)
    if confirmed_only:
        detailed = [r for r in detailed if r.get('email_confirmed') is not False]
    
    emails = {r['email'] for r in detailed}
    
    # Add imported emails (always included — admin's responsibility)
    if imported_emails:
        for imp in imported_emails:
            email = imp.get('email', '').strip().lower()
            if email and '@' in email:
                emails.add(email)
    
    # Remove manually excluded
    if excluded_emails:
        excluded_set = {e.strip().lower() for e in excluded_emails if e.strip()}
        emails -= excluded_set
    
    return emails


def _collect_newsletter_recipients_detailed(target_langs, target_form_ids):
    """Collect detailed recipient info from form submissions.
    
    Returns list of dicts: [{email, name, source, form_id, email_confirmed}, ...]
    Already excludes unsubscribed and spam.
    
    email_confirmed values:
        True  — confirmed via double opt-in
        False — pending (waiting for confirmation click)
        None  — form doesn't use double opt-in (not applicable)
    """
    # Get forms with email fields
    query = Form.query.filter_by(is_active=True)
    
    if target_form_ids:
        query = query.filter(Form.id.in_(target_form_ids))
    
    if target_langs:
        query = query.filter(Form.language.in_(target_langs))
    
    forms = query.all()
    
    # Map form_id -> (email_field_name, name_field_name, form_title)
    form_field_map = {}
    for f in forms:
        fields = f.get_fields()
        email_field = None
        name_field = None
        for field in fields:
            if field.get('type') == 'email' and not email_field:
                email_field = field.get('name', 'email')
            if field.get('type') == 'text' and field.get('name', '').lower() in ('name', 'full_name', 'fullname', 'first_name', 'firstname', 'your_name') and not name_field:
                name_field = field.get('name')
            # Also check label for name-like fields
            if field.get('type') == 'text' and not name_field:
                label = (field.get('label') or '').lower()
                if any(w in label for w in ('name', 'имя', 'nom', 'nombre', 'nome', 'name')):
                    name_field = field.get('name')
        if email_field:
            form_field_map[f.id] = {
                'email_field': email_field,
                'name_field': name_field,
                'title': f.title,
                'language': f.language
            }
    
    if not form_field_map:
        return []
    
    # Collect from submissions
    recipients_map = {}  # email -> {email, name, source, form_id, email_confirmed}
    form_ids = list(form_field_map.keys())
    submissions = FormSubmission.query.filter(
        FormSubmission.form_id.in_(form_ids),
        FormSubmission.status != 'spam'
    ).all()
    
    for sub in submissions:
        info = form_field_map.get(sub.form_id)
        if not info:
            continue
        data = sub.get_data()
        email = data.get(info['email_field'], '').strip().lower()
        if not email or '@' not in email:
            continue
        
        name = ''
        if info['name_field']:
            name = data.get(info['name_field'], '').strip()
        
        # Keep first occurrence (or update name/status if we find a better one)
        if email not in recipients_map:
            recipients_map[email] = {
                'email': email,
                'name': name,
                'source': info['title'],
                'form_id': sub.form_id,
                'email_confirmed': sub.email_confirmed
            }
        else:
            existing = recipients_map[email]
            if not existing['name'] and name:
                existing['name'] = name
            # Upgrade confirmation status: True > None > False
            old = existing['email_confirmed']
            new = sub.email_confirmed
            if new is True or (new is None and old is False):
                existing['email_confirmed'] = new
    
    # Remove unsubscribed
    unsubscribed_raw = Settings.get('newsletter_unsubscribed', '')
    if unsubscribed_raw:
        unsubscribed = {e.strip().lower() for e in unsubscribed_raw.split(',') if e.strip()}
        for email in unsubscribed:
            recipients_map.pop(email, None)
    
    return list(recipients_map.values())


def _get_recipient_stats(email_forms):
    """Get recipient count per language and per form for display in compose page"""
    from collections import defaultdict
    lang_stats = defaultdict(int)
    form_stats = {}
    
    for f in email_forms:
        fields = f.get_fields()
        email_field_name = None
        for field in fields:
            if field.get('type') == 'email':
                email_field_name = field.get('name', 'email')
                break
        if not email_field_name:
            continue
        
        subs = FormSubmission.query.filter(
            FormSubmission.form_id == f.id,
            FormSubmission.status != 'spam'
        ).all()
        
        seen = set()
        for sub in subs:
            data = sub.get_data()
            email = data.get(email_field_name, '').strip().lower()
            if email and '@' in email and email not in seen:
                seen.add(email)
                lang_stats[f.language] += 1
        
        form_stats[f.id] = len(seen)
    
    return dict(lang_stats), dict(form_stats)


# --- Public API: Submit Form ---
@app.route('/api/forms/<int:form_id>/submit', methods=['POST'])
@csrf.exempt  # Public API — uses honeypot + rate limiting instead of CSRF
@limiter.limit("5 per minute")
def api_form_submit(form_id):
    """Public API endpoint for form submission (AJAX from public site)."""
    form = Form.query.get_or_404(form_id)
    
    if not form.is_active:
        return jsonify({'success': False, 'error': t('api.form_not_accepting')}), 400
    
    settings = form.get_settings()
    fields = form.get_fields()
    
    # Anti-spam: honeypot check
    if settings.get('anti_spam') == 'honeypot':
        honeypot_value = request.form.get('_hp_website', '')
        if not honeypot_value and request.is_json:
            honeypot_value = (request.json or {}).get('_hp_website', '')
        if honeypot_value:
            # Bot detected — return success to not reveal the trap
            return jsonify({'success': True, 'message': settings.get('success_message', t('forms.default_thank_you'))})
    
    # Rate limiting by IP
    limit = settings.get('limit_per_ip', 0)
    if limit and limit > 0:
        recent_count = FormSubmission.query.filter(
            FormSubmission.form_id == form_id,
            FormSubmission.ip_address == request.remote_addr,
            FormSubmission.created_at >= utc_now() - timedelta(hours=1)
        ).count()
        if recent_count >= limit:
            return jsonify({'success': False, 'error': t('api.too_many_submissions')}), 429
    
    # Collect form data
    is_json = request.is_json
    submission_data = {}
    errors = []
    
    for field in fields:
        fname = field['name']
        ftype = field['type']
        required = field.get('required', False)
        
        if is_json:
            value = (request.json or {}).get(fname, '')
        else:
            if ftype == 'checkbox' and field.get('options'):
                value = request.form.getlist(fname)
            else:
                value = request.form.get(fname, '')
        
        # Validate required
        form_lang = form.language or Settings.get('default_language', 'en')
        if required and not value:
            errors.append(pt_for('form_field_required', form_lang, label=field["label"]))
            continue
        
        # Basic validation by type
        if value and ftype == 'email' and '@' not in str(value):
            errors.append(pt_for('form_field_invalid_email', form_lang, label=field["label"]))
        
        if value and ftype == 'consent' and str(value).lower() not in ('true', 'on', '1', 'yes'):
            errors.append(pt_for('form_field_required', form_lang, label=field["label"]))
            continue
        
        submission_data[fname] = value
    
    if errors:
        return jsonify({'success': False, 'errors': errors}), 400
    
    # Store submission
    submission = None
    if settings.get('store_submissions', True):
        submission = FormSubmission(
            form_id=form_id,
            ip_address=request.remote_addr,
            user_agent=request.headers.get('User-Agent', '')[:500],
            referrer=request.headers.get('Referer', '')[:500]
        )
        submission.set_data(submission_data)
        db.session.add(submission)
        db.session.commit()
    
    # Send notification email to admin
    if settings.get('notify_on_submit') and settings.get('notify_email'):
        try:
            _send_form_notification(form, submission_data, settings['notify_email'])
        except Exception as e:
            # Don't fail the submission if email fails
            app.logger.error(f'Form notification email failed: {e}')
    
    # Confirmation email / Double opt-in for visitor
    # Find the email value from submitted data
    visitor_email = None
    for field in fields:
        if field.get('type') == 'email' and field.get('required'):
            visitor_email = submission_data.get(field.get('name', ''), '').strip()
            if visitor_email:
                break
    
    if visitor_email and submission:
        use_double_opt_in = settings.get('double_opt_in', False)
        use_confirmation = settings.get('confirmation_email', False)
        
        if use_double_opt_in:
            # Check if this email was already confirmed in any previous submission
            already_confirmed = FormSubmission.query.filter(
                FormSubmission.email_confirmed == True,
                FormSubmission.data.like(f'%{visitor_email}%')
            ).first()
            
            if already_confirmed:
                # Email already verified — auto-confirm and send simple thank-you (no link needed)
                submission.email_confirmed = True
                submission.confirmed_at = already_confirmed.confirmed_at
                db.session.commit()
                try:
                    _send_confirmation_email(form, submission, visitor_email, double_opt_in=False)
                except Exception as e:
                    app.logger.error(f'Confirmation email failed (already verified): {e}')
            else:
                # First time: generate token, send confirmation link
                token = submission.generate_confirmation_token()
                db.session.commit()
                try:
                    _send_confirmation_email(form, submission, visitor_email, double_opt_in=True)
                except Exception as e:
                    app.logger.error(f'Double opt-in email failed: {e}')
        elif use_confirmation:
            # Simple confirmation: just send a thank-you email, no token
            try:
                _send_confirmation_email(form, submission, visitor_email, double_opt_in=False)
            except Exception as e:
                app.logger.error(f'Confirmation email failed: {e}')
    
    response = {'success': True, 'message': settings.get('success_message', t('forms.default_thank_you'))}
    if settings.get('redirect_url'):
        response['redirect'] = settings['redirect_url']
    
    return jsonify(response)


# --- API: Get form structure (for AI agents) ---
@app.route('/api/forms/<int:form_id>', methods=['GET'])
@csrf.exempt  # Public read-only API
def api_form_get(form_id):
    """Get form structure — useful for AI agents and headless rendering."""
    form = Form.query.get_or_404(form_id)
    if not form.is_active:
        return jsonify({'success': False, 'error': t('api.form_inactive')}), 404
    
    return jsonify({
        'success': True,
        'form': {
            'id': form.id,
            'title': form.title,
            'description': form.description,
            'form_type': form.form_type,
            'fields': form.get_fields(),
            'settings': {
                'submit_label': form.get_settings().get('submit_label', t('forms.default_submit')),
                'success_message': form.get_settings().get('success_message', ''),
            }
        }
    })


# --- API: List forms (for Page Builder dropdown) ---
@app.route('/api/forms/list')
@login_required
def api_forms_list():
    """API endpoint for Page Builder — returns list of active forms with fields for preview."""
    forms = Form.query.filter_by(is_active=True).order_by(Form.title).all()
    return jsonify({
        'forms': [{
            'id': f.id,
            'title': f.title,
            'description': f.description or '',
            'form_type': f.form_type,
            'language': f.language or 'en',
            'fields_count': len(f.get_fields()),
            'fields': f.get_fields(),
            'settings': f.get_settings()
        } for f in forms]
    })


@app.route('/api/forms/<int:form_id>')
@login_required
def api_form_detail(form_id):
    """API endpoint for Page Builder — returns single form data for canvas preview."""
    form = Form.query.get_or_404(form_id)
    return jsonify({
        'id': form.id,
        'title': form.title,
        'description': form.description or '',
        'form_type': form.form_type,
        'fields_count': len(form.get_fields()),
        'fields': form.get_fields(),
        'settings': form.get_settings()
    })


# --- Helper: Form Templates ---
def get_form_template(form_type):
    """Return pre-built field templates for common form types."""
    templates = {
        'contact': {
            'fields': [
                {'id': 'f1', 'type': 'text', 'name': 'name', 'label': t('forms.tpl_contact_name'), 'placeholder': t('forms.tpl_contact_name_placeholder'), 'required': True, 'width': 'half'},
                {'id': 'f2', 'type': 'email', 'name': 'email', 'label': t('forms.tpl_contact_email'), 'placeholder': t('forms.tpl_contact_email_placeholder'), 'required': True, 'width': 'half'},
                {'id': 'f3', 'type': 'text', 'name': 'subject', 'label': t('forms.tpl_contact_subject'), 'placeholder': t('forms.tpl_contact_subject_placeholder'), 'required': False, 'width': 'full'},
                {'id': 'f4', 'type': 'textarea', 'name': 'message', 'label': t('forms.tpl_contact_message'), 'placeholder': t('forms.tpl_contact_message_placeholder'), 'required': True, 'width': 'full'},
                {'id': 'f5', 'type': 'consent', 'name': 'privacy', 'label': t('forms.tpl_contact_privacy'), 'consent_text': t('forms.tpl_contact_consent'), 'required': True, 'width': 'full'},
            ],
            'settings': {
                'submit_label': t('forms.tpl_contact_submit'),
                'success_message': t('forms.tpl_contact_success'),
                'notify_on_submit': True,
                'store_submissions': True,
                'anti_spam': 'honeypot',
            }
        },
        'survey': {
            'fields': [
                {'id': 'f1', 'type': 'text', 'name': 'name', 'label': t('forms.tpl_survey_name'), 'placeholder': '', 'required': False, 'width': 'full'},
                {'id': 'f2', 'type': 'radio', 'name': 'satisfaction', 'label': t('forms.tpl_survey_satisfaction'), 'options': [t('forms.tpl_survey_opt_very_satisfied'), t('forms.tpl_survey_opt_satisfied'), t('forms.tpl_survey_opt_neutral'), t('forms.tpl_survey_opt_dissatisfied'), t('forms.tpl_survey_opt_very_dissatisfied')], 'required': True, 'width': 'full'},
                {'id': 'f3', 'type': 'rating', 'name': 'rating', 'label': t('forms.tpl_survey_rating'), 'max_rating': 5, 'required': True, 'width': 'full'},
                {'id': 'f4', 'type': 'checkbox', 'name': 'improvements', 'label': t('forms.tpl_survey_improvements'), 'options': [t('forms.tpl_survey_opt_speed'), t('forms.tpl_survey_opt_quality'), t('forms.tpl_survey_opt_communication'), t('forms.tpl_survey_opt_price'), t('forms.tpl_survey_opt_support')], 'required': False, 'width': 'full'},
                {'id': 'f5', 'type': 'textarea', 'name': 'comments', 'label': t('forms.tpl_survey_comments'), 'placeholder': t('forms.tpl_survey_comments_placeholder'), 'required': False, 'width': 'full'},
            ],
            'settings': {
                'submit_label': t('forms.tpl_survey_submit'),
                'success_message': t('forms.tpl_survey_success'),
                'notify_on_submit': True,
                'store_submissions': True,
                'anti_spam': 'honeypot',
            }
        },
        'subscription': {
            'fields': [
                {'id': 'f1', 'type': 'email', 'name': 'email', 'label': t('forms.tpl_sub_email'), 'placeholder': t('forms.tpl_sub_email_placeholder'), 'required': True, 'width': 'full'},
                {'id': 'f2', 'type': 'text', 'name': 'name', 'label': t('forms.tpl_sub_name'), 'placeholder': t('forms.tpl_sub_name_placeholder'), 'required': False, 'width': 'full'},
                {'id': 'f3', 'type': 'consent', 'name': 'consent', 'label': t('forms.tpl_sub_consent_label'), 'consent_text': t('forms.tpl_sub_consent_text'), 'required': True, 'width': 'full'},
            ],
            'settings': {
                'submit_label': t('forms.tpl_sub_submit'),
                'success_message': t('forms.tpl_sub_success'),
                'notify_on_submit': True,
                'store_submissions': True,
                'anti_spam': 'honeypot',
                'double_opt_in': True,
            }
        },
        'custom': {
            'fields': [
                {'id': 'f1', 'type': 'text', 'name': 'field_1', 'label': t('forms.tpl_custom_field'), 'placeholder': '', 'required': False, 'width': 'full'},
            ],
            'settings': {
                'submit_label': t('forms.tpl_custom_submit'),
                'success_message': t('forms.tpl_custom_success'),
                'notify_on_submit': True,
                'store_submissions': True,
                'anti_spam': 'honeypot',
            }
        }
    }
    return templates.get(form_type, templates['custom'])


# --- Helper: Send notification email ---
def _send_form_notification(form, data, to_email):
    """Send email notification to admin when a form is submitted."""
    # Use the same SMTP settings as the rest of the app
    smtp_host = Settings.get('smtp_host')
    if not smtp_host:
        return  # No email configured
    
    import smtplib
    from email.mime.text import MIMEText
    from email.mime.multipart import MIMEMultipart
    
    smtp_port = int(Settings.get('smtp_port', '587'))
    smtp_user = Settings.get('smtp_username', '')
    smtp_pass = decrypt_smtp_password(Settings.get('smtp_password_encrypted', ''))
    from_email = Settings.get('smtp_from_email', smtp_user)
    from_name = Settings.get('smtp_from_name', 'AliothPress')
    use_tls = Settings.get('smtp_use_tls', 'true') == 'true'
    use_ssl = Settings.get('smtp_use_ssl', 'false') == 'true'
    
    # Determine admin recipient's preferred language
    recipient = User.query.filter_by(email=to_email).first()
    lang = recipient.admin_language if recipient else Settings.get('default_language', 'en')
    
    # Build email body
    fields = form.get_fields()
    field_labels = {f['name']: f['label'] for f in fields}
    
    lines = [f'<h2>{t_for("forms.email_new_submission", lang, title=html_escape(form.title))}</h2>', '<table style="border-collapse:collapse;width:100%">']
    for key, value in data.items():
        if key.startswith('_'):
            continue
        label = html_escape(field_labels.get(key, key))
        if isinstance(value, list):
            value = ', '.join(str(v) for v in value)
        value_safe = html_escape(str(value))
        lines.append(f'<tr><td style="padding:8px;border:1px solid #ddd;font-weight:bold;width:30%">{label}</td>')
        lines.append(f'<td style="padding:8px;border:1px solid #ddd">{value_safe}</td></tr>')
    lines.append('</table>')
    
    html_body = '\n'.join(lines)
    
    msg = MIMEMultipart('alternative')
    msg['Subject'] = t_for('forms.email_subject_submission', lang, title=form.title)
    msg['From'] = f'{from_name} <{from_email}>'
    msg['To'] = to_email
    msg.attach(MIMEText(html_body, 'html'))
    
    try:
        if use_ssl:
            server = smtplib.SMTP_SSL(smtp_host, smtp_port, timeout=10)
        else:
            server = smtplib.SMTP(smtp_host, smtp_port, timeout=10)
            if use_tls:
                server.starttls()
        
        if smtp_user and smtp_pass:
            server.login(smtp_user, smtp_pass)
        
        server.sendmail(from_email, [to_email], msg.as_string())
        server.quit()
    except Exception as e:
        app.logger.error(f'Form notification failed: {e}')


# ============================================================================
# PAGE BUILDER
# ============================================================================

def _extract_theme_css_vars(theme_name):
    """Extract --ap-* CSS custom properties from a theme's theme.css.
    
    Returns two dicts: (dark_vars, light_vars) where each maps
    variable names to their values. Parses :root{} for dark 
    and .light-theme{} for light.
    """
    import re
    theme_path = os.path.join(app.static_folder, 'css', 'themes', theme_name, 'theme.css')
    if not os.path.isfile(theme_path):
        return {}, {}
    
    try:
        with open(theme_path, 'r', encoding='utf-8') as f:
            css = f.read()
    except Exception:
        return {}, {}
    
    def parse_block(text):
        """Extract --ap-* variable declarations from a CSS block."""
        props = {}
        for match in re.finditer(r'(--ap-[a-z0-9-]+)\s*:\s*([^;]+);', text):
            props[match.group(1)] = match.group(2).strip()
        return props
    
    # Parse :root { ... } — dark vars (or default)
    dark_vars = {}
    root_match = re.search(r':root\s*\{([^}]+)\}', css)
    if root_match:
        dark_vars = parse_block(root_match.group(1))
    
    # Parse .light-theme { ... } — light vars
    light_vars = {}
    light_match = re.search(r'\.light-theme\s*\{([^}]+)\}', css)
    if light_match:
        light_vars = parse_block(light_match.group(1))
    
    return dark_vars, light_vars


def _builder_preview_theme_css():
    """CSS that scopes the ACTIVE public theme's --ap-* variables to the
    read-only builder preview inside the admin editor (.ae-visual).

    The static bridge in alioth-editor.css maps --ap-* onto the admin
    palette so blocks are never unstyled. This function layers the real
    theme values ON TOP (higher specificity: body[data-theme] adds an
    attribute + element vs the bridge's two classes), keyed on the
    admin's resolved light/dark mode. Result: the preview shows the same
    colors, buttons and section backgrounds as the public site, and
    follows a theme switch automatically — the bridge stays underneath
    as a fallback for any variable the theme doesn't define.

    Returns '' when the theme's variables cannot be extracted (missing
    or unparseable theme.css); the admin-palette bridge then remains the
    sole styling — the previous behaviour — so injection can never make
    the preview worse than before.
    """
    active_theme = effective_active_theme()
    dark_vars, light_vars = _extract_theme_css_vars(active_theme)
    if not dark_vars and not light_vars:
        return ''
    # Same cascade rule as the builder canvas (admin_page_builder):
    # themes are dark-first — :root holds dark defaults, .light-theme
    # holds light overrides — so light mode inherits dark as a base.
    combined_light_vars = {**dark_vars, **light_vars}
    parts = []
    if dark_vars:
        parts.append('body[data-theme="dark"] .ae-editor .ae-visual {')
        for k, v in dark_vars.items():
            parts.append(f'  {k}: {v};')
        parts.append('}')
    if combined_light_vars:
        parts.append('body[data-theme="light"] .ae-editor .ae-visual {')
        for k, v in combined_light_vars.items():
            parts.append(f'  {k}: {v};')
        parts.append('}')
    return '\n'.join(parts)


@app.context_processor
def _inject_builder_preview_theme_css():
    """Expose builder_preview_theme_css() to templates. Called only from
    pages/edit.html and pages/translate.html inside their use_builder
    branches, so the theme.css read happens solely on builder pages."""
    return {'builder_preview_theme_css': _builder_preview_theme_css}


@app.route('/admin/pages/<int:page_id>/builder')
@login_required
def admin_page_builder(page_id):
    """Page builder visual editor"""
    page = Page.query.get_or_404(page_id)
    # Get media library images for the image picker
    images = Image.query.order_by(Image.uploaded_at.desc()).limit(100).all()
    
    # Get active site theme colors for canvas preview
    active_theme = effective_active_theme()
    default_mode = Settings.get('default_theme_mode', 'auto')
    dark_vars, light_vars = _extract_theme_css_vars(active_theme)
    
    # Build CSS string with theme vars scoped to the canvas
    # Dark mode vars go into .pb-canvas (default for dark-first themes)
    # Light mode vars go into .pb-canvas.pb-canvas--light
    theme_css_parts = []
    
    # Themes are dark-first: :root contains dark mode defaults, .light-theme
    # contains overrides for light mode. To mimic the public site's CSS cascade
    # inside the canvas, light vars must inherit from dark vars as a base —
    # otherwise variables declared only in :root (like --ap-color-white) would
    # be missing from .pb-canvas--theme-light scope.
    combined_light_vars = {**dark_vars, **light_vars}
    
    if dark_vars:
        theme_css_parts.append('.pb-canvas--theme-dark {')
        for k, v in dark_vars.items():
            theme_css_parts.append(f'  {k}: {v};')
        theme_css_parts.append('}')
    
    if combined_light_vars:
        theme_css_parts.append('.pb-canvas--theme-light {')
        for k, v in combined_light_vars.items():
            theme_css_parts.append(f'  {k}: {v};')
        theme_css_parts.append('}')
    
    theme_css = '\n'.join(theme_css_parts)
    
    # Pass builder translations as JSON for JS
    builder_i18n = {}
    builder_keys = [
        'save', 'saving', 'saved', 'save_failed', 'network_error', 'drag_hint', 'loading',
        'block_settings', 'layout', 'alignment', 'left', 'center', 'right', 'justify',
        'top_margin', 'bottom_margin', 'none', 'css_class', 'anchor_id',
        'text_content', 'level', 'image_url', 'image_url_hint', 'alt_text',
        'caption', 'link_url', 'size', 'size_full', 'size_medium', 'size_small',
        'button_text', 'button_url', 'style', 'style_primary', 'style_outline',
        'style_ghost', 'style_solid', 'style_dashed', 'style_dotted', 'style_gradient',
        'size_sm', 'size_md', 'size_lg', 'size_auto', 'target', 'target_self', 'target_blank',
        'height_px', 'columns_layout', 'columns_hint', 'video_url', 'video_url_hint',
        'video_title', 'aspect_ratio', 'ratio_cinematic', 'ratio_auto', 'html_code', 'quote_text', 'author',
        'code_content', 'code_language', 'audio_url', 'audio_title',
        'table_headers', 'table_rows', 'add_item', 'remove', 'add_tab',
        'gallery_images_hint', 'gallery_images_format', 'gallery_columns',
        'background', 'bg_none', 'bg_color', 'bg_gradient', 'bg_image',
        'bg_value', 'padding', 'padding_sm', 'padding_md', 'padding_lg', 'padding_xl',
        'text', 'page_builder', 'preview', 'basic', 'structure', 'media', 'advanced',
        'heading', 'image', 'button', 'divider', 'spacer', 'columns', 'section',
        'video', 'gallery', 'audio', 'quote', 'table', 'accordion', 'tabs',
        'html', 'code', 'form', 'form_select', 'form_no_forms', 'form_show_title',
        'form_show_description', 'form_fields', 'form_submit_default',
        'form_search_placeholder', 'form_all_languages', 'form_new',
        'click_select_form', 'slideshow',
        'convert_from_editor_title', 'convert_from_editor_warning', 'convert_from_editor_ok',
        'bg_color_pick', 'bg_gradient_value', 'bg_image_url', 'bg_image_hint',
        'bg_overlay',
        'bg_light_mode', 'bg_light_color_pick',
        'bg_light_gradient_value', 'bg_light_image_url', 'bg_light_overlay',
        'bg_dark_mode', 'bg_dark_color_pick', 'bg_dark_gradient_value',
        'bg_dark_image_url', 'bg_dark_overlay', 'bg_theme_default',
        'text_color', 'text_color_auto', 'text_color_light',
        'text_color_dark',
        'min_height', 'min_height_auto', 'min_height_50_screen', 'min_height_75_screen', 'min_height_full_screen',
        'section_layout', 'section_full_width',
        'section_full_width_hint', 'section_content_width',
        'section_width_narrow', 'section_width_default', 'section_width_wide',
        'section_width_full', 'slideshow_mode', 'slideshow_mode_image',
        'slideshow_mode_hero', 'slideshow_animation', 'animation_slide', 'animation_fade',
        'slideshow_autoplay',
        'slideshow_interval', 'slideshow_arrows', 'slideshow_dots',
        'slideshow_overlay', 'slideshow_slides', 'slideshow_add_slide', 'slideshow_remove_slide',
        'choose_image', 'gallery_add_from_library', 'gallery_pick_title',
        'gallery_add_selected',
        # Canvas previews & inline editing
        'gallery_images', 'table_rows_count', 'accordion_items', 'tabs_count',
        'delete_block_confirm', 'delete_item_confirm', 'empty_hint',
        'default_heading', 'default_text', 'default_button',
        'default_accordion_title', 'default_tab',
        'click_select_image', 'add_block', 'section_full_width_badge',
        'table_add_row', 'table_add_col', 'table_width', 'column_alignment',
        'table_insert_row', 'column_width', 'column_width_hint', 'total',
        'alt_missing', 'alt_text_library_hint',
        'content_alignment', 'alignment_auto', 'block_position',
        'slideshow_add_block', 'slide_select_hint',
        'accordion_content', 'tab_content',
        'html_executes', 'text_edit_canvas_hint',
        # Video gallery
        'video_gallery', 'video_gallery_add', 'video_gallery_add_title',
        'video_gallery_add_btn', 'video_gallery_empty', 'video_gallery_videos',
        'video_gallery_title_fetched',
        'video_meta_fetch_failed',
        # Toolbar actions
        'drag', 'move_up', 'move_down', 'duplicate', 'delete', 'undo', 'redo',
        'visibility', 'visibility_all', 'visibility_desktop', 'visibility_mobile',
        'scroll_animation', 'animation_fade_in', 'animation_slide_up',
        # Text formatting toolbar
        'bold', 'italic', 'underline', 'strikethrough',
        'align_left', 'align_center', 'align_right', 'align_justify',
        'bullet_list', 'numbered_list', 'link', 'clear_formatting',
        # Accordion icon labels
        'icon_check', 'icon_cross', 'icon_star', 'icon_feature',
        'icon_tip', 'icon_warning', 'icon_plus', 'icon_minus', 'icon_dot',
        # Logo ticker
        'ticker', 'ticker_speed', 'ticker_slow', 'ticker_medium', 'ticker_fast',
        'ticker_pause_hover', 'ticker_grayscale', 'ticker_height', 'ticker_images',
        'ticker_direction', 'ticker_dir_rtl', 'ticker_dir_ltr',
        # AI assistant
        'ai_thinking', 'ai_welcome', 'ai_generated', 'ai_no_blocks', 'ai_blocks_loaded',
        # Misc
        'error_loading_forms', 'unsaved_warning',
    ]
    for key in builder_keys:
        builder_i18n[key] = t('builder.' + key)
    
    # Unified media picker strings — reused from the admin-wide namespaces so
    # the picker looks and behaves identically everywhere without adding any
    # new translation keys.
    builder_i18n['picker_search_placeholder'] = t('media.search_placeholder')
    # Themed pbDialog (replaces native prompt/confirm) reuses the editor's
    # OK/Cancel strings — present in every admin language file already.
    builder_i18n['ok'] = t('aeditor.ok')
    builder_i18n['cancel'] = t('aeditor.cancel')
    builder_i18n['picker_no_results'] = t('media.no_results')
    builder_i18n['picker_prev'] = t('pagination.prev')
    builder_i18n['picker_next'] = t('pagination.next')
    builder_i18n['picker_loading'] = t('common.loading')
    builder_i18n['picker_error'] = t('media.error_loading_retry')
    
    return render_template('admin/pages/builder.html', page=page, images=images,
                         builder_i18n_json=json.dumps(builder_i18n, ensure_ascii=False),
                         theme_css=theme_css,
                         active_theme=active_theme,
                         default_mode=default_mode)


# --- Editor-content → builder-blocks seed conversion -------------------------
#
# When the builder opens a page whose content was written in the regular
# editor, the content is converted to blocks for the canvas preview (and, if
# the user confirms the save, becomes the page's builder layout). A faithful
# conversion must keep the document structure: top-level <h1>–<h6> become
# dedicated Heading blocks — dumping them into a Text block would get them
# demoted to <p> by the text-block heading guard (see
# _demote_headings_in_text_blocks / block_renderer._render_text), losing the
# page's heading outline. Everything between headings stays as Text blocks
# with its HTML untouched (lists, links, images, tables).
#
# Block alignment mirrors the legacy rendering: the legacy path wraps content
# in a plain left-flowing .ap-prose, so seeded blocks get alignment 'left'
# ('right' for RTL languages, matching the dir= logic in public/base.html).
# The old seed forced 'center', which re-centered the whole page and broke
# list bullet layout — the exact "saved differently" complaint.
#
# Input domain: page.content is bleach-sanitized (sanitize_html) — tags are
# balanced and comment/script-free — so a lightweight tag-depth scan is
# reliable here. Anything unexpected falls back to the previous single-text-
# block behaviour (minus the forced centering), never worse than before.

# Void elements never change nesting depth.
_SEED_VOID_TAGS = frozenset((
    'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link',
    'meta', 'param', 'source', 'track', 'wbr'))
# One HTML tag; quoted attribute values are consumed wholesale so a '>'
# inside quotes can't end the match early.
_SEED_TAG_RE = re.compile(
    r'<(/?)([a-zA-Z][a-zA-Z0-9]*)((?:[^>"\']|"[^"]*"|\'[^\']*\')*?)(/?)>')
_SEED_HEADING_RE = re.compile(
    r'<[hH]([1-6])(?:\s(?:[^>"\']|"[^"]*"|\'[^\']*\')*)?>(.*?)</[hH]\1\s*>',
    re.DOTALL)


def _seed_html_depth(fragment):
    """Net tag-nesting depth change across an HTML fragment (opens − closes,
    ignoring void and self-closed tags)."""
    depth = 0
    for m in _SEED_TAG_RE.finditer(fragment):
        if m.group(2).lower() in _SEED_VOID_TAGS or m.group(4):
            continue
        depth += -1 if m.group(1) else 1
    return depth


def _seed_blocks_from_editor_content(content, language):
    """Convert regular-editor HTML into a list of builder blocks.

    Top-level headings → Heading blocks (inline markup stripped, since the
    Heading block stores plain text); the runs between them → Text blocks
    with their HTML verbatim. Headings nested inside other elements (e.g. a
    blockquote) are left inside their Text block — pulling them out would
    tear the parent element apart. Returns at least one block for non-empty
    content; on any parsing surprise falls back to one Text block holding
    the whole content unchanged.
    """
    import uuid as _uuid
    from html import unescape as _html_unescape
    from block_renderer import _is_rich_html_empty

    align = 'right' if (language or '') in ('ar', 'he') else 'left'

    def _mk(btype, data):
        return {'id': str(_uuid.uuid4())[:8], 'type': btype, 'data': data,
                'settings': {'alignment': align}}

    blocks = []
    try:
        last = 0      # start of the pending text chunk
        pos = 0       # scan position for incremental depth tracking
        depth = 0     # tag depth at `pos`
        for m in _SEED_HEADING_RE.finditer(content):
            depth += _seed_html_depth(content[pos:m.start()])
            pos = m.start()
            if depth != 0:
                # Nested heading — keep it inside the surrounding Text chunk
                # (the text-block guard will demote it, same as today).
                continue
            chunk = content[last:m.start()]
            if not _is_rich_html_empty(chunk):
                blocks.append(_mk('text', {'html': chunk.strip()}))
            text = _html_unescape(re.sub(r'<[^>]+>', '', m.group(2))).strip()
            if text:
                blocks.append(_mk('heading',
                                  {'text': text, 'level': int(m.group(1))}))
            # The heading itself is balanced — skip past it, depth unchanged.
            pos = m.end()
            last = m.end()
        tail = content[last:]
        if not _is_rich_html_empty(tail):
            blocks.append(_mk('text', {'html': tail.strip()}))
    except Exception:
        blocks = []
    if not blocks:
        blocks = [_mk('text', {'html': content})]
    return blocks


@app.route('/api/pages/<int:page_id>/blocks', methods=['GET'])
@login_required
def api_get_page_blocks(page_id):
    """Get page blocks JSON (for builder UI).
    
    Preloads form field data (_fields, _settings, etc.) so that the builder
    canvas can render form previews immediately without a separate fetch.
    """
    page = Page.query.get_or_404(page_id)
    blocks_data = {'blocks': [], 'version': 1}
    if page.blocks:
        try:
            blocks_data = json.loads(page.blocks)
        except (ValueError, TypeError):
            pass
    
    # Seed blocks from TinyMCE content when opening builder for the first time.
    # This handles the case where content was written in TinyMCE (new/translate)
    # before the user switched to Page Builder.
    #
    # The seed is deliberately NOT persisted: merely opening the builder must
    # never convert an editor-managed page (GET stays side-effect free). The
    # page only becomes builder-managed when the user saves (PUT), and for
    # pages that already hold editor-authored content the builder UI asks for
    # confirmation first — see the converts_on_save flag below. Re-seeding on
    # every open is idempotent because page.content does not change until the
    # first builder save.
    converts_on_save = (not (page.use_builder or False)
                        and bool(page.content and page.content.strip()))
    if not blocks_data.get('blocks') and page.content and page.content.strip():
        blocks_data = {
            'blocks': _seed_blocks_from_editor_content(page.content,
                                                       page.language),
            'version': 1
        }
    
    # Preload form preview data into form blocks
    _preload_form_data(blocks_data.get('blocks', []))
    
    return jsonify({
        'success': True,
        'page_id': page.id,
        'title': page.title,
        'use_builder': page.use_builder or False,
        'converts_on_save': converts_on_save,
        'blocks': blocks_data
    })


def _preload_form_data(blocks):
    """Recursively inject _fields/_settings into form blocks for builder preview."""
    for b in (blocks or []):
        if b.get('type') == 'form' and b.get('data', {}).get('form_id'):
            form = Form.query.get(b['data']['form_id'])
            if form:
                fields = form.get_fields()
                b['data']['_fields'] = fields
                b['data']['_settings'] = form.get_settings()
                b['data']['_description'] = form.description or ''
                b['data']['_form_title_raw'] = form.title or ''
                b['data']['_form_title'] = '{} ({}, {} fields)'.format(
                    form.title or '', form.form_type or '', len(fields)
                )
        # Recurse into nested containers
        if b.get('type') == 'columns' and b.get('data', {}).get('columns'):
            for col in b['data']['columns']:
                _preload_form_data(col.get('blocks', []))
        if b.get('type') == 'section' and b.get('data', {}).get('blocks'):
            _preload_form_data(b['data']['blocks'])
        if b.get('type') == 'slideshow' and b.get('data', {}).get('slides'):
            for slide in b['data']['slides']:
                _preload_form_data(slide.get('blocks', []))


# Regex for forcing target="_blank" on anchor tags in text and table blocks.
# Applied at save-time so the database stores self-contained HTML: the
# renderer stays dumb, migrations/exports carry correct data.
_ANCHOR_OPEN_RE = re.compile(r'<a\b([^>]*)>', re.IGNORECASE)


def _force_blank_target_on_links(html):
    """Add target="_blank" rel="noopener noreferrer" to every <a> tag that
    does not already declare a target. Leaves existing target values alone.
    Only touches opening <a> tags — closing tags, href values, and tag
    contents are not modified.

    Internal links are left alone (same tab):
      - in-page anchors: href starts with '#'
      - same-origin paths: href starts with '/' but NOT '//' (protocol-relative
        URLs resolve to another host, so those are treated as external).
    """
    if not html or '<a' not in html.lower():
        return html

    def _rewrite(match):
        attrs = match.group(1)
        if re.search(r'\btarget\s*=', attrs, re.IGNORECASE):
            return match.group(0)
        href_match = re.search(r'\bhref\s*=\s*["\']([^"\']*)["\']', attrs, re.IGNORECASE)
        if not href_match:
            return match.group(0)
        href = href_match.group(1).strip().lower()
        # Internal: in-page anchor or same-origin absolute path.
        if href.startswith('#') or (href.startswith('/') and not href.startswith('//')):
            return match.group(0)
        rel_match = re.search(r'\brel\s*=\s*["\']([^"\']*)["\']', attrs, re.IGNORECASE)
        if rel_match:
            existing_rel = rel_match.group(1).lower().split()
            needed = [t for t in ('noopener', 'noreferrer') if t not in existing_rel]
            if needed:
                new_rel = ' '.join(rel_match.group(1).split() + needed)
                attrs = attrs[:rel_match.start(1)] + new_rel + attrs[rel_match.end(1):]
            return f'<a{attrs} target="_blank">'
        return f'<a{attrs} target="_blank" rel="noopener noreferrer">'

    return _ANCHOR_OPEN_RE.sub(_rewrite, html)


# Anchor IDs must be safe for use as both the `id` attribute of the wrapping
# element AND the fragment portion of a URL (so `#anchor` links work). We use
# the same shape as the page/post slug normalizer in templates/admin/posts/
# edit.html:cleanSlugInput — kebab-case ASCII — for predictable UX across the
# whole admin: anywhere a user types something that becomes a URL-addressable
# identifier, the rules are the same.
_ANCHOR_ID_INVALID_RE = re.compile(r'[^a-z0-9-]')


def _sanitize_anchor_id(raw):
    """Normalize a user-typed anchor_id to a URL-fragment-safe form.

    Empty string is returned unchanged — that means "no anchor" and is
    handled by the renderer (no id attribute emitted).
    """
    if not raw or not isinstance(raw, str):
        return ''
    v = raw.strip().lower()
    v = re.sub(r'\s+', '-', v)
    v = _ANCHOR_ID_INVALID_RE.sub('', v)
    v = re.sub(r'-+', '-', v).strip('-')
    return v


def _sanitize_anchor_ids_in_blocks(blocks):
    """Walk the block tree and normalize every settings.anchor_id in place.
    Same recursion pattern as _apply_blank_target_to_blocks.
    """
    if not isinstance(blocks, list):
        return
    for block in blocks:
        if not isinstance(block, dict):
            continue
        settings = block.get('settings')
        if isinstance(settings, dict) and 'anchor_id' in settings:
            settings['anchor_id'] = _sanitize_anchor_id(settings.get('anchor_id'))
        btype = block.get('type', '')
        data = block.get('data', {}) or {}
        if btype == 'columns':
            for col in data.get('columns', []):
                _sanitize_anchor_ids_in_blocks(col.get('blocks', []))
        elif btype == 'section':
            _sanitize_anchor_ids_in_blocks(data.get('blocks', []))
        elif btype == 'slideshow':
            for slide in data.get('slides', []):
                _sanitize_anchor_ids_in_blocks(slide.get('blocks', []))


_HEADING_TAG_RE = re.compile(r'<(/?)[hH][1-6]((?:\s[^>]*)?)>')


# --- Default alignment for blocks new to a page ------------------------------
#
# Server-side twin of defaultAlignFor() in the builder UI: content-flow
# blocks default to 'left' ('right' on RTL pages), visual/decorative blocks
# to 'center'. Applied on save (PUT) ONLY to blocks whose id is new to the
# page and that carry no alignment of their own — so agents composing blocks
# without settings get the same starting alignment a human gets in the UI.
# Blocks already stored on the page are never touched, even when they lack
# an alignment: those legacy blocks keep rendering through the long-standing
# '|| center' fallbacks, and re-saving a page must not change how it looks.

_CONTENT_FLOW_BLOCK_TYPES = frozenset((
    'heading', 'text', 'quote', 'table', 'accordion', 'tabs', 'code',
    'html', 'form'))


def _default_alignment_for(btype, language):
    if btype in _CONTENT_FLOW_BLOCK_TYPES:
        return 'right' if (language or '') in ('ar', 'he') else 'left'
    return 'center'


def _walk_block_tree(blocks, fn):
    """Call fn(block) for every block in the tree, recursing into columns,
    sections, and slideshow slides — same traversal as the other block
    normalizers above."""
    if not isinstance(blocks, list):
        return
    for block in blocks:
        if not isinstance(block, dict):
            continue
        fn(block)
        btype = block.get('type', '')
        data = block.get('data', {}) or {}
        if btype == 'columns':
            for col in data.get('columns', []) or []:
                if isinstance(col, dict):
                    _walk_block_tree(col.get('blocks', []), fn)
        elif btype == 'section':
            _walk_block_tree(data.get('blocks', []), fn)
        elif btype == 'slideshow':
            for slide in data.get('slides', []) or []:
                if isinstance(slide, dict):
                    _walk_block_tree(slide.get('blocks', []), fn)


def _collect_block_ids(blocks):
    """Set of all block ids currently stored in a block tree."""
    ids = set()

    def _add(block):
        bid = block.get('id')
        if bid:
            ids.add(bid)

    _walk_block_tree(blocks, _add)
    return ids


def _apply_default_alignment_to_new_blocks(blocks, existing_ids, language):
    """Give blocks that are new to the page (id absent from existing_ids)
    and have no alignment the same per-type default the builder UI uses."""

    def _fill(block):
        bid = block.get('id')
        if bid and bid in existing_ids:
            return
        settings = block.get('settings')
        if not isinstance(settings, dict):
            settings = {}
            block['settings'] = settings
        if not settings.get('alignment'):
            settings['alignment'] = _default_alignment_for(
                block.get('type', ''), language)

    _walk_block_tree(blocks, _fill)


def _demote_headings_in_text_blocks(blocks):
    """Walk the block tree and demote stray <h1>-<h6> to <p> in every rich
    Text block's html (preserving text and attributes). Headings belong to the
    dedicated Heading block; one pasted into a Text block (the inline editor
    shows it like normal text, so the author can't tell) would emit a real
    heading and break the page heading order — which PageSpeed flags. This
    cleans the stored JSON on save; block_renderer applies the same guard at
    render time. The raw 'html' embed block is intentionally left untouched.
    Same recursion as _sanitize_anchor_ids_in_blocks."""
    if not isinstance(blocks, list):
        return
    for block in blocks:
        if not isinstance(block, dict):
            continue
        btype = block.get('type', '')
        data = block.get('data', {}) or {}
        if btype == 'text' and isinstance(data.get('html'), str):
            data['html'] = _HEADING_TAG_RE.sub(r'<\1p\2>', data['html'])
        elif btype == 'columns':
            for col in data.get('columns', []):
                _demote_headings_in_text_blocks(col.get('blocks', []))
        elif btype == 'section':
            _demote_headings_in_text_blocks(data.get('blocks', []))
        elif btype == 'slideshow':
            for slide in data.get('slides', []):
                _demote_headings_in_text_blocks(slide.get('blocks', []))


def _apply_blank_target_to_blocks(blocks):
    """Walk the block tree and force target="_blank" on links inside text
    and table blocks. Recurses into columns, sections, and slideshow slides.
    Mutates the list in place.
    """
    if not isinstance(blocks, list):
        return
    for block in blocks:
        if not isinstance(block, dict):
            continue
        btype = block.get('type', '')
        data = block.get('data', {}) or {}
        if btype == 'text' and data.get('html'):
            data['html'] = _force_blank_target_on_links(data['html'])
        elif btype == 'table':
            headers = data.get('headers', [])
            for i, h in enumerate(headers):
                if h:
                    headers[i] = _force_blank_target_on_links(h)
            for row in data.get('rows', []):
                for i, cell in enumerate(row):
                    if cell:
                        row[i] = _force_blank_target_on_links(cell)
        # Recurse into nested structures
        elif btype == 'columns':
            for col in data.get('columns', []):
                _apply_blank_target_to_blocks(col.get('blocks', []))
        elif btype == 'section':
            _apply_blank_target_to_blocks(data.get('blocks', []))
        elif btype == 'slideshow':
            for slide in data.get('slides', []):
                _apply_blank_target_to_blocks(slide.get('blocks', []))


@app.route('/api/pages/<int:page_id>/blocks', methods=['PUT'])
@csrf.exempt  # API endpoint — protected by @login_required + SameSite cookie
@login_required
def api_save_page_blocks(page_id):
    """Save page blocks JSON from builder"""
    page = Page.query.get_or_404(page_id)
    data = request.get_json()
    
    if not data or 'blocks' not in data:
        return jsonify({'success': False, 'error': t('api.no_blocks_data')}), 400
    
    # Conversion guard — server-side twin of the builder-UI confirmation.
    # A page whose content was written in the regular editor only becomes
    # builder-managed after an explicit opt-in: the builder UI sends
    # convert=true after the user confirms its dialog, and API clients
    # (WebMCP agents, scripts) must pass it consciously. Without this, a
    # stray set_page_blocks call — e.g. under WebMCP autopilot, which
    # auto-approves the generic client-side confirm — would silently flip
    # an editor page into the builder. Empty drafts (no editor content)
    # are exempt, so agent flows that create fresh builder pages and
    # immediately PUT their blocks keep working unchanged.
    if (not (page.use_builder or False)) and page.content \
            and page.content.strip() and not bool(data.get('convert')):
        return jsonify({'success': False,
                        'error': t('api.convert_requires_confirm'),
                        'requires_convert': True}), 409
    
    # Normalize JSON structure
    blocks_data = data['blocks']
    if not isinstance(blocks_data, dict) or 'blocks' not in blocks_data:
        blocks_data = {'blocks': blocks_data if isinstance(blocks_data, list) else [], 'version': 1}
    
    # Force all links in text and table blocks to open in a new tab.
    # Buttons keep their per-link target setting from the sidebar and are
    # unaffected. Runs once on save; the DB stores the final HTML.
    #
    # Validation guard: the builder UI always sends well-formed trees, but
    # API clients (agents, scripts) may not. A malformed block structure
    # must produce a clean 400 for the caller, never a 500.
    try:
        _apply_blank_target_to_blocks(blocks_data.get('blocks', []))

        # Give blocks that are new to this page the same default alignment a
        # human gets in the builder UI (content blocks left / right on RTL,
        # visual blocks center). Blocks already stored on the page keep their
        # data byte-for-byte, so re-saving never changes existing appearance.
        existing_ids = set()
        if page.blocks:
            try:
                existing_ids = _collect_block_ids(
                    json.loads(page.blocks).get('blocks', []))
            except (ValueError, TypeError):
                pass
        _apply_default_alignment_to_new_blocks(
            blocks_data.get('blocks', []), existing_ids, page.language)

        # Normalize user-typed anchor IDs to URL-fragment-safe form. Defense in
        # depth: the panel input already runs cleanAnchorIdInput() on blur, but
        # anything that bypasses the UI (translation flow, future API clients)
        # still gets clean data on save.
        _sanitize_anchor_ids_in_blocks(blocks_data.get('blocks', []))

        # Demote any stray heading tags pasted into Text blocks (see function doc).
        _demote_headings_in_text_blocks(blocks_data.get('blocks', []))

        serialized_blocks = json.dumps(blocks_data)

        # Also generate flat HTML content for SEO / search / fallback
        from block_renderer import render_blocks_to_html
        rendered_html = render_blocks_to_html(serialized_blocks)
    except (AttributeError, TypeError, KeyError, ValueError):
        return jsonify({'success': False, 'error': t('api.no_blocks_data')}), 400

    page.blocks = serialized_blocks
    page.use_builder = True
    page.updated_at = utc_now()
    page.content = rendered_html
    
    db.session.commit()
    
    AuditLog.log('edit_page_builder', 'page', page.id,
                 _ad('updated_builder', title=page.title),
                 user=current_user, ip=request.remote_addr)
    db.session.commit()
    
    return jsonify({'success': True, 'page_id': page.id})


# ============================================================================
# SLUG VALIDATION API
# ============================================================================

@app.route('/admin/api/check-slug', methods=['POST'])
@login_required
def admin_api_check_slug():
    """Check if a slug is already taken"""
    data = request.get_json()
    slug = data.get('slug', '').strip()
    content_type = data.get('content_type', 'post')  # 'post' or 'page'
    exclude_id = data.get('exclude_id')  # exclude current item when editing
    
    if not slug:
        return jsonify({'available': False, 'message': t('api.slug_empty')})
    
    # Validate slug format
    import re
    clean_slug = slugify(slug)
    if clean_slug != slug:
        return jsonify({
            'available': False, 
            'message': t('api.slug_invalid_chars', slug=clean_slug)
        })
    
    if content_type == 'post':
        query = Post.query.filter_by(slug=slug)
        if exclude_id:
            query = query.filter(Post.id != int(exclude_id))
        exists = query.first()
        # Also check in Pages
        if not exists:
            exists = Page.query.filter_by(slug=slug).first()
    else:
        query = Page.query.filter_by(slug=slug)
        if exclude_id:
            query = query.filter(Page.id != int(exclude_id))
        exists = query.first()
        # Also check in Posts
        if not exists:
            exists = Post.query.filter_by(slug=slug).first()
    
    if exists:
        return jsonify({
            'available': False, 
            'message': t('api.slug_taken', slug=slug, title=exists.title, language=exists.language.upper())
        })
    
    return jsonify({'available': True, 'message': t('api.slug_available')})


@app.route('/admin/api/search-titles', methods=['GET'])
@login_required
def admin_api_search_titles():
    """Live search — returns matching post/page/form titles for autocomplete.
    Matches anywhere in the title (not just prefix)."""
    q = request.args.get('q', '').strip()
    content_type = request.args.get('content_type', 'post')  # 'post', 'page', or 'form'
    
    if not q or len(q) < 2:
        return jsonify({'results': []})
    
    search_pattern = f"%{q}%"
    
    if content_type == 'post':
        items = Post.query.filter(Post.title.ilike(search_pattern)) \
            .order_by(Post.updated_at.desc()).limit(10).all()
        results = [{
            'id': item.id,
            'title': item.title,
            'slug': item.slug,
            'status': item.status,
            'language': item.language
        } for item in items]
    elif content_type == 'form':
        items = Form.query.filter(Form.title.ilike(search_pattern)) \
            .order_by(Form.updated_at.desc()).limit(10).all()
        results = [{
            'id': item.id,
            'title': item.title,
            'form_type': item.form_type,
            'language': item.language
        } for item in items]
    else:
        items = Page.query.filter(Page.title.ilike(search_pattern)) \
            .order_by(Page.updated_at.desc()).limit(10).all()
        results = [{
            'id': item.id,
            'title': item.title,
            'slug': item.slug,
            'status': item.status,
            'language': item.language
        } for item in items]
    
    return jsonify({'results': results})


# ============================================================================
# TRANSLATION API — AI-powered translation that creates a new post/page
# ============================================================================

# ============================================================================
# BUILDER BLOCKS TRANSLATION HELPERS
# ============================================================================

def _extract_builder_texts(blocks, prefix=''):
    """Extract translatable text fields from page builder blocks.
    
    Returns a list of dicts like:
        [{"id": "0", "type": "heading", "text": "Hello"}, 
         {"id": "1.columns.0.0", "type": "text", "html": "<p>...</p>"}, ...]
    
    The "id" encodes the path so we can put translations back.
    """
    results = []
    for i, block in enumerate(blocks):
        block_id = f"{prefix}{i}"
        block_type = block.get('type', '')
        data = block.get('data', {})
        entry = {'id': block_id, 'type': block_type}
        has_text = False
        
        if block_type == 'heading' and data.get('text'):
            entry['text'] = data['text']
            has_text = True
        elif block_type == 'text' and data.get('html'):
            entry['html'] = data['html']
            has_text = True
        elif block_type == 'button' and data.get('text'):
            entry['text'] = data['text']
            has_text = True
        elif block_type == 'image':
            if data.get('alt'):
                entry['alt'] = data['alt']
                has_text = True
            if data.get('caption'):
                entry['caption'] = data['caption']
                has_text = True
        elif block_type == 'quote':
            if data.get('text'):
                entry['text'] = data['text']
                has_text = True
            if data.get('author'):
                entry['author'] = data['author']
                has_text = True
        elif block_type == 'table':
            if data.get('headers'):
                entry['headers'] = data['headers']
                has_text = True
            if data.get('rows'):
                entry['rows'] = data['rows']
                has_text = True
        elif block_type == 'accordion' and data.get('items'):
            entry['items'] = [{'title': item.get('title', ''), 'content': item.get('content', '')} 
                             for item in data['items']]
            has_text = True
        elif block_type == 'tabs' and data.get('tabs'):
            entry['tabs'] = [{'title': tab.get('title', ''), 'content': tab.get('content', '')} 
                            for tab in data['tabs']]
            has_text = True
        elif block_type == 'video' and data.get('title'):
            entry['title'] = data['title']
            has_text = True
        elif block_type == 'audio' and data.get('title'):
            entry['title'] = data['title']
            has_text = True
        elif block_type == 'slideshow' and data.get('slides'):
            entry['slides'] = []
            for slide in data['slides']:
                slide_entry = {}
                if slide.get('alt'):
                    slide_entry['alt'] = slide['alt']
                # Nested blocks in hero slides — extract their texts
                slide_blocks = slide.get('blocks', [])
                if slide_blocks:
                    slide_entry['blocks'] = _extract_builder_texts(slide_blocks, f"{block_id}.slide.")
                if slide_entry:
                    entry['slides'].append(slide_entry)
                    has_text = True
                else:
                    entry['slides'].append({})
        
        if has_text:
            results.append(entry)
        
        # Recurse into columns
        if block_type == 'columns' and data.get('columns'):
            for col_idx, col in enumerate(data['columns']):
                col_blocks = col.get('blocks', [])
                if col_blocks:
                    results.extend(_extract_builder_texts(col_blocks, f"{block_id}.c{col_idx}."))
        
        # Recurse into sections
        if block_type == 'section' and data.get('blocks'):
            results.extend(_extract_builder_texts(data['blocks'], f"{block_id}.s."))
    
    return results


def _apply_builder_texts(blocks, translated_texts, prefix=''):
    """Apply translated texts back into builder blocks using the id path."""
    # Build a lookup by id
    text_map = {t['id']: t for t in translated_texts}
    
    for i, block in enumerate(blocks):
        block_id = f"{prefix}{i}"
        block_type = block.get('type', '')
        data = block.get('data', {})
        
        if block_id in text_map:
            tr = text_map[block_id]
            
            if block_type == 'heading' and 'text' in tr:
                data['text'] = tr['text']
            elif block_type == 'text' and 'html' in tr:
                data['html'] = tr['html']
            elif block_type == 'button' and 'text' in tr:
                data['text'] = tr['text']
            elif block_type == 'image':
                if 'alt' in tr:
                    data['alt'] = tr['alt']
                if 'caption' in tr:
                    data['caption'] = tr['caption']
            elif block_type == 'quote':
                if 'text' in tr:
                    data['text'] = tr['text']
                if 'author' in tr:
                    data['author'] = tr['author']
            elif block_type == 'table':
                if 'headers' in tr:
                    data['headers'] = tr['headers']
                if 'rows' in tr:
                    data['rows'] = tr['rows']
            elif block_type == 'accordion' and 'items' in tr:
                for idx, item in enumerate(tr['items']):
                    if idx < len(data.get('items', [])):
                        if 'title' in item:
                            data['items'][idx]['title'] = item['title']
                        if 'content' in item:
                            data['items'][idx]['content'] = item['content']
            elif block_type == 'tabs' and 'tabs' in tr:
                for idx, tab in enumerate(tr['tabs']):
                    if idx < len(data.get('tabs', [])):
                        if 'title' in tab:
                            data['tabs'][idx]['title'] = tab['title']
                        if 'content' in tab:
                            data['tabs'][idx]['content'] = tab['content']
            elif block_type == 'video' and 'title' in tr:
                data['title'] = tr['title']
            elif block_type == 'audio' and 'title' in tr:
                data['title'] = tr['title']
            elif block_type == 'slideshow' and 'slides' in tr:
                for idx, slide_tr in enumerate(tr['slides']):
                    if idx < len(data.get('slides', [])):
                        if 'alt' in slide_tr:
                            data['slides'][idx]['alt'] = slide_tr['alt']
                        # Apply translated texts to nested blocks in hero slides
                        if 'blocks' in slide_tr and data['slides'][idx].get('blocks'):
                            _apply_builder_texts(data['slides'][idx]['blocks'], slide_tr['blocks'], f"{block_id}.slide.")
        
        # Recurse into columns
        if block_type == 'columns' and data.get('columns'):
            for col_idx, col in enumerate(data['columns']):
                col_blocks = col.get('blocks', [])
                if col_blocks:
                    _apply_builder_texts(col_blocks, translated_texts, f"{block_id}.c{col_idx}.")
        
        # Recurse into sections
        if block_type == 'section' and data.get('blocks'):
            _apply_builder_texts(data['blocks'], translated_texts, f"{block_id}.s.")


# ============================================================================
# COPY AS TRANSLATION (no AI — full copy with original texts)
# ============================================================================

@app.route('/admin/api/copy-as-translation', methods=['POST'])
@login_required
@limiter.limit("20 per minute")
def admin_api_copy_as_translation():
    """Create a translation copy WITHOUT AI — keeps all original texts for manual editing.
    Copies everything including Page Builder structure, all SEO fields, images, etc."""
    data = request.get_json()
    content_type = data.get('content_type', 'post')  # 'post', 'page', or 'newsletter'
    source_id = data.get('source_id')
    target_lang_code = data.get('target_lang')
    
    if not source_id or not target_lang_code:
        return jsonify({'error': t('api.missing_source_or_lang')}), 400
    
    # Get source
    if content_type == 'post':
        source = Post.query.get(source_id)
    elif content_type == 'newsletter':
        source = Newsletter.query.get(source_id)
    else:
        source = Page.query.get(source_id)
    
    if not source:
        return jsonify({'error': t('api.source_not_found')}), 404
    
    # Check if translation already exists
    if content_type == 'newsletter':
        existing = Newsletter.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    else:
        ModelClass = Post if content_type == 'post' else Page
        existing = ModelClass.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    
    if existing:
        target_lang_name = LANG_CODE_TO_NAME.get(target_lang_code, target_lang_code)
        return jsonify({
            'error': t('api.translation_already_exists', language=target_lang_name),
            'existing_id': existing.id
        }), 409
    
    target_lang_name = LANG_CODE_TO_NAME.get(target_lang_code, target_lang_code)
    
    # --- Newsletter copy (no slug needed) ---
    if content_type == 'newsletter':
        new_item = Newsletter(
            subject=source.subject,
            content=source.content,
            language=target_lang_code,
            translation_group=source.translation_group,
            status='draft',
            user_id=current_user.id,
            target_languages=source.target_languages,
            target_form_ids=source.target_form_ids,
            confirmed_only=source.confirmed_only,
        )
        db.session.add(new_item)
        db.session.commit()
        edit_url = url_for('admin_newsletter_edit', newsletter_id=new_item.id)
        
        return jsonify({
            'success': True,
            'message': t('api.translation_copy_created', language=target_lang_name),
            'edit_url': edit_url,
            'new_id': new_item.id
        })
    
    # --- POST / PAGE copy (need slug) ---
    ModelClass = Post if content_type == 'post' else Page
    translated_slug = slugify(source.title) + '-' + target_lang_code
    base_slug = translated_slug
    counter = 1
    while ModelClass.query.filter_by(slug=translated_slug).first():
        translated_slug = f"{base_slug}-{counter}"
        counter += 1
    
    if content_type == 'post':
        new_item = Post(
            title=source.title,
            slug=translated_slug,
            content=source.content,
            excerpt=source.excerpt,
            language=target_lang_code,
            translation_group=source.translation_group,
            status='draft',
            user_id=current_user.id,
            meta_title=source.meta_title,
            meta_description=source.meta_description,
            meta_keywords=source.meta_keywords,
            og_title=source.og_title,
            og_description=source.og_description,
            og_image=source.og_image,
            twitter_title=source.twitter_title,
            twitter_description=source.twitter_description,
            twitter_image=source.twitter_image,
            featured_image=source.featured_image,
            featured_image_alt=source.featured_image_alt,
            featured_image_caption=source.featured_image_caption,
            tags=source.tags,
            category=source.category,
            schema_type=source.schema_type,
            schema_author=source.schema_author,
            schema_publisher=source.schema_publisher,
        )
        db.session.add(new_item)
        db.session.commit()
        edit_url = url_for('admin_posts_edit', post_id=new_item.id)
    else:
        new_item = Page(
            title=source.title,
            slug=translated_slug,
            content=source.content,
            excerpt=source.excerpt,
            language=target_lang_code,
            translation_group=source.translation_group,
            is_posts_page=source.is_posts_page,
            status='draft',
            template=source.template,
            user_id=current_user.id,
            meta_title=source.meta_title,
            meta_description=source.meta_description,
            meta_keywords=source.meta_keywords,
            og_title=source.og_title,
            og_description=source.og_description,
            og_image=source.og_image,
            twitter_title=source.twitter_title,
            twitter_description=source.twitter_description,
            twitter_image=source.twitter_image,
            featured_image=source.featured_image,
            featured_image_alt=source.featured_image_alt,
            featured_image_caption=source.featured_image_caption,
            tags=source.tags,
            category=source.category,
            schema_type=source.schema_type,
            schema_author=source.schema_author,
            schema_publisher=source.schema_publisher,
        )
        # Copy Page Builder data as-is
        if getattr(source, 'use_builder', False) and source.blocks:
            new_item.blocks = source.blocks
            new_item.use_builder = True
        
        db.session.add(new_item)
        db.session.commit()
        edit_url = url_for('admin_pages_edit', page_id=new_item.id)
    
    return jsonify({
        'success': True,
        'message': t('api.translation_copy_created', language=target_lang_name),
        'edit_url': edit_url,
        'new_id': new_item.id
    })


# ============================================================================
# DUPLICATE PAGE / POST (independent copy, no translation link)
# ============================================================================

@app.route('/admin/api/duplicate', methods=['POST'])
@login_required
@limiter.limit("20 per minute")
def admin_api_duplicate():
    """Create an independent copy of a page, post, newsletter, or form.
    New copy gets its own translation_group, status=draft, auto-generated slug,
    and title prefixed with '(Copy)'. Copies all content including Page Builder blocks."""
    data = request.get_json()
    content_type = data.get('content_type', 'page')  # 'post', 'page', 'newsletter', or 'form'
    source_id = data.get('source_id')
    
    if not source_id:
        return jsonify({'error': t('api.missing_source_id')}), 400
    
    if content_type == 'post':
        source = Post.query.get(source_id)
    elif content_type == 'newsletter':
        source = Newsletter.query.get(source_id)
    elif content_type == 'form':
        source = Form.query.get(source_id)
    else:
        source = Page.query.get(source_id)
    
    if not source:
        return jsonify({'error': t('api.source_not_found')}), 404
    
    # New title with (Copy) prefix
    copy_title = f"(Copy) {source.title if content_type != 'newsletter' else source.subject}"
    
    # Auto-generate unique slug from the new title (not needed for newsletter and form)
    if content_type not in ('newsletter', 'form'):
        new_slug = slugify(copy_title)
        if not new_slug:
            new_slug = slugify(source.title) + '-copy'
        new_slug = ensure_unique_slug(new_slug, Post if content_type == 'post' else Page)
    
    # New independent translation_group
    import uuid
    new_translation_group = str(uuid.uuid4())
    
    if content_type == 'form':
        new_item = Form(
            title=copy_title,
            description=source.description,
            language=source.language,
            translation_group=new_translation_group,
            form_type=source.form_type,
            fields=source.fields,
            settings=source.settings,
            is_active=False,
            user_id=current_user.id,
        )
        db.session.add(new_item)
        db.session.commit()
        
        AuditLog.log('duplicate_form', 'form', new_item.id,
                     _ad('duplicated_form', title=source.title),
                     user=current_user)
        db.session.commit()
        
        edit_url = url_for('admin_forms_edit', form_id=new_item.id)
    elif content_type == 'newsletter':
        new_item = Newsletter(
            subject=copy_title,
            content=source.content,
            language=source.language,
            translation_group=new_translation_group,
            status='draft',
            user_id=current_user.id,
            target_languages=source.target_languages,
            target_form_ids=source.target_form_ids,
            confirmed_only=source.confirmed_only,
        )
        db.session.add(new_item)
        db.session.commit()
        
        AuditLog.log('duplicate_newsletter', 'newsletter', new_item.id,
                     _ad('duplicated_newsletter', title=source.subject),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        edit_url = url_for('admin_newsletter_edit', newsletter_id=new_item.id)
    elif content_type == 'post':
        new_item = Post(
            title=copy_title,
            slug=new_slug,
            content=source.content,
            excerpt=source.excerpt,
            language=source.language,
            translation_group=new_translation_group,
            status='draft',
            user_id=current_user.id,
            meta_title=source.meta_title,
            meta_description=source.meta_description,
            meta_keywords=source.meta_keywords,
            og_title=source.og_title,
            og_description=source.og_description,
            og_image=source.og_image,
            twitter_title=source.twitter_title,
            twitter_description=source.twitter_description,
            twitter_image=source.twitter_image,
            featured_image=source.featured_image,
            featured_image_alt=source.featured_image_alt,
            featured_image_caption=source.featured_image_caption,
            tags=source.tags,
            category=source.category,
            schema_type=source.schema_type,
            schema_author=source.schema_author,
            schema_publisher=source.schema_publisher,
        )
        db.session.add(new_item)
        db.session.commit()
        
        AuditLog.log('duplicate_post', 'post', new_item.id,
                     _ad('duplicated_post', title=source.title),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        edit_url = url_for('admin_posts_edit', post_id=new_item.id)
    else:
        new_item = Page(
            title=copy_title,
            slug=new_slug,
            content=source.content,
            excerpt=source.excerpt,
            language=source.language,
            translation_group=new_translation_group,
            is_posts_page=source.is_posts_page,
            status='draft',
            template=source.template,
            user_id=current_user.id,
            meta_title=source.meta_title,
            meta_description=source.meta_description,
            meta_keywords=source.meta_keywords,
            og_title=source.og_title,
            og_description=source.og_description,
            og_image=source.og_image,
            twitter_title=source.twitter_title,
            twitter_description=source.twitter_description,
            twitter_image=source.twitter_image,
            featured_image=source.featured_image,
            featured_image_alt=source.featured_image_alt,
            featured_image_caption=source.featured_image_caption,
            tags=source.tags,
            category=source.category,
            schema_type=source.schema_type,
            schema_author=source.schema_author,
            schema_publisher=source.schema_publisher,
        )
        # Copy Page Builder data
        if getattr(source, 'use_builder', False) and source.blocks:
            new_item.blocks = source.blocks
            new_item.use_builder = True
        
        db.session.add(new_item)
        db.session.commit()
        
        AuditLog.log('duplicate_page', 'page', new_item.id,
                     _ad('duplicated_page', title=source.title),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        edit_url = url_for('admin_pages_edit', page_id=new_item.id)
    
    return jsonify({
        'success': True,
        'message': t('api.duplicate_created', title=copy_title),
        'edit_url': edit_url,
        'new_id': new_item.id
    })



# ============================================================================
# AI PROVIDER — Multi-provider abstraction (Anthropic, DeepSeek, Gemini)
# ============================================================================

def get_ai_config():
    """Get active AI provider, API key, and model. Returns (provider, api_key, error)."""
    provider = Settings.get('ai_provider', 'anthropic')
    
    if provider == 'deepseek':
        api_key = decrypt_smtp_password(Settings.get('deepseek_api_key_encrypted', ''))
    elif provider == 'gemini':
        api_key = decrypt_smtp_password(Settings.get('gemini_api_key_encrypted', ''))
    else:  # anthropic (default)
        provider = 'anthropic'
        api_key = decrypt_smtp_password(Settings.get('anthropic_api_key_encrypted', ''))
        if not api_key:
            api_key = Settings.get('anthropic_api_key', '')  # Legacy plaintext fallback
    
    if not api_key or len(api_key) < 10:
        return provider, None, t('api.api_key_not_configured')
    
    return provider, api_key, None


def call_ai(system_prompt, user_message, max_tokens=4096, timeout=90):
    """Call the active AI provider. Returns (text, error).
    
    Supports: Anthropic Claude, DeepSeek, Google Gemini.
    All use the same system_prompt + user_message interface.
    """
    import requests as http_requests
    
    provider, api_key, error = get_ai_config()
    if error:
        return None, error
    
    try:
        if provider == 'anthropic':
            response = http_requests.post(
                'https://api.anthropic.com/v1/messages',
                headers={
                    'x-api-key': api_key,
                    'anthropic-version': '2023-06-01',
                    'content-type': 'application/json'
                },
                json={
                    'model': Settings.get('ai_model_anthropic', 'claude-sonnet-4-20250514'),
                    'max_tokens': max_tokens,
                    'system': system_prompt,
                    'messages': [{'role': 'user', 'content': user_message}]
                },
                timeout=timeout
            )
            if response.status_code != 200:
                detail = ''
                try:
                    detail = response.json().get('error', {}).get('message', '')
                except Exception:
                    pass
                return None, t('api.api_error_detail', code=response.status_code, detail=detail)
            return response.json()['content'][0]['text'], None
        
        elif provider == 'deepseek':
            response = http_requests.post(
                'https://api.deepseek.com/v1/chat/completions',
                headers={
                    'Authorization': f'Bearer {api_key}',
                    'Content-Type': 'application/json'
                },
                json={
                    'model': Settings.get('ai_model_deepseek', 'deepseek-chat'),
                    'max_tokens': max_tokens,
                    'messages': [
                        {'role': 'system', 'content': system_prompt},
                        {'role': 'user', 'content': user_message}
                    ]
                },
                timeout=timeout
            )
            if response.status_code != 200:
                detail = ''
                try:
                    detail = response.json().get('error', {}).get('message', '')
                except Exception:
                    pass
                return None, t('api.api_error_detail', code=response.status_code, detail=detail)
            return response.json()['choices'][0]['message']['content'], None
        
        elif provider == 'gemini':
            model = Settings.get('ai_model_gemini', 'gemini-2.0-flash')
            response = http_requests.post(
                f'https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}',
                headers={'Content-Type': 'application/json'},
                json={
                    'system_instruction': {'parts': [{'text': system_prompt}]},
                    'contents': [{'role': 'user', 'parts': [{'text': user_message}]}],
                    'generationConfig': {'maxOutputTokens': max_tokens}
                },
                timeout=timeout
            )
            if response.status_code != 200:
                detail = ''
                try:
                    detail = response.json().get('error', {}).get('message', '')
                except Exception:
                    pass
                return None, t('api.api_error_detail', code=response.status_code, detail=detail)
            return response.json()['candidates'][0]['content']['parts'][0]['text'], None
        
        else:
            return None, t('api.unknown_provider', provider=provider)
    
    except Exception as e:
        return None, str(e)


@app.route('/admin/api/translate', methods=['POST'])
@login_required
@limiter.limit("20 per minute")
def admin_api_translate():
    """AI translate: creates a NEW post/page as a translation, linked by translation_group.
    This is called via AJAX from the edit page sidebar."""
    
    # Get active AI provider and key
    provider, api_key, key_error = get_ai_config()
    if key_error:
        return jsonify({'error': key_error}), 400
    
    data = request.get_json()
    content_type = data.get('content_type', 'post')  # 'post', 'page', 'form', or 'newsletter'
    source_id = data.get('source_id')
    target_lang_code = data.get('target_lang')
    
    if not source_id or not target_lang_code:
        return jsonify({'error': t('api.missing_source_or_lang')}), 400
    
    # Get source content
    if content_type == 'post':
        source = Post.query.get(source_id)
    elif content_type == 'form':
        source = Form.query.get(source_id)
    elif content_type == 'newsletter':
        source = Newsletter.query.get(source_id)
    else:
        source = Page.query.get(source_id)
    
    if not source:
        return jsonify({'error': t('api.source_not_found')}), 404
    
    # Check if translation already exists
    existing = None
    if content_type == 'post':
        existing = Post.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    elif content_type == 'form':
        existing = Form.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    elif content_type == 'newsletter':
        existing = Newsletter.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    else:
        existing = Page.query.filter_by(
            translation_group=source.translation_group, language=target_lang_code
        ).first()
    
    if existing:
        return jsonify({'error': t('api.translation_already_exists', language=LANG_CODE_TO_NAME.get(target_lang_code, target_lang_code)),
                       'existing_id': existing.id}), 409
    
    source_lang_name = LANG_CODE_TO_NAME.get(source.language, source.language)
    target_lang_name = LANG_CODE_TO_NAME.get(target_lang_code, target_lang_code)
    instructions = data.get('instructions', '').strip()
    
    # --- FORM TRANSLATION (special: translate fields JSON + settings) ---
    if content_type == 'form':
        fields_to_translate = {
            'title': source.title,
        }
        if source.description:
            fields_to_translate['description'] = source.description
        
        # Include form fields for translation (labels, placeholders, options, consent_text, help_text)
        source_fields = source.get_fields()
        fields_to_translate['form_fields'] = source_fields
        
        # Include translatable settings
        source_settings = source.get_settings()
        fields_to_translate['settings'] = {
            'submit_label': source_settings.get('submit_label', t('forms.default_submit')),
            'success_message': source_settings.get('success_message', ''),
        }
        
        instructions_line = f"\nAdditional instructions from user: {instructions}" if instructions else ""
        system_prompt = f"""You are a professional translator. Translate ALL the following from {source_lang_name} to {target_lang_name}.{instructions_line}

For "form_fields": translate ONLY the user-facing text: "label", "placeholder", "help_text", "consent_text", and each string in "options" arrays. Do NOT change "id", "name", "type", "required", "width", "max_rating", "validation" or any structural keys.
For "settings": translate "submit_label" and "success_message".
For "title" and "description": translate normally.

Return ONLY a valid JSON object with these keys: title, description (if present), form_fields, settings.
No other text, no markdown backticks."""

        user_message = json.dumps(fields_to_translate, ensure_ascii=False)
        
        try:
            ai_text, ai_error = call_ai(system_prompt, user_message, max_tokens=4096, timeout=120)
            if ai_error:
                return jsonify({'error': ai_error}), 500
            
            ai_text = ai_text.strip()
            if ai_text.startswith('```'):
                ai_text = ai_text.split('\n', 1)[1] if '\n' in ai_text else ai_text[3:]
                if ai_text.endswith('```'):
                    ai_text = ai_text[:-3]
                ai_text = ai_text.strip()
            
            translated = json.loads(ai_text)
            
            translated_title = translated.get('title', source.title)
            translated_slug = slugify(translated_title) or slugify(source.title) + '-' + target_lang_code
            base_slug = translated_slug
            counter = 1
            while Form.query.filter_by(slug=translated_slug).first():
                translated_slug = f"{base_slug}-{counter}"
                counter += 1
            
            # Merge translated settings back into full settings
            full_settings = source.get_settings()
            translated_settings = translated.get('settings', {})
            full_settings['submit_label'] = translated_settings.get('submit_label', full_settings['submit_label'])
            full_settings['success_message'] = translated_settings.get('success_message', full_settings['success_message'])
            
            new_form = Form(
                title=translated_title,
                slug=translated_slug,
                description=translated.get('description', source.description or ''),
                form_type=source.form_type,
                language=target_lang_code,
                translation_group=source.translation_group,
                is_active=source.is_active,
                user_id=current_user.id,
            )
            new_form.set_fields(translated.get('form_fields', source_fields))
            new_form.set_settings(full_settings)
            
            db.session.add(new_form)
            db.session.commit()
            
            edit_url = url_for('admin_forms_edit', form_id=new_form.id)
            return jsonify({
                'success': True,
                'message': t('api.form_translation_created', language=target_lang_name),
                'edit_url': edit_url,
                'new_id': new_form.id
            })
            
        except json.JSONDecodeError:
            return jsonify({'error': t('api.could_not_parse')}), 500
        except Exception as e:
            return jsonify({'error': str(e)}), 500
    
    # --- NEWSLETTER TRANSLATION (simpler: just subject + HTML content) ---
    if content_type == 'newsletter':
        fields_to_translate = {
            'subject': source.subject,
            'content': source.content,
        }
        
        instructions_line = f"\nAdditional instructions from user: {instructions}" if instructions else ""
        system_prompt = f"""You are a professional translator. Translate the following from {source_lang_name} to {target_lang_name}.{instructions_line}

"subject" is the email subject line — keep it concise and compelling.
"content" is HTML email content — translate all text but preserve all HTML tags, attributes, links, and structure exactly.

Return ONLY a valid JSON object with keys: subject, content.
No other text, no markdown backticks."""
        
        user_message = json.dumps(fields_to_translate, ensure_ascii=False)
        
        try:
            ai_text, ai_error = call_ai(system_prompt, user_message, max_tokens=4096, timeout=120)
            if ai_error:
                return jsonify({'error': ai_error}), 500
            
            ai_text = ai_text.strip()
            if ai_text.startswith('```'):
                ai_text = ai_text.split('\n', 1)[1] if '\n' in ai_text else ai_text[3:]
                if ai_text.endswith('```'):
                    ai_text = ai_text[:-3]
                ai_text = ai_text.strip()
            
            translated = json.loads(ai_text)
            
            new_item = Newsletter(
                subject=translated.get('subject', source.subject),
                content=translated.get('content', source.content),
                language=target_lang_code,
                translation_group=source.translation_group,
                status='draft',
                user_id=current_user.id,
                target_languages=source.target_languages,
                target_form_ids=source.target_form_ids,
                confirmed_only=source.confirmed_only,
            )
            
            db.session.add(new_item)
            db.session.commit()
            
            edit_url = url_for('admin_newsletter_edit', newsletter_id=new_item.id)
            
            return jsonify({
                'success': True,
                'message': t('api.newsletter_translation_created', language=target_lang_name),
                'edit_url': edit_url,
                'new_id': new_item.id
            })
            
        except json.JSONDecodeError:
            return jsonify({'error': t('api.could_not_parse')}), 500
        except Exception as e:
            return jsonify({'error': str(e)}), 500
    
    # --- POST / PAGE TRANSLATION ---
    # Translate title, content, excerpt via AI
    fields_to_translate = {
        'title': source.title,
        'content': source.content,
    }
    if source.excerpt:
        fields_to_translate['excerpt'] = source.excerpt
    if source.meta_title:
        fields_to_translate['meta_title'] = source.meta_title
    if source.meta_description:
        fields_to_translate['meta_description'] = source.meta_description
    if source.og_title:
        fields_to_translate['og_title'] = source.og_title
    if source.og_description:
        fields_to_translate['og_description'] = source.og_description
    if source.twitter_title:
        fields_to_translate['twitter_title'] = source.twitter_title
    if source.twitter_description:
        fields_to_translate['twitter_description'] = source.twitter_description
    if source.featured_image_alt:
        fields_to_translate['featured_image_alt'] = source.featured_image_alt
    if source.featured_image_caption:
        fields_to_translate['featured_image_caption'] = source.featured_image_caption
    
    # --- PAGE BUILDER: extract translatable texts from blocks ---
    has_builder_blocks = False
    source_blocks_data = None
    if content_type == 'page' and getattr(source, 'use_builder', False) and source.blocks:
        try:
            source_blocks_data = json.loads(source.blocks)
            blocks_texts = _extract_builder_texts(source_blocks_data.get('blocks', []))
            if blocks_texts:
                has_builder_blocks = True
                fields_to_translate['builder_texts'] = blocks_texts
        except (ValueError, TypeError):
            pass
    
    # Build a single AI request to translate all fields at once (efficient!)
    instructions_line = f"\nAdditional instructions from user: {instructions}" if instructions else ""
    builder_instruction = ""
    if has_builder_blocks:
        builder_instruction = """
For "builder_texts": this is an array of objects extracted from a page builder. Each object has an "id" (block index path — DO NOT change it) and text fields like "text", "html", "caption", "alt", "author", "title", "button_text", "items", "headers", "rows", "tabs", "slides".
Translate ONLY the text values. Preserve all HTML tags in "html" and "content" fields. Keep "id" values exactly as they are.
Return "builder_texts" as an array with the same structure, same "id" values, and translated text fields."""
    system_prompt = f"""You are a professional translator. Translate ALL the following fields from {source_lang_name} to {target_lang_name}.
Preserve all HTML formatting in content fields. Keep the same tone and style.{instructions_line}{builder_instruction}
Return ONLY a valid JSON object with the translated fields. No other text, no markdown backticks.
The JSON keys must match exactly: {', '.join(fields_to_translate.keys())}"""
    
    user_message = json.dumps(fields_to_translate, ensure_ascii=False)
    
    try:
        ai_text, ai_error = call_ai(
            system_prompt, user_message,
            max_tokens=16384 if has_builder_blocks else 4096,
            timeout=180 if has_builder_blocks else 120
        )
        if ai_error:
            return jsonify({'error': ai_error}), 500
        
        # Parse translated fields
        # Strip possible markdown backticks
        ai_text = ai_text.strip()
        if ai_text.startswith('```'):
            ai_text = ai_text.split('\n', 1)[1] if '\n' in ai_text else ai_text[3:]
            if ai_text.endswith('```'):
                ai_text = ai_text[:-3]
            ai_text = ai_text.strip()
        
        translated = json.loads(ai_text)
        
        # Create new post/page with translated content
        translated_title = translated.get('title', source.title)
        translated_slug = slugify(translated_title) or slugify(source.title) + '-' + target_lang_code
        
        if content_type == 'post':
            # Ensure slug uniqueness
            base_slug = translated_slug
            counter = 1
            while Post.query.filter_by(slug=translated_slug).first():
                translated_slug = f"{base_slug}-{counter}"
                counter += 1
            
            new_item = Post(
                title=translated_title,
                slug=translated_slug,
                content=translated.get('content', source.content),
                excerpt=translated.get('excerpt', source.excerpt),
                language=target_lang_code,
                translation_group=source.translation_group,
                status='draft',  # Always start as draft — user should review
                user_id=current_user.id,
                meta_title=translated.get('meta_title', ''),
                meta_description=translated.get('meta_description', ''),
                meta_keywords=source.meta_keywords,  # keywords don't need translation
                og_title=translated.get('og_title', ''),
                og_description=translated.get('og_description', ''),
                og_image=source.og_image,  # images stay the same
                twitter_title=translated.get('twitter_title', ''),
                twitter_description=translated.get('twitter_description', ''),
                twitter_image=source.twitter_image,
                featured_image=source.featured_image,
                featured_image_alt=translated.get('featured_image_alt', source.featured_image_alt),
                featured_image_caption=translated.get('featured_image_caption', source.featured_image_caption),
                tags=source.tags,
                category=source.category,
                schema_type=source.schema_type,
                schema_author=source.schema_author,
                schema_publisher=source.schema_publisher,
                ai_generated=True,
            )
        else:
            # Page
            base_slug = translated_slug
            counter = 1
            while Page.query.filter_by(slug=translated_slug).first():
                translated_slug = f"{base_slug}-{counter}"
                counter += 1
            
            new_item = Page(
                title=translated_title,
                slug=translated_slug,
                content=translated.get('content', source.content),
                excerpt=translated.get('excerpt', source.excerpt),
                language=target_lang_code,
                translation_group=source.translation_group,
                is_posts_page=source.is_posts_page,
                status='draft',
                template=source.template,
                    user_id=current_user.id,
                meta_title=translated.get('meta_title', ''),
                meta_description=translated.get('meta_description', ''),
                meta_keywords=source.meta_keywords,
                og_title=translated.get('og_title', ''),
                og_description=translated.get('og_description', ''),
                og_image=source.og_image,
                twitter_title=translated.get('twitter_title', ''),
                twitter_description=translated.get('twitter_description', ''),
                twitter_image=source.twitter_image,
                featured_image=source.featured_image,
                featured_image_alt=translated.get('featured_image_alt', source.featured_image_alt),
                featured_image_caption=translated.get('featured_image_caption', source.featured_image_caption),
                tags=source.tags,
                category=source.category,
                schema_type=source.schema_type,
                schema_author=source.schema_author,
                schema_publisher=source.schema_publisher,
                ai_generated=True,
            )
            
            # Copy Page Builder data and apply translated texts
            if has_builder_blocks and source_blocks_data:
                translated_blocks = json.loads(json.dumps(source_blocks_data))  # deep copy
                translated_builder_texts = translated.get('builder_texts', [])
                if translated_builder_texts:
                    _apply_builder_texts(translated_blocks.get('blocks', []), translated_builder_texts)
                # Keep stored data clean even when the source predates the guard.
                _demote_headings_in_text_blocks(translated_blocks.get('blocks', []))
                new_item.blocks = json.dumps(translated_blocks)
                new_item.use_builder = True
                # Re-generate flat HTML from translated blocks for SEO/fallback
                try:
                    from block_renderer import render_blocks_to_html
                    new_item.content = render_blocks_to_html(new_item.blocks)
                except Exception:
                    pass  # keep AI-translated content as fallback
            elif content_type == 'page' and getattr(source, 'use_builder', False) and source.blocks:
                # Builder exists but no texts to translate (e.g. only images/spacers)
                new_item.blocks = source.blocks
                new_item.use_builder = True
        
        db.session.add(new_item)
        db.session.commit()
        
        # Return the new item's edit URL
        if content_type == 'post':
            edit_url = url_for('admin_posts_edit', post_id=new_item.id)
        else:
            edit_url = url_for('admin_pages_edit', page_id=new_item.id)
        
        return jsonify({
            'success': True,
            'message': t('api.ai_translation_created', language=target_lang_name),
            'edit_url': edit_url,
            'new_id': new_item.id
        })
        
    except json.JSONDecodeError:
        return jsonify({'error': t('api.could_not_parse')}), 500
    except Exception as e:
        return jsonify({'error': str(e)}), 500


# ============================================================================
# MEDIA / IMAGES MANAGEMENT
# ============================================================================

def allowed_file(filename):
    """Check if file extension is allowed"""
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS']


def process_uploaded_image(filepath, filename, upload_folder):
    """Process uploaded image: resize, strip EXIF, generate WebP/AVIF variants.
    
    Returns dict with:
        - width, height: original dimensions (after max resize)
        - file_size: size of processed original
        - dominant_color: hex color for placeholder
        - variants: dict of all generated variants
        - processed_filepath: path to the (possibly resized) original
    """
    result = {
        'width': None,
        'height': None,
        'file_size': 0,
        'dominant_color': None,
        'variants': {}
    }
    
    try:
        img = PILImage.open(filepath)
    except Exception:
        # Not a processable image (e.g. SVG) — return basic info.
        # For SVG specifically, parse the XML to extract intrinsic
        # dimensions so PageSpeed gets explicit width/height on <img>.
        result['file_size'] = os.path.getsize(filepath)
        if filename.lower().endswith('.svg'):
            svg_w, svg_h = _extract_svg_dimensions(filepath)
            if svg_w and svg_h:
                result['width'] = svg_w
                result['height'] = svg_h
        return result
    
    # Handle orientation from EXIF (auto-rotate)
    try:
        from PIL import ImageOps
        img = ImageOps.exif_transpose(img)
    except Exception:
        pass
    
    # Convert to RGB if needed (RGBA → RGB with white background for JPEG/WebP)
    original_mode = img.mode
    if img.mode == 'RGBA':
        background = PILImage.new('RGB', img.size, (255, 255, 255))
        background.paste(img, mask=img.split()[3])
        img_rgb = background
    elif img.mode != 'RGB':
        img_rgb = img.convert('RGB')
    else:
        img_rgb = img
    
    # --- Extract dominant color (from small thumbnail for speed) ---
    try:
        thumb = img_rgb.copy()
        thumb.thumbnail((40, 40))
        pixels = list(thumb.getdata())
        avg_r = sum(p[0] for p in pixels) // len(pixels)
        avg_g = sum(p[1] for p in pixels) // len(pixels)
        avg_b = sum(p[2] for p in pixels) // len(pixels)
        result['dominant_color'] = f'#{avg_r:02x}{avg_g:02x}{avg_b:02x}'
    except Exception:
        result['dominant_color'] = '#888888'
    
    # --- Resize if larger than max dimension ---
    max_dim = app.config.get('IMAGE_MAX_DIMENSION', 2560)
    if img_rgb.width > max_dim or img_rgb.height > max_dim:
        img_rgb.thumbnail((max_dim, max_dim), PILImage.LANCZOS)
        # Also resize RGBA version if we have transparency
        if original_mode == 'RGBA':
            img.thumbnail((max_dim, max_dim), PILImage.LANCZOS)
    
    result['width'] = img_rgb.width
    result['height'] = img_rgb.height
    
    # --- Save processed original (stripped of EXIF, resized) ---
    name, ext = os.path.splitext(filename)
    ext_lower = ext.lower()
    
    if ext_lower in ['.jpg', '.jpeg']:
        img_rgb.save(filepath, 'JPEG', quality=app.config.get('IMAGE_QUALITY_ORIGINAL', 85), optimize=True)
    elif ext_lower == '.png':
        # Keep PNG as PNG (might have transparency)
        if original_mode == 'RGBA':
            img.save(filepath, 'PNG', optimize=True)
        else:
            img_rgb.save(filepath, 'PNG', optimize=True)
    elif ext_lower == '.webp':
        img_rgb.save(filepath, 'WEBP', quality=app.config.get('IMAGE_QUALITY_WEBP', 82))
    elif ext_lower == '.gif':
        # Don't process GIFs (may be animated)
        result['file_size'] = os.path.getsize(filepath)
        return result
    
    result['file_size'] = os.path.getsize(filepath)
    
    # --- Store original info in variants ---
    result['variants']['original'] = {
        'url': f'/static/uploads/{filename}',
        'width': result['width'],
        'height': result['height'],
        'size': result['file_size']
    }
    
    # --- Generate responsive variants (WebP + AVIF) ---
    variant_widths = app.config.get('IMAGE_VARIANT_WIDTHS', [400, 800, 1200, 1920])
    quality_webp = app.config.get('IMAGE_QUALITY_WEBP', 82)
    quality_avif = app.config.get('IMAGE_QUALITY_AVIF', 68)
    
    # WebP and AVIF both support alpha. For RGBA originals (PNG logos,
    # icons, screenshots with transparency) we generate variants from
    # the alpha-preserving source so transparency is kept. Opaque images
    # use the flat RGB version. OG variants below stay flat regardless
    # because social previews need a solid background.
    variant_source = img if original_mode == 'RGBA' else img_rgb
    
    for fmt, quality, pil_format in [('webp', quality_webp, 'WEBP'), ('avif', quality_avif, 'AVIF')]:
        fmt_variants = {}
        
        for target_width in variant_widths:
            # Skip sizes larger than original
            if target_width >= variant_source.width:
                # Save at original size as the largest variant
                variant_img = variant_source.copy()
                actual_width = variant_source.width
            else:
                # Calculate proportional height
                ratio = target_width / variant_source.width
                target_height = int(variant_source.height * ratio)
                variant_img = variant_source.resize((target_width, target_height), PILImage.LANCZOS)
                actual_width = target_width
            
            # Generate variant filename
            variant_filename = f"{name}_{actual_width}w.{fmt}"
            variant_path = os.path.join(upload_folder, variant_filename)
            
            try:
                variant_img.save(variant_path, pil_format, quality=quality)
                variant_size = os.path.getsize(variant_path)
                
                fmt_variants[str(actual_width)] = {
                    'url': f'/static/uploads/{variant_filename}',
                    'width': variant_img.width,
                    'height': variant_img.height,
                    'size': variant_size
                }
            except Exception as e:
                # AVIF might not be supported in older Pillow versions — skip silently
                app.logger.warning(f'Could not generate {fmt} variant at {actual_width}w: {e}')
                continue
            
            # If we saved at original size, no need for larger breakpoints
            if actual_width == variant_source.width:
                break
        
        if fmt_variants:
            result['variants'][fmt] = fmt_variants
    
    # --- Generate OG/Twitter crop (1200×630) for social sharing ---
    # Only if image is large enough (at least 1200px wide or 630px tall)
    og_width, og_height = 1200, 630
    if img_rgb.width >= 600 and img_rgb.height >= 315:
        og_variants = {}
        
        # Calculate crop to 1200:630 aspect ratio (≈1.905:1) — center crop
        target_ratio = og_width / og_height  # 1.905
        img_ratio = img_rgb.width / img_rgb.height
        
        if img_ratio > target_ratio:
            # Image is wider — crop sides
            new_w = int(img_rgb.height * target_ratio)
            left = (img_rgb.width - new_w) // 2
            crop_box = (left, 0, left + new_w, img_rgb.height)
        else:
            # Image is taller — crop top/bottom
            new_h = int(img_rgb.width / target_ratio)
            top = (img_rgb.height - new_h) // 2
            crop_box = (0, top, img_rgb.width, top + new_h)
        
        # Use the alpha-preserving source for RGBA originals so OG WebP/AVIF
        # variants stay transparent (matching the responsive variants logic
        # above). JPG is generated only for opaque sources further below —
        # JPEG cannot store an alpha channel.
        og_source = img if original_mode == 'RGBA' else img_rgb
        og_img = og_source.crop(crop_box)
        
        # Resize to exactly 1200×630 (or smaller if original is small)
        final_w = min(og_width, og_img.width)
        final_h = int(final_w / target_ratio)
        og_img = og_img.resize((final_w, final_h), PILImage.LANCZOS)
        
        for fmt, quality, pil_format in [('webp', quality_webp, 'WEBP'), ('avif', quality_avif, 'AVIF')]:
            og_filename = f"{name}_og.{fmt}"
            og_path = os.path.join(upload_folder, og_filename)
            try:
                og_img.save(og_path, pil_format, quality=quality)
                og_size = os.path.getsize(og_path)
                og_variants[fmt] = {
                    'url': f'/static/uploads/{og_filename}',
                    'width': og_img.width,
                    'height': og_img.height,
                    'size': og_size
                }
            except Exception:
                pass
        
        # Also save as JPEG for maximum compatibility (Facebook/LinkedIn) —
        # but ONLY for opaque originals. Transparent PNGs cannot become
        # JPEGs without losing the alpha channel; templates fall back to
        # WebP/AVIF via Image.get_og_meta() in that case.
        if original_mode != 'RGBA':
            og_jpg_filename = f"{name}_og.jpg"
            og_jpg_path = os.path.join(upload_folder, og_jpg_filename)
            try:
                og_img.save(og_jpg_path, 'JPEG', quality=85, optimize=True)
                og_variants['jpg'] = {
                    'url': f'/static/uploads/{og_jpg_filename}',
                    'width': og_img.width,
                    'height': og_img.height,
                    'size': os.path.getsize(og_jpg_path)
                }
            except Exception:
                pass
        
        if og_variants:
            result['variants']['og'] = og_variants
    
    return result


@app.route('/admin/media')
@login_required
def admin_media_library():
    """Media library - list all images"""
    page = get_page()
    per_page = get_per_page()
    search_query = request.args.get('q', '').strip()
    
    query = Image.query
    if search_query:
        pattern = f'%{escape_like(search_query)}%'
        query = query.filter(
            db.or_(
                Image.alt_text.ilike(pattern, escape='\\'),
                Image.original_filename.ilike(pattern, escape='\\'),
                # Stored (sanitized/renamed) file name too — it's what shows
                # in image URLs, so people search by it as well.
                Image.filename.ilike(pattern, escape='\\'),
                Image.title.ilike(pattern, escape='\\'),
                Image.description.ilike(pattern, escape='\\')
            )
        )
    
    images = query.order_by(Image.uploaded_at.desc()).paginate(
        page=page, per_page=per_page, error_out=False
    )
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': img.id,
                'url': img.public_url,
                'alt_text': img.alt_text or '',
                'original_filename': img.original_filename or img.filename,
                'file_size': img.file_size,
            } for img in images.items],
            'page': images.page,
            'pages': images.pages,
            'total': images.total,
            'has_prev': images.has_prev,
            'has_next': images.has_next,
            'prev_num': images.prev_num,
            'next_num': images.next_num,
            'per_page': per_page,
        })
    
    return render_template('admin/media/library.html', images=images, search_query=search_query, per_page=per_page)


@app.route('/admin/media/upload', methods=['GET', 'POST'])
@login_required
def admin_media_upload():
    """Upload new image with automatic optimization"""
    if request.method == 'POST':
        # Agent / WebMCP callers ask for JSON via _format=json and get a clean
        # machine-readable contract on every branch (same route, no parallel copy).
        wants_json = _is_agent_json()

        # Check if file is present
        if 'file' not in request.files:
            if wants_json:
                return jsonify({'success': False, 'error': 'no_file'}), 400
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        file = request.files['file']
        
        if file.filename == '':
            if wants_json:
                return jsonify({'success': False, 'error': 'no_file'}), 400
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        # Check alt text (MANDATORY!)
        alt_text = request.form.get('alt_text')
        if not alt_text:
            if wants_json:
                return jsonify({'success': False, 'error': 'alt_text_required'}), 400
            flash(t('flash.alt_text_required'), 'error')
            return redirect(request.url)
        
        if file and allowed_file(file.filename):
            # Safe name: extension survives non-Latin originals, length is
            # capped under the 255-byte filesystem limit (see helper).
            name, ext = os.path.splitext(make_upload_filename(file.filename))
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            filename = f"{name}_{timestamp}{ext}"
            # Guard against same-second collisions between different uploads.
            filename = unique_upload_path(app.config['UPLOAD_FOLDER'], filename)
            
            # Save original file first
            filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
            file.save(filepath)
            
            # Sanitize SVG files — remove <script>, event handlers, etc.
            if ext.lower() == '.svg':
                sanitize_svg(filepath)
            
            # Process image: resize, strip EXIF, generate WebP/AVIF variants
            processed = process_uploaded_image(filepath, filename, app.config['UPLOAD_FOLDER'])
            
            # Generate clean public slug from the ORIGINAL user filename
            # (so 'My Sunset Photo.jpg' → slug 'my-sunset-photo', public URL /my-sunset-photo.jpg).
            # Uniqueness is checked against Post/Page/File/Image via _generate_image_slug.
            download_slug = _generate_image_slug(file.filename)
            
            # Create database record
            image = Image(
                filename=filename,
                original_filename=file.filename,
                url=f'/static/uploads/{filename}',
                download_slug=download_slug,
                alt_text=alt_text,
                title=request.form.get('title'),
                caption=request.form.get('caption'),
                description=request.form.get('description'),
                width=processed['width'],
                height=processed['height'],
                file_size=processed['file_size'],
                mime_type=file.content_type,
                dominant_color=processed['dominant_color'],
                user_id=current_user.id
            )
            image.set_variants(processed['variants'])
            
            db.session.add(image)
            db.session.flush()
            AuditLog.log('upload_image', 'media', image.id,
                         _ad('uploaded_image', filename=image.original_filename or image.filename),
                         current_user, request.remote_addr)
            db.session.commit()
            
            if wants_json:
                return jsonify({
                    'success': True,
                    'id': image.id,
                    'url': image.public_url,
                    'alt_text': image.alt_text,
                    'width': image.width,
                    'height': image.height,
                    'file_size': image.file_size,
                })
            flash(t('flash.image_uploaded'), 'success')
            return _safe_return_to('admin_media_library')
        else:
            allowed = ', '.join(sorted(app.config['ALLOWED_EXTENSIONS'])).upper()
            if wants_json:
                return jsonify({'success': False, 'error': 'invalid_file_type',
                                'allowed': sorted(app.config['ALLOWED_EXTENSIONS'])}), 400
            flash(t('flash.invalid_file_type', formats=allowed, size=app.config['MAX_CONTENT_LENGTH'] // (1024*1024)), 'error')
            return redirect(request.url)
    
    return render_template('admin/media/upload.html')


@app.route('/admin/media/<int:image_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_media_edit(image_id):
    """Edit image metadata (alt text, title, caption, description, public slug)."""
    image = Image.query.get_or_404(image_id)
    
    if request.method == 'POST':
        alt_text = request.form.get('alt_text')
        
        if not alt_text:
            flash(t('flash.alt_text_required_short'), 'error')
            return render_template('admin/media/edit.html', image=image)
        
        image.alt_text = alt_text
        image.title = request.form.get('title')
        image.caption = request.form.get('caption')
        image.description = request.form.get('description')
        
        # ── Slug editing ────────────────────────────────────────────────
        # User can rewrite the public URL slug. Pasted garbage is sanitized
        # through slugify(); collisions get auto-resolved with -N counter;
        # any change creates a 301 redirect from the old URL so external
        # links keep working.
        raw_slug = request.form.get('download_slug', '').strip()
        new_slug = _sanitize_image_slug_input(raw_slug)
        
        if new_slug and new_slug != image.download_slug:
            # If the cleaned slug is already taken (by a post, page, file or
            # another image), auto-disambiguate instead of rejecting — same
            # UX as posts/pages.
            final_slug = new_slug
            counter = 2
            while True:
                taken = False
                if Image.query.filter(
                    Image.download_slug == final_slug,
                    Image.id != image.id
                ).first():
                    taken = True
                if not taken and Post.query.filter_by(slug=final_slug).first():
                    taken = True
                if not taken and Page.query.filter_by(slug=final_slug).first():
                    taken = True
                if not taken:
                    conflict_file = File.query.filter(
                        File.download_slug.like(f'{final_slug}.%')
                    ).first()
                    if conflict_file:
                        taken = True
                if not taken:
                    break
                final_slug = f'{new_slug}-{counter}'
                counter += 1
            
            if image.download_slug and image.download_slug != final_slug:
                # Create a 301 redirect from the base URL of the old slug to
                # the new one. Variant URLs (/-800w, /-og, etc.) are not
                # redirected individually — they're only referenced from our
                # own templates which re-render with the new slug automatically.
                old_public_path = f'{image.download_slug}.{image.original_extension}'
                new_public_path = f'{final_slug}.{image.original_extension}'
                Redirect.create_redirect(
                    old_public_path, new_public_path, 'image', image.id
                )
                if final_slug != new_slug:
                    flash(t('flash.slug_exists'), 'warning')
            
            image.download_slug = final_slug
        
        AuditLog.log('edit_image', 'media', image.id,
                     _ad('edited_image', filename=image.original_filename or image.filename),
                     current_user, request.remote_addr)
        db.session.commit()
        return _inline_saved(t('flash.image_updated'), 'admin_media_library')
    
    return render_template('admin/media/edit.html', image=image)


# ────────────────────────────────────────────────────────────────────────────
# Asset Usage Check & Cleanup — helper functions
# ────────────────────────────────────────────────────────────────────────────

def _find_image_usage(image):
    """Find all places where an image is referenced across the site.
    
    Accepts an `Image` object (not a URL) so that it can check every short
    public URL the image can be referenced by — base, width variants, and
    OG crop — plus the legacy long path as a safety net.
    Returns list of dicts: [{type, id, title, fields}, ...]
    """
    if image is None:
        return []
    
    base_urls = list(image.get_all_public_urls())
    if image.url and image.url not in base_urls:
        base_urls.append(image.url)
    
    if not base_urls:
        return []
    
    # Expand each base URL into http/https/relative variants.
    variants = []
    for base in base_urls:
        for v in _get_url_variants(base):
            if v not in variants:
                variants.append(v)
    patterns = _content_url_sql_patterns(variants)
    
    results = []
    
    # Posts — direct fields (exact match against any variant) + HTML content
    content_filters = [Post.content.like(p) for p in patterns]
    posts = Post.query.filter(
        db.or_(
            Post.featured_image.in_(variants),
            Post.og_image.in_(variants),
            Post.twitter_image.in_(variants),
            *content_filters
        )
    ).all()
    for p in posts:
        fields = []
        if p.featured_image in variants:
            fields.append('featured_image')
        if p.og_image in variants:
            fields.append('og_image')
        if p.twitter_image in variants:
            fields.append('twitter_image')
        if _any_variant_in(p.content, variants):
            fields.append('content')
        if fields:
            results.append({
                'type': 'post', 'id': p.id,
                'title': p.title, 'fields': fields
            })
    
    # Pages — direct fields + HTML content + builder blocks
    content_filters = [Page.content.like(p) for p in patterns]
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages = Page.query.filter(
        db.or_(
            Page.featured_image.in_(variants),
            Page.og_image.in_(variants),
            Page.twitter_image.in_(variants),
            *(content_filters + blocks_filters)
        )
    ).all()
    for p in pages:
        fields = []
        if p.featured_image in variants:
            fields.append('featured_image')
        if p.og_image in variants:
            fields.append('og_image')
        if p.twitter_image in variants:
            fields.append('twitter_image')
        if _any_variant_in(p.content, variants):
            fields.append('content')
        if _any_variant_in(p.blocks, variants):
            fields.append('page_builder')
        if fields:
            results.append({
                'type': 'page', 'id': p.id,
                'title': p.title, 'fields': fields
            })
    
    # Newsletters — HTML content
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if _any_variant_in(n.content, variants):
            results.append({
                'type': 'newsletter', 'id': n.id,
                'title': n.subject, 'fields': ['content']
            })
    
    # Site Settings — exact match against any variant
    settings_fields = []
    if Settings.get('site_logo', '') in variants:
        settings_fields.append('site_logo')
    if Settings.get('default_og_image', '') in variants:
        settings_fields.append('default_og_image')
    if settings_fields:
        results.append({
            'type': 'settings', 'id': 0,
            'title': 'Site Settings', 'fields': settings_fields
        })
    
    return results


def _find_form_usage(form_id):
    """Find all pages where a form is embedded in page builder blocks.
    Returns list of dicts: [{type, id, title, fields}, ...]
    """
    if not form_id:
        return []
    
    results = []
    pages = Page.query.filter(Page.blocks.like(f'%form_id%')).all()
    
    for p in pages:
        if not p.blocks:
            continue
        try:
            blocks_data = json.loads(p.blocks)
            block_list = blocks_data.get('blocks', []) if isinstance(blocks_data, dict) else blocks_data
            if _blocks_contain_form_id(block_list, int(form_id)):
                results.append({
                    'type': 'page', 'id': p.id,
                    'title': p.title, 'fields': ['page_builder']
                })
        except (json.JSONDecodeError, TypeError, ValueError):
            pass
    
    return results


def _blocks_contain_form_id(blocks, form_id):
    """Recursively check if any block references the given form_id."""
    for block in blocks:
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        if block_type == 'form' and data.get('form_id') == form_id:
            return True
        elif block_type == 'columns':
            for col in data.get('columns', []):
                if _blocks_contain_form_id(col.get('blocks', []), form_id):
                    return True
        elif block_type == 'section':
            if _blocks_contain_form_id(data.get('blocks', []), form_id):
                return True
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('blocks') and _blocks_contain_form_id(slide['blocks'], form_id):
                    return True
    return False


def _find_file_usage(file_url):
    """Find all places where a file URL is referenced (posts, pages content).
    Handles both relative and absolute URLs.
    """
    if not file_url:
        return []
    
    results = []
    variants = _get_url_variants(file_url)
    patterns = _content_url_sql_patterns(variants)
    
    post_filters = [Post.content.like(p) for p in patterns]
    posts = Post.query.filter(db.or_(*post_filters)).all()
    for p in posts:
        if _any_variant_in(p.content, variants):
            results.append({
                'type': 'post', 'id': p.id,
                'title': p.title, 'fields': ['content']
            })
    
    content_filters = [Page.content.like(p) for p in patterns]
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages = Page.query.filter(
        db.or_(*(content_filters + blocks_filters))
    ).all()
    for p in pages:
        fields = []
        if _any_variant_in(p.content, variants):
            fields.append('content')
        if _any_variant_in(p.blocks, variants):
            fields.append('page_builder')
        if fields:
            results.append({
                'type': 'page', 'id': p.id,
                'title': p.title, 'fields': fields
            })
    
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if _any_variant_in(n.content, variants):
            results.append({
                'type': 'newsletter', 'id': n.id,
                'title': n.subject, 'fields': ['content']
            })
    
    # Menu items — custom links pointing to this file (legacy `url` +
    # multilingual `urls_json`).
    for mi in _find_menu_link_items_by_url_variants(variants):
        results.append({
            'type': 'menu_item', 'id': mi.id,
            'title': f'{mi.label} ({mi.menu_location})',
            'fields': ['link']
        })
    
    return results


# ── Cleanup functions ──

def _cleanup_image_references(image):
    """Remove all references to a deleted image from the entire site.
    
    Accepts an `Image` object (not just a URL) so that it can collect every
    short public URL the image can be referenced by — base slug, responsive
    variants and the OG crop — and scrub all of them from post/page content,
    builder blocks, newsletters and Settings. Also scrubs the legacy long
    `/static/uploads/...` path as a safety net.
    """
    if image is None:
        return
    
    # Collect every URL by which this image could be referenced anywhere.
    # The short URLs come from the model (base + width variants + og crops);
    # the legacy long URL is kept as a safety net for content inserted
    # via direct HTML pasting.
    base_urls = list(image.get_all_public_urls())
    if image.url and image.url not in base_urls:
        base_urls.append(image.url)
    
    if not base_urls:
        return
    
    # Expand each base URL into http/https/relative variants so we catch
    # absolute URLs hard-coded into editor content.
    all_variants = []
    for base in base_urls:
        for v in _get_url_variants(base):
            if v not in all_variants:
                all_variants.append(v)
    
    patterns = _content_url_sql_patterns(all_variants)
    
    # Build one big <img src="…"> regex matching any variant in one pass.
    escaped = [re.escape(v) for v in all_variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    img_re = re.compile(
        r'<img[^>]*src=["\']' + url_alt + r'["\'][^>]*/?>',
    )
    
    # Posts — clear direct fields (exact match against any variant)
    posts = Post.query.filter(
        db.or_(
            Post.featured_image.in_(all_variants),
            Post.og_image.in_(all_variants),
            Post.twitter_image.in_(all_variants),
        )
    ).all()
    for p in posts:
        if p.featured_image in all_variants:
            p.featured_image = ''
            p.featured_image_alt = ''
            p.featured_image_caption = ''
        if p.og_image in all_variants:
            p.og_image = ''
        if p.twitter_image in all_variants:
            p.twitter_image = ''
    
    # Posts — remove <img> from HTML content (any variant)
    post_content_filters = [Post.content.like(p) for p in patterns]
    posts_with_content = Post.query.filter(db.or_(*post_content_filters)).all()
    for p in posts_with_content:
        if p.content and _any_variant_in(p.content, all_variants):
            p.content = img_re.sub('', p.content)
    
    # Pages — clear direct fields
    pages = Page.query.filter(
        db.or_(
            Page.featured_image.in_(all_variants),
            Page.og_image.in_(all_variants),
            Page.twitter_image.in_(all_variants),
        )
    ).all()
    for p in pages:
        if p.featured_image in all_variants:
            p.featured_image = ''
            p.featured_image_alt = ''
            p.featured_image_caption = ''
        if p.og_image in all_variants:
            p.og_image = ''
        if p.twitter_image in all_variants:
            p.twitter_image = ''
    
    # Pages — remove <img> from HTML content
    page_content_filters = [Page.content.like(p) for p in patterns]
    pages_with_content = Page.query.filter(db.or_(*page_content_filters)).all()
    for p in pages_with_content:
        if p.content and _any_variant_in(p.content, all_variants):
            p.content = img_re.sub('', p.content)
    
    # Pages — clean builder blocks
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages_with_blocks = Page.query.filter(db.or_(*blocks_filters)).all()
    for p in pages_with_blocks:
        if p.blocks:
            try:
                blocks_data = json.loads(p.blocks)
                block_list = blocks_data.get('blocks', []) if isinstance(blocks_data, dict) else []
                _clean_image_from_blocks(block_list, all_variants)
                p.blocks = json.dumps(blocks_data, ensure_ascii=False)
            except (json.JSONDecodeError, TypeError):
                pass
    
    # Newsletters — remove <img> from content
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if n.content and _any_variant_in(n.content, all_variants):
            n.content = img_re.sub('', n.content)
    
    # Settings — exact match
    if Settings.get('site_logo', '') in all_variants:
        Settings.set('site_logo', '')
    if Settings.get('default_og_image', '') in all_variants:
        Settings.set('default_og_image', '')


def _clean_image_from_blocks(blocks, url_variants):
    """Recursively clean image URL from builder blocks.
    
    Args:
        blocks: list of builder block dicts
        url_variants: list of URL strings to match (relative + absolute)
    """
    url_set = set(url_variants)
    
    escaped = [re.escape(v) for v in url_variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    img_re = re.compile(
        r'<img[^>]*src=["\']' + url_alt + r'["\'][^>]*/?>',
    )
    
    for block in blocks:
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        if block_type == 'image' and data.get('url') in url_set:
            data['url'] = ''
            data['alt'] = ''
        elif block_type == 'gallery':
            data['images'] = [
                img for img in data.get('images', [])
                if img.get('url') not in url_set
            ]
        elif block_type == 'ticker':
            data['images'] = [
                img for img in data.get('images', [])
                if img.get('url') not in url_set
            ]
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('image') in url_set:
                    slide['image'] = ''
                if slide.get('blocks'):
                    _clean_image_from_blocks(slide['blocks'], url_variants)
        elif block_type == 'section':
            # Clean background image
            if data.get('background_image') in url_set:
                data['background_image'] = ''
            if data.get('background_type') == 'image' and data.get('background_value') in url_set:
                data['background_value'] = ''
                data['background_type'] = 'none'
            # Clean dark mode background image (was missed before — same
            # pattern of bug as the ticker/table cleanup gaps fixed earlier)
            if data.get('background_type_dark') == 'image' and data.get('background_value_dark') in url_set:
                data['background_value_dark'] = ''
                data['background_type_dark'] = 'color'
            # Recurse into nested blocks
            _clean_image_from_blocks(data.get('blocks', []), url_variants)
        elif block_type == 'columns':
            for col in data.get('columns', []):
                _clean_image_from_blocks(col.get('blocks', []), url_variants)
        elif block_type == 'text':
            # Rich text HTML may contain <img> tags
            if data.get('html') and _any_variant_in(data['html'], url_variants):
                data['html'] = img_re.sub('', data['html'])
        elif block_type == 'accordion':
            for item in data.get('items', []):
                if item.get('content') and _any_variant_in(item['content'], url_variants):
                    item['content'] = img_re.sub('', item['content'])
        elif block_type == 'tabs':
            for tab in data.get('tabs', []):
                if tab.get('content') and _any_variant_in(tab['content'], url_variants):
                    tab['content'] = img_re.sub('', tab['content'])


def _cleanup_form_references(form_id):
    """Remove form blocks from page builder where form_id matches."""
    if not form_id:
        return
    
    pages = Page.query.filter(Page.blocks.like(f'%form_id%')).all()
    for p in pages:
        if not p.blocks:
            continue
        try:
            blocks_data = json.loads(p.blocks)
            block_list = blocks_data.get('blocks', []) if isinstance(blocks_data, dict) else []
            if _blocks_contain_form_id(block_list, int(form_id)):
                _clean_form_from_blocks(block_list, int(form_id))
                p.blocks = json.dumps(blocks_data, ensure_ascii=False)
        except (json.JSONDecodeError, TypeError, ValueError):
            pass


def _clean_form_from_blocks(blocks, form_id):
    """Recursively remove form blocks with matching form_id."""
    i = 0
    while i < len(blocks):
        block = blocks[i]
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        if block_type == 'form' and data.get('form_id') == form_id:
            blocks.pop(i)
            continue
        
        if block_type == 'columns':
            for col in data.get('columns', []):
                _clean_form_from_blocks(col.get('blocks', []), form_id)
        elif block_type == 'section':
            _clean_form_from_blocks(data.get('blocks', []), form_id)
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('blocks'):
                    _clean_form_from_blocks(slide['blocks'], form_id)
        
        i += 1


def _cleanup_file_references(file_url):
    """Remove all references to a deleted file from the entire site.
    Handles both relative and absolute URLs.
    Cleans <a> links and button-blocks pointing to this file URL.
    """
    if not file_url:
        return
    
    variants = _get_url_variants(file_url)
    patterns = _content_url_sql_patterns(variants)
    
    # Build regex alternation for all URL variants
    escaped = [re.escape(v) for v in variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    link_re = re.compile(
        r'<a[^>]*href=["\']' + url_alt + r'["\'][^>]*>.*?</a>',
        flags=re.DOTALL
    )
    
    # Posts — remove <a> links to this file from HTML content
    post_filters = [Post.content.like(p) for p in patterns]
    posts = Post.query.filter(db.or_(*post_filters)).all()
    for p in posts:
        if p.content and _any_variant_in(p.content, variants):
            p.content = link_re.sub('', p.content)
    
    # Pages — remove <a> links from HTML content
    content_filters = [Page.content.like(p) for p in patterns]
    pages_content = Page.query.filter(db.or_(*content_filters)).all()
    for p in pages_content:
        if p.content and _any_variant_in(p.content, variants):
            p.content = link_re.sub('', p.content)
    
    # Pages — clean builder blocks (button blocks with file links)
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages_blocks = Page.query.filter(db.or_(*blocks_filters)).all()
    for p in pages_blocks:
        if p.blocks:
            try:
                blocks_data = json.loads(p.blocks)
                block_list = blocks_data.get('blocks', []) if isinstance(blocks_data, dict) else []
                _clean_file_from_blocks(block_list, variants)
                p.blocks = json.dumps(blocks_data, ensure_ascii=False)
            except (json.JSONDecodeError, TypeError):
                pass
    
    # Newsletters — remove <a> links from content
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if n.content and _any_variant_in(n.content, variants):
            n.content = link_re.sub('', n.content)
    
    # Menu items — custom links pointing to this file. Per-language scrub of
    # multilingual Custom Links (urls_json) + delete of legacy single-url ones.
    _cleanup_menu_links_by_url_variants(variants)


def _clean_file_from_blocks(blocks, url_variants):
    """Recursively clean file URL from builder blocks (buttons, links, HTML content).
    
    Args:
        blocks: list of builder block dicts
        url_variants: list of URL strings to match (relative + absolute)
    """
    url_set = set(url_variants)
    
    escaped = [re.escape(v) for v in url_variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    # Capture inner text in group 1 — we unwrap matching links
    # (keep visible text, only remove the <a> wrapper). Buttons are removed
    # entirely above; this regex only applies to rich-text and table cells.
    link_re = re.compile(
        r'<a[^>]*href=["\']' + url_alt + r'["\'][^>]*>(.*?)</a>',
        flags=re.DOTALL
    )
    
    i = 0
    while i < len(blocks):
        block = blocks[i]
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        # Button block linking to this file — remove the block
        if block_type == 'button' and data.get('url') in url_set:
            blocks.pop(i)
            continue
        
        # Any block with a generic 'url' or 'link' field
        if data.get('url') in url_set:
            data['url'] = ''
        if data.get('link') in url_set:
            data['link'] = ''
        
        # Rich text HTML may contain <a> links to this file —
        # unwrap them, preserving the visible link text.
        if block_type == 'text':
            if data.get('html') and _any_variant_in(data['html'], url_variants):
                data['html'] = link_re.sub(r'\1', data['html'])
        elif block_type == 'accordion':
            for item in data.get('items', []):
                if item.get('content') and _any_variant_in(item['content'], url_variants):
                    item['content'] = link_re.sub(r'\1', item['content'])
        elif block_type == 'tabs':
            for tab in data.get('tabs', []):
                if tab.get('content') and _any_variant_in(tab['content'], url_variants):
                    tab['content'] = link_re.sub(r'\1', tab['content'])
        elif block_type == 'table':
            # Headers + cells are HTML strings; unwrap matching links,
            # keep visible text. Sanitizer allows <a href> only.
            headers = data.get('headers', [])
            for hi, h in enumerate(headers):
                if h and _any_variant_in(h, url_variants):
                    headers[hi] = link_re.sub(r'\1', h)
            for row in data.get('rows', []):
                for ci, cell in enumerate(row):
                    if cell and _any_variant_in(cell, url_variants):
                        row[ci] = link_re.sub(r'\1', cell)
        
        # Recurse into nested structures
        if block_type == 'columns':
            for col in data.get('columns', []):
                _clean_file_from_blocks(col.get('blocks', []), url_variants)
        elif block_type == 'section':
            _clean_file_from_blocks(data.get('blocks', []), url_variants)
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('blocks'):
                    _clean_file_from_blocks(slide['blocks'], url_variants)
        
        i += 1


# ── Content (post/page) internal link detection & cleanup ──

def _get_url_variants(relative_url):
    """Build all possible URL variants for a relative URL.
    
    Users may insert links as relative (/path) or absolute
    (https://mysite.com/path, http://mysite.com/path).
    Returns a list of all variants to check.
    """
    if not relative_url:
        return []
    
    variants = [relative_url]
    
    try:
        base = request.url_root.rstrip('/')
        if base:
            absolute = base + relative_url
            variants.append(absolute)
            # Also cover the opposite protocol (http↔https)
            if base.startswith('https://'):
                variants.append(base.replace('https://', 'http://', 1) + relative_url)
            elif base.startswith('http://'):
                variants.append(base.replace('http://', 'https://', 1) + relative_url)
    except RuntimeError:
        pass  # No request context
    
    # Deduplicate while preserving order
    seen = set()
    unique = []
    for v in variants:
        if v not in seen:
            seen.add(v)
            unique.append(v)
    return unique


def _get_content_url_variants(slug):
    """Build URL variants for a content slug (/slug, https://domain/slug, etc.)."""
    if not slug:
        return []
    return _get_url_variants(f'/{slug}')


def _content_url_sql_patterns(variants):
    """Build SQL LIKE patterns for all URL variants."""
    return [f'%{v}%' for v in variants]


def _any_variant_in(text, variants):
    """Check if any URL variant appears in text."""
    if not text:
        return False
    return any(v in text for v in variants)


def _menu_url_matches_variant(stored_url, variants):
    """True if a stored menu URL points at (a URL variant of) deleted content.

    Compares only the PATH portion of the stored URL — the ?query and #anchor
    are dropped first, and one trailing slash is tolerated. So deleting /docs
    removes a link to /docs, /docs/, /docs#faq, /docs?x=1 (anchors/queries on
    the same page), but never /docs-team or /docs/sub (different paths).
    Substring matching is deliberately avoided to prevent those false hits.
    """
    if not stored_url:
        return False
    s = stored_url.strip()
    # Keep only the path: drop fragment and query, then any trailing slash.
    s_path = s.split('#', 1)[0].split('?', 1)[0].rstrip('/')
    return any(s_path == v.rstrip('/') for v in variants)


def _delete_menu_subtree(item_ids):
    """Delete the given menu items together with all of their descendants.

    Used when a menu item is removed by cascade (its target page/post/file was
    deleted): the whole branch is removed rather than promoting children to the
    top level or leaving them with a dangling parent_id. Recursive, so
    multi-level submenus are fully cleared. Returns the number of rows deleted.
    """
    if not item_ids:
        return 0
    to_delete = set()
    frontier = set(item_ids)
    # Walk down the tree collecting every descendant id.
    while frontier:
        to_delete |= frontier
        children = MenuItem.query.filter(
            MenuItem.parent_id.in_(frontier)
        ).all()
        frontier = {c.id for c in children if c.id not in to_delete}
    MenuItem.query.filter(
        MenuItem.id.in_(to_delete)
    ).delete(synchronize_session=False)
    return len(to_delete)


def _custom_menu_candidates_by_url(variants):
    """Custom MenuItems possibly referencing any variant (LIKE prefilter only).

    A broad SQL prefilter over both the legacy `url` column and `urls_json`,
    restricted to custom links (page_id/post_id NULL). Callers MUST verify each
    hit with `_menu_url_matches_variant` — the LIKE is intentionally loose so it
    also surfaces anchor/query forms (/docs#faq) for exact path verification.
    Deduped by id.
    """
    if not variants:
        return []
    like_conds = []
    for v in variants:
        like_conds.append(MenuItem.url.like(f'%{v}%'))
        like_conds.append(MenuItem.urls_json.like(f'%{v}%'))
    rows = MenuItem.query.filter(
        MenuItem.page_id.is_(None),
        MenuItem.post_id.is_(None),
        db.or_(*like_conds),
    ).all()
    return rows


def _find_menu_link_items_by_url_variants(variants):
    """Return custom MenuItem objects linking to any of the given URL variants.

    Covers BOTH the multilingual Custom Links feature (per-language URLs in
    `urls_json`) and legacy single-url custom links (`url` column), including
    anchor/query forms (e.g. /docs#faq matches a deletion of /docs).
    Page/post-linked items (page_id/post_id set) are intentionally excluded —
    those are matched by FK in the content cleanup/detection paths. Deduped.
    """
    import json as _json
    if not variants:
        return []
    found = {}
    for mi in _custom_menu_candidates_by_url(variants):
        matched = _menu_url_matches_variant(mi.url, variants)
        if not matched and mi.urls_json:
            try:
                urls = _json.loads(mi.urls_json)
            except (ValueError, TypeError):
                urls = None
            if isinstance(urls, dict):
                matched = any(
                    _menu_url_matches_variant(u, variants) for u in urls.values()
                )
        if matched:
            found[mi.id] = mi
    return list(found.values())


def _cleanup_menu_links_by_url_variants(variants):
    """Per-language scrub of custom menu links pointing to deleted content/files.

    For every multilingual Custom Link whose `urls_json` references one of the
    `variants` (matched per-language, anchors/queries included), remove only the
    affected language entries from `urls_json` and `labels_json`:
      - if at least one language survives, the item is kept and the legacy
        `url`/`label` fallback is refreshed to a surviving language if needed;
      - if no language survives, the whole item — and its entire submenu branch
        — is deleted (see _delete_menu_subtree).
    Legacy single-url custom links (no urls_json) that match are deleted with
    their branch too.

    This mirrors the per-language behaviour of page/post-linked items (which
    vanish for the specific language only). Page/post-linked items are NOT
    touched here — the caller removes those by page_id/post_id FK.
    Returns the number of menu items affected (edited or deleted, excl. swept
    descendants).
    """
    import json as _json
    if not variants:
        return 0

    affected = 0
    subtree_roots = []   # ids of items to delete together with their descendants

    for mi in _custom_menu_candidates_by_url(variants):
        # ── Multilingual custom link ───────────────────────────────────────
        if mi.urls_json:
            try:
                urls = _json.loads(mi.urls_json) or {}
            except (ValueError, TypeError):
                urls = {}
            if not isinstance(urls, dict):
                urls = {}
            try:
                labels = _json.loads(mi.labels_json) if mi.labels_json else {}
            except (ValueError, TypeError):
                labels = {}
            if not isinstance(labels, dict):
                labels = {}

            removed_langs = [
                code for code, u in urls.items()
                if _menu_url_matches_variant(u, variants)
            ]
            if not removed_langs:
                continue

            for code in removed_langs:
                urls.pop(code, None)
                labels.pop(code, None)

            if not urls:
                subtree_roots.append(mi.id)
            else:
                mi.urls_json = _json.dumps(urls)
                mi.labels_json = _json.dumps(labels) if labels else None
                if _menu_url_matches_variant(mi.url, variants):
                    surviving = next(iter(urls))
                    mi.url = urls[surviving]
                    mi.label = labels.get(surviving, mi.label)
            affected += 1

        # ── Legacy single-url custom link (no urls_json) ───────────────────
        elif _menu_url_matches_variant(mi.url, variants):
            subtree_roots.append(mi.id)
            affected += 1

    # Delete fully-removed items together with their submenu branches.
    _delete_menu_subtree(subtree_roots)

    return affected


def _pick_reanchor_target(survivors):
    """Choose which surviving translation a menu item should re-anchor to.

    Deterministic and independent of how many languages a site has active:
      1. the site's default language, if a survivor exists in it;
      2. the first active language (in configured order) that has a survivor;
      3. otherwise the survivor with the smallest id (stable fallback).
    """
    if not survivors:
        return None
    by_lang = {s.language: s for s in survivors}

    default_lang = Settings.get('default_language', 'en')
    if default_lang in by_lang:
        return by_lang[default_lang]

    # Configured active-language order (the setting is an ordered CSV; the
    # set-returning helper loses order, so read the raw value here).
    active_raw = Settings.get('active_languages', 'en') or ''
    for code in (c.strip() for c in active_raw.split(',')):
        if code and code in by_lang:
            return by_lang[code]

    return sorted(survivors, key=lambda s: s.id)[0]


def _reanchor_or_delete_content_menu_links(content_type, content_id):
    """Fix menu items anchored (via page_id/post_id) to a page/post being deleted.

    Handles both re-anchoring and submenu children for page/post links:
      - if the deleted content still has published translation siblings, every
        affected menu item is RE-ANCHORED to the best surviving translation
        (picked by _pick_reanchor_target) so the entry keeps working in the
        remaining languages — its per-language labels and submenu are untouched;
      - if no translation survives, the item and its whole submenu branch are
        deleted (see _delete_menu_subtree).

    Must run while the content row still exists (cleanup happens before the
    actual delete/commit in the route), so its translation_group is readable.
    """
    if content_type == 'post':
        Model, fk_col = Post, 'post_id'
    elif content_type == 'page':
        Model, fk_col = Page, 'page_id'
    else:
        return
    if not content_id:
        return

    items = MenuItem.query.filter_by(**{fk_col: content_id}).all()
    if not items:
        return

    obj = Model.query.get(content_id)
    survivors = []
    if obj and obj.translation_group:
        survivors = Model.query.filter(
            Model.translation_group == obj.translation_group,
            Model.id != content_id,
            Model.status == 'published',
        ).all()

    if survivors:
        target = _pick_reanchor_target(survivors)
        for mi in items:
            setattr(mi, fk_col, target.id)
            mi.url = f'/{target.slug}'
            # label / labels_json stay valid: resolve_single resolves the
            # display label per visitor language, independent of the anchor row.
    else:
        _delete_menu_subtree([mi.id for mi in items])


def _find_content_usage(slug, exclude_type=None, exclude_id=None):
    """Find all places that link to /<slug> across the site.
    
    Checks: other posts/pages (HTML content + builder blocks),
    menu items, and newsletters.
    Handles both relative (/slug) and absolute (https://domain/slug) URLs.
    
    Args:
        slug: URL slug of the content being deleted
        exclude_type: 'post' or 'page' — skip the item being deleted
        exclude_id: ID of the item being deleted
    
    Returns list of dicts: [{type, id, title, fields}, ...]
    """
    if not slug:
        return []
    
    results = []
    variants = _get_content_url_variants(slug)
    if not variants:
        return []
    
    # Build SQL OR conditions for all URL variants
    patterns = _content_url_sql_patterns(variants)
    
    # Posts — HTML content containing links to this slug
    post_filters = [Post.content.like(p) for p in patterns]
    posts = Post.query.filter(db.or_(*post_filters)).all()
    for p in posts:
        if exclude_type == 'post' and p.id == exclude_id:
            continue
        if _any_variant_in(p.content, variants):
            results.append({
                'type': 'post', 'id': p.id,
                'title': p.title, 'fields': ['content']
            })
    
    # Pages — HTML content + builder blocks
    content_filters = [Page.content.like(p) for p in patterns]
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages = Page.query.filter(
        db.or_(*(content_filters + blocks_filters))
    ).all()
    for p in pages:
        if exclude_type == 'page' and p.id == exclude_id:
            continue
        fields = []
        if _any_variant_in(p.content, variants):
            fields.append('content')
        if _any_variant_in(p.blocks, variants):
            fields.append('page_builder')
        if fields:
            results.append({
                'type': 'page', 'id': p.id,
                'title': p.title, 'fields': fields
            })
    
    # Menu items — custom links by URL (legacy `url` + multilingual `urls_json`,
    # all variants ± trailing slash) and page/post-linked items by FK.
    menu_items = _find_menu_link_items_by_url_variants(variants)
    
    if exclude_type == 'post':
        menu_items += MenuItem.query.filter_by(post_id=exclude_id).all()
    elif exclude_type == 'page':
        menu_items += MenuItem.query.filter_by(page_id=exclude_id).all()
    
    # Deduplicate menu items
    seen_menu_ids = set()
    for mi in menu_items:
        if mi.id not in seen_menu_ids:
            seen_menu_ids.add(mi.id)
            results.append({
                'type': 'menu_item', 'id': mi.id,
                'title': f'{mi.label} ({mi.menu_location})',
                'fields': ['link']
            })
    
    # Newsletters
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if _any_variant_in(n.content, variants):
            results.append({
                'type': 'newsletter', 'id': n.id,
                'title': n.subject, 'fields': ['content']
            })
    
    return results


def _cleanup_content_references(slug, content_type=None, content_id=None):
    """Remove all internal links to /<slug> from the entire site.
    
    Handles both relative (/slug) and absolute (https://domain/slug) URLs.
    - HTML content: unwraps <a> tags (keeps the link text, removes the href)
    - Builder button blocks: removes blocks pointing to this URL
    - Builder text/accordion/tabs: unwraps <a> tags in rich HTML
    - Menu items: deletes items linking to this content
    """
    if not slug:
        return
    
    variants = _get_content_url_variants(slug)
    if not variants:
        return
    
    # Build a single regex alternation for all URL variants
    escaped = [re.escape(v) for v in variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    unwrap_re = re.compile(
        r'<a[^>]*href=["\']' + url_alt + r'(?:/)?["\'][^>]*>(.*?)</a>',
        flags=re.DOTALL
    )
    
    # SQL patterns for all variants
    patterns = _content_url_sql_patterns(variants)
    
    # Posts — unwrap links in HTML content
    post_filters = [Post.content.like(p) for p in patterns]
    posts = Post.query.filter(db.or_(*post_filters)).all()
    for p in posts:
        if content_type == 'post' and p.id == content_id:
            continue
        if p.content and _any_variant_in(p.content, variants):
            p.content = unwrap_re.sub(r'\1', p.content)
    
    # Pages — unwrap links in HTML content
    content_filters = [Page.content.like(p) for p in patterns]
    pages_content = Page.query.filter(db.or_(*content_filters)).all()
    for p in pages_content:
        if content_type == 'page' and p.id == content_id:
            continue
        if p.content and _any_variant_in(p.content, variants):
            p.content = unwrap_re.sub(r'\1', p.content)
    
    # Pages — clean builder blocks
    blocks_filters = [Page.blocks.like(p) for p in patterns]
    pages_blocks = Page.query.filter(db.or_(*blocks_filters)).all()
    for p in pages_blocks:
        if content_type == 'page' and p.id == content_id:
            continue
        if p.blocks:
            try:
                blocks_data = json.loads(p.blocks)
                block_list = blocks_data.get('blocks', []) if isinstance(blocks_data, dict) else []
                _clean_content_link_from_blocks(block_list, variants)
                p.blocks = json.dumps(blocks_data, ensure_ascii=False)
            except (json.JSONDecodeError, TypeError):
                pass
    
    # Menu items — custom links pointing to this content (per-language scrub of
    # multilingual Custom Links + legacy single-url links, with subtree delete).
    _cleanup_menu_links_by_url_variants(variants)
    # Page/post-linked items (page_id/post_id): re-anchor to a surviving
    # translation when one exists, otherwise delete the item and its submenu.
    _reanchor_or_delete_content_menu_links(content_type, content_id)
    
    # Newsletters — unwrap links in content
    nl_filters = [Newsletter.content.like(p) for p in patterns]
    newsletters = Newsletter.query.filter(db.or_(*nl_filters)).all()
    for n in newsletters:
        if n.content and _any_variant_in(n.content, variants):
            n.content = unwrap_re.sub(r'\1', n.content)


def _clean_content_link_from_blocks(blocks, url_variants):
    """Recursively remove internal links from builder blocks.
    
    Args:
        blocks: list of builder block dicts
        url_variants: list of URL strings to match (relative + absolute)
    
    - Button blocks with matching url → remove entire block
    - Text/accordion/tabs → unwrap <a> tags (keep text)
    - Image blocks with link_url → clear the link
    """
    # Build set for O(1) exact-match lookups (include trailing-slash forms)
    url_set = set()
    for v in url_variants:
        url_set.add(v)
        url_set.add(v.rstrip('/') + '/')
    
    # Build regex alternation for HTML <a> tag unwrapping
    escaped = [re.escape(v) for v in url_variants]
    url_alt = '(?:' + '|'.join(escaped) + ')'
    unwrap_re = re.compile(
        r'<a[^>]*href=["\']' + url_alt + r'(?:/)?["\'][^>]*>(.*?)</a>',
        flags=re.DOTALL
    )
    
    i = 0
    while i < len(blocks):
        block = blocks[i]
        data = block.get('data', {})
        block_type = block.get('type', '')
        
        # Button block linking to this content → remove the block
        if block_type == 'button' and data.get('url') in url_set:
            blocks.pop(i)
            continue
        
        # Image block with link to this content → clear the link
        if block_type == 'image' and data.get('link_url') in url_set:
            data['link_url'] = ''
        
        # Rich text HTML — unwrap <a> tags
        if block_type == 'text':
            if data.get('html') and _any_variant_in(data['html'], url_variants):
                data['html'] = unwrap_re.sub(r'\1', data['html'])
        elif block_type == 'accordion':
            for item in data.get('items', []):
                if item.get('content') and _any_variant_in(item['content'], url_variants):
                    item['content'] = unwrap_re.sub(r'\1', item['content'])
        elif block_type == 'tabs':
            for tab in data.get('tabs', []):
                if tab.get('content') and _any_variant_in(tab['content'], url_variants):
                    tab['content'] = unwrap_re.sub(r'\1', tab['content'])
        elif block_type == 'table':
            # Headers + cells are HTML strings; unwrap matching links,
            # keep visible text. Sanitizer allows <a href> only.
            headers = data.get('headers', [])
            for hi, h in enumerate(headers):
                if h and _any_variant_in(h, url_variants):
                    headers[hi] = unwrap_re.sub(r'\1', h)
            for row in data.get('rows', []):
                for ci, cell in enumerate(row):
                    if cell and _any_variant_in(cell, url_variants):
                        row[ci] = unwrap_re.sub(r'\1', cell)
        
        # Recurse into nested structures
        if block_type == 'columns':
            for col in data.get('columns', []):
                _clean_content_link_from_blocks(col.get('blocks', []), url_variants)
        elif block_type == 'section':
            _clean_content_link_from_blocks(data.get('blocks', []), url_variants)
        elif block_type == 'slideshow':
            for slide in data.get('slides', []):
                if slide.get('blocks'):
                    _clean_content_link_from_blocks(slide['blocks'], url_variants)
        
        i += 1


@app.route('/admin/media/<int:image_id>/delete', methods=['POST'])
@login_required
def admin_media_delete(image_id):
    """Delete image and ALL its variants, cleaning up references site-wide."""
    image = Image.query.get_or_404(image_id)
    
    # Clean up all references BEFORE deleting the row — the helper reads
    # `image.download_slug` and `image.variants` to build the full list of
    # short URLs that might be linked across content.
    _cleanup_image_references(image)
    
    # Delete ALL files: original + all generated variants
    all_paths = image.get_all_file_paths(app.config['UPLOAD_FOLDER'])
    for fpath in all_paths:
        try:
            if os.path.exists(fpath):
                os.remove(fpath)
        except OSError:
            pass  # File already gone or permission issue — not critical
    
    AuditLog.log('delete_image', 'media', image.id,
                 _ad('deleted_image', filename=image.original_filename or image.filename),
                 current_user, request.remote_addr)
    db.session.delete(image)
    db.session.commit()
    
    flash(t('flash.image_deleted'), 'success')
    return redirect(url_for('admin_media_library'))


@app.route('/admin/media/api/list')
@login_required
def admin_media_api_list():
    """API endpoint to get list of images for the unified image picker.
    
    Returns the CLEAN SHORT URL as `url`, not the internal /static/uploads
    path — so when the picker inserts `<img src="...">` into the editor,
    it's already the short URL. Admins never see long paths anywhere.
    
    Supports the same live search as the Media Library page (`?q=` over
    alt text, original + stored filename, title and description) plus
    pagination.
    Response shape is backwards-compatible: without a `page` argument the
    endpoint returns the historical plain ARRAY (WebMCP agent tools and
    older callers rely on it); with `?page=N` it returns an object
    `{items, page, pages, total, has_prev, has_next, prev_num, next_num}`.
    """
    query = Image.query
    
    q = request.args.get('q', '').strip()
    if q:
        pattern = f'%{escape_like(q)}%'
        query = query.filter(
            db.or_(
                Image.alt_text.ilike(pattern, escape='\\'),
                Image.original_filename.ilike(pattern, escape='\\'),
                # Stored (sanitized/renamed) file name too — it's what shows
                # in image URLs, so people search by it as well.
                Image.filename.ilike(pattern, escape='\\'),
                Image.title.ilike(pattern, escape='\\'),
                Image.description.ilike(pattern, escape='\\')
            )
        )
    
    query = query.order_by(Image.uploaded_at.desc())
    
    def _row(img):
        return {
            'id': img.id,
            'url': img.public_url,
            # Small WebP variant for the picker grid — the modal shows dozens
            # of images at ~200px, so loading originals (often multi-MB) is
            # wasteful. Falls back to public_url for images without variants
            # (e.g. SVG).
            'thumb': img.get_smallest_url('webp'),
            'alt_text': img.alt_text,
            'title': img.title or '',
            'filename': img.filename,
            'original_filename': img.original_filename or img.filename,
            'width': img.width,
            'height': img.height,
            'file_size': img.file_size,
        }
    
    if 'page' in request.args:
        per_page = min(max(1, request.args.get('per_page', 20, type=int)), 100)
        images = query.paginate(page=get_page(), per_page=per_page, error_out=False)
        return jsonify({
            'items': [_row(img) for img in images.items],
            'page': images.page,
            'pages': images.pages,
            'total': images.total,
            'has_prev': images.has_prev,
            'has_next': images.has_next,
            'prev_num': images.prev_num,
            'next_num': images.next_num,
            'per_page': per_page,
        })
    
    # Legacy shape: full flat array.
    return jsonify([_row(img) for img in query.all()])


# ============================================================================
# FILE MANAGER — Documents (PDF, DOC, XLS, ZIP, etc.)
# ============================================================================

# Allowed document extensions (separate from image ALLOWED_EXTENSIONS)
ALLOWED_FILE_EXTENSIONS = {'pdf', 'doc', 'docx', 'xls', 'xlsx', 'zip', 'rar', 'txt', 'csv'}
MAX_FILE_SIZE = 50 * 1024 * 1024  # 50 MB


def allowed_document(filename):
    """Check if file extension is allowed for document uploads"""
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_FILE_EXTENSIONS


def _generate_file_slug(original_filename):
    """Generate a unique, clean download slug from original filename.
    
    Uses python-slugify (same as posts/pages) for proper transliteration:
      'Report Q3.pdf'    → 'report-q3.pdf'
      'Мой файл.docx'   → 'moj-fajl.docx'
      '報告書.pdf'        → 'bao-gao-shu.pdf'
      '   .pdf'          → 'file.pdf' (fallback)
    
    Checks uniqueness across File, Post, Page AND Image — no URL collisions.
    """
    # Split name and extension
    if '.' in original_filename:
        raw_name = original_filename.rsplit('.', 1)[0]
        ext = '.' + original_filename.rsplit('.', 1)[1].lower()
    else:
        raw_name = original_filename
        ext = '.bin'
    
    # Slugify the name part (handles CJK, Cyrillic, Arabic, etc.)
    name = slugify(raw_name)
    
    # Fallback if slugify returns empty (e.g. only special chars)
    if not name:
        name = 'file'
    
    def _slug_is_taken(s):
        """Check if slug is taken by ANY content: files, posts, pages, or images."""
        if File.query.filter_by(download_slug=s).first():
            return True
        if Post.query.filter_by(slug=s).first():
            return True
        if Page.query.filter_by(slug=s).first():
            return True
        # A file URL like '/report.jpg' would collide with image '/report.jpg'
        # (image base slug 'report' + ext 'jpg'). Only matching extension counts.
        if '.' in s:
            bare, sext = s.rsplit('.', 1)
            img = Image.query.filter_by(download_slug=bare).first()
            if img and img.original_extension == sext.lower():
                return True
        return False
    
    # Try base slug first
    slug = f"{name}{ext}"
    if not _slug_is_taken(slug):
        return slug
    
    # Add counter if taken
    counter = 2
    while True:
        slug = f"{name}-{counter}{ext}"
        if not _slug_is_taken(slug):
            return slug
        counter += 1


# ============================================================================
# IMAGE SHORT URLS — clean public URL scheme for media library images
# ============================================================================
#
# Every image uploaded to the library gets a `download_slug` (just the base
# name, no extension). This slug is used to build short public URLs:
#
#   /sunset.jpg              → original file
#   /sunset-400w.webp        → 400px wide WebP variant
#   /sunset-800w.webp        → 800px wide WebP variant
#   /sunset-1200w.avif       → 1200px wide AVIF variant
#   /sunset-og.jpg           → 1200×630 OG social-sharing crop
#
# The physical files on disk keep their existing timestamped names; this is
# purely a public-facing URL scheme. Inside <picture>, <source srcset>, og:image,
# schema.org, sitemap, RSS and llms.txt — every URL is short. Long paths to
# /static/uploads/... never leak into HTML, which eliminates the "duplicate
# content" worry and makes right-click "Copy image address" return clean URLs
# in every browser.
#
# The catch-all route `/<slug>` tries post → page → redirect → file → image in
# that order; images are last because they're the only category whose URLs
# must contain a dot.


# Image extensions we serve via short URLs. Kept in sync with what
# process_uploaded_image() actually generates.
_IMAGE_SHORT_URL_EXTENSIONS = frozenset(
    ('jpg', 'jpeg', 'png', 'webp', 'avif', 'gif', 'svg')
)

# Parses a width suffix like '-800w'. The base slug never ends in this pattern
# because slugify strips trailing digits+letters in this combination, and our
# image slug generator additionally refuses base slugs matching this shape.
_IMAGE_WIDTH_SUFFIX_RE = re.compile(r'^(.+)-(\d+)w$')


def _parse_image_short_url(path):
    """Parse a short image URL path (without leading slash) into its parts.
    
    Returns a tuple (base_slug, variant, ext), or None if the path doesn't
    look like a valid image short URL.
    
    `variant` is one of:
        None   — original file (/sunset.jpg)
        int N  — responsive variant at width N (/sunset-800w.webp)
        'og'   — OG social-sharing crop (/sunset-og.jpg)
    
    Examples:
        'sunset.jpg'          → ('sunset', None, 'jpg')
        'sunset-800w.webp'    → ('sunset', 800, 'webp')
        'my-photo-1200w.avif' → ('my-photo', 1200, 'avif')
        'sunset-og.jpg'       → ('sunset', 'og', 'jpg')
        'my-photo.png'        → ('my-photo', None, 'png')
        'report.pdf'          → None (extension not in image set)
        'about-us'            → None (no extension)
    """
    if '.' not in path:
        return None
    base, ext = path.rsplit('.', 1)
    ext = ext.lower()
    if ext not in _IMAGE_SHORT_URL_EXTENSIONS:
        return None
    if not base:
        return None
    
    # -og suffix for social-sharing crop
    if base.endswith('-og'):
        base_slug = base[:-3]
        if not base_slug:
            return None
        return (base_slug, 'og', ext)
    
    # -NNNw suffix for responsive variant
    m = _IMAGE_WIDTH_SUFFIX_RE.match(base)
    if m:
        return (m.group(1), int(m.group(2)), ext)
    
    # No suffix → original
    return (base, None, ext)


def _serve_image_short_url(path):
    """Look up an image by its short URL path and return a Flask response.
    
    Returns None if the path doesn't map to a stored image variant — caller
    (catch-all route) should fall through to 404.
    """
    parsed = _parse_image_short_url(path)
    if parsed is None:
        return None
    
    base_slug, variant, ext = parsed
    image = Image.query.filter_by(download_slug=base_slug).first()
    if not image:
        return None
    
    variants_data = image.get_variants()
    upload_folder = app.config['UPLOAD_FOLDER']
    
    # Decide which physical file to serve.
    if variant is None:
        # Original — only serve if the requested extension matches the real one.
        if ext != image.original_extension:
            return None
        filename = image.filename
    elif variant == 'og':
        og_variants = variants_data.get('og', {})
        v = og_variants.get(ext)
        if not v:
            return None
        # variants JSON stores long /static/uploads/...  URLs; we just need
        # the basename to feed send_from_directory.
        filename = v['url'].rsplit('/', 1)[-1]
    else:
        # Responsive variant at width N — the requested format must match the
        # stored format (WebP or AVIF); we don't transcode on the fly.
        fmt_variants = variants_data.get(ext, {})
        v = fmt_variants.get(str(variant))
        if not v:
            return None
        filename = v['url'].rsplit('/', 1)[-1]
    
    response = send_from_directory(upload_folder, filename)
    # Variants are byte-immutable: if the user re-uploads the image they get
    # a new timestamped filename on disk and (if they edit the slug) a new
    # slug too. So we can cache forever without staleness risk.
    response.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
    response.headers['Vary'] = 'Accept'
    return response


def _generate_image_slug(original_filename, exclude_image_id=None):
    """Generate a unique, clean base slug for a media-library image.
    
    Unlike file slugs, image slugs have NO extension — the extension is
    appended when the URL is rendered (e.g. '/sunset.jpg', '/sunset-800w.webp').
    The slug is checked for uniqueness against posts, pages, files and other
    images to prevent any URL collision across the whole `/<slug>` namespace.
    """
    if '.' in original_filename:
        raw_name = original_filename.rsplit('.', 1)[0]
    else:
        raw_name = original_filename
    
    name = slugify(raw_name)
    if not name:
        name = 'image'
    
    # A base slug that ends in '-NNNw' would be ambiguous with a width suffix.
    # Very unlikely in practice (slugify produces it only if the original
    # filename was literally 'something-400w'), but we guard anyway.
    if _IMAGE_WIDTH_SUFFIX_RE.match(name) or name.endswith('-og'):
        name = f'{name}-img'
    
    def _image_slug_is_taken(s):
        q = Image.query.filter_by(download_slug=s)
        if exclude_image_id:
            q = q.filter(Image.id != exclude_image_id)
        if q.first():
            return True
        # A post/page with the same bare slug would confuse admins even though
        # /sunset (post) and /sunset.jpg (image) don't technically clash at the
        # URL level. Treat them as reserved.
        if Post.query.filter_by(slug=s).first():
            return True
        if Page.query.filter_by(slug=s).first():
            return True
        # A file with download_slug 'sunset.jpg' would clash with image slug
        # 'sunset' only if the image's extension is 'jpg'. Different extensions
        # produce different URLs and are safe to coexist.
        file_conflict = File.query.filter(
            File.download_slug.like(f'{s}.%')
        ).first()
        if file_conflict:
            return True
        return False
    
    if not _image_slug_is_taken(name):
        return name
    
    counter = 2
    while True:
        candidate = f'{name}-{counter}'
        if not _image_slug_is_taken(candidate):
            return candidate
        counter += 1


def _sanitize_image_slug_input(raw):
    """Sanitize a user-edited image slug from a form field.
    
    Runs the same `slugify()` pipeline as posts/pages/files — so any
    pasted garbage (spaces, dots, quotes, emoji, etc.) gets filtered.
    Returns an empty string if the result is empty after slugify.
    """
    if not raw:
        return ''
    cleaned = slugify(raw)
    # Never allow a form submission to produce an ambiguous URL shape.
    if _IMAGE_WIDTH_SUFFIX_RE.match(cleaned) or cleaned.endswith('-og'):
        cleaned = f'{cleaned}-img'
    return cleaned


# ─────────────────────────────────────────────────────────────────────────────
# Video thumbnail proxy & oEmbed metadata
#
# Purpose: privacy. AliothPress never sends visitor data (IP, User-Agent,
# Referer) to YouTube/Vimeo. All video thumbnails are fetched server-side
# once, cached on disk, and served from our own domain. Iframes only load
# after a user click (facade pattern — see block_renderer._render_video
# and _render_video_gallery).
#
# Admin title auto-fetch also goes through our server (previously went
# through noembed.com, which was a third-party dependency).
# ─────────────────────────────────────────────────────────────────────────────

_YOUTUBE_ID_RE = re.compile(r'^[A-Za-z0-9_\-]{11}$')
_VIMEO_ID_RE = re.compile(r'^\d+$')


def _validate_video_id(source, video_id):
    """Strict per-provider ID validation — prevents SSRF via the proxy."""
    if not video_id:
        return False
    if source == 'youtube':
        return bool(_YOUTUBE_ID_RE.match(video_id))
    if source == 'vimeo':
        return bool(_VIMEO_ID_RE.match(video_id))
    return False


def _video_thumb_fallback():
    """Neutral SVG placeholder returned when a provider fetch fails.
    Dark gradient — works in both light and dark themes (decorative,
    not theme-dependent; matches the look of the previous
    ap-video-gallery__vimeo-placeholder)."""
    svg = (
        '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 360" '
        'preserveAspectRatio="xMidYMid slice">'
        '<defs><linearGradient id="g" x1="0" y1="0" x2="1" y2="1">'
        '<stop offset="0%" stop-color="#1a1a2e"/>'
        '<stop offset="100%" stop-color="#0f0f1a"/>'
        '</linearGradient></defs>'
        '<rect width="640" height="360" fill="url(#g)"/>'
        '<circle cx="320" cy="180" r="48" fill="none" '
        'stroke="rgba(255,255,255,0.25)" stroke-width="3"/>'
        '<polygon points="306,160 306,200 340,180" fill="rgba(255,255,255,0.25)"/>'
        '</svg>'
    )
    response = make_response(svg)
    response.headers['Content-Type'] = 'image/svg+xml'
    response.headers['Cache-Control'] = 'public, max-age=300'
    return response


def _fetch_url_bytes(url, max_bytes=2 * 1024 * 1024, timeout=5):
    """Fetch a URL with a size cap and timeout. Returns bytes or None on any failure."""
    try:
        req = Request(url, headers={'User-Agent': 'AliothPress/1.0'})
        with urlopen(req, timeout=timeout) as resp:
            if getattr(resp, 'status', 200) != 200:
                return None
            content = resp.read(max_bytes + 1)
            if len(content) > max_bytes:
                return None
            return content
    except (URLError, HTTPError, socket.timeout, OSError, ValueError):
        return None


def _fetch_oembed(source, video_id):
    """Server-side oEmbed lookup against the official provider endpoints.
    No third-party proxies. Returns dict or None."""
    if source == 'youtube':
        url = 'https://www.youtube.com/oembed?' + urlencode({
            'url': f'https://www.youtube.com/watch?v={video_id}',
            'format': 'json',
        })
    elif source == 'vimeo':
        url = 'https://vimeo.com/api/oembed.json?' + urlencode({
            'url': f'https://vimeo.com/{video_id}',
        })
    else:
        return None
    
    raw = _fetch_url_bytes(url, max_bytes=64 * 1024)
    if not raw:
        return None
    try:
        return json.loads(raw.decode('utf-8'))
    except (json.JSONDecodeError, UnicodeDecodeError):
        return None


@app.route('/media/video-thumb/<source>/<video_id>.<ext>')
def media_video_thumb(source, video_id, ext):
    """Public endpoint. Proxies and caches a video thumbnail on disk.
    
    First request → fetch from provider, save to
      static/uploads/video-thumbs/{source}_{id}.jpg
    Subsequent requests → served directly as a static file with long cache.
    
    On any failure → returns the neutral SVG placeholder.
    
    Strict validation on source and ID prevents SSRF.
    """
    if source not in ('youtube', 'vimeo'):
        abort(404)
    if ext.lower() not in ('jpg', 'jpeg'):
        abort(404)
    if not _validate_video_id(source, video_id):
        abort(404)
    
    cache_dir = os.path.join(app.static_folder, 'uploads', 'video-thumbs')
    try:
        os.makedirs(cache_dir, exist_ok=True)
    except OSError:
        return _video_thumb_fallback()
    
    cache_name = f'{source}_{video_id}.jpg'
    cache_path = os.path.join(cache_dir, cache_name)
    
    # Cache hit
    if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
        response = send_from_directory(cache_dir, cache_name, mimetype='image/jpeg')
        response.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
        return response
    
    # Cache miss — resolve provider URL
    provider_url = None
    if source == 'youtube':
        provider_url = f'https://i.ytimg.com/vi/{video_id}/hqdefault.jpg'
    else:  # vimeo
        meta = _fetch_oembed('vimeo', video_id)
        if meta:
            provider_url = meta.get('thumbnail_url')
    
    if not provider_url:
        return _video_thumb_fallback()
    
    content = _fetch_url_bytes(provider_url)
    if not content:
        return _video_thumb_fallback()
    
    try:
        with open(cache_path, 'wb') as f:
            f.write(content)
    except OSError:
        return _video_thumb_fallback()
    
    response = send_from_directory(cache_dir, cache_name, mimetype='image/jpeg')
    response.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
    return response


@app.route('/api/video-meta')
@login_required
def api_video_meta():
    """Admin-only. Replaces the client-side noembed.com dependency.
    
    Admin browser → our server → official YouTube/Vimeo oEmbed.
    Admin IP never reaches Google/Vimeo directly.
    
    Query:   ?source=youtube|vimeo&id=<video_id>
    Returns: { "title": "...", "thumb_url": "/media/video-thumb/..." }
             or { "error": "..." } with status 400/502
    """
    source = request.args.get('source', '')
    video_id = request.args.get('id', '')
    
    if source not in ('youtube', 'vimeo') or not _validate_video_id(source, video_id):
        return jsonify({'error': 'invalid_params'}), 400
    
    meta = _fetch_oembed(source, video_id)
    if not meta:
        return jsonify({'error': 'fetch_failed'}), 502
    
    title = (meta.get('title') or '').strip()
    if not title:
        return jsonify({'error': 'no_title'}), 502
    
    return jsonify({
        'title': title,
        'thumb_url': f'/media/video-thumb/{source}/{video_id}.jpg',
    })


@app.route('/admin/files')
@login_required
def admin_files_list():
    """File manager — list all uploaded documents"""
    page = get_page()
    per_page = get_per_page()
    q = request.args.get('q', '').strip()
    
    query = File.query
    
    if q:
        search_pattern = f"%{q}%"
        query = query.filter(
            db.or_(
                File.title.ilike(search_pattern),
                File.original_filename.ilike(search_pattern),
                File.description.ilike(search_pattern)
            )
        )
    
    files = query.order_by(File.uploaded_at.desc()).paginate(
        page=page, per_page=per_page, error_out=False
    )
    
    # JSON mode for live search
    if request.args.get('format') == 'json':
        return jsonify({
            'items': [{
                'id': f.id,
                'title': f.title or f.original_filename,
                'original_filename': f.original_filename or f.filename,
                'download_slug': f.download_slug,
                'extension': f.get_extension(),
                'icon': f.get_icon(),
                'type_label': f.get_type_label(),
                'size': f.human_size() if f.file_size else None,
            } for f in files.items],
            'page': files.page,
            'pages': files.pages,
            'total': files.total,
            'has_prev': files.has_prev,
            'has_next': files.has_next,
            'prev_num': files.prev_num,
            'next_num': files.next_num,
            'per_page': per_page,
        })
    
    return render_template('admin/files/list.html', files=files, search_query=q, per_page=per_page)


@app.route('/admin/files/upload', methods=['GET', 'POST'])
@login_required
def admin_files_upload():
    """Upload new document file"""
    if request.method == 'POST':
        if 'file' not in request.files:
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        file = request.files['file']
        
        if file.filename == '':
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        if file and allowed_document(file.filename):
            # Check file size
            file.seek(0, 2)  # Seek to end
            size = file.tell()
            file.seek(0)  # Seek back to start
            
            if size > MAX_FILE_SIZE:
                flash(t('flash.file_too_large', size=MAX_FILE_SIZE // (1024 * 1024)), 'error')
                return redirect(request.url)
            
            # Safe name with timestamp: extension survives non-Latin
            # originals, length capped under the 255-byte filesystem limit.
            name, ext = os.path.splitext(make_upload_filename(file.filename))
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            filename = f"{name}_{timestamp}{ext}"
            
            # Ensure files directory exists
            files_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'files')
            os.makedirs(files_dir, exist_ok=True)
            
            # Guard against same-second collisions between different uploads.
            filename = unique_upload_path(files_dir, filename)
            
            # Save file
            filepath = os.path.join(files_dir, filename)
            file.save(filepath)
            
            # Get actual file size from disk
            actual_size = os.path.getsize(filepath)
            
            # Generate unique download slug from original filename
            # e.g. "report.pdf", and if taken → "report-2.pdf"
            download_slug = _generate_file_slug(file.filename)
            
            # Create database record
            doc = File(
                filename=filename,
                original_filename=file.filename,
                url=f'/static/uploads/files/{filename}',
                download_slug=download_slug,
                title=request.form.get('title', '').strip() or file.filename,
                description=request.form.get('description', '').strip(),
                file_size=actual_size,
                mime_type=file.content_type,
                user_id=current_user.id
            )
            
            db.session.add(doc)
            db.session.commit()
            
            AuditLog.log('file_uploaded', 'file', doc.id,
                         _ad('file_uploaded', name=file.filename),
                         user=current_user, ip=request.remote_addr)
            
            flash(t('flash.file_uploaded'), 'success')
            return _safe_return_to('admin_files_list')
        else:
            allowed = ', '.join(sorted(ALLOWED_FILE_EXTENSIONS)).upper()
            flash(t('flash.invalid_file_type', formats=allowed, size=MAX_FILE_SIZE // (1024 * 1024)), 'error')
            return redirect(request.url)
    
    return render_template('admin/files/upload.html')


@app.route('/admin/files/<int:file_id>/edit', methods=['GET', 'POST'])
@login_required
def admin_files_edit(file_id):
    """Edit file metadata (title, description)"""
    doc = File.query.get_or_404(file_id)
    
    if request.method == 'POST':
        doc.title = request.form.get('title', '').strip() or doc.original_filename
        doc.description = request.form.get('description', '').strip()
        
        db.session.commit()
        return _inline_saved(t('flash.file_updated'), 'admin_files_list')
    
    return render_template('admin/files/edit.html', file=doc)


@app.route('/admin/files/<int:file_id>/delete', methods=['POST'])
@login_required
def admin_files_delete(file_id):
    """Delete a document file, cleaning up references site-wide."""
    doc = File.query.get_or_404(file_id)
    
    # Clean up all references before deleting
    # Must clean BOTH the internal storage URL and the public download slug,
    # because builder/content may reference either one
    _cleanup_file_references(doc.url)
    if doc.download_slug:
        _cleanup_file_references(f'/{doc.download_slug}')
    
    # Delete physical file
    files_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'files')
    filepath = os.path.join(files_dir, doc.filename)
    try:
        if os.path.exists(filepath):
            os.remove(filepath)
    except OSError:
        pass
    
    AuditLog.log('file_deleted', 'file', doc.id,
                 _ad('file_deleted', name=doc.original_filename),
                 user=current_user, ip=request.remote_addr)
    
    db.session.delete(doc)
    db.session.commit()
    
    flash(t('flash.file_deleted'), 'success')
    return redirect(url_for('admin_files_list'))


@app.route('/admin/files/api/list')
@login_required
def admin_files_api_list():
    """API endpoint — list files as JSON (for future pickers)"""
    files = File.query.order_by(File.uploaded_at.desc()).all()
    
    return jsonify([{
        'id': f.id,
        'url': f'/{f.download_slug}',
        'title': f.title or '',
        'filename': f.original_filename or f.filename,
        'download_slug': f.download_slug,
        'file_size': f.file_size,
        'mime_type': f.mime_type,
        'extension': f.get_extension(),
        'icon': f.get_icon(),
    } for f in files])


@app.route('/admin/api/search-files', methods=['GET'])
@login_required
def admin_api_search_files():
    """Live search — returns matching file titles for autocomplete"""
    q = request.args.get('q', '').strip()
    
    if not q or len(q) < 2:
        return jsonify({'results': []})
    
    search_pattern = f"%{q}%"
    items = File.query.filter(
        db.or_(
            File.title.ilike(search_pattern),
            File.original_filename.ilike(search_pattern)
        )
    ).order_by(File.uploaded_at.desc()).limit(10).all()
    
    results = [{
        'id': item.id,
        'title': item.title or item.original_filename,
        'extension': item.get_extension(),
        'icon': item.get_icon(),
        'size': item.human_size(),
    } for item in items]
    
    return jsonify({'results': results})


# ────────────────────────────────────────────────────────────────────────────
# Asset Usage Check API
# ────────────────────────────────────────────────────────────────────────────

@app.route('/admin/api/check-usage', methods=['POST'])
@login_required
def admin_api_check_usage():
    """Check where an asset (image, form, file, post, page) is used across the site.
    
    Request JSON: { "type": "image|form|file|post|page", "id": <int> }
    Response JSON: { "usage": [...], "count": <int> }
    """
    data = request.get_json()
    if not data:
        return jsonify({'error': 'Invalid request'}), 400
    
    asset_type = data.get('type', '')
    asset_id = data.get('id')
    
    if not asset_type or not asset_id:
        return jsonify({'error': 'Missing type or id'}), 400
    
    usage = []
    
    if asset_type == 'image':
        image = Image.query.get(asset_id)
        if image:
            usage = _find_image_usage(image)
    
    elif asset_type == 'form':
        usage = _find_form_usage(asset_id)
    
    elif asset_type == 'file':
        file_obj = File.query.get(asset_id)
        if file_obj:
            usage = _find_file_usage(file_obj.url)
            # Also check the public download slug URL (used in builder/content)
            if file_obj.download_slug:
                slug_url = f'/{file_obj.download_slug}'
                if slug_url != file_obj.url:
                    slug_usage = _find_file_usage(slug_url)
                    # Merge without duplicates
                    seen = {(u['type'], u['id']) for u in usage}
                    for u in slug_usage:
                        if (u['type'], u['id']) not in seen:
                            usage.append(u)
    
    elif asset_type == 'post':
        post = Post.query.get(asset_id)
        if post:
            usage = _find_content_usage(post.slug, 'post', post.id)
    
    elif asset_type == 'page':
        page = Page.query.get(asset_id)
        if page:
            usage = _find_content_usage(page.slug, 'page', page.id)
    
    return jsonify({'usage': usage, 'count': len(usage)})


# ============================================================================
# FAVICON MANAGER
# ============================================================================

@app.route('/admin/media/favicon', methods=['GET', 'POST'])
@login_required
def admin_favicon_manager():
    """Favicon Manager — upload one image, auto-generate all required sizes.
    
    Generates:
      - favicon.ico (multi-size: 16×16, 32×32, 48×48)
      - favicon-96x96.png
      - favicon-192x192.png  (Android home screen)
      - favicon-512x512.png  (PWA splash screen)
      - apple-touch-icon.png (180×180, iOS)
      - site.webmanifest
      - SVG favicon (if source is SVG — passed through as-is)
    
    All files are saved to /static/uploads/favicon/ and paths stored in Settings.
    """
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    if request.method == 'POST':
        if 'favicon_source' not in request.files:
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        file = request.files['favicon_source']
        if file.filename == '':
            flash(t('flash.no_file_selected'), 'error')
            return redirect(request.url)
        
        # Validate file type
        ext = file.filename.rsplit('.', 1)[-1].lower() if '.' in file.filename else ''
        if ext not in ('png', 'jpg', 'jpeg', 'svg', 'webp'):
            flash(t('flash.favicon_invalid_format'), 'error')
            return redirect(request.url)
        
        # Create favicon output directory
        favicon_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'favicon')
        os.makedirs(favicon_dir, exist_ok=True)
        
        # Save source file temporarily
        source_path = os.path.join(favicon_dir, f'source.{ext}')
        file.save(source_path)
        
        generated = {}
        
        try:
            if ext == 'svg':
                # SVG → sanitize, copy as favicon SVG, but still generate raster sizes
                sanitize_svg(source_path)
                import shutil
                svg_dest = os.path.join(favicon_dir, 'favicon.svg')
                shutil.copy2(source_path, svg_dest)
                generated['favicon_svg'] = '/static/uploads/favicon/favicon.svg'
                
                # Try to rasterize SVG for ICO and PNG sizes
                try:
                    import cairosvg
                    # Render SVG to a large PNG first, then resize
                    large_png = os.path.join(favicon_dir, '_temp_512.png')
                    cairosvg.svg2png(url=source_path, write_to=large_png, 
                                     output_width=512, output_height=512)
                    img = PILImage.open(large_png)
                except Exception:
                    # cairosvg not available — try Pillow directly
                    try:
                        img = PILImage.open(source_path)
                        if img.width < 64:
                            # SVG opened as tiny raster — not useful
                            raise ValueError('SVG rasterization too small')
                    except Exception:
                        # Cannot rasterize SVG — save SVG only, warn user
                        flash(t('flash.favicon_svg_only'), 'warning')
                        Settings.set('favicon_svg', generated['favicon_svg'])
                        db.session.commit()
                        return redirect(request.url)
            else:
                # Raster image (PNG, JPG, WebP)
                img = PILImage.open(source_path)
            
            # Ensure RGB/RGBA
            if img.mode not in ('RGB', 'RGBA'):
                img = img.convert('RGBA')
            
            # ── Generate all sizes ──
            
            sizes = {
                'favicon-16x16.png': 16,
                'favicon-32x32.png': 32,
                'favicon-48x48.png': 48,
                'favicon-96x96.png': 96,
                'apple-touch-icon.png': 180,
                'favicon-192x192.png': 192,
                'favicon-512x512.png': 512,
            }
            
            for filename, size in sizes.items():
                resized = img.copy()
                resized.thumbnail((size, size), PILImage.LANCZOS)
                
                # Create exact square canvas (centered)
                canvas = PILImage.new('RGBA', (size, size), (0, 0, 0, 0))
                offset_x = (size - resized.width) // 2
                offset_y = (size - resized.height) // 2
                canvas.paste(resized, (offset_x, offset_y))
                
                # Save as PNG
                out_path = os.path.join(favicon_dir, filename)
                canvas.save(out_path, 'PNG', optimize=True)
            
            # ── Generate favicon.ico (multi-size) ──
            ico_sizes = []
            for s in [16, 32, 48]:
                ico_img = img.copy()
                ico_img.thumbnail((s, s), PILImage.LANCZOS)
                canvas = PILImage.new('RGBA', (s, s), (0, 0, 0, 0))
                ox = (s - ico_img.width) // 2
                oy = (s - ico_img.height) // 2
                canvas.paste(ico_img, (ox, oy))
                ico_sizes.append(canvas)
            
            ico_path = os.path.join(favicon_dir, 'favicon.ico')
            ico_sizes[0].save(ico_path, format='ICO', 
                              sizes=[(s.width, s.height) for s in ico_sizes],
                              append_images=ico_sizes[1:])
            
            # ── Build paths dict ──
            base_url = '/static/uploads/favicon'
            generated.update({
                'favicon_ico': f'{base_url}/favicon.ico',
                'favicon_96': f'{base_url}/favicon-96x96.png',
                'favicon_192': f'{base_url}/favicon-192x192.png',
                'favicon_512': f'{base_url}/favicon-512x512.png',
                'apple_touch_icon': f'{base_url}/apple-touch-icon.png',
            })
            
            # ── Generate site.webmanifest ──
            site_name = Settings.get('site_name', 'AliothPress')
            site_tagline = Settings.get('site_tagline', '')
            theme_color = Settings.get('theme_primary_color', '#2563eb')
            
            manifest = {
                'name': site_name,
                'short_name': site_name[:12] if len(site_name) > 12 else site_name,
                'description': site_tagline,
                'icons': [
                    {
                        'src': f'{base_url}/favicon-192x192.png',
                        'sizes': '192x192',
                        'type': 'image/png'
                    },
                    {
                        'src': f'{base_url}/favicon-512x512.png',
                        'sizes': '512x512',
                        'type': 'image/png'
                    }
                ],
                'theme_color': theme_color,
                'background_color': '#ffffff',
                'display': 'standalone',
                'start_url': '/'
            }
            
            manifest_path = os.path.join(favicon_dir, 'site.webmanifest')
            with open(manifest_path, 'w', encoding='utf-8') as f:
                json.dump(manifest, f, indent=2, ensure_ascii=False)
            
            generated['site_webmanifest'] = f'{base_url}/site.webmanifest'
            
            # ── Save all paths to Settings ──
            for key, value in generated.items():
                Settings.set(key, value)
            db.session.commit()
            
            # Clean up temp files
            for tmp in ['source.png', 'source.jpg', 'source.jpeg', 'source.svg', 
                        'source.webp', '_temp_512.png']:
                tmp_path = os.path.join(favicon_dir, tmp)
                if os.path.exists(tmp_path):
                    try:
                        os.remove(tmp_path)
                    except OSError:
                        pass
            
            AuditLog.log('update_favicon', 'settings', None,
                         _ad('favicon_updated', sizes=", ".join(generated.keys())),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
            
            flash(t('flash.favicon_generated'), 'success')
        
        except Exception as e:
            flash(t('flash.favicon_generation_failed', error=str(e)), 'error')
        
        return redirect(request.url)
    
    # GET — show current favicons
    current_favicons = {
        'favicon_ico': Settings.get('favicon_ico', ''),
        'favicon_96': Settings.get('favicon_96', ''),
        'favicon_192': Settings.get('favicon_192', ''),
        'favicon_512': Settings.get('favicon_512', ''),
        'favicon_svg': Settings.get('favicon_svg', ''),
        'apple_touch_icon': Settings.get('apple_touch_icon', ''),
        'site_webmanifest': Settings.get('site_webmanifest', ''),
    }
    
    return render_template('admin/media/favicon.html', favicons=current_favicons)


@app.route('/admin/media/favicon/delete', methods=['POST'])
@login_required
def admin_favicon_delete():
    """Delete all generated favicons and clear settings"""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    # Remove all favicon files from disk
    favicon_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'favicon')
    if os.path.isdir(favicon_dir):
        import shutil
        shutil.rmtree(favicon_dir, ignore_errors=True)
    
    # Clear all favicon settings
    favicon_keys = [
        'favicon_ico', 'favicon_96', 'favicon_192', 'favicon_512',
        'favicon_svg', 'apple_touch_icon', 'site_webmanifest'
    ]
    for key in favicon_keys:
        setting = Settings.query.get(key)
        if setting:
            db.session.delete(setting)
    db.session.commit()
    
    AuditLog.log('delete_favicon', 'settings', None,
                 _ad('favicon_deleted'),
                 user=current_user, ip=request.remote_addr)
    db.session.commit()
    
    flash(t('flash.favicon_deleted'), 'success')
    return redirect(url_for('admin_favicon_manager'))


@app.route('/favicon.ico')
def favicon_ico():
    """Serve favicon.ico from root URL.
    
    Google, Slack, Telegram and many crawlers request /favicon.ico directly.
    This route serves the generated file or returns 404 silently.
    """
    favicon_path = Settings.get('favicon_ico', '')
    if favicon_path:
        # favicon_path is like /static/uploads/favicon/favicon.ico
        # Extract the actual file path on disk
        file_dir = os.path.join(app.config['UPLOAD_FOLDER'], 'favicon')
        file_name = 'favicon.ico'
        if os.path.exists(os.path.join(file_dir, file_name)):
            return send_from_directory(file_dir, file_name, 
                                       mimetype='image/x-icon',
                                       max_age=86400)
    # No favicon generated — return empty 204 (avoids 404 noise in logs)
    return '', 204


# ============================================================================
# SETTINGS ROUTES
# ============================================================================

@app.route('/admin/settings', methods=['GET', 'POST'])
@login_required
def admin_settings():
    """Site settings page"""
    if request.method == 'POST':
        # Update settings
        settings_to_update = [
    'site_name', 'site_tagline', 'site_description', 'site_logo',
    'site_logo_width', 'site_logo_height',
    'homepage_group',
    'google_analytics_id', 'google_tag_manager_id', 'custom_analytics_code',
    'cookie_consent_enabled',
    'default_language', 'active_languages', 'show_language_switcher',
    'site_timezone',
    'footer_copyright', 'footer_license', 'footer_additional', 'footer_columns',
    'social_facebook', 'social_twitter', 'social_linkedin', 'social_github', 'social_youtube', 'social_instagram',
    'org_sameas_wikipedia', 'org_sameas_wikidata', 'org_sameas_crunchbase', 'org_legal_name', 'org_founding_year',
    'smtp_host', 'smtp_port', 'smtp_username', 'smtp_password',
    'smtp_security', 'smtp_from_email', 'smtp_from_name',
    'twofa_site_enabled', 'twofa_email_fallback',
    'license_key'
]
        
        for key in settings_to_update:
            # active_languages is a multi-checkbox — needs getlist
            if key == 'active_languages':
                langs = request.form.getlist('active_languages')
                # Invariant: the default language must always be active.
                # Guards against selecting an inactive default (its pages
                # would 404, including a default-language homepage) and
                # against submitting an empty active set.
                dl = (request.form.get('default_language')
                      or Settings.get('default_language', 'en') or '').strip()
                if dl and dl not in langs:
                    langs.append(dl)
                value = ','.join(langs)
            elif key == 'show_language_switcher':
                value = 'true' if request.form.get('show_language_switcher') else 'false'
            elif key == 'site_timezone':
                # Validate against the IANA database; anything unknown falls
                # back to the previously saved value (or UTC) so a tampered
                # form can never break date rendering site-wide.
                value = request.form.get('site_timezone', '').strip() or 'UTC'
                if value != 'UTC':
                    try:
                        ZoneInfo(value)
                    except Exception:
                        value = Settings.get('site_timezone', 'UTC') or 'UTC'
            elif key == 'cookie_consent_enabled':
                value = 'true' if request.form.get('cookie_consent_enabled') else 'false'
            elif key == 'smtp_password':
                # Encrypt SMTP password before storing
                raw_password = request.form.get('smtp_password', '')
                if raw_password:
                    # Only update if user entered a new password (not the placeholder)
                    value = encrypt_smtp_password(raw_password)
                    key = 'smtp_password_encrypted'
                else:
                    continue  # Skip — keep existing encrypted password
            elif key == 'smtp_security':
                # Radio button: 'tls' or 'ssl' → store as two separate keys
                security = request.form.get('smtp_security', 'tls')
                Settings.query.filter_by(key='smtp_use_tls').delete()
                Settings.query.filter_by(key='smtp_use_ssl').delete()
                db.session.add(Settings(key='smtp_use_tls', value='true' if security == 'tls' else 'false'))
                db.session.add(Settings(key='smtp_use_ssl', value='true' if security == 'ssl' else 'false'))
                continue  # Already handled
            elif key == 'license_key':
                raw_key = request.form.get('license_key', '').strip()
                if raw_key:
                    # ── Step 1: Save encrypted key to DB FIRST ──
                    # _compute_license_signature() reads license_key_encrypted
                    # from the database, so it MUST exist before we call
                    # _save_license_result(), otherwise the HMAC signature
                    # will be computed against an empty string and
                    # _verify_license_integrity() will always fail.
                    encrypted = encrypt_smtp_password(raw_key)
                    Settings.set('license_key_encrypted', encrypted)
                    
                    # ── Step 2: Validate against update server ──
                    try:
                        validator = LicenseValidator(app.config.get('UPDATE_SERVER_URL', ''))
                        site_url = request.url_root.rstrip('/')
                        result = validator.validate(raw_key, site_url)
                        _save_license_result(result)
                        if result.get('valid'):
                            AuditLog.log('activate_license', 'settings', None,
                                         _ad('license_activated', platform=result.get('platform', 'unknown')),
                                         user=current_user, ip=request.remote_addr)
                        elif result.get('error'):
                            flash(result['error'], 'error')
                    except Exception:
                        _save_license_invalid()
                        flash(t('flash.license_validation_failed'), 'warning')
                    continue  # Already saved — skip the generic loop write
                else:
                    continue  # Skip — keep existing key
            else:
                value = request.form.get(key, '')
            setting = Settings.query.filter_by(key=key).first()
            if setting:
                setting.value = value
            else:
                setting = Settings(key=key, value=value)
                db.session.add(setting)

        db.session.commit()
        
        AuditLog.log('edit_settings', 'settings', None,
                     _ad('settings_updated'),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        flash(t('flash.settings_saved'), 'success')
        return redirect(url_for('admin_settings'))
    
    # Get current settings
    settings = {}
    for s in Settings.query.all():
        settings[s.key] = s.value
    
    # Parse active languages (comma-separated)
    active_langs = settings.get('active_languages', 'en').split(',')
    
    # Get all pages for homepage selector
    homepage_pages = Page.query.filter_by(status='published').order_by(Page.title).all()
    
    # Decrypt license key for eye-toggle display
    _license_key_raw = ''
    _enc = Settings.get('license_key_encrypted', '')
    if _enc:
        try:
            _license_key_raw = decrypt_smtp_password(_enc)
        except Exception:
            pass
    
    return render_template('admin/settings.html', 
                         settings=settings, 
                         available_languages=AVAILABLE_LANGUAGES,
                         active_langs=active_langs,
                         timezones=timezone_choices(),
                         homepage_pages=homepage_pages,
                         smtp_has_password=bool(settings.get('smtp_password_encrypted', '')),
                         license_info=get_license_status(Settings.get),
                         license_key_raw=_license_key_raw,
                         cms_version=get_current_version(),
                         ssl_domain=Settings.get('site_domain', ''),
                         ssl_enabled=Settings.get('ssl_enabled', 'false') == 'true')

# ============================================================================
# EMAIL FUNCTIONALITY
# ============================================================================

# ── License & Update API routes ──────────────────────────────────────────────

@app.route('/admin/license/deactivate', methods=['POST'])
@login_required
def admin_license_deactivate():
    """Deactivate current license key"""
    if not current_user.can_access_settings():
        return jsonify({'success': False, 'error': t('api.access_denied')}), 403
    
    try:
        encrypted_key = Settings.get('license_key_encrypted', '')
        if encrypted_key:
            raw_key = decrypt_smtp_password(encrypted_key)
            validator = LicenseValidator(app.config.get('UPDATE_SERVER_URL', ''))
            deact_result = validator.deactivate(raw_key, request.url_root.rstrip('/'))
            if not deact_result.get('success') and deact_result.get('error'):
                return jsonify({'success': False, 'error': deact_result['error']})
        
        # Clear all license data
        for k in ['license_key_encrypted', 'license_valid', 'license_platform', 'license_plan', 'license_last_checked']:
            setting = Settings.query.get(k)
            if setting:
                db.session.delete(setting)
        db.session.commit()
        
        AuditLog.log('deactivate_license', 'settings', None,
                     _ad('license_deactivated'),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})


@app.route('/admin/update/check', methods=['POST'])
@login_required
def admin_update_check():
    """Check for available updates"""
    if not current_user.can_access_settings():
        return jsonify({'error': t('api.access_denied')}), 403
    
    try:
        encrypted_key = Settings.get('license_key_encrypted', '')
        raw_key = decrypt_smtp_password(encrypted_key) if encrypted_key else ''
        
        checker = UpdateChecker(app.config.get('UPDATE_SERVER_URL', ''))
        result = checker.check(raw_key)
        
        # Update last checked timestamp
        Settings.set('license_last_checked', to_site_time(utc_now()).strftime('%Y-%m-%d %H:%M'))
        db.session.commit()
        
        return jsonify(result)
    except Exception as e:
        return jsonify({'error': str(e), 'update_available': False})


@app.route('/admin/update/perform', methods=['POST'])
@login_required
def admin_update_perform():
    """Download and apply update"""
    if not current_user.can_access_settings():
        return jsonify({'success': False, 'error': t('api.access_denied')}), 403
    
    try:
        encrypted_key = Settings.get('license_key_encrypted', '')
        raw_key = decrypt_smtp_password(encrypted_key) if encrypted_key else ''
        
        update_server = app.config.get('UPDATE_SERVER_URL', '')
        
        # Step 1: Check what's available
        checker = UpdateChecker(update_server)
        update_info = checker.check(raw_key)
        
        if not update_info.get('update_available'):
            return jsonify({'success': False, 'error': t('settings.license_up_to_date')})
        
        # No download_url means the update server requires a valid license
        if not update_info.get('download_url'):
            return jsonify({'success': False, 'error': t('settings.license_free_note_short')})
        
        # Step 2: Create backup
        performer = UpdatePerformer(update_server)
        backup_path = performer.create_pre_update_backup()
        if not backup_path:
            return jsonify({'success': False, 'error': t('api.backup_failed')})
        
        AuditLog.log('update_backup', 'system', None,
                     _ad('update_backup'),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        # Step 3: Download
        zip_path = performer.download_update(
            update_info['download_url'],
            update_info.get('checksum', ''),
            license_key=raw_key
        )
        if not zip_path:
            return jsonify({'success': False, 'error': t('api.download_failed')})
        
        # Step 4: Apply
        success, message = performer.apply_update(zip_path, update_info['latest_version'])
        
        if success:
            AuditLog.log('update_cms', 'system', None,
                         _ad('updated_cms', version=update_info['latest_version']),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
            
            # Schedule graceful restart so the new code is loaded
            restart_method, restart_msg = performer.schedule_restart(delay_seconds=4)
            needs_manual = restart_method == 'manual'
            
            return jsonify({
                'success': True,
                'message': message,
                'restart_msg': restart_msg,
                'needs_manual_restart': needs_manual,
                'auto_reload_delay': 6 if not needs_manual else 0,
            })
        else:
            # Rollback
            performer.rollback(backup_path)
            AuditLog.log('update_failed', 'system', None,
                         _ad('update_rolled_back', error=message),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
            return jsonify({'success': False, 'error': message})
    
    except Exception as e:
        AuditLog.log('update_failed', 'system', None,
                     _ad('update_error', error=str(e)),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        return jsonify({'success': False, 'error': str(e)})

# ============================================================================
# SSL / HTTPS SETUP — Let's Encrypt via admin panel
# ============================================================================

import socket
import subprocess


@app.route('/admin/ssl/setup', methods=['POST'])
@login_required
@limiter.limit("5 per hour")
def admin_ssl_setup():
    """Enable HTTPS via Let's Encrypt for a custom domain.
    
    Security:
    - Only owner/admin can access (can_access_settings)
    - Domain is validated with strict regex (no shell injection possible)
    - DNS is checked to confirm domain points to this server
    - Actual work is done by /usr/local/bin/aliothpress-ssl (runs as root via sudoers)
    - Flask never writes to nginx config directly
    - Rate limited to 5/hour (Let's Encrypt has its own limits too)
    - Logged to audit log
    """
    if not current_user.can_access_settings():
        return jsonify({'success': False, 'error': t('api.access_denied')}), 403
    
    data = request.get_json()
    if not data:
        return jsonify({'success': False, 'error': t('api.no_data_provided')}), 400
    
    domain = data.get('domain', '').strip().lower()
    
    # ── Strict domain validation (prevents shell injection) ─────────
    if not domain:
        return jsonify({'success': False, 'error': t('ssl.error_empty_domain')})
    
    if not re.match(r'^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*\.[a-z]{2,}$', domain):
        return jsonify({'success': False, 'error': t('ssl.error_invalid_domain')})
    
    if len(domain) > 253:
        return jsonify({'success': False, 'error': t('ssl.error_domain_too_long')})
    
    # ── DNS check — domain must resolve to this server ──────────────
    try:
        resolved_ip = socket.gethostbyname(domain)
    except socket.gaierror:
        return jsonify({'success': False, 'error': t('ssl.error_dns_failed', domain=domain)})
    
    # Get this server's public IP (must be IPv4 to match gethostbyname above)
    server_ip = None
    try:
        import requests as http_requests
        server_ip = http_requests.get('https://api.ipify.org', timeout=5).text.strip()
    except Exception:
        try:
            import requests as http_requests
            server_ip = http_requests.get('https://ifconfig.me', timeout=5, headers={'Accept': 'text/plain'}).text.strip()
        except Exception:
            pass
    
    if server_ip and resolved_ip != server_ip:
        return jsonify({
            'success': False,
            'error': t('ssl.error_dns_mismatch', domain=domain,
                       resolved=resolved_ip, server=server_ip)
        })
    
    # ── Check that the SSL helper script exists ─────────────────────
    ssl_helper = '/usr/local/bin/aliothpress-ssl'
    if not os.path.isfile(ssl_helper):
        return jsonify({
            'success': False,
            'error': t('ssl.error_not_cloud')
        })
    
    # ── Run the SSL helper script via sudo ──────────────────────────
    admin_email = current_user.email
    
    try:
        result = subprocess.run(
            ['sudo', ssl_helper, domain, admin_email],
            capture_output=True, text=True, timeout=120
        )
        
        output = (result.stdout + result.stderr).strip()
        
        if result.returncode != 0:
            # Parse error from helper script
            error_msg = output
            if 'ERROR:' in error_msg:
                error_msg = error_msg.split('ERROR:', 1)[1].strip()
            
            # Friendly messages for common certbot errors
            error_lower = error_msg.lower()
            if 'dns_server_ip_unknown' in error_lower:
                error_msg = t('ssl.error_server_ip_unknown')
            elif 'dns_apex_mismatch' in error_lower:
                error_msg = t('ssl.error_dns_apex_mismatch', domain=domain)
            elif 'dns_www_missing' in error_lower:
                error_msg = t('ssl.error_dns_www_missing', domain=domain)
            elif 'too many certificates' in error_lower or 'rate limit' in error_lower:
                error_msg = t('ssl.error_rate_limit')
            elif 'connection refused' in error_lower or 'could not connect' in error_lower:
                error_msg = t('ssl.error_firewall', domain=domain)
            elif 'unauthorized' in error_lower or 'challenge failed' in error_lower:
                error_msg = t('ssl.error_challenge', domain=domain)
            
            AuditLog.log('ssl_setup_failed', 'settings', None,
                         _ad('ssl_failed', domain=domain, error=error_msg[:200]),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
            
            return jsonify({'success': False, 'error': error_msg[:500]})
        
        # ── Success — save domain in settings ───────────────────────
        Settings.set('site_domain', domain)
        Settings.set('ssl_enabled', 'true')
        db.session.commit()
        
        AuditLog.log('ssl_setup', 'settings', None,
                     _ad('ssl_enabled', domain=domain),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        return jsonify({
            'success': True,
            'message': t('ssl.success', domain=domain)
        })
    
    except subprocess.TimeoutExpired:
        AuditLog.log('ssl_setup_failed', 'settings', None,
                     _ad('ssl_timeout', domain=domain),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        return jsonify({'success': False, 'error': t('ssl.error_timeout')})
    
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})

def get_smtp_settings():
    """Get SMTP settings from database"""
    return {
        'host': Settings.get('smtp_host', ''),
        'port': int(Settings.get('smtp_port', '587')),
        'username': Settings.get('smtp_username', ''),
        'password': decrypt_smtp_password(Settings.get('smtp_password_encrypted', '')),
        'use_tls': Settings.get('smtp_use_tls', 'true') == 'true',
        'use_ssl': Settings.get('smtp_use_ssl', 'false') == 'true',
        'from_email': Settings.get('smtp_from_email', ''),
        'from_name': Settings.get('smtp_from_name', ''),
    }


def encrypt_smtp_password(plain_password):
    """Encrypt sensitive data for storage in database.
    Uses Fernet symmetric encryption (AES-128-CBC) with SECRET_KEY as basis.
    """
    if not plain_password:
        return ''
    from cryptography.fernet import Fernet
    import base64
    # Derive a Fernet-compatible key from SECRET_KEY
    key = base64.urlsafe_b64encode(
        hashlib.sha256(app.config['SECRET_KEY'].encode()).digest()
    )
    f = Fernet(key)
    return f.encrypt(plain_password.encode()).decode()


def decrypt_smtp_password(encrypted_password):
    """Decrypt sensitive data from database"""
    if not encrypted_password:
        return ''
    try:
        if encrypted_password.startswith('b64:'):
            # Legacy fallback — migrate to Fernet on next save
            import base64
            return base64.b64decode(encrypted_password[4:].encode()).decode()
        from cryptography.fernet import Fernet
        import base64
        key = base64.urlsafe_b64encode(
            hashlib.sha256(app.config['SECRET_KEY'].encode()).digest()
        )
        f = Fernet(key)
        return f.decrypt(encrypted_password.encode()).decode()
    except Exception:
        return ''


def send_email(to_email, subject, body_html, body_text=None):
    """Send email using SMTP settings from database.
    
    Args:
        to_email: recipient email address
        subject: email subject
        body_html: HTML body content
        body_text: plain text body (auto-generated from HTML if not provided)
    
    Returns:
        (success: bool, message: str)
    """
    import smtplib
    from email.mime.text import MIMEText
    from email.mime.multipart import MIMEMultipart
    
    smtp = get_smtp_settings()
    
    # Validate settings
    if not smtp['host'] or not smtp['from_email']:
        return False, 'SMTP not configured. Please fill in Email Settings.'
    
    # Build message
    msg = MIMEMultipart('alternative')
    msg['From'] = f"{smtp['from_name']} <{smtp['from_email']}>" if smtp['from_name'] else smtp['from_email']
    msg['To'] = to_email
    msg['Subject'] = subject
    
    # Plain text fallback
    if not body_text:
        import re
        body_text = re.sub(r'<[^>]+>', '', body_html)
        body_text = re.sub(r'\s+', ' ', body_text).strip()
    
    msg.attach(MIMEText(body_text, 'plain', 'utf-8'))
    msg.attach(MIMEText(body_html, 'html', 'utf-8'))
    
    try:
        if smtp['use_ssl']:
            # SSL connection (port 465 typically)
            server = smtplib.SMTP_SSL(smtp['host'], smtp['port'], timeout=15)
        else:
            server = smtplib.SMTP(smtp['host'], smtp['port'], timeout=15)
            if smtp['use_tls']:
                server.starttls()
        
        if smtp['username'] and smtp['password']:
            server.login(smtp['username'], smtp['password'])
        
        server.sendmail(smtp['from_email'], to_email, msg.as_string())
        server.quit()
        
        return True, t('flash.email_sent_ok')
    except smtplib.SMTPAuthenticationError:
        return False, t('flash.smtp_auth_failed')
    except smtplib.SMTPConnectError:
        return False, t('flash.smtp_connect_failed', host=smtp['host'], port=smtp['port'])
    except smtplib.SMTPRecipientsRefused:
        return False, t('flash.smtp_recipient_rejected', email=to_email)
    except Exception as e:
        return False, t('flash.smtp_error', error=str(e))


@app.route('/admin/email/test', methods=['POST'])
@login_required
def admin_email_test():
    """Send a test email to verify SMTP settings"""
    test_address = request.form.get('test_email', current_user.email)
    is_ajax = request.headers.get('X-Requested-With') == 'XMLHttpRequest'
    
    if not test_address:
        if is_ajax:
            return jsonify({'success': False, 'message': t('flash.email_address_required')})
        flash(t('flash.email_address_required'), 'error')
        return redirect(url_for('admin_settings'))
    
    site_name = Settings.get('site_name', 'AliothPress')
    powered_by = '' if _verify_license_integrity() else ' &bull; Powered by <a href="https://aliothpress.com" style="color: #a0aec0; text-decoration: underline;">AliothPress</a>'
    
    subject = t('email.test_subject', site_name=site_name)
    body_html = f"""
    <div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 600px; margin: 0 auto; padding: 40px 20px;">
        <div style="background: #667eea; color: white; padding: 30px; border-radius: 12px 12px 0 0; text-align: center;">
            <h1 style="margin: 0; font-size: 24px;">{site_name}</h1>
            <p style="margin: 8px 0 0; opacity: 0.9;">{t('email.test_subtitle')}</p>
        </div>
        <div style="background: white; padding: 30px; border: 1px solid #e2e8f0; border-top: none; border-radius: 0 0 12px 12px;">
            <p style="font-size: 16px; color: #2d3748;">{t('email.test_body')}</p>
            <p style="font-size: 14px; color: #718096;">{t('email.test_body_detail', site_name=site_name)}</p>
            <hr style="border: none; border-top: 1px solid #e2e8f0; margin: 20px 0;">
            <p style="font-size: 12px; color: #a0aec0; text-align: center;">{t('email.sent_by', site_name=site_name)}{powered_by}</p>
        </div>
    </div>
    """
    
    success, message = send_email(test_address, subject, body_html)
    
    if is_ajax:
        return jsonify({
            'success': success,
            'message': t('flash.email_test_sent', email=test_address) if success else message
        })
    
    if success:
        flash(t('flash.email_test_sent', email=test_address), 'success')
    else:
        flash(t('flash.email_test_failed', message=message), 'error')
    
    return redirect(url_for('admin_settings'))


# ============================================================================
# ROLE-BASED ACCESS CONTROL
# ============================================================================

def admin_required(f):
    """Decorator: only owner and admin roles can access this route"""
    @wraps(f)
    @login_required
    def decorated_function(*args, **kwargs):
        if not current_user.can_access_settings():
            flash(t('flash.permission_denied'), 'error')
            return redirect(url_for('admin_dashboard'))
        return f(*args, **kwargs)
    return decorated_function


def owner_required(f):
    """Decorator: only the site owner can access this route"""
    @wraps(f)
    @login_required
    def decorated_function(*args, **kwargs):
        if not current_user.is_owner():
            flash(t('flash.owner_only'), 'error')
            return redirect(url_for('admin_dashboard'))
        return f(*args, **kwargs)
    return decorated_function


# ============================================================================
# USER MANAGEMENT ROUTES
# ============================================================================

@app.route('/admin/users')
@admin_required
def admin_users_list():
    """List all users"""
    users = User.query.order_by(User.created_at.asc()).all()
    return render_template('admin/users.html', users=users)


@app.route('/admin/users/invite', methods=['POST'])
@admin_required
def admin_users_invite():
    """Send invitation email to a new user"""
    import secrets
    
    email = request.form.get('invite_email', '').strip()
    role = request.form.get('invite_role', 'editor')
    
    if not email:
        flash(t('flash.invite_email_required'), 'error')
        return redirect(url_for('admin_users_list'))
    
    # Server-side validation: `type="email"` in the browser checks format but
    # not length — a 212-char address passes the form and used to crash on
    # the VARCHAR(120) column. The generic length safety net would only
    # silently truncate it into a wrong address, so reject explicitly.
    if len(email) > 120 or not re.fullmatch(r'[^@\s]+@[^@\s]+\.[^@\s]+', email):
        flash(t('flash.invite_email_invalid'), 'error')
        return redirect(url_for('admin_users_list'))
    
    # Check if email already exists
    if User.query.filter_by(email=email).first():
        flash(t('flash.invite_email_exists', email=email), 'error')
        return redirect(url_for('admin_users_list'))
    
    # Validate role
    if role not in ('admin', 'editor'):
        role = 'editor'
    
    # Only owner can invite admins
    if role == 'admin' and not current_user.is_owner():
        flash(t('flash.invite_admin_owner_only'), 'error')
        return redirect(url_for('admin_users_list'))
    
    # Generate invite token
    token = secrets.token_urlsafe(32)
    expires = utc_now() + timedelta(days=7)
    
    # Create placeholder user with invite token
    invite_user = User(
        username=f'_pending_{token[:24]}',
        email=email,
        password=generate_password_hash(secrets.token_urlsafe(16)),  # Random, will be reset
        role=role,
        is_admin=(role in ('admin', 'owner')),
        is_active_account=False,  # Not active until they accept
        invite_token=token,
        invite_expires=expires,
        admin_language=Settings.get('default_language', 'en'),
    )
    
    db.session.add(invite_user)
    db.session.flush()
    AuditLog.log('invite_user', 'user', invite_user.id,
                 _ad('invited_user', email=email, role=role),
                 current_user, request.remote_addr)
    db.session.commit()
    
    # Build invite URL
    invite_url = request.url_root.rstrip('/') + url_for('accept_invite', token=token)
    site_name = Settings.get('site_name', 'AliothPress')
    inviter_name = current_user.get_display_name()
    lang = current_user.admin_language or 'en'
    role_label = t_for('email.role_admin', lang) if role == 'admin' else t_for('email.role_editor', lang)
    powered_by = '' if _verify_license_integrity() else ' &bull; Powered by <a href="https://aliothpress.com" style="color: #a0aec0; text-decoration: underline;">AliothPress</a>'
    
    # Send invitation email (in inviter's language — best guess for invitee)
    subject = t_for('email.invite_subject', lang, site_name=site_name)
    body_html = f"""
    <div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 600px; margin: 0 auto; padding: 40px 20px;">
        <div style="background: #667eea; color: white; padding: 30px; border-radius: 12px 12px 0 0; text-align: center;">
            <h1 style="margin: 0; font-size: 24px;">{site_name}</h1>
            <p style="margin: 8px 0 0; opacity: 0.9;">{t_for('email.invite_subtitle', lang)}</p>
        </div>
        <div style="background: white; padding: 30px; border: 1px solid #e2e8f0; border-top: none; border-radius: 0 0 12px 12px;">
            <p style="font-size: 16px; color: #2d3748;">{t_for('email.invite_body', lang, inviter=inviter_name, site_name=site_name, role=role_label)}</p>
            <p style="font-size: 14px; color: #718096; margin: 16px 0;">{t_for('email.invite_click_hint', lang)}</p>
            <div style="text-align: center; margin: 24px 0;">
                <a href="{invite_url}" style="display: inline-block; padding: 14px 32px; background: #667eea; color: white; text-decoration: none; border-radius: 8px; font-weight: 600; font-size: 16px;">{t_for('email.invite_cta', lang)}</a>
            </div>
            <p style="font-size: 13px; color: #a0aec0;">{t_for('email.invite_expires', lang)}</p>
            <hr style="border: none; border-top: 1px solid #e2e8f0; margin: 20px 0;">
            <p style="font-size: 12px; color: #a0aec0; text-align: center;">{t_for('email.sent_by', lang, site_name=site_name)}{powered_by}</p>
        </div>
    </div>
    """
    
    success, message = send_email(email, subject, body_html)
    
    if success:
        flash(t('flash.invite_sent', email=email), 'success')
    else:
        flash(t('flash.invite_email_failed', message=message, url=invite_url), 'warning')
    
    return redirect(url_for('admin_users_list'))


@app.route('/invite/<token>', methods=['GET', 'POST'])
def accept_invite(token):
    """Accept invitation and set up account"""
    user = User.query.filter_by(invite_token=token).first()
    
    if not user:
        flash(t('flash.invite_invalid'), 'error')
        return redirect(url_for('admin_login'))
    
    if user.invite_expires and user.invite_expires < utc_now():
        flash(t('flash.invite_expired'), 'error')
        return redirect(url_for('admin_login'))
    
    if request.method == 'POST':
        username = request.form.get('username', '').strip()
        password = request.form.get('password', '')
        confirm_password = request.form.get('confirm_password', '')
        display_name = request.form.get('display_name', '').strip()
        
        # Validation
        if not username or not password:
            flash(t('flash.username_password_required'), 'error')
            return render_template('admin/accept_invite.html', user=user, token=token)
        
        if len(password) < 8:
            flash(t('flash.password_min_length'), 'error')
            return render_template('admin/accept_invite.html', user=user, token=token)
        
        is_valid, pwd_error = validate_password(password)
        if not is_valid:
            flash(pwd_error, 'error')
            return render_template('admin/accept_invite.html', user=user, token=token)
        
        if password != confirm_password:
            flash(t('flash.passwords_mismatch'), 'error')
            return render_template('admin/accept_invite.html', user=user, token=token)
        
        # Check username uniqueness
        existing = User.query.filter(User.username == username, User.id != user.id).first()
        if existing:
            flash(t('flash.username_taken'), 'error')
            return render_template('admin/accept_invite.html', user=user, token=token)
        
        # Activate account
        user.username = username
        user.password = generate_password_hash(password)
        user.display_name = display_name or username
        user.is_active_account = True
        user.invite_token = None
        user.invite_expires = None
        
        AuditLog.log('accept_invite', 'user', user.id,
                     _ad('accepted_invite', name=user.get_display_name()),
                     user, request.remote_addr)
        db.session.commit()
        
        flash(t('flash.account_created'), 'success')
        return redirect(url_for('admin_login'))
    
    return render_template('admin/accept_invite.html', user=user, token=token)


@app.route('/admin/users/<int:user_id>/edit', methods=['POST'])
@admin_required
def admin_users_edit(user_id):
    """Change user role"""
    user = User.query.get_or_404(user_id)
    new_role = request.form.get('role', 'editor')
    
    # Cannot change owner's role
    if user.is_owner():
        flash(t('flash.cannot_change_owner_role'), 'error')
        return redirect(url_for('admin_users_list'))
    
    # Only owner can promote to admin
    if new_role == 'admin' and not current_user.is_owner():
        flash(t('flash.owner_assign_admin_only'), 'error')
        return redirect(url_for('admin_users_list'))
    
    if new_role in ('admin', 'editor'):
        user.role = new_role
        user.is_admin = (new_role == 'admin')
        AuditLog.log('change_user_role', 'user', user.id,
                     _ad('changed_user_role', name=user.get_display_name(), role=new_role),
                     current_user, request.remote_addr)
        db.session.commit()
        flash(t('flash.role_changed', name=user.get_display_name(), role=new_role), 'success')
    
    return redirect(url_for('admin_users_list'))


@app.route('/admin/users/<int:user_id>/delete', methods=['POST'])
@admin_required
def admin_users_delete(user_id):
    """Delete a user (cannot delete owner or self)"""
    user = User.query.get_or_404(user_id)
    
    if user.is_owner():
        flash(t('flash.cannot_delete_owner'), 'error')
        return redirect(url_for('admin_users_list'))
    
    if user.id == current_user.id:
        flash(t('flash.cannot_delete_self'), 'error')
        return redirect(url_for('admin_users_list'))
    
    name = user.get_display_name()
    AuditLog.log('delete_user', 'user', user.id,
                 _ad('deleted_user', name=name),
                 current_user, request.remote_addr)
    db.session.delete(user)
    db.session.commit()
    flash(t('flash.user_deleted', name=name), 'success')
    return redirect(url_for('admin_users_list'))


@app.route('/admin/profile', methods=['GET', 'POST'])
@login_required
def admin_profile():
    """Edit own profile (display name, bio, avatar) — available to all roles"""
    if request.method == 'POST':
        current_user.display_name = request.form.get('display_name', '').strip()
        current_user.bio = request.form.get('bio', '').strip()
        current_user.avatar_url = request.form.get('avatar_url', '').strip()
        
        # AEO — Author entity links
        if hasattr(current_user, 'linkedin_url'):
            current_user.linkedin_url = request.form.get('linkedin_url', '').strip()
        if hasattr(current_user, 'website_url'):
            current_user.website_url = request.form.get('website_url', '').strip()
        if hasattr(current_user, 'github_url'):
            current_user.github_url = request.form.get('github_url', '').strip()
        
        # Admin interface language
        new_lang = request.form.get('admin_language', 'en').strip()
        if new_lang in LANG_CODE_TO_NAME:
            current_user.admin_language = new_lang
        
        # Admin interface theme
        new_theme = request.form.get('admin_theme', 'light').strip()
        if new_theme in ('light', 'dark', 'system'):
            current_user.admin_theme = new_theme
        
        # Allow changing email
        new_email = request.form.get('email', '').strip()
        if new_email and new_email != current_user.email:
            existing = User.query.filter(User.email == new_email, User.id != current_user.id).first()
            if existing:
                flash(t('flash.email_in_use'), 'error')
                return redirect(url_for('admin_profile'))
            current_user.email = new_email
        
        # Allow changing password (requires current password for security)
        new_password = request.form.get('new_password', '')
        confirm_password = request.form.get('confirm_password', '')
        if new_password:
            current_password = request.form.get('current_password', '')
            if not check_password_hash(current_user.password, current_password):
                flash(t('flash.current_password_wrong'), 'error')
                return redirect(url_for('admin_profile'))
            is_valid, pwd_error = validate_password(new_password)
            if not is_valid:
                flash(pwd_error, 'error')
                return redirect(url_for('admin_profile'))
            if new_password != confirm_password:
                flash(t('flash.passwords_mismatch'), 'error')
                return redirect(url_for('admin_profile'))
            current_user.password = generate_password_hash(new_password)
            # Invalidate all other sessions by incrementing version
            current_user.session_version = (current_user.session_version or 0) + 1
            session['_session_version'] = current_user.session_version
        
        AuditLog.log('update_profile', 'user', current_user.id,
                     _ad('updated_profile'),
                     current_user, request.remote_addr)
        db.session.commit()
        flash(t('flash.profile_updated'), 'success')
        return redirect(url_for('admin_profile'))
    
    return render_template('admin/profile.html', available_languages=AVAILABLE_LANGUAGES)


# ============================================================================
# MENU PICKER API (paginated content for menu builder)
# ============================================================================

@app.route('/admin/api/menu-picker', methods=['GET'])
@login_required
def admin_api_menu_picker():
    """Paginated JSON endpoint for the menu content picker (pages/posts)."""
    content_type = request.args.get('type', 'pages')  # 'pages' or 'posts'
    search = request.args.get('search', '').strip()
    lang = request.args.get('lang', 'all')
    sort = request.args.get('sort', '')          # title-asc, title-desc, date-asc, date-desc
    page = get_page()
    per_page = request.args.get('per_page', 20, type=int)
    per_page = min(max(1, per_page), 100)

    if content_type == 'posts':
        query = Post.query.filter_by(status='published')
    else:
        query = Page.query.filter_by(status='published')

    # Search
    if search:
        query = query.filter(
            (Post if content_type == 'posts' else Page).title.ilike(f'%{search}%')
        )

    # Language filter
    if lang and lang != 'all':
        query = query.filter_by(language=lang)

    # Sort
    Model = Post if content_type == 'posts' else Page
    if sort == 'title-desc':
        query = query.order_by(Model.title.desc())
    elif sort == 'date-asc':
        query = query.order_by(Model.published_at.asc().nullslast())
    elif sort == 'date-desc':
        query = query.order_by(Model.published_at.desc().nullslast())
    else:
        query = query.order_by(Model.title.asc())

    # Paginate
    pagination = query.paginate(page=page, per_page=per_page, error_out=False)

    items = []
    for item in pagination.items:
        published_str = ''
        if item.published_at:
            published_str = to_site_time(item.published_at).strftime('%b %d, %Y')
        items.append({
            'id': item.id,
            'title': item.title,
            'slug': item.slug,
            'language': item.language,
            'published_at': published_str,
            'translation_group': item.translation_group or '',
        })

    return jsonify({
        'items': items,
        'page': pagination.page,
        'per_page': per_page,
        'total': pagination.total,
        'total_pages': pagination.pages,
    })


# ============================================================================
# MENU BUILDER ROUTE
# ============================================================================

@app.route('/admin/menus', methods=['GET', 'POST'])
@login_required
def admin_menus():
    """Menu builder — supports pages, posts, and multilingual custom labels with submenus"""
    if request.method == 'POST':
        import json as _json
        location = request.form.get('menu_location', 'header')
        active_codes = _get_active_language_codes()

        # ── 1. Delete old items for this location ──────────────────────────────
        # Delete children first (foreign key), then parents
        all_old = MenuItem.query.filter_by(menu_location=location).all()
        old_ids = {item.id for item in all_old}
        # Nullify parent_id on children that point into this location before delete
        MenuItem.query.filter(
            MenuItem.parent_id.in_(old_ids)
        ).update({MenuItem.parent_id: None}, synchronize_session=False)
        MenuItem.query.filter_by(menu_location=location).delete(synchronize_session=False)
        db.session.flush()

        # ── 2. Helper: build one MenuItem from form data ──────────────────────
        def _build_item(item_type, content_id, order_idx, parent_db_id=None):
            """Create and add a MenuItem; return it (not yet committed)."""
            labels = {}
            prefix = f'{item_type}_{content_id}'
            custom_label = request.form.get(f'label_{prefix}', '').strip()

            if item_type == 'page':
                obj = Page.query.get(content_id)
                if not obj:
                    return None
                if custom_label:
                    labels[obj.language] = custom_label
                # Collect custom labels from translation siblings (via DB lookup)
                if obj.translation_group:
                    siblings = Page.query.filter(
                        Page.translation_group == obj.translation_group,
                        Page.id != obj.id,
                        Page.status == 'published'
                    ).all()
                    for sibling in siblings:
                        lbl = request.form.get(f'label_page_{sibling.id}', '').strip()
                        if lbl:
                            labels[sibling.language] = lbl
                item = MenuItem(
                    menu_location=location,
                    label=custom_label or obj.title,
                    url=f'/{obj.slug}',
                    page_id=obj.id,
                    page_type='page',
                    language=obj.language,
                    order=order_idx,
                    parent_id=parent_db_id,
                    labels_json=_json.dumps(labels) if labels else None,
                )
            elif item_type == 'post':
                obj = Post.query.get(content_id)
                if not obj:
                    return None
                if custom_label:
                    labels[obj.language] = custom_label
                # Collect custom labels from translation siblings (via DB lookup)
                if obj.translation_group:
                    siblings = Post.query.filter(
                        Post.translation_group == obj.translation_group,
                        Post.id != obj.id,
                        Post.status == 'published'
                    ).all()
                    for sibling in siblings:
                        lbl = request.form.get(f'label_post_{sibling.id}', '').strip()
                        if lbl:
                            labels[sibling.language] = lbl
                item = MenuItem(
                    menu_location=location,
                    label=custom_label or obj.title,
                    url=f'/{obj.slug}',
                    post_id=obj.id,
                    page_type='post',
                    language=obj.language,
                    order=order_idx,
                    parent_id=parent_db_id,
                    labels_json=_json.dumps(labels) if labels else None,
                )
            elif item_type == 'custom':
                # Custom links are identified by their temp_id (no DB id yet
                # at submit time). Form fields are keyed by temp_id:
                #   custom_url_<tid>_<lang>   — per-language URL
                #   custom_label_<tid>_<lang> — per-language label
                #   custom_target_<tid>      — _self / _blank
                tid = content_id  # for custom items, "id" is the temp_id
                urls = {}
                labels_per_lang = {}
                # Whitelisted URL schemes; anything else is dropped. Internal
                # paths must start with a single '/', not '//' (protocol-relative
                # URLs would resolve against the current scheme to an external host).
                explicit_schemes = ('http://', 'https://', 'mailto:', 'tel:')
                def _is_allowed_url(s):
                    s_low = s.lower()
                    if s_low.startswith(explicit_schemes):
                        return True
                    # Internal/relative path: single '/' only, not '//'
                    if s_low.startswith('/') and not s_low.startswith('//'):
                        return True
                    return False
                for code in active_codes:
                    u = request.form.get(f'custom_url_{tid}_{code}', '').strip()
                    l = request.form.get(f'custom_label_{tid}_{code}', '').strip()
                    if u and not _is_allowed_url(u):
                        u = ''   # silently drop unsafe URL
                    if u and l:
                        urls[code] = u
                        labels_per_lang[code] = l
                if not urls:
                    # Nothing usable submitted — skip this item entirely
                    return None
                custom_target = request.form.get(f'custom_target_{tid}', '_self')
                if custom_target not in ('_self', '_blank'):
                    custom_target = '_self'
                # Pick any language as the legacy `url`/`label` (for fallback
                # in resolve_single when urls_json is read by old code paths).
                # Prefer default_lang, else first available.
                default_lang_local = Settings.get('default_language', 'en')
                fallback_lang = (default_lang_local if default_lang_local in urls
                                 else next(iter(urls)))
                item = MenuItem(
                    menu_location=location,
                    label=labels_per_lang[fallback_lang],
                    url=urls[fallback_lang],
                    page_type='custom',
                    target=custom_target,
                    order=order_idx,
                    parent_id=parent_db_id,
                    labels_json=_json.dumps(labels_per_lang),
                    urls_json=_json.dumps(urls),
                )
            else:
                return None

            db.session.add(item)
            return item

        # ── 3. Parse the ordered item list submitted by the drag-and-drop UI ──
        # The form sends a JSON array: menu_structure = '[{"type":"page","id":"3","parent":null}, ...]'
        # Each entry has: type (page|post|custom), id (content id), parent
        # (temp client-side id or null), temp_id (client-side id of the row).
        #
        # Depth-agnostic BFS build (2.3.4): materializes the tree tier by tier
        # up to MENU_MAX_DEPTH. Entries submitted deeper than the cap are
        # clamped: they attach to their deepest allowed ancestor, so no client
        # payload can create chains beyond the configured depth. Entries whose
        # parent chain never reaches a top-level row (orphans, cycles) are
        # dropped. Translation-group dedup applies within each SIBLING group —
        # the same per-scope rule the two-tier code always had, generalized.
        structure_raw = request.form.get('menu_structure', '')
        if structure_raw:
            try:
                structure = _json.loads(structure_raw)
            except (ValueError, TypeError):
                structure = []
            if not isinstance(structure, list):
                structure = []

            max_depth = app.config.get('MENU_MAX_DEPTH', 3)
            temp_to_item = {}   # temp_id (str) → flushed MenuItem

            entries_by_parent = {}   # parent temp_id ('' for top) → [entry, ...]
            for entry in structure:
                if not isinstance(entry, dict):
                    continue
                p = str(entry.get('parent') or '')
                entries_by_parent.setdefault(p, []).append(entry)

            def _create_sibling_level(entries, parent_db_id, depth):
                """Create one sibling group; returns [(temp_id, db_item), ...]."""
                created = []
                order_idx = 0
                added_groups_page = set()
                added_groups_post = set()
                for entry in entries:
                    itype = entry.get('type')
                    cid = str(entry.get('id', ''))
                    tid = str(entry.get('temp_id', ''))

                    # Skip duplicate translation groups within this sibling group
                    if itype == 'page':
                        obj = Page.query.get(cid)
                        if obj and obj.translation_group:
                            if obj.translation_group in added_groups_page:
                                continue
                            added_groups_page.add(obj.translation_group)
                    elif itype == 'post':
                        obj = Post.query.get(cid)
                        if obj and obj.translation_group:
                            if obj.translation_group in added_groups_post:
                                continue
                            added_groups_post.add(obj.translation_group)

                    item = _build_item(itype, cid, order_idx,
                                       parent_db_id=parent_db_id)
                    if item:
                        db.session.flush()   # get item.id for the next tier
                        if tid:
                            temp_to_item[tid] = item
                        created.append((tid, item))
                        order_idx += 1
                return created

            # Tier 0
            frontier = _create_sibling_level(entries_by_parent.get('', []),
                                             None, 0)
            consumed = {''}
            depth = 1
            # Tiers 1 .. max_depth-1: children of the previous tier
            while frontier and depth < max_depth:
                next_frontier = []
                for parent_tid, parent_item in frontier:
                    if not parent_tid or parent_tid in consumed:
                        continue
                    consumed.add(parent_tid)
                    kids = entries_by_parent.get(parent_tid, [])
                    if kids:
                        next_frontier.extend(_create_sibling_level(
                            kids, parent_item.id, depth))
                frontier = next_frontier
                depth += 1

            # Depth clamp: entries hanging under the LAST created tier would
            # exceed the cap — re-attach them to their (already created)
            # parent's parent... i.e. flatten them INTO the deepest allowed
            # tier by creating them as siblings appended under the deepest
            # allowed ancestor. Concretely: their declared parent sits at the
            # cap boundary, so they are created at the boundary tier attached
            # to that parent's own parent — keeping content instead of
            # silently dropping it.
            for parent_tid, parent_item in frontier:
                if not parent_tid or parent_tid in consumed:
                    continue
                consumed.add(parent_tid)
                too_deep = entries_by_parent.get(parent_tid, [])
                if too_deep:
                    _create_sibling_level(too_deep, parent_item.parent_id,
                                          max_depth - 1)

        else:
            # ── Fallback: old flat form format (backwards compatibility) ──────
            added_page_groups = set()
            added_post_groups = set()
            selected_pages = request.form.getlist('pages')
            order_idx = 0
            for page_id in selected_pages:
                page = Page.query.get(page_id)
                if not page:
                    continue
                if page.translation_group and page.translation_group in added_page_groups:
                    continue
                labels = {}
                custom_label = request.form.get(f'label_page_{page_id}', '').strip()
                if custom_label:
                    labels[page.language] = custom_label
                if page.translation_group:
                    for other_page_id in selected_pages:
                        if other_page_id == page_id:
                            continue
                        other_page = Page.query.get(other_page_id)
                        if other_page and other_page.translation_group == page.translation_group:
                            other_label = request.form.get(f'label_page_{other_page_id}', '').strip()
                            if other_label:
                                labels[other_page.language] = other_label
                menu_item = MenuItem(
                    menu_location=location,
                    label=custom_label or page.title,
                    url=f'/{page.slug}',
                    page_id=page.id,
                    page_type='page',
                    language=page.language,
                    order=order_idx,
                    labels_json=_json.dumps(labels) if labels else None,
                )
                db.session.add(menu_item)
                order_idx += 1
                if page.translation_group:
                    added_page_groups.add(page.translation_group)

            selected_posts = request.form.getlist('posts')
            for post_id in selected_posts:
                post = Post.query.get(post_id)
                if not post:
                    continue
                if post.translation_group and post.translation_group in added_post_groups:
                    continue
                labels = {}
                custom_label = request.form.get(f'label_post_{post_id}', '').strip()
                if custom_label:
                    labels[post.language] = custom_label
                if post.translation_group:
                    for other_post_id in selected_posts:
                        if other_post_id == post_id:
                            continue
                        other_post = Post.query.get(other_post_id)
                        if other_post and other_post.translation_group == post.translation_group:
                            other_label = request.form.get(f'label_post_{other_post_id}', '').strip()
                            if other_label:
                                labels[other_post.language] = other_label
                menu_item = MenuItem(
                    menu_location=location,
                    label=custom_label or post.title,
                    url=f'/{post.slug}',
                    post_id=post.id,
                    page_type='post',
                    language=post.language,
                    order=order_idx,
                    labels_json=_json.dumps(labels) if labels else None,
                )
                db.session.add(menu_item)
                order_idx += 1
                if post.translation_group:
                    added_post_groups.add(post.translation_group)

        AuditLog.log('save_menu', 'menu', None,
                     _ad('saved_menu', location=location),
                     current_user, request.remote_addr)
        db.session.commit()
        flash(t('flash.menu_updated', location=location.capitalize()), 'success')
        if request.form.get('_format') == 'json':
            return jsonify({'success': True, 'location': location})
        return redirect(url_for('admin_menus'))

    # ── GET: render menu builder ──────────────────────────────────────────────
    # Pages/posts are now loaded via AJAX (/admin/api/menu-picker)

    # Top-level items only (parent_id is None)
    header_items = MenuItem.query.filter_by(
        menu_location='header', parent_id=None
    ).order_by(MenuItem.order).all()
    footer_items = MenuItem.query.filter_by(
        menu_location='footer', parent_id=None
    ).order_by(MenuItem.order).all()

    # Build labels lookup: {location: {page_id/post_id: {lang: label}}}
    import json as _json

    def _all_items_flat(location_str):
        # Ordered by saved position so the builder renders submenu children in
        # the same sequence as the public site (id is a stable tie-breaker).
        return MenuItem.query.filter_by(menu_location=location_str).order_by(
            MenuItem.order, MenuItem.id
        ).all()

    menu_labels = {'header': {}, 'footer': {}}
    for loc in ('header', 'footer'):
        for item in _all_items_flat(loc):
            labels = {}
            if item.labels_json:
                try:
                    labels = _json.loads(item.labels_json)
                except (ValueError, TypeError):
                    pass
            if item.page_id:
                page = item.page
                if page and page.translation_group:
                    all_versions = Page.query.filter_by(translation_group=page.translation_group).all()
                    for v in all_versions:
                        lang_label = labels.get(v.language, '')
                        if v.id == item.page_id and not lang_label:
                            if item.label != page.title:
                                lang_label = item.label
                        menu_labels[loc][f'page_{v.id}'] = lang_label
                else:
                    if item.label != (page.title if page else ''):
                        menu_labels[loc][f'page_{item.page_id}'] = item.label
            elif item.post_id:
                post = item.post
                if post and post.translation_group:
                    all_versions = Post.query.filter_by(translation_group=post.translation_group).all()
                    for v in all_versions:
                        lang_label = labels.get(v.language, '')
                        if v.id == item.post_id and not lang_label:
                            if item.label != post.title:
                                lang_label = item.label
                        menu_labels[loc][f'post_{v.id}'] = lang_label
                else:
                    if item.label != (post.title if post else ''):
                        menu_labels[loc][f'post_{item.post_id}'] = item.label

    # Build checked sets: include translation siblings
    checked = {'header': {'pages': set(), 'posts': set()}, 'footer': {'pages': set(), 'posts': set()}}
    checked_groups = {'header': {'pages': [], 'posts': []}, 'footer': {'pages': [], 'posts': []}}
    for loc in ('header', 'footer'):
        _seen_pg = set()
        _seen_po = set()
        for item in _all_items_flat(loc):
            if item.page_id and item.page:
                page = item.page
                checked[loc]['pages'].add(page.id)
                if page.translation_group:
                    siblings = Page.query.filter_by(
                        translation_group=page.translation_group, status='published'
                    ).all()
                    for s in siblings:
                        checked[loc]['pages'].add(s.id)
                    if page.translation_group not in _seen_pg:
                        checked_groups[loc]['pages'].append(page.translation_group)
                        _seen_pg.add(page.translation_group)
            elif item.post_id and item.post:
                post = item.post
                checked[loc]['posts'].add(post.id)
                if post.translation_group:
                    siblings = Post.query.filter_by(
                        translation_group=post.translation_group, status='published'
                    ).all()
                    for s in siblings:
                        checked[loc]['posts'].add(s.id)
                    if post.translation_group not in _seen_po:
                        checked_groups[loc]['posts'].append(post.translation_group)
                        _seen_po.add(post.translation_group)

    # Build parent_map: item_id → [child MenuItem, ...] for the menu builder UI
    parent_map = {'header': {}, 'footer': {}}
    for loc in ('header', 'footer'):
        for item in _all_items_flat(loc):
            if item.parent_id is not None:
                parent_map[loc].setdefault(item.parent_id, []).append(item)

    # List of files for the Custom Links picker (file → URL helper)
    files_list = File.query.order_by(File.uploaded_at.desc()).all()

    # Pre-parse custom-link JSON fields server-side so the template can pass
    # them through |tojson safely. The previous template-side approach
    # ({{ item.urls_json | safe }}) injected raw DB JSON into a <script>
    # block, which would break the page if a URL or label ever contained a
    # quote, backslash, or </script> sequence. Same shape as before, just
    # built in Python and passed as a Python dict.
    custom_links_data = {'header': {}, 'footer': {}}
    for loc in ('header', 'footer'):
        # _all_items_flat() returns BOTH top-level and nested items, which
        # matters because a custom link can be dragged into a submenu — if
        # we iterated only header_items (top-level), nested custom links
        # would lose their data on the next page load.
        for item in _all_items_flat(loc):
            if item.page_type != 'custom':
                continue
            try:
                urls = _json.loads(item.urls_json) if item.urls_json else {}
            except (ValueError, TypeError):
                urls = {}
            # Defensive: a row could legitimately parse as JSON yet not be a
            # dict — e.g. legacy data or 'null' / '"text"' / '42' / '[]'
            # written by some earlier code path. menus.html later passes
            # this to Object.keys(data.urls).forEach(...) which throws on
            # null and silently iterates string indices on a string. Force
            # a dict here regardless of what the DB held.
            if not isinstance(urls, dict):
                urls = {}
            try:
                labels = _json.loads(item.labels_json) if item.labels_json else {}
            except (ValueError, TypeError):
                labels = {}
            if not isinstance(labels, dict):
                labels = {}
            # Key must match the row's data-id rendered in the template:
            # nested rows use 'srv-child-<id>', top-level rows use 'srv-<id>'.
            srv_key = f'srv-child-{item.id}' if item.parent_id else f'srv-{item.id}'
            custom_links_data[loc][srv_key] = {
                'urls': urls,
                'labels': labels,
                'target': item.target or '_self',
            }

    return render_template('admin/menus.html',
                           header_items=header_items,
                           footer_items=footer_items,
                           menu_labels=menu_labels,
                           checked=checked,
                           checked_groups=checked_groups,
                           parent_map=parent_map,
                           files_list=files_list,
                           custom_links_data=custom_links_data,
                           available_languages=AVAILABLE_LANGUAGES)

# ============================================================================
# AI ASSISTANT
# ============================================================================

@app.route('/admin/ai', methods=['GET', 'POST'])
@login_required
def admin_ai():
    """AI Assistant - settings and tools (multi-provider)"""
    if request.method == 'POST':
        action = request.form.get('action')
        
        if action == 'save_ai_config':
            # Save active provider
            provider = request.form.get('ai_provider', 'anthropic')
            if provider in ('anthropic', 'deepseek', 'gemini'):
                Settings.set('ai_provider', provider)
            
            # Save API keys (only if provided — empty = keep existing)
            for prov in ('anthropic', 'deepseek', 'gemini'):
                key_field = f'{prov}_api_key'
                key_value = request.form.get(key_field, '').strip()
                if key_value:
                    Settings.set(f'{prov}_api_key_encrypted', encrypt_smtp_password(key_value))
            
            # Legacy cleanup: remove old plaintext Anthropic key
            old_plain = Settings.query.get('anthropic_api_key')
            if old_plain:
                db.session.delete(old_plain)
                db.session.commit()
            
            flash(t('flash.api_key_saved'), 'success')
            return redirect(url_for('admin_ai'))

        if action == 'save_webmcp':
            # Agent access (WebMCP) — owner only. Powerful capability.
            if current_user.is_owner():
                for wkey in ('webmcp_admin_enabled', 'webmcp_public_enabled', 'webmcp_polyfill'):
                    Settings.set(wkey, 'true' if request.form.get(wkey) else 'false')
                token = request.form.get('webmcp_origin_trial_token', '').strip()
                Settings.set('webmcp_origin_trial_token', token)
                AuditLog.log('edit_settings', 'webmcp', None,
                             _ad('settings_updated'),
                             user=current_user, ip=request.remote_addr)
                flash(t('webmcp.saved'), 'success')
            else:
                flash(t('flash.owner_only'), 'error')
            return redirect(url_for('admin_ai'))
    
    # Get current config
    active_provider = Settings.get('ai_provider', 'anthropic')
    
    # Check which providers have keys
    providers = {}
    for prov in ('anthropic', 'deepseek', 'gemini'):
        key = decrypt_smtp_password(Settings.get(f'{prov}_api_key_encrypted', ''))
        if prov == 'anthropic' and not key:
            key = Settings.get('anthropic_api_key', '')  # Legacy fallback
        has_key = bool(key and len(key) > 10)
        providers[prov] = {
            'has_key': has_key,
            'preview': key[:8] + '...' if has_key else ''
        }
    
    # Active provider must have a key
    has_api_key = providers.get(active_provider, {}).get('has_key', False)
    api_key_preview = providers.get(active_provider, {}).get('preview', '')
    
    return render_template('admin/ai.html',
        has_api_key=has_api_key,
        api_key_preview=api_key_preview,
        active_provider=active_provider,
        providers=providers
    )


@app.route('/admin/ai/generate', methods=['POST'])
@login_required
@limiter.limit("20 per minute")
def admin_ai_generate():
    """AI content generation API endpoint — unified for all AI tasks"""
    
    # Check AI provider is configured
    provider, api_key, key_error = get_ai_config()
    if key_error:
        return jsonify({'error': key_error}), 400
    
    data = request.get_json()
    task = data.get('task', 'generate')
    content = data.get('content', '')
    prompt = data.get('prompt', '')
    source_lang = data.get('source_lang', 'en')
    target_lang = data.get('target_lang', 'de')
    title = data.get('title', '')
    language = data.get('language', 'English')
    content_type = data.get('content_type', 'page')  # page or post
    conversation_history = data.get('conversation_history', '')
    
    # Build system prompt based on task
    if task == 'translate':
        system_prompt = f"""You are a professional translator. Translate the following content from {source_lang} to {target_lang}. 
Preserve all HTML formatting. Keep the same tone and style. Only return the translated text, nothing else."""
        user_message = content
        
    elif task == 'seo':
        system_prompt = f"""You are an SEO expert. The user's interface language is {language} — answer and generate all content in {language}.
Based on the content provided and user's instructions, generate optimized SEO fields.
Return ONLY a valid JSON object with these fields:
- "title" (if user wants title change)
- "meta_title" (max 70 chars)
- "meta_description" (max 160 chars)
- "meta_keywords" (comma-separated)
- "og_title"
- "og_description"
- "ai_message" (your comment/advice to the user, in {language})
No markdown, no code fences. Return ONLY the JSON object."""
        user_message = f"Title: {title}\n\nContent: {content[:2000]}\n\nUser instructions: {prompt}" if prompt else f"Title: {title}\n\nContent: {content[:2000]}"
        
    elif task in ('generate', 'generate_full'):
        system_prompt = f"""You are a professional content writer and web designer. The user's language is {language} — write ALL content in {language}.

Generate a complete {content_type} based on the user's description. Return ONLY a valid JSON object with these fields:
- "title": compelling page/post title
- "content": full HTML content (use h2, h3, p, ul, li, blockquote, etc.)
- "excerpt": short summary (1-2 sentences)
- "meta_title": SEO title (max 70 chars)
- "meta_description": SEO description (max 160 chars)
- "meta_keywords": comma-separated keywords
- "og_title": social sharing title
- "og_description": social sharing description
- "faq": array of 3-5 Q&A objects like [{{"question": "...", "answer": "..."}}, ...] — frequently asked questions related to the topic. Answers should be concise (1-3 sentences).
- "ai_message": your comment to the user — suggestions about images, improvements, etc. Write this in {language}.

IMPORTANT:
- Write substantial, high-quality content (at least 3-5 sections with headings)
- ANSWER-FIRST STRUCTURE (AEO): Start the content with a concise 2-3 sentence definition or direct answer to the topic. Place this BEFORE any introduction or context. AI systems cite content from the first 150 words most frequently.
- If the user's prompt is vague, include "ai_message" asking for clarification instead of generating thin content
- Note in "ai_message" where the user should add images or videos manually
- No markdown code fences. Return ONLY the JSON object."""
        
        context_parts = []
        if title:
            context_parts.append(f"Existing title: {title}")
        if content:
            context_parts.append(f"Existing content (first 500 chars): {content[:500]}")
        if conversation_history:
            context_parts.append(f"Previous conversation:\n{conversation_history}")
        
        context = '\n\n'.join(context_parts)
        user_message = f"{context}\n\nUser request: {prompt}" if context else prompt
    
    elif task == 'builder_blocks':
        system_prompt = f"""You are a professional web page designer. The user's language is {language} — write ALL text content in {language}.

Generate a page layout as JSON blocks for a visual page builder. Return ONLY a valid JSON object with:
- "blocks": array of block objects
- "ai_message": your comment to the user (in {language}) — mention where they should add images/videos manually

Each block has: "id" (random string like "b1a2b3c"), "type", "data", "settings"

Available block types and their data:

BASIC BLOCKS:
- heading: {{"text": "...", "level": 2}}
- text: {{"html": "<p>...</p>"}}
- image: {{"url": "", "alt": "description of needed image", "caption": "...", "size": "full"}}
- button: {{"text": "...", "url": "#", "style": "primary|outline|ghost", "size": "sm|md|lg", "target": "_self|_blank"}}
- divider: {{"style": "solid|dashed|dotted|gradient"}}
- spacer: {{"height": "48"}}

STRUCTURE BLOCKS:
- columns: {{"layout": "1/2+1/2", "columns": [{{"blocks": [...]}}, {{"blocks": [...]}}]}}
  Layout options: "1", "1/2+1/2", "1/3+1/3+1/3", "2/3+1/3", "1/3+2/3", "1/4+1/4+1/4+1/4"
  Columns can contain nested blocks (any type except columns/section/slideshow)
- section: {{"background_type": "color|gradient|image|none", "background_value": "#1a1a2e", "background_image": "", "background_overlay": "rgba(0,0,0,0.5)", "background_type_dark": "color|gradient|image|none", "background_value_dark": "#0f172a", "background_overlay_dark": "", "padding": "sm|md|lg|xl", "text_color": "auto|light|dark", "min_height": "auto", "full_width": false, "content_width": "default|narrow|wide|full", "blocks": [...]}}
  Section wraps nested blocks with a background. Use text_color "light" for dark backgrounds, "dark" for light backgrounds.
  Nested blocks go in the "blocks" array (any type except section).
- slideshow: {{"mode": "hero", "slides": [{{"image": "", "alt": "", "blocks": [...]}}], "animation": "slide|fade", "autoplay": true, "interval": 5, "show_arrows": true, "show_dots": true, "aspect_ratio": "16/9|4/3|21/9|1/1|auto", "overlay": "rgba(0,0,0,0.4)", "padding": "lg", "text_color": "light|dark|auto", "full_width": false, "content_width": "default|narrow|wide|full"}}
  Hero mode: background image + nested content blocks on each slide. Image mode: simple image carousel.
  Each slide has "image" (url), "alt", and "blocks" array for hero mode content (heading, text, button, etc.).

MEDIA BLOCKS:
- video: {{"source": "youtube|vimeo|self", "url": "", "embed_id": "", "title": "", "aspect_ratio": "16/9"}}
- gallery: {{"images": [{{"url": "", "alt": "", "caption": ""}}], "layout": "grid", "columns": 3, "gap": "md", "lightbox": true}}
- video_gallery: {{"videos": [{{"url": "", "embed_id": "", "source": "youtube|vimeo", "title": ""}}], "columns": 2, "gap": "md"}}
  Grid of video thumbnails (YouTube/Vimeo). Facade pattern: no iframes on load, click opens lightbox player. embed_id is the YouTube video ID or Vimeo numeric ID.
- audio: {{"url": "", "title": ""}}
- ticker: {{"images": [{{"url": "", "alt": "Company logo"}}], "speed": "slow|medium|fast", "pause_on_hover": true, "grayscale": true, "height": "sm|md|lg|xl", "direction": "rtl|ltr"}}
  Logo ticker: infinite horizontal scroll of logos/icons. Full viewport width. Grayscale effect makes logos muted until hover. direction "rtl" scrolls left (default), "ltr" scrolls right. Leave image urls empty, fill alt with descriptions.

ADVANCED BLOCKS:
- quote: {{"text": "...", "author": "..."}}
- accordion: {{"items": [{{"title": "Q", "content": "<p>A</p>"}}], "allow_multiple": false}}
- tabs: {{"tabs": [{{"title": "Tab 1", "content": "<p>Content</p>"}}, {{"title": "Tab 2", "content": "<p>Content</p>"}}]}}
- table: {{"headers": ["Col1","Col2"], "rows": [["a","b"]], "striped": true, "responsive": true, "table_width": "full|auto", "column_alignments": ["left",""], "column_widths": ["30",""]}}
  Cell content supports inline HTML: <b>bold</b>, <i>italic</i>, <a href="...">links</a>. column_alignments: per-column text alignment (left|center|right|justify), empty string for default. column_widths: per-column width in percent (e.g. "30" for 30%), empty for auto. Use table_width "full" for comparison tables, "auto" for compact data.
- code: {{"code": "...", "language": "javascript"}}
- html: {{"code": "<!-- raw HTML -->"}}
- form: {{"form_id": 1, "show_title": true, "show_description": true}}

Settings (optional for all blocks): {{"alignment": "left|center|right", "content_alignment": "left|center|right", "margin_top": "none|sm|md|lg|xl", "margin_bottom": "none|sm|md|lg|xl", "css_class": "", "anchor_id": ""}}

DESIGN PRINCIPLES:
- Structure pages with SECTIONS as visual containers. Alternate between "none" background (inherits site theme), colored backgrounds, and gradients to create rhythm and visual separation.
- For dark-themed sites, use deep colors for section backgrounds: #0d1117, #111827, #1a1a2e, #0f172a, rgba(99,102,241,0.08). For light-themed sites, use: #f8fafc, #f1f5f9, #eef2ff, #faf5ff, #ffffff.
- Every section should have purpose: hero (attention), features (value), social proof (trust), CTA (action), details (information).
- Use "content_width": "narrow" for text-heavy sections (about, testimonials) and "wide" or "default" for feature grids and galleries.

LANDING PAGE STRUCTURE (when user asks for a landing page or similar):
1. HERO — slideshow in hero mode with overlay, large heading (level 1), subtitle text, and 1-2 buttons (primary + outline)
2. FEATURES/BENEFITS — section with 3 or 4 columns, each with a heading (level 3) and short text
3. ABOUT/DETAILS — section with 2 columns (2/3+1/3 or 1/2+1/2): text block + image placeholder
4. TESTIMONIALS/SOCIAL PROOF — section with colored background, quotes or cards in columns
5. FAQ — section with accordion block for common questions
6. CTA — section with gradient background, centered heading, text, and a prominent button
7. CONTACT — section with a form block (if forms available) or columns with contact info

Not every page needs all 7 sections. Match the structure to what the user describes. A simple "About" page might only need 2-3 sections.

CONTENT QUALITY:
- Write substantial, realistic text — not "Lorem ipsum" placeholders. Each paragraph should be 2-4 sentences.
- Headings should be specific and compelling, not generic ("Our Services" → "Solutions That Scale With Your Business").
- Button text should be action-oriented: "Get Started", "Learn More", "Request a Demo" — not just "Click Here".
- If the user's request is vague, generate a reasonable page AND include suggestions in "ai_message" for what they might want to customize.

IMPORTANT:
- For images, leave url empty but fill alt with a description of what image is needed (e.g. "Team photo of diverse professionals in a modern office").
- Include "ai_message" telling the user what images/videos to add manually and suggesting improvements.
- For form blocks, use one of the available forms listed below. If no forms exist, suggest the user create one first.
- No markdown code fences. Return ONLY the JSON object."""

        # Inject active theme colors into the prompt so AI uses theme-aware section backgrounds
        active_theme = effective_active_theme()
        dark_vars, light_vars = _extract_theme_css_vars(active_theme)
        if dark_vars:
            bg = dark_vars.get('--ap-color-bg', '')
            surface = dark_vars.get('--ap-color-surface', '')
            primary = dark_vars.get('--ap-color-primary', '')
            if bg or surface or primary:
                system_prompt += f"\n\nActive site theme colors (dark mode): bg={bg}, surface={surface}, primary={primary}. Use these as reference when choosing section background colors — pick shades that harmonize with the theme."

        # Inject available forms into the prompt
        active_forms = Form.query.filter_by(is_active=True).order_by(Form.title).all()
        if active_forms:
            forms_info = "\n".join([f"  - id: {f.id}, title: \"{f.title}\", type: {f.form_type}, language: {f.language}, translation_group: {f.translation_group}" for f in active_forms])
            system_prompt += f"\n\nAvailable forms (use form_id in form blocks):\n{forms_info}\nForms with the same translation_group are translations of each other. Always pick the form whose language matches the page language ({language})."
        else:
            system_prompt += "\n\nNo forms available yet. If user asks for a form, tell them to create one first in Forms → Create New Form."
        
        context_parts = []
        if title:
            context_parts.append(f"Page title: {title}")
        if conversation_history:
            context_parts.append(f"Previous conversation:\n{conversation_history}")
        context = '\n\n'.join(context_parts)
        user_message = f"{context}\n\nUser request: {prompt}" if context else prompt
        
    elif task == 'generate_form':
        system_prompt = f"""You are a professional form designer and UX expert. The user's language is {language} — write ALL text content (labels, placeholders, help texts, consent texts, options, submit button, success message) in {language}.

Generate a complete form structure based on the user's description. Return ONLY a valid JSON object with these fields:
- "title": form title
- "description": short description shown above the form
- "fields": array of field objects, each with:
  - "id": random string like "f1a2b3c"
  - "type": one of "text", "email", "phone", "number", "textarea", "select", "radio", "checkbox", "date", "url", "rating", "consent", "hidden"
  - "name": internal field name (lowercase, no spaces, use underscores)
  - "label": human-readable label
  - "placeholder": placeholder text (where applicable)
  - "required": true/false
  - "width": "full" or "half"
  - "help_text": optional help text below the field
  - "options": array of strings (for select, radio, checkbox types only)
  - "max_rating": number 3-10 (for rating type only)
  - "consent_text": text for consent checkbox (for consent type only)
- "settings": object with:
  - "submit_label": submit button text
  - "success_message": message shown after successful submission
  - "notify_on_submit": true/false
  - "store_submissions": true
  - "anti_spam": "honeypot"
- "ai_message": your comment to the user (in {language}) — suggestions, improvements, what to configure manually

IMPORTANT:
- Create well-thought-out, user-friendly forms with logical field ordering
- Use appropriate field types (email for emails, phone for phone numbers, etc.)
- Include proper validation (required fields where needed)
- For contact forms, always include a consent/GDPR checkbox
- For surveys, use ratings, radio buttons, and checkboxes appropriately
- Use "half" width for short fields (name, phone, date) and "full" for longer ones (message, textarea)
- No markdown code fences. Return ONLY the JSON object."""

        context_parts = []
        if title:
            context_parts.append(f"Form title: {title}")
        if content:
            context_parts.append(f"Existing form description: {content[:500]}")
        if conversation_history:
            context_parts.append(f"Previous conversation:\n{conversation_history}")
        context = '\n\n'.join(context_parts)
        user_message = f"{context}\n\nUser request: {prompt}" if context else prompt

    elif task == 'chat':
        # Universal AI assistant — can handle any task in one conversation
        active_forms = Form.query.filter_by(is_active=True).order_by(Form.title).all()
        forms_info = ""
        if active_forms:
            forms_info = "\n".join([f"  - id: {f.id}, title: \"{f.title}\", type: {f.form_type}, language: {f.language}, translation_group: {f.translation_group}" for f in active_forms])
            forms_info = f"\n\nAvailable forms in the CMS:\n{forms_info}\nForms with the same translation_group are translations of each other. When inserting a form into a page, always pick the form whose language matches the page language."
        
        system_prompt = f"""You are a professional AI assistant for a multilingual CMS called AliothPress. The user's primary language is {language}. Always respond in the same language the user writes in.

You help with ALL aspects of content management in one conversation:

1. **Content writing** — articles, blog posts, landing page text, product descriptions
2. **Translation** — translate content between any languages, preserving HTML formatting
3. **SEO optimization** — generate meta titles, descriptions, keywords, Open Graph tags
4. **Page Builder layouts** — create structured page layouts as JSON blocks
5. **Form design** — design contact forms, surveys, registrations

RESPONSE FORMAT:
- For casual conversation, questions, or advice: respond naturally in plain text.
- When the user asks you to GENERATE structured content (SEO fields, full article, form, builder blocks), return a JSON object. The JSON must include an "ai_message" field with your comment/advice to the user.
- For SEO: return JSON with fields: title, meta_title, meta_description, meta_keywords, og_title, og_description, ai_message
- For content generation: return JSON with: title, content (HTML), excerpt, meta_title, meta_description, meta_keywords, og_title, og_description, ai_message  
- For form design: return JSON with: title, description, fields (array), settings, ai_message
- For builder blocks: return JSON with: blocks (array), ai_message
- For translation: just return the translated text directly.

IMPORTANT:
- You can switch between tasks naturally within the same conversation
- Remember context from earlier in the conversation
- If the user's request is vague, ask clarifying questions instead of guessing
- Be concise but thorough
- No markdown code fences around JSON — return raw JSON when generating structured data
{forms_info}"""

        # Build proper message with conversation history
        context_parts = []
        if title:
            context_parts.append(f"Page/post title: {title}")
        if content:
            context_parts.append(f"Content: {content[:2000]}")
        if conversation_history:
            context_parts.append(f"Conversation so far:\n{conversation_history}")
        context = '\n\n'.join(context_parts)
        user_message = f"{context}\n\nUser: {prompt}" if context else prompt

    else:
        return jsonify({'error': t('api.unknown_task')}), 400
    
    try:
        ai_text, ai_error = call_ai(
            system_prompt, user_message,
            max_tokens=16384 if task == 'builder_blocks' else 4096,
            timeout=180 if task == 'builder_blocks' else 90
        )
        if ai_error:
            return jsonify({'error': ai_error}), 500
        
        return jsonify({'result': ai_text, 'task': task})
        
    except Exception as e:
        return jsonify({'error': str(e)}), 500


# ============================================================================
# SEO TOOLS: SITEMAP & ROBOTS.TXT
# ============================================================================

@app.route('/sitemap.xml')
def sitemap_xml():
    """Auto-generated XML sitemap with hreflang for multilingual content"""
    from flask import Response
    xml_content, _stats = _build_sitemap_xml()
    return Response(xml_content, mimetype='application/xml')


def _build_sitemap_xml():
    """Build the sitemap XML. Returns (xml_string, stats).

    stats is a per-section ledger of emitted <url> entries:
      {'homepage': n, 'posts': n, 'pages': n, 'listings': n, 'total': n}
    The admin SEO Tools card shows these numbers — counting at emission
    time means the card can never drift from the real file, no matter
    what sections the sitemap grows in the future."""
    stats = {'homepage': 0, 'posts': 0, 'pages': 0, 'listings': 0}
    
    base_url = request.url_root.rstrip('/')
    default_lang = Settings.get('default_language', 'en')
    active_codes = _get_active_language_codes()
    
    # Collect all published content in active languages, grouped by translation_group
    posts = Post.query.filter(
        Post.status == 'published',
        Post.language.in_(list(active_codes))
    ).order_by(Post.updated_at.desc()).all()
    pages = Page.query.filter(
        Page.status == 'published',
        Page.language.in_(list(active_codes))
    ).order_by(Page.updated_at.desc()).all()
    
    # Group by translation_group for hreflang links
    post_groups = {}
    for post in posts:
        group = post.translation_group or f'post_standalone_{post.id}'
        if group not in post_groups:
            post_groups[group] = []
        post_groups[group].append(post)
    
    page_groups = {}
    for page in pages:
        group = page.translation_group or f'page_standalone_{page.id}'
        if group not in page_groups:
            page_groups[group] = []
        page_groups[group].append(page)
    
    # Build XML
    xml_parts = ['<?xml version="1.0" encoding="UTF-8"?>']
    xml_parts.append('<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>')
    xml_parts.append('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
    xml_parts.append('        xmlns:xhtml="http://www.w3.org/1999/xhtml">')
    
    # NOTE on x-default:
    # x-default is declared in each page's <head> (in post.html / page.html),
    # pointing to the page itself. We intentionally OMIT x-default here in
    # the sitemap to avoid duplicating the signal and to avoid imposing any
    # "dominant language" on the sitemap level. The sitemap stays a pure map
    # of content and its language versions; x-default fallback routing is
    # handled at the per-page <head> level.
    
    # Homepage — one <url> per language, all cross-linked via hreflang.
    # Same equal-weight pattern used for all other multilingual pages.
    homepage_group = Settings.get('homepage_group', '')
    homepage_pages = []
    if homepage_group:
        homepage_pages = Page.query.filter(
            Page.translation_group == homepage_group,
            Page.status == 'published',
            Page.language.in_(list(active_codes))
        ).all()
    
    def _hp_url(language):
        """Default language → /, others → /?lang=xx"""
        if language == default_lang:
            return f'{base_url}/'
        return f'{base_url}/?lang={language}'
    
    if homepage_pages and len(homepage_pages) > 1:
        homepage_lastmod = max(p.updated_at for p in homepage_pages).strftime("%Y-%m-%d")
        
        # Same ordering rule as the posts-mode branch below: the default
        # language leads the block (its clean / is the canonical spelling),
        # the rest follow AVAILABLE_LANGUAGES order. Without this the block
        # came out in DB row order — i.e. whatever order the translations
        # happened to be created in.
        _lang_rank = {l['code']: i for i, l in enumerate(AVAILABLE_LANGUAGES)}
        homepage_pages.sort(key=lambda p: (p.language != default_lang,
                                           _lang_rank.get(p.language, len(_lang_rank))))
        
        for hp in homepage_pages:
            stats['homepage'] += 1
            xml_parts.append('  <url>')
            xml_parts.append(f'    <loc>{_hp_url(hp.language)}</loc>')
            xml_parts.append(f'    <lastmod>{homepage_lastmod}</lastmod>')
            xml_parts.append('    <changefreq>daily</changefreq>')
            xml_parts.append('    <priority>1.0</priority>')
            for other in homepage_pages:
                xml_parts.append(f'    <xhtml:link rel="alternate" hreflang="{other.language}" href="{_hp_url(other.language)}"/>')
            xml_parts.append('  </url>')
    elif not homepage_group and len(active_codes) > 1:
        # Posts-mode homepage (no builder page configured) on a multilingual
        # site: index() serves /?lang=xx for EVERY active language — with a
        # graceful content fallback, so none of them is a soft-404 — and the
        # public language switcher links all of them. The sitemap must
        # mirror that same set, cross-linked via hreflang, exactly like the
        # builder-homepage branch above. Before this branch existed, only /
        # was emitted here and the language versions of the homepage were
        # invisible to crawlers.
        latest = posts[0].updated_at if posts else utc_now()
        hp_lastmod = latest.strftime("%Y-%m-%d")
        # Default language first (its clean / opens the homepage block),
        # then the rest in AVAILABLE_LANGUAGES order.
        ordered_codes = [l['code'] for l in AVAILABLE_LANGUAGES if l['code'] in active_codes]
        if default_lang in ordered_codes:
            ordered_codes.remove(default_lang)
        ordered_codes.insert(0, default_lang)
        for lang in ordered_codes:
            stats['homepage'] += 1
            xml_parts.append('  <url>')
            xml_parts.append(f'    <loc>{_hp_url(lang)}</loc>')
            xml_parts.append(f'    <lastmod>{hp_lastmod}</lastmod>')
            xml_parts.append('    <changefreq>daily</changefreq>')
            xml_parts.append('    <priority>1.0</priority>')
            for other in ordered_codes:
                xml_parts.append(f'    <xhtml:link rel="alternate" hreflang="{other}" href="{_hp_url(other)}"/>')
            xml_parts.append('  </url>')
    else:
        # Single-language or no homepage group — one entry without hreflang
        stats['homepage'] += 1
        xml_parts.append('  <url>')
        xml_parts.append(f'    <loc>{base_url}/</loc>')
        if homepage_pages:
            xml_parts.append(f'    <lastmod>{homepage_pages[0].updated_at.strftime("%Y-%m-%d")}</lastmod>')
        else:
            xml_parts.append(f'    <lastmod>{utc_now().strftime("%Y-%m-%d")}</lastmod>')
        xml_parts.append('    <changefreq>daily</changefreq>')
        xml_parts.append('    <priority>1.0</priority>')
        xml_parts.append('  </url>')
    
    # Posts — grouped by translation_group
    for group_id, group_posts in post_groups.items():
        for post in group_posts:
            stats['posts'] += 1
            xml_parts.append('  <url>')
            xml_parts.append(f'    <loc>{base_url}/{post.slug}</loc>')
            xml_parts.append(f'    <lastmod>{post.updated_at.strftime("%Y-%m-%d")}</lastmod>')
            xml_parts.append('    <changefreq>weekly</changefreq>')
            xml_parts.append('    <priority>0.8</priority>')
            if len(group_posts) > 1:
                for version in group_posts:
                    xml_parts.append(f'    <xhtml:link rel="alternate" hreflang="{version.language}" href="{base_url}/{version.slug}"/>')
            xml_parts.append('  </url>')
    
    # Pages — grouped by translation_group (skip homepage pages, already included above)
    for group_id, group_pages in page_groups.items():
        # Skip homepage pages — they are already listed as /
        if homepage_group and group_pages and group_pages[0].translation_group == homepage_group:
            continue
        for page in group_pages:
            stats['pages'] += 1
            xml_parts.append('  <url>')
            xml_parts.append(f'    <loc>{base_url}/{page.slug}</loc>')
            xml_parts.append(f'    <lastmod>{page.updated_at.strftime("%Y-%m-%d")}</lastmod>')
            xml_parts.append('    <changefreq>monthly</changefreq>')
            xml_parts.append('    <priority>0.6</priority>')
            if len(group_pages) > 1:
                for version in group_pages:
                    xml_parts.append(f'    <xhtml:link rel="alternate" hreflang="{version.language}" href="{base_url}/{version.slug}"/>')
            xml_parts.append('  </url>')
    
    # ── Tag & category listing pages + their hubs ────────────────────────
    # Listing pages are legitimate index pages, so they belong in the map —
    # but only their canonical spellings: first page only (?page=N is
    # discovered by crawling the pagination links), default language on the
    # clean URL, other languages via ?lang=. Only language versions that
    # actually list something are included — an empty listing would be a
    # soft-404 in the map. Tag/category names may contain spaces and
    # non-ASCII, hence quote(); GSC shows raw-space URLs otherwise.
    from urllib.parse import quote
    
    def _listing_url(path, language):
        if language == default_lang:
            return f'{base_url}{path}'
        return f'{base_url}{path}?lang={language}'
    
    # name -> {language: latest updated_at}
    tag_langs = {}
    cat_langs = {}
    for post in posts:
        for tg in (post.get_tags_list() or []):
            d = tag_langs.setdefault(tg, {})
            if post.language not in d or post.updated_at > d[post.language]:
                d[post.language] = post.updated_at
        if post.category:
            d = cat_langs.setdefault(post.category, {})
            if post.language not in d or post.updated_at > d[post.language]:
                d[post.language] = post.updated_at
    for page in pages:
        for tg in (page.get_tags_list() or []):
            d = tag_langs.setdefault(tg, {})
            if page.language not in d or page.updated_at > d[page.language]:
                d[page.language] = page.updated_at
    
    def _emit_listing_group(path, lang_lastmod, changefreq, priority):
        # Same ordering convention as the rest of the sitemap: the default
        # language (clean URL) first, then the other languages alphabetically.
        langs = sorted(lang_lastmod.keys(), key=lambda l: (l != default_lang, l))
        for language in langs:
            stats['listings'] += 1
            xml_parts.append('  <url>')
            xml_parts.append(f'    <loc>{_listing_url(path, language)}</loc>')
            xml_parts.append(f'    <lastmod>{lang_lastmod[language].strftime("%Y-%m-%d")}</lastmod>')
            xml_parts.append(f'    <changefreq>{changefreq}</changefreq>')
            xml_parts.append(f'    <priority>{priority}</priority>')
            if len(langs) > 1:
                for other in langs:
                    xml_parts.append(f'    <xhtml:link rel="alternate" hreflang="{other}" href="{_listing_url(path, other)}"/>')
            xml_parts.append('  </url>')
    
    # Hubs (/tag/ and /category/) — one per language that has any listings.
    hub_tag_langs = {}
    for d in tag_langs.values():
        for language, ts in d.items():
            if language not in hub_tag_langs or ts > hub_tag_langs[language]:
                hub_tag_langs[language] = ts
    hub_cat_langs = {}
    for d in cat_langs.values():
        for language, ts in d.items():
            if language not in hub_cat_langs or ts > hub_cat_langs[language]:
                hub_cat_langs[language] = ts
    if hub_tag_langs:
        _emit_listing_group('/tag/', hub_tag_langs, 'weekly', '0.5')
    if hub_cat_langs:
        _emit_listing_group('/category/', hub_cat_langs, 'weekly', '0.5')
    
    for cat_name in sorted(cat_langs.keys(), key=str.lower):
        _emit_listing_group(f'/category/{quote(cat_name, safe="")}',
                            cat_langs[cat_name], 'weekly', '0.5')
    for tag_name in sorted(tag_langs.keys(), key=str.lower):
        _emit_listing_group(f'/tag/{quote(tag_name, safe="")}',
                            tag_langs[tag_name], 'weekly', '0.4')
    
    xml_parts.append('</urlset>')
    
    stats['total'] = sum(stats.values())
    return '\n'.join(xml_parts), stats


@app.route('/robots.txt')
def robots_txt():
    """Serve robots.txt from settings (editable in admin)"""
    from flask import Response
    
    content = Settings.get('robots_txt', 'User-agent: *\nDisallow: /admin/\nDisallow: /api/\nAllow: /api/public/\nDisallow: /setup/\nDisallow: /unsubscribe\nDisallow: /confirm-email\nDisallow: /invite/\nAllow: /')
    
    # Auto-append sitemap URL if not already present
    if 'Sitemap:' not in content:
        sitemap_url = request.url_root.rstrip('/') + '/sitemap.xml'
        content = content.rstrip('\n') + f'\n\nSitemap: {sitemap_url}'
    
    # Note: We do NOT auto-append a "Llms-txt:" directive to robots.txt.
    # The llmstxt.org spec proposed this directive, but Google's robots.txt
    # parser flags it as "Unknown directive" (it's not in the official Robots
    # Exclusion Protocol), causing a warning in PageSpeed Insights and Search
    # Console. AI crawlers (GPTBot, ClaudeBot, PerplexityBot, etc.) discover
    # /llms.txt by convention — same way search engines find /robots.txt —
    # so no explicit reference is needed.
    
    return Response(content, mimetype='text/plain')


@app.route('/llms.txt')
def llms_txt():
    """Auto-generated llms.txt — AI-friendly site summary in Markdown.
    Spec: https://llmstxt.org/
    Multilingual: groups content by language with translated section headers.
    """
    from flask import Response

    base_url = request.url_root.rstrip('/')
    site_name = Settings.get('site_name', 'AliothPress')
    site_desc = Settings.get('site_description', '')
    site_tagline = Settings.get('site_tagline', '')
    default_lang = Settings.get('default_language', 'en')

    lines = []

    # H1 — required by spec
    lines.append(f'# {site_name}')
    lines.append('')

    # Blockquote — site summary
    summary = site_desc or site_tagline
    if summary:
        lines.append(f'> {summary}')
        lines.append('')

    # Get active languages
    active_codes = _get_active_language_codes()

    # --- Posts ---
    posts = Post.query.filter(
        Post.status == 'published',
        Post.language.in_(list(active_codes))
    ).order_by(Post.published_at.desc()).limit(50).all()

    if posts:
        lines.append(f'## {pt_for("llms_posts", default_lang)}')
        lines.append('')

        # Group posts by language
        posts_by_lang = {}
        for post in posts:
            lang = post.language or default_lang
            if lang not in posts_by_lang:
                posts_by_lang[lang] = []
            posts_by_lang[lang].append(post)

        # Show default language first, then the rest
        lang_order = [default_lang] + [l for l in sorted(posts_by_lang.keys()) if l != default_lang]

        for lang in lang_order:
            if lang not in posts_by_lang:
                continue
            lang_name = LANG_CODE_TO_NAME.get(lang, lang)
            if len(posts_by_lang) > 1:
                lines.append(f'### {lang_name}')
            for post in posts_by_lang[lang]:
                title = post.meta_title or post.title
                desc = post.meta_description or post.excerpt or ''
                if len(desc) > 120:
                    desc = desc[:117].rsplit(' ', 1)[0] + '...'
                line = f'- [{title}]({base_url}/{post.slug})'
                if desc:
                    line += f': {desc}'
                lines.append(line)
            lines.append('')

    # --- Pages ---
    pages = Page.query.filter(
        Page.status == 'published',
        Page.language.in_(list(active_codes))
    ).order_by(Page.updated_at.desc()).all()

    content_pages = []
    legal_pages = []
    for page in pages:
        if page.translation_group and page.translation_group.startswith('default_'):
            legal_pages.append(page)
        else:
            content_pages.append(page)

    if content_pages:
        lines.append(f'## {pt_for("llms_pages", default_lang)}')
        lines.append('')

        # Group pages by language
        pages_by_lang = {}
        homepage_group = Settings.get('homepage_group', '')
        for page in content_pages:
            lang = page.language or default_lang
            if lang not in pages_by_lang:
                pages_by_lang[lang] = []
            pages_by_lang[lang].append(page)

        lang_order = [default_lang] + [l for l in sorted(pages_by_lang.keys()) if l != default_lang]

        for lang in lang_order:
            if lang not in pages_by_lang:
                continue
            lang_name = LANG_CODE_TO_NAME.get(lang, lang)
            if len(pages_by_lang) > 1:
                lines.append(f'### {lang_name}')
            for page in pages_by_lang[lang]:
                title = page.meta_title or page.title
                desc = page.meta_description or page.excerpt or ''
                if len(desc) > 120:
                    desc = desc[:117].rsplit(' ', 1)[0] + '...'
                # Homepage pages use / instead of /slug
                if homepage_group and page.translation_group == homepage_group:
                    page_url = f'{base_url}/?lang={page.language}' if page.language != default_lang else f'{base_url}/'
                else:
                    page_url = f'{base_url}/{page.slug}'
                line = f'- [{title}]({page_url})'
                if desc:
                    line += f': {desc}'
                lines.append(line)
            lines.append('')

    if legal_pages:
        lines.append(f'## {pt_for("llms_legal", default_lang)}')
        lines.append('')
        for page in legal_pages:
            lines.append(f'- [{page.title}]({base_url}/{page.slug})')
        lines.append('')

    content = '\n'.join(lines)
    return Response(content, mimetype='text/plain; charset=utf-8')


# ============================================================================
# RSS FEED — Multilingual, SEO-friendly
# ============================================================================

@app.route('/feed')
@app.route('/feed/<lang_code>')
def rss_feed(lang_code=None):
    """RSS 2.0 feed with multilingual support.
    
    /feed           — posts in default language
    /feed/de        — posts in German
    /feed/es        — posts in Spanish
    
    Includes proper XML escaping, publication dates, categories (tags),
    featured images as enclosures, and hreflang alternate links.
    """
    from flask import Response
    from html import escape as h
    
    base_url = request.url_root.rstrip('/')
    default_lang = Settings.get('default_language', 'en')
    lang = lang_code or default_lang
    active_codes = _get_active_language_codes()
    site_name = Settings.get('site_name', 'AliothPress')
    site_tagline = Settings.get('site_tagline', '')
    
    # If a specific language was requested but it's deactivated — 404
    if lang_code and lang_code not in active_codes:
        return render_template('public/404.html',
                               all_versions=_build_hub_lang_versions(request.path)), 404
    
    # Get published posts for this language (active languages only)
    posts = Post.query.filter_by(
        status='published', language=lang
    ).order_by(Post.published_at.desc()).limit(20).all()
    
    # If no posts in this language, fall back to default
    if not posts and lang != default_lang:
        posts = Post.query.filter_by(
            status='published', language=default_lang
        ).order_by(Post.published_at.desc()).limit(20).all()
        lang = default_lang
    
    # Build date for lastBuildDate
    last_build = posts[0].published_at if posts else utc_now()
    
    # Get active languages for alternate links
    active_langs = _get_active_languages()
    
    xml = ['<?xml version="1.0" encoding="UTF-8"?>']
    xml.append('<rss version="2.0"')
    xml.append('     xmlns:atom="http://www.w3.org/2005/Atom"')
    xml.append('     xmlns:dc="http://purl.org/dc/elements/1.1/"')
    xml.append('     xmlns:media="http://search.yahoo.com/mrss/">')
    xml.append('<channel>')
    xml.append(f'  <title>{h(site_name)}</title>')
    xml.append(f'  <link>{base_url}</link>')
    xml.append(f'  <description>{h(site_tagline)}</description>')
    xml.append(f'  <language>{lang}</language>')
    xml.append(f'  <lastBuildDate>{last_build.strftime("%a, %d %b %Y %H:%M:%S +0000")}</lastBuildDate>')
    xml.append(f'  <generator>AliothPress</generator>')
    
    # Self-referencing atom:link (required for valid RSS)
    feed_url = f'{base_url}/feed/{lang}' if lang_code else f'{base_url}/feed'
    xml.append(f'  <atom:link href="{h(feed_url)}" rel="self" type="application/rss+xml"/>')
    
    # Alternate language feeds
    for al in active_langs:
        if al['code'] != lang:
            xml.append(f'  <atom:link href="{base_url}/feed/{al["code"]}" rel="alternate" hreflang="{al["code"]}" type="application/rss+xml"/>')
    
    # Site logo as channel image
    site_logo = Settings.get('site_logo', '')
    if site_logo:
        logo_url = site_logo if site_logo.startswith('http') else f'{base_url}{site_logo}'
        xml.append('  <image>')
        xml.append(f'    <url>{h(logo_url)}</url>')
        xml.append(f'    <title>{h(site_name)}</title>')
        xml.append(f'    <link>{base_url}</link>')
        xml.append('  </image>')
    
    for post in posts:
        post_url = f'{base_url}/{post.slug}'
        pub_date = (post.published_at or post.created_at).strftime('%a, %d %b %Y %H:%M:%S +0000')
        
        xml.append('  <item>')
        xml.append(f'    <title>{h(post.title)}</title>')
        xml.append(f'    <link>{post_url}</link>')
        xml.append(f'    <guid isPermaLink="true">{post_url}</guid>')
        xml.append(f'    <pubDate>{pub_date}</pubDate>')
        
        # Author
        author_name = post.schema_author or (post.author.get_display_name() if post.author else '')
        if author_name:
            xml.append(f'    <dc:creator>{h(author_name)}</dc:creator>')
        
        # Description (excerpt or truncated content)
        desc = post.excerpt or post.meta_description or ''
        if not desc and post.content:
            # Strip HTML and truncate
            import re
            plain = re.sub(r'<[^>]+>', '', post.content)
            desc = plain[:300].rsplit(' ', 1)[0] + '...' if len(plain) > 300 else plain
        if desc:
            xml.append(f'    <description><![CDATA[{desc}]]></description>')
        
        # Tags as categories
        if post.tags:
            for tag in post.get_tags_list():
                xml.append(f'    <category>{h(tag)}</category>')
        
        # Featured image as media:content
        if post.featured_image:
            img_url = post.featured_image if post.featured_image.startswith('http') else f'{base_url}{post.featured_image}'
            xml.append(f'    <media:content url="{h(img_url)}" medium="image"/>')
        
        xml.append('  </item>')
    
    xml.append('</channel>')
    xml.append('</rss>')
    
    xml_content = '\n'.join(xml)
    return Response(xml_content, mimetype='application/rss+xml; charset=utf-8')


def _mask_env_secrets(env_content):
    """Mask sensitive values in .env content for backup safety."""
    sensitive_keys = [
        'SECRET_KEY', 'GITHUB_TOKEN', 'ANTHROPIC_API_KEY',
        'HUGGINGFACE_TOKEN', 'DATABASE_URI'
    ]
    lines = env_content.split('\n')
    masked = []
    for line in lines:
        for key in sensitive_keys:
            if line.strip().startswith(f'{key}='):
                line = f'{key}=REDACTED_IN_BACKUP'
                break
        masked.append(line)
    return '\n'.join(masked)


@app.route('/admin/backup', methods=['GET', 'POST'])
@login_required
def admin_backup():
    """Backup system — create downloadable archive of DB + uploads"""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    if request.method == 'POST':
        import zipfile
        import tempfile
        import shutil
        
        try:
            # Determine what to backup
            project_dir = os.path.dirname(os.path.abspath(__file__))
            upload_folder = app.config['UPLOAD_FOLDER']
            db_uri = app.config['SQLALCHEMY_DATABASE_URI']
            
            # Create backup filename with timestamp
            timestamp = utc_now().strftime('%Y%m%d_%H%M%S')
            backup_filename = f'aliothpress_backup_{timestamp}.zip'
            
            # Create temp directory for backup
            temp_dir = tempfile.mkdtemp()
            zip_path = os.path.join(temp_dir, backup_filename)
            
            with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
                
                # 1. Backup database
                if 'sqlite' in db_uri:
                    # SQLite: find the actual database file path
                    from sqlalchemy import inspect as sa_inspect
                    db_path = db.engine.url.database
                    if db_path and not os.path.isabs(db_path):
                        db_path = os.path.join(project_dir, db_path)
                    if db_path and os.path.exists(db_path):
                        zf.write(db_path, 'database/aliothpress.db')
                    else:
                        # Fallback: try instance folder
                        instance_path = os.path.join(project_dir, 'instance', 'aliothpress.db')
                        if os.path.exists(instance_path):
                            zf.write(instance_path, 'database/aliothpress.db')
                else:
                    # PostgreSQL: dump using pg_dump
                    import subprocess
                    import re
                    
                    # Parse URI: postgresql://user:pass@host:port/dbname
                    match = re.match(
                        r'postgresql://([^:]+):([^@]*)@([^:]+):(\d+)/(.+)', db_uri
                    )
                    if match:
                        pg_user, pg_pass, pg_host, pg_port, pg_dbname = match.groups()
                        dump_path = os.path.join(temp_dir, 'database.sql')
                        
                        env = os.environ.copy()
                        env['PGPASSWORD'] = pg_pass
                        
                        result = subprocess.run(
                            ['pg_dump', '-h', pg_host, '-p', pg_port,
                             '-U', pg_user, '-d', pg_dbname, '-f', dump_path],
                            env=env, capture_output=True, text=True, timeout=120
                        )
                        
                        if result.returncode == 0 and os.path.exists(dump_path):
                            zf.write(dump_path, 'database/aliothpress.sql')
                        else:
                            # If pg_dump fails, add error note
                            zf.writestr('database/DUMP_ERROR.txt',
                                       f'pg_dump failed: {result.stderr}')
                
                # 2. Backup uploads folder
                if os.path.exists(upload_folder):
                    for root, dirs, files in os.walk(upload_folder):
                        for file in files:
                            file_path = os.path.join(root, file)
                            arcname = os.path.join(
                                'uploads',
                                os.path.relpath(file_path, upload_folder)
                            )
                            zf.write(file_path, arcname)
                
                # 2b. Backup installed plugins (their settings/licenses live
                # in the DB, but the code lives on disk — without this a
                # restore on a fresh server would lose the plugin files)
                plugins_folder = os.path.join(project_dir, 'plugins')
                if os.path.exists(plugins_folder):
                    for root, dirs, files in os.walk(plugins_folder):
                        dirs[:] = [d for d in dirs if d != '__pycache__']
                        for file in files:
                            file_path = os.path.join(root, file)
                            arcname = os.path.join(
                                'plugins',
                                os.path.relpath(file_path, plugins_folder)
                            )
                            zf.write(file_path, arcname)
                
                # 3. Backup .env (with sensitive values masked for security)
                env_path = os.path.join(project_dir, '.env')
                if os.path.exists(env_path):
                    with open(env_path, 'r') as f:
                        env_content = f.read()
                    # Mask sensitive values
                    masked_env = _mask_env_secrets(env_content)
                    zf.writestr('config/.env', masked_env)
            
            # Log the backup
            zip_size = os.path.getsize(zip_path)
            size_mb = round(zip_size / (1024 * 1024), 1)
            AuditLog.log('create_backup', 'backup', None,
                         _ad('created_backup', size=str(size_mb)),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
            
            # Send file to browser for download
            response = send_from_directory(
                temp_dir, backup_filename,
                as_attachment=True,
                download_name=backup_filename
            )
            
            # Clean up temp dir after response is sent
            @response.call_on_close
            def cleanup():
                shutil.rmtree(temp_dir, ignore_errors=True)
            
            return response
            
        except Exception as e:
            flash(t('flash.backup_failed', error=str(e)), 'error')
            AuditLog.log('backup_failed', 'backup', None,
                         _ad('backup_failed', error=str(e)),
                         user=current_user, ip=request.remote_addr)
            db.session.commit()
    
    # GET — show backup page with info
    project_dir = os.path.dirname(os.path.abspath(__file__))
    upload_folder = app.config['UPLOAD_FOLDER']
    db_uri = app.config['SQLALCHEMY_DATABASE_URI']
    
    # Calculate uploads size
    uploads_size = 0
    uploads_count = 0
    if os.path.exists(upload_folder):
        for root, dirs, files in os.walk(upload_folder):
            for file in files:
                uploads_size += os.path.getsize(os.path.join(root, file))
                uploads_count += 1
    
    # Database info
    db_type = 'PostgreSQL' if 'postgresql' in db_uri else 'SQLite'
    db_size = 0
    if 'sqlite' in db_uri:
        db_path = db.engine.url.database
        if db_path and not os.path.isabs(db_path):
            db_path = os.path.join(project_dir, db_path)
        if not db_path or not os.path.exists(db_path):
            db_path = os.path.join(project_dir, 'instance', 'aliothpress.db')
        if os.path.exists(db_path):
            db_size = os.path.getsize(db_path)
    
    return render_template('admin/backup.html',
                         db_type=db_type,
                         db_size=db_size,
                         uploads_size=uploads_size,
                         uploads_count=uploads_count,
                         post_count=Post.query.count(),
                         page_count=Page.query.count(),
                         image_count=Image.query.count())


@app.route('/admin/backup/restore', methods=['POST'])
@login_required
def admin_backup_restore():
    """Restore site from a backup ZIP file"""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    import zipfile
    import tempfile
    import shutil
    
    file = request.files.get('backup_file')
    if not file or not file.filename.endswith('.zip'):
        flash(t('flash.backup_upload_zip'), 'error')
        return redirect(url_for('admin_backup'))
    
    try:
        # Save uploaded ZIP to temp location
        temp_dir = tempfile.mkdtemp()
        zip_path = os.path.join(temp_dir, 'backup.zip')
        file.save(zip_path)
        
        # Validate it's a real ZIP
        if not zipfile.is_zipfile(zip_path):
            flash(t('flash.backup_invalid_zip'), 'error')
            shutil.rmtree(temp_dir, ignore_errors=True)
            return redirect(url_for('admin_backup'))
        
        extract_dir = os.path.join(temp_dir, 'extracted')
        
        with zipfile.ZipFile(zip_path, 'r') as zf:
            # Security: check for path traversal (hardened)
            for name in zf.namelist():
                safe_name = os.path.normpath(name)
                if safe_name.startswith('..') or os.path.isabs(safe_name):
                    flash(t('flash.backup_suspicious'), 'error')
                    shutil.rmtree(temp_dir, ignore_errors=True)
                    return redirect(url_for('admin_backup'))
                # Verify extracted path stays within extract_dir
                full_path = os.path.realpath(os.path.join(extract_dir, name))
                if not full_path.startswith(os.path.realpath(extract_dir)):
                    flash(t('flash.backup_suspicious'), 'error')
                    shutil.rmtree(temp_dir, ignore_errors=True)
                    return redirect(url_for('admin_backup'))
            
            zf.extractall(extract_dir)
        
        project_dir = os.path.dirname(os.path.abspath(__file__))
        upload_folder = app.config['UPLOAD_FOLDER']
        db_uri = app.config['SQLALCHEMY_DATABASE_URI']
        
        restored_items = []
        
        # 1. Restore database
        if 'sqlite' in db_uri:
            backup_db = os.path.join(extract_dir, 'database', 'aliothpress.db')
            if os.path.exists(backup_db):
                # Find current DB path
                db_path = db.engine.url.database
                if db_path and not os.path.isabs(db_path):
                    db_path = os.path.join(project_dir, db_path)
                if not db_path or not os.path.exists(db_path):
                    db_path = os.path.join(project_dir, 'instance', 'aliothpress.db')
                
                if db_path:
                    # Close current connections
                    db.session.remove()
                    db.engine.dispose()
                    
                    # Replace database file
                    os.makedirs(os.path.dirname(db_path), exist_ok=True)
                    shutil.copy2(backup_db, db_path)
                    restored_items.append('Database')
        else:
            # PostgreSQL: restore from SQL dump
            backup_sql = os.path.join(extract_dir, 'database', 'aliothpress.sql')
            if os.path.exists(backup_sql):
                import subprocess
                import re
                
                match = re.match(
                    r'postgresql://([^:]+):([^@]*)@([^:]+):(\d+)/(.+)', db_uri
                )
                if match:
                    pg_user, pg_pass, pg_host, pg_port, pg_dbname = match.groups()
                    
                    env = os.environ.copy()
                    env['PGPASSWORD'] = pg_pass
                    
                    # Drop and recreate all tables, then restore
                    result = subprocess.run(
                        ['psql', '-h', pg_host, '-p', pg_port,
                         '-U', pg_user, '-d', pg_dbname, '-f', backup_sql],
                        env=env, capture_output=True, text=True, timeout=120
                    )
                    
                    if result.returncode == 0:
                        restored_items.append('Database (PostgreSQL)')
                    else:
                        flash(t('flash.db_restore_warning', warning=result.stderr[:200]), 'warning')
        
        # 2. Restore uploads
        backup_uploads = os.path.join(extract_dir, 'uploads')
        if os.path.exists(backup_uploads):
            # Ensure upload folder exists
            os.makedirs(upload_folder, exist_ok=True)
            
            # Copy files (overwrite existing)
            file_count = 0
            for root, dirs, files in os.walk(backup_uploads):
                for fname in files:
                    src = os.path.join(root, fname)
                    rel_path = os.path.relpath(src, backup_uploads)
                    dst = os.path.join(upload_folder, rel_path)
                    os.makedirs(os.path.dirname(dst), exist_ok=True)
                    shutil.copy2(src, dst)
                    file_count += 1
            restored_items.append(f'Uploads ({file_count} files)')
        
        # 2b. Restore plugins (owner only — plugin files are executable
        # code, so restoring them is gated like plugin installation is.
        # The ZIP was already path-traversal-checked above.)
        backup_plugins = os.path.join(extract_dir, 'plugins')
        if os.path.exists(backup_plugins):
            if current_user.is_owner():
                plugins_folder = os.path.join(project_dir, 'plugins')
                os.makedirs(plugins_folder, exist_ok=True)
                plugin_file_count = 0
                for root, dirs, files in os.walk(backup_plugins):
                    dirs[:] = [d for d in dirs if d != '__pycache__']
                    for fname in files:
                        src = os.path.join(root, fname)
                        rel_path = os.path.relpath(src, backup_plugins)
                        dst = os.path.join(plugins_folder, rel_path)
                        os.makedirs(os.path.dirname(dst), exist_ok=True)
                        shutil.copy2(src, dst)
                        plugin_file_count += 1
                restored_items.append(f'Plugins ({plugin_file_count} files)')
            else:
                restored_items.append('Plugins SKIPPED (owner role required)')
        
        # 3. Restore .env (optional — only if present in backup)
        backup_env = os.path.join(extract_dir, 'config', '.env')
        if os.path.exists(backup_env):
            env_path = os.path.join(project_dir, '.env')
            shutil.copy2(backup_env, env_path)
            restored_items.append('Configuration (.env)')
        
        # Clean up temp files
        shutil.rmtree(temp_dir, ignore_errors=True)
        
        # Log the restore
        AuditLog.log('restore_backup', 'backup', None,
                     _ad('restored_backup', items=", ".join(restored_items)),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
        
        if restored_items:
            flash(t('flash.backup_restored', items=', '.join(restored_items)), 'success')
        else:
            flash(t('flash.backup_no_data'), 'warning')
        
    except Exception as e:
        flash(t('flash.backup_restore_failed', error=str(e)), 'error')
        AuditLog.log('restore_failed', 'backup', None,
                     _ad('restore_failed', error=str(e)),
                     user=current_user, ip=request.remote_addr)
        db.session.commit()
    
    return redirect(url_for('admin_backup'))


# Actions whose audit entries carry a meaningful IP address. Only these
# show the IP in the audit log view; content actions never display one.
AUDIT_SECURITY_ACTIONS = frozenset([
    'login', 'login_failed', 'login_blocked', 'logout',
    'password_reset_request', 'password_reset', 'password_reset_email_failed',
    '2fa_enabled', '2fa_disabled', '2fa_reset_by_admin', '2fa_login_failed',
    '2fa_recovery_used', '2fa_email_code', '2fa_bypass_file',
])


@app.route('/admin/audit-log')
@login_required
def admin_audit_log():
    """View audit log — security and activity monitoring"""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    page = get_page()
    per_page = get_per_page(50)
    
    # Filter options
    action_filter = request.args.get('action', '')
    actor_filter = request.args.get('actor', '')   # '', 'human', 'agent'
    
    query = AuditLog.query.order_by(AuditLog.created_at.desc())
    
    if action_filter:
        query = query.filter(AuditLog.action == action_filter)
    if actor_filter == 'agent':
        query = query.filter(AuditLog.via_agent.is_(True))
    elif actor_filter == 'human':
        query = query.filter(AuditLog.via_agent.is_(False))
    
    logs = query.paginate(page=page, per_page=per_page, error_out=False)
    
    # Get unique actions for the filter dropdown
    all_actions = db.session.query(AuditLog.action).distinct().order_by(AuditLog.action).all()
    action_list = [a[0] for a in all_actions]
    
    return render_template('admin/audit_log.html', 
                         logs=logs, action_filter=action_filter,
                         actor_filter=actor_filter,
                         action_list=action_list, per_page=per_page,
                         SECURITY_ACTIONS=AUDIT_SECURITY_ACTIONS)


# ============================================================================
# AEO HELPERS — Schema.org sameAs arrays for Entity Graph
# ============================================================================

def _build_org_sameas():
    """Collect all non-empty organization sameAs URLs for Schema.org."""
    keys = [
        'social_facebook', 'social_twitter', 'social_linkedin',
        'social_github', 'social_youtube', 'social_instagram',
        'org_sameas_wikipedia', 'org_sameas_wikidata', 'org_sameas_crunchbase'
    ]
    urls = [Settings.get(k, '') for k in keys]
    return [v for v in urls if v]


def _build_author_sameas(user):
    """Collect all non-empty author sameAs URLs for Schema.org."""
    urls = []
    for attr in ('linkedin_url', 'website_url', 'github_url'):
        val = getattr(user, attr, None)
        if val:
            urls.append(val)
    return urls


@app.route('/admin/tools', methods=['GET', 'POST'])
@login_required
def admin_tools():
    """SEO Tools — sitemap preview, robots.txt editor"""
    if request.method == 'POST':
        robots_content = request.form.get('robots_txt', '')
        Settings.set('robots_txt', robots_content)
        flash(t('flash.robots_updated'), 'success')
        return redirect(url_for('admin_tools'))
    
    # Get current robots.txt
    robots_content = Settings.get('robots_txt', 'User-agent: *\nDisallow: /admin/\nDisallow: /api/\nAllow: /api/public/\nDisallow: /setup/\nDisallow: /unsubscribe\nDisallow: /confirm-email\nDisallow: /invite/\nAllow: /')
    
    # Sitemap numbers come from the SAME builder that serves /sitemap.xml,
    # counted at emission time — so the card is exact by construction.
    # The old formula (1 + posts + pages) over-counted whenever the homepage
    # is a static multilingual page: its language versions sat inside the
    # page count AND were re-added as the flat "+1".
    _sitemap_xml, sitemap_stats = _build_sitemap_xml()
    sitemap_count = sitemap_stats['total']
    
    # llms.txt counts (content stats, not sitemap rows)
    post_count = Post.query.filter_by(status='published').count()
    page_count = Page.query.filter_by(status='published').count()
    llms_post_count = min(post_count, 50)
    llms_page_count = page_count
    
    return render_template('admin/tools.html', 
                         robots_content=robots_content,
                         sitemap_count=sitemap_count,
                         sitemap_stats=sitemap_stats,
                         post_count=post_count,
                         page_count=page_count,
                         llms_post_count=llms_post_count,
                         llms_page_count=llms_page_count)


# ============================================================================
# THEMES MANAGEMENT
# ============================================================================

THEME_NAME_RE = re.compile(r'^[a-z0-9-]{2,50}$')
MAX_THEME_ZIP_BYTES = 10 * 1024 * 1024        # 10 MB compressed
MAX_THEME_UNPACKED_BYTES = 30 * 1024 * 1024   # 30 MB unpacked
THEME_ALLOWED_EXTENSIONS = {
    'css', 'svg', 'png', 'jpg', 'jpeg', 'webp', 'gif', 'ico',
    'woff', 'woff2', 'ttf', 'otf', 'txt', 'md',
}

# Themes shipped with the CMS. Deleting them is pointless (the next core
# update restores their files), so they are not deletable — only custom
# (uploaded) themes are. When a new built-in theme ships with a release,
# add its folder name here.
BUILTIN_THEMES = {
    'amber-blossom', 'corporate-clean', 'crimson-velvet', 'editorial-press',
    'emerald-forest', 'frozen-moss', 'handmade-story', 'midnight-botanist',
    'nebula-rose', 'neon-launch', 'ocean-whispers', 'oceanic-depth',
    'solar-flare', 'stellar-origin', 'volcanic-ash',
}


@app.route('/admin/themes/upload', methods=['POST'])
@login_required
def admin_themes_upload():
    """Install a theme from an uploaded ZIP.

    A theme is a folder with theme.css (required) and optional preview
    image / fonts. Static assets only — no code — so unlike plugins this
    needs no license gating, no owner role, and no restart: the theme
    appears in the list immediately. Files are written by the service
    user, so future CMS updates never stumble over ownership.

    Accepted layouts: theme.css at the ZIP root (folder name derived from
    the ZIP filename) or a single top-level folder containing theme.css.
    """
    import zipfile as _zipfile
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))

    f = request.files.get('theme_zip')
    if not f or not f.filename:
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    # Size cap (compressed)
    f.stream.seek(0, os.SEEK_END)
    if f.stream.tell() > MAX_THEME_ZIP_BYTES:
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))
    f.stream.seek(0)

    try:
        zf = _zipfile.ZipFile(f.stream)
    except _zipfile.BadZipFile:
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    names = [n.replace('\\', '/') for n in zf.namelist()]
    # Path traversal + size-bomb checks
    total_unpacked = 0
    for info in zf.infolist():
        n = info.filename.replace('\\', '/')
        if n.startswith('/') or '..' in n.split('/'):
            flash(t('themes.upload_err'), 'error')
            return redirect(url_for('admin_themes'))
        total_unpacked += info.file_size
    if total_unpacked > MAX_THEME_UNPACKED_BYTES:
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    # Layout: theme.css at root, or inside a single top-level folder
    roots = {n.split('/', 1)[0] for n in names if n.strip('/')}
    if 'theme.css' in names:
        prefix = ''
        raw_name = os.path.splitext(os.path.basename(f.filename))[0]
    elif len(roots) == 1 and (next(iter(roots)) + '/theme.css') in names:
        prefix = next(iter(roots)) + '/'
        raw_name = next(iter(roots))
    else:
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    # Sanitize folder name: lowercase, spaces/underscores -> hyphens
    theme_name = re.sub(r'[^a-z0-9-]', '',
                        raw_name.strip().lower().replace(' ', '-').replace('_', '-'))
    if not THEME_NAME_RE.match(theme_name):
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    themes_dir = os.path.join(app.static_folder, 'css', 'themes')
    target = os.path.join(themes_dir, theme_name)
    replacing = os.path.isdir(target)
    tmp_target = target + '.uploading'

    import shutil as _shutil
    if os.path.exists(tmp_target):
        _shutil.rmtree(tmp_target, ignore_errors=True)
    os.makedirs(tmp_target, exist_ok=True)

    try:
        for info in zf.infolist():
            n = info.filename.replace('\\', '/')
            rel = n[len(prefix):] if prefix and n.startswith(prefix) else (n if not prefix else None)
            if not rel or rel.endswith('/'):
                continue
            ext = rel.rsplit('.', 1)[-1].lower() if '.' in rel else ''
            base = os.path.basename(rel)
            if ext not in THEME_ALLOWED_EXTENSIONS or base.startswith('.'):
                continue  # skip junk (.DS_Store, executables, etc.)
            dest = os.path.normpath(os.path.join(tmp_target, rel))
            if not dest.startswith(os.path.normpath(tmp_target) + os.sep):
                raise ValueError('zip-slip attempt')
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            with zf.open(info) as src, open(dest, 'wb') as out:
                _shutil.copyfileobj(src, out)
        if not os.path.isfile(os.path.join(tmp_target, 'theme.css')):
            raise ValueError('theme.css missing after extraction')
        if replacing:
            _shutil.rmtree(target)
        os.rename(tmp_target, target)
    except Exception:
        _shutil.rmtree(tmp_target, ignore_errors=True)
        app.logger.exception('Theme upload failed')
        flash(t('themes.upload_err'), 'error')
        return redirect(url_for('admin_themes'))

    AuditLog.log('upload_theme', 'theme', None, theme_name,
                 user=current_user, ip=request.remote_addr)
    db.session.commit()
    flash(t('themes.upload_replaced' if replacing else 'themes.upload_ok',
            name=theme_name), 'success')
    return redirect(url_for('admin_themes'))


@app.route('/admin/themes/<theme_name>/delete', methods=['POST'])
@login_required
def admin_themes_delete(theme_name):
    """Delete a theme folder. The active theme cannot be deleted.
    Themes shipped with the CMS reappear on the next core update."""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    if not THEME_NAME_RE.match(theme_name or ''):
        flash(t('flash.theme_not_found'), 'error')
        return redirect(url_for('admin_themes'))
    if theme_name in BUILTIN_THEMES:
        # No delete button exists for built-ins; guard direct POSTs too.
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_themes'))
    if theme_name == Settings.get('active_theme', 'stellar-origin'):
        flash(t('themes.cannot_delete_active'), 'error')
        return redirect(url_for('admin_themes'))
    target = os.path.join(app.static_folder, 'css', 'themes', theme_name)
    if not os.path.isdir(target):
        flash(t('flash.theme_not_found'), 'error')
        return redirect(url_for('admin_themes'))
    import shutil as _shutil
    _shutil.rmtree(target, ignore_errors=True)
    AuditLog.log('delete_theme', 'theme', None, theme_name,
                 user=current_user, ip=request.remote_addr)
    db.session.commit()
    flash(t('themes.deleted_ok', name=theme_name), 'success')
    return redirect(url_for('admin_themes'))


@app.route('/admin/themes', methods=['GET', 'POST'])
@login_required
def admin_themes():
    """Theme management — browse and activate themes"""
    if not current_user.can_access_settings():
        flash(t('flash.access_denied'), 'error')
        return redirect(url_for('admin_dashboard'))
    
    if request.method == 'POST':
        # Custom CSS form — independent from theme/mode forms (separate <form>)
        if 'site_custom_css' in request.form:
            custom_css_input = request.form.get('site_custom_css', '')
            is_valid, err_key = _validate_custom_css(custom_css_input)
            if not is_valid:
                # Preserve rejected input across redirect so admin doesn't lose work
                session['_custom_css_rejected'] = custom_css_input
                flash(t(err_key), 'error')
                return redirect(url_for('admin_themes'))
            
            old_value = Settings.get('site_custom_css', '')
            if old_value != custom_css_input:
                Settings.set('site_custom_css', custom_css_input)
                db.session.commit()
                AuditLog.log('edit_custom_css', 'settings', None,
                             _ad('custom_css_updated'),
                             user=current_user, ip=request.remote_addr)
                db.session.commit()
            flash(t('flash.custom_css_saved'), 'success')
            return redirect(url_for('admin_themes'))
        
        selected_theme = request.form.get('active_theme', '')
        default_mode = request.form.get('default_theme_mode', '')
        mode_lock = request.form.get('color_mode_lock', '')
        # Set to True if admin actually changed mode_lock or default_theme_mode.
        # Used at the end of this branch to bump theme_settings_version, which
        # tells the public inline-script to reset visitors' saved preference
        # so the new admin choice takes effect for everyone on next visit.
        theme_choice_changed = False
        
        # Save mode lock (controls whether the public site lets visitors switch
        # between light and dark, or forces a single mode and hides the toggle).
        if mode_lock in ('none', 'light', 'dark'):
            old_lock = Settings.get('color_mode_lock', 'none')
            if old_lock != mode_lock:
                Settings.set('color_mode_lock', mode_lock)
                db.session.commit()
                AuditLog.log('change_mode_lock', 'settings', None,
                             _ad('changed_mode_lock', old_lock=old_lock, new_lock=mode_lock),
                             user=current_user, ip=request.remote_addr)
                db.session.commit()
                theme_choice_changed = True
        
        # If lock is light/dark, force default_theme_mode to match. Without
        # this guard a browser with disabled JS could submit lock=light but
        # default_theme_mode=auto, leaving the two settings inconsistent.
        if mode_lock in ('light', 'dark'):
            default_mode = mode_lock
        
        # Save default display mode (always, even without theme change)
        if default_mode in ('dark', 'light', 'auto'):
            old_mode = Settings.get('default_theme_mode', 'auto')
            if old_mode != default_mode:
                Settings.set('default_theme_mode', default_mode)
                db.session.commit()
                AuditLog.log('change_theme_mode', 'settings', None,
                             _ad('changed_theme_mode', old_mode=old_mode, new_mode=default_mode),
                             user=current_user, ip=request.remote_addr)
                db.session.commit()
                flash(t('flash.display_mode_changed'), 'success')
                theme_choice_changed = True
        
        # Bump version so visitors' localStorage 'ap-theme' is reset on next
        # visit. Without this, a visitor who once clicked the theme toggle keeps
        # seeing their old choice forever, even after the admin switched the
        # default — because their localStorage value wins over default_theme_mode
        # in the inline-script. The version is a Unix timestamp; the script
        # compares it to its own stored copy and clears 'ap-theme' on mismatch.
        if theme_choice_changed:
            Settings.set('theme_settings_version', str(int(utc_now().timestamp())))
            db.session.commit()
        
        # Activate selected theme
        if selected_theme:
            theme_dir = os.path.join(app.static_folder, 'css', 'themes', selected_theme)
            if os.path.isdir(theme_dir) and os.path.isfile(os.path.join(theme_dir, 'theme.css')):
                Settings.set('active_theme', selected_theme)
                db.session.commit()
                
                AuditLog.log('change_theme', 'settings', None,
                             _ad('activated_theme', theme=selected_theme),
                             user=current_user, ip=request.remote_addr)
                db.session.commit()
                
                flash(t('flash.theme_activated', name=selected_theme), 'success')
            else:
                flash(t('flash.theme_not_found'), 'error')
        
        return redirect(url_for('admin_themes'))
    
    # Scan themes directory
    themes_dir = os.path.join(app.static_folder, 'css', 'themes')
    themes = []
    
    if os.path.isdir(themes_dir):
        for name in sorted(os.listdir(themes_dir)):
            theme_path = os.path.join(themes_dir, name)
            if os.path.isdir(theme_path) and os.path.isfile(os.path.join(theme_path, 'theme.css')):
                # Read first comment block from theme.css for description
                description = ''
                has_light_mode = False
                try:
                    with open(os.path.join(theme_path, 'theme.css'), 'r', encoding='utf-8') as f:
                        content = f.read(2000)  # Read just the header
                        # Extract description from CSS comment
                        if '/*' in content and '*/' in content:
                            comment = content[content.index('/*')+2:content.index('*/')]
                            # Find "Concept:" line
                            for line in comment.split('\n'):
                                line = line.strip().lstrip('* ').strip()
                                if line.lower().startswith('concept:'):
                                    description = line[8:].strip()
                                    break
                        # Check if .light-theme exists
                        has_light_mode = '.light-theme' in content or '.light-theme' in f.read()
                except Exception:
                    pass
                
                # Check for preview image
                preview = ''
                for ext in ['png', 'jpg', 'webp', 'svg']:
                    preview_file = os.path.join(theme_path, f'preview.{ext}')
                    if os.path.isfile(preview_file):
                        preview = url_for('static', filename=f'css/themes/{name}/preview.{ext}')
                        break
                
                # Use i18n key if available, fallback to CSS comment
                i18n_key = f'themes.desc_{name.replace("-", "_")}'
                i18n_desc = t(i18n_key)
                
                themes.append({
                    'name': name,
                    'display_name': name.replace('-', ' ').replace('_', ' ').title(),
                    'description': i18n_desc if i18n_desc != i18n_key else description,
                    'has_light_mode': has_light_mode,
                    'preview': preview,
                    # Built-in themes ship with the CMS and would reappear
                    # after the next core update anyway, so the UI offers
                    # deletion only for custom (uploaded) themes.
                    'is_builtin': name in BUILTIN_THEMES,
                })
    
    active_theme = effective_active_theme()
    default_mode = Settings.get('default_theme_mode', 'auto')
    mode_lock = Settings.get('color_mode_lock', 'none')
    
    # Recover rejected input from a failed save, otherwise load saved value
    custom_css = session.pop('_custom_css_rejected', None)
    if custom_css is None:
        custom_css = Settings.get('site_custom_css', '')
    
    return render_template('admin/themes.html',
                         themes=themes,
                         active_theme=active_theme,
                         default_mode=default_mode,
                         mode_lock=mode_lock,
                         custom_css=custom_css)

# ============================================================================
# WEBMCP — AGENT TOOLS (manifest + read/list live in webmcp.py)
# ============================================================================

import webmcp as _webmcp
_webmcp.init_app(app, csrf=csrf, login_required=login_required, limiter=limiter)

import twofa as _twofa_mod
_twofa_mod.init_app(
    app, db=db, User=User, Settings=Settings, AuditLog=AuditLog,
    RecoveryCode=TwoFactorRecoveryCode, LoginAttempt=LoginAttempt,
    login_user=login_user, login_required=login_required,
    admin_required=admin_required, limiter=limiter, send_email=send_email,
    encrypt=encrypt_smtp_password, decrypt=decrypt_smtp_password,
    validate_next_url=validate_next_url, audit_detail=_ad, utc_now=utc_now,
    license_ok=_verify_license_integrity)


# ============================================================================
# RUN APPLICATION
# ============================================================================

# Initialize database tables — must run both via gunicorn and python app.py
init_database()

# ============================================================================
# PLUGIN SYSTEM
# ============================================================================
# Loads plugins from the plugins/ directory. Requires a valid core license
# (plugins are a licensed-installation feature); plugins may additionally
# carry their own license, validated against the same update server with
# the plugin's product_id. Must run AFTER init_database() so Settings are
# readable, and after all core routes/decorators exist.

import plugin_system as _plugin_system

_plugin_system.init_plugin_system(
    app, db,
    settings_get=Settings.get,
    settings_set=Settings.set,
    settings_set_many=Settings.set_many,
    is_core_licensed=_verify_license_integrity,
    encrypt=encrypt_smtp_password,
    decrypt=decrypt_smtp_password,
    license_validator_factory=lambda: LicenseValidator(
        app.config.get('UPDATE_SERVER_URL', '')),
)

if __name__ == '__main__':
    print()
    print('=' * 52)
    print('  AliothPress is running!')
    print()
    print('  Site:  http://localhost:5000/')
    print('  Admin: http://localhost:5000/admin')
    print('  Login: http://localhost:5000/admin/login')
    print('=' * 52)
    print()
    
    app.run(debug=False, host='0.0.0.0', port=5000)
