#!/usr/bin/env python3
"""Guides drafts -> resources/content/guides/guides.json (SEO plan T301).

Input: one plain-text copy per Google Doc draft (GDxx.txt): front matter between --- lines,
the English article, a ===== line, then the Simplified Chinese article. checker_notes and
aeo_missing stay out of the output. Every entry is written as a draft (status "draft").

Usage: python3 scripts/seo/guides_from_drafts.py [drafts_dir] [out_file]
"""
import glob
import json
import os
import re
import sys

REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
SRC = sys.argv[1] if len(sys.argv) > 1 else '/tmp/cg'
OUT = sys.argv[2] if len(sys.argv) > 2 else '/tmp/cg/guides.json'
TOPICS = ['Buying basics', 'Costs & tax', 'Foreign buyers & MM2H', 'Market & areas', 'Compare projects', 'Using PropertyLab']
# GD01 predates the front-matter format.
FALLBACK = {'GD01': {'id': 'GD01', 'topic': 'Foreign buyers & MM2H', 'slug': 'can-foreigners-buy-property-in-malaysia',
                     'related_terms': '[freehold, leasehold, strata-title, stamp-duty, rpgt]'}}
MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december']
SKIP = {'on this page', '本页目录', '目录'}
FAQ = {'faq', 'faqs', 'frequently asked questions', 'common questions', '常见问题'}
NEXT = {'next step', 'next steps', '下一步'}
SOURCES = {'sources', '资料来源', '来源', '参考资料'}
UL = re.compile(r'^[-*•]\s+')
OL = re.compile(r'^\d+[.)]\s+')


def unescape(s):
    return re.sub(r'\\([\\`*_{}\[\]()#+\-.!|>~=])', r'\1', s)


def clean(s):
    s = re.sub(r'(?<![*\w])\*(?![*\s])([^*\n]+?)(?<![*\s])\*(?![*\w])', r'\1', s.strip())
    return re.sub(r'\s+', ' ', s)


def as_list(v):
    v = (v or '').strip()
    if v.startswith('[') and v.endswith(']'):
        v = v[1:-1]
    return [x.strip().strip('\'"') for x in v.split(',') if x.strip().strip('\'"')]


def front_matter(lines):
    if not lines or lines[0].strip() != '---':
        return {}, lines
    meta = {}
    for i in range(1, len(lines)):
        if lines[i].strip() == '---':
            return meta, lines[i + 1:]
        m = re.match(r'^([a-z_]+):\s*(.*)$', lines[i])
        if m:
            meta[m.group(1)] = '' if m.group(2).strip() == '|' else m.group(2).strip()
    return meta, []


def split_body(lines):
    en, zh, cur = [], [], None
    for line in lines:
        s = line.strip()
        if re.match(r'^={5,}$', s):
            if cur is en:
                cur = None
            continue
        if cur is None:
            if s.startswith('# '):
                cur = zh if en else en
            else:
                continue
        cur.append(line)
    return en, zh


def take_run(lines, i, pred):
    run, n = [], len(lines)
    while i < n:
        s = lines[i].strip()
        if not s:
            j = i
            while j < n and not lines[j].strip():
                j += 1
            if j < n and pred(lines[j].strip()):
                i = j
                continue
            break
        if not pred(s):
            break
        run.append(s)
        i += 1
    return run, i


def cells(row):
    return [clean(c) for c in row.strip().strip('|').split('|')]


def is_rule(row):
    return '-' in row and set(row.replace(' ', '')) <= set('|-:')


def parse_trust(s, d):
    m = re.search(r'(\d+)\s*-?\s*minute', s) or re.search(r'约\s*(\d+)\s*分钟', s)
    if m:
        d['readMinutes'] = int(m.group(1))
    m = re.search(r'Checked by:?\s*([^·]+)', s) or re.search(r'核查人[:：]\s*([^·]+)', s)
    if m and not m.group(1).strip().startswith('['):
        d['reviewer'] = m.group(1).strip()
    m = re.search(r'(\d{1,2})\s+([A-Za-z]+)\s+(\d{4})', s)
    if m and m.group(2).lower() in MONTHS:
        d['updated'] = '%s-%02d-%02d' % (m.group(3), MONTHS.index(m.group(2).lower()) + 1, int(m.group(1)))
    m = re.search(r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日', s)
    if m:
        d['updated'] = '%s-%02d-%02d' % (m.group(1), int(m.group(2)), int(m.group(3)))


def parse_source(text):
    url = None
    md = re.search(r'\[([^\]]+)\]\((https?://[^)\s]+)\)', text)
    if md:
        url = md.group(2)
        text = text.replace(md.group(0), md.group(1))
    else:
        m = re.search(r'https?://[^\s，、；;（）()<>]+', text)
        if m:
            url = m.group(0).rstrip('.,;:)」』')
            text = text.replace(m.group(0), ' ')
    acc = re.search(r'(?:checked|accessed)\s*(\d{4}-\d{2}-\d{2})', text, re.I) or re.search(r'(\d{4}-\d{2}-\d{2})\s*查阅', text)
    title = re.sub(r'\s*[—–-]?\s*(?:checked|accessed)\s*\d{4}-\d{2}-\d{2}\.?', '', text, flags=re.I)
    title = re.sub(r'[（(]\s*\d{4}-\d{2}-\d{2}\s*查阅\s*[）)]', '', title)
    title = re.sub(r'\s+', ' ', title).strip(' ,，、;；.。—–-')
    return {'title': title, 'url': url, 'accessed': acc.group(1) if acc else None}


def parse_body(lines):
    d = {'title': None, 'answer': '', 'intro': [], 'sections': [], 'faqs': [], 'sources': [],
         'cta': None, 'readMinutes': None, 'reviewer': None, 'updated': None}
    link = re.compile(r'\*{0,2}\[([^\]]+)\](?:\([^)]*\))?\*{0,2}')
    mode, sec, i, n = 'intro', None, 0, len(lines)
    while i < n:
        s = lines[i].strip()
        i += 1
        if not s:
            continue
        if s.startswith('# ') and d['title'] is None:
            d['title'] = clean(s[2:])
            continue
        if s.startswith('## '):
            heading = clean(s[3:])
            key = re.sub(r'[\s:：?？]+$', '', heading).lower()
            if key in SKIP:
                mode = 'skip'
            elif key in FAQ:
                mode = 'faq'
            elif key in NEXT:
                mode = 'cta'
            elif key in SOURCES:
                mode = 'sources'
            else:
                mode, sec = 'section', {'heading': heading, 'blocks': []}
                d['sections'].append(sec)
            continue
        if mode == 'skip':
            continue
        if mode == 'intro':
            if re.match(r'^(Updated|更新)', s):
                parse_trust(s, d)
                continue
            m = re.match(r'^\*\*\s*(Short answer|简短回答|简答)\s*[:：]?\s*\*\*\s*[:：]?\s*(.*)$', s)
            if m:
                d['answer'] = clean(m.group(2))
                continue
        if mode == 'faq':
            m = re.match(r'^\*\*(.+)\*\*$', s) or re.match(r'^###\s+(.+)$', s)
            if m:
                d['faqs'].append({'q': clean(m.group(1)), 'a': ''})
            elif d['faqs']:
                faq = d['faqs'][-1]
                faq['a'] = (faq['a'] + ' ' + clean(OL.sub('', UL.sub('', s)))).strip()
            continue
        if mode == 'cta':
            m = link.search(s)
            cta = d['cta'] or {'text': '', 'label': None}
            cta['text'] = (cta['text'] + ' ' + clean(link.sub('', s))).strip()
            if m:
                cta['label'] = clean(m.group(1))
            d['cta'] = cta
            continue
        if mode == 'sources':
            if UL.match(s):
                d['sources'].append(parse_source(clean(UL.sub('', s))))
            continue
        blocks = sec['blocks'] if mode == 'section' else d['intro']
        if s.startswith('### '):
            blocks.append({'type': 'h3', 'text': clean(s[4:])})
        elif s.startswith('|'):
            rows, i = take_run(lines, i - 1, lambda x: x.startswith('|'))
            rows = [r for r in rows if not is_rule(r)]
            blocks.append({'type': 'table', 'head': cells(rows[0]), 'rows': [cells(r) for r in rows[1:]]})
        elif UL.match(s) or OL.match(s):
            pat = UL if UL.match(s) else OL
            items, i = take_run(lines, i - 1, lambda x, p=pat: bool(p.match(x)))
            blocks.append({'type': 'ul' if pat is UL else 'ol', 'items': [clean(pat.sub('', x)) for x in items]})
        else:
            blocks.append({'type': 'p', 'text': clean(s)})
    return d


def load_projects():
    path = os.path.join(REPO, 'resources/content/project-pages/my.json')
    try:
        with open(path, encoding='utf-8') as fh:
            data = json.load(fh)
    except (OSError, ValueError):
        return {}
    pages = data.get('pages', {}) if isinstance(data, dict) else {}
    return {slug: (page or {}).get('name') for slug, page in pages.items()}


def related_projects(names, known):
    out = []
    for name in names:
        slug = re.sub(r'[^a-z0-9]+', '-', name.lower().split('/projects/')[-1]).strip('-')
        if slug:
            shown = known.get(slug) or (name if ' ' in name else slug.replace('-', ' ').title())
            out.append({'slug': slug, 'name': shown, 'url': '/my/projects/' + slug})
    return out


def build(path, known, report):
    fid = os.path.basename(path).split('.')[0]
    with open(path, encoding='utf-8') as fh:
        text = fh.read()
    text = text.replace('\r\n', '\n').replace(' ', ' ').replace('​', '').lstrip('﻿\n ')
    lines = unescape(text).split('\n')
    meta, rest = front_matter(lines)
    if not meta:
        meta = dict(FALLBACK.get(fid, {}))
        for line in lines[:30]:
            m = re.match(r'^Meta title[^:]*:\s*(.+)$', line)
            if m:
                meta['meta_title'] = m.group(1).strip()
            m = re.match(r'^Meta description[^:]*:\s*(.+)$', line)
            if m:
                meta['meta_description'] = m.group(1).strip()
            m = re.search(r'Semrush MY[^:]*:\s*([\d,]+)/month', line)
            if m:
                meta['volume'] = m.group(1)
        rest = lines
    en_lines, zh_lines = split_body(rest)
    en = parse_body(en_lines)
    zh = parse_body(zh_lines) if zh_lines else None
    volume = str(meta.get('volume', '')).replace(',', '')
    minutes = str(meta.get('read_minutes', ''))
    entry = {
        'id': meta.get('id') or fid,
        'slug': meta.get('slug'),
        'status': 'draft',
        'topic': meta.get('topic'),
        'title': en['title'],
        'titleZh': (zh or {}).get('title') or meta.get('title_zh') or None,
        'short': meta.get('summary') or meta.get('meta_description') or '',
        'answer': en['answer'],
        'intro': en['intro'],
        'sections': en['sections'],
        'faqs': en['faqs'],
        'sources': en['sources'],
        'cta': en['cta'],
        'readMinutes': en['readMinutes'] or (int(minutes) if minutes.isdigit() else None),
        'reviewer': en['reviewer'] or (zh or {}).get('reviewer'),
        'related': as_list(meta.get('related_terms')),
        'relatedProjects': related_projects(as_list(meta.get('related_projects')), known),
        'updated': en['updated'] or '2026-11-09',
        'checkedBy': None,
        'seoTitle': meta.get('meta_title') or None,
        'seoDescription': meta.get('meta_description') or None,
        'volume': int(volume) if volume.isdigit() else None,
    }
    if zh:
        entry['zh'] = {k: zh[k] for k in ('title', 'answer', 'intro', 'sections', 'faqs', 'sources', 'cta')}
        entry['zh']['readMinutes'] = zh['readMinutes'] or entry['readMinutes']
        entry['zh']['updated'] = zh['updated'] or entry['updated']
    problems = ['missing ' + k for k in ('slug', 'topic', 'title', 'answer') if not entry.get(k)]
    if entry['topic'] not in TOPICS:
        problems.append('unknown topic %r' % entry['topic'])
    if entry['seoTitle'] and len(entry['seoTitle']) > 60:
        problems.append('seoTitle %d chars' % len(entry['seoTitle']))
    if entry['seoDescription'] and len(entry['seoDescription']) > 155:
        problems.append('seoDescription %d chars' % len(entry['seoDescription']))
    if not zh:
        problems.append('no zh')
    problems += ['project not in my.json: ' + p['slug'] for p in entry['relatedProjects'] if p['slug'] not in known]
    z = entry.get('zh') or {'sections': [], 'faqs': [], 'sources': []}
    report.append('%s %-42s sec=%d faq=%d src=%d intro=%d | zh sec=%d faq=%d src=%d | ans=%dw | proj=%d | %s' % (
        entry['id'], entry['slug'], len(entry['sections']), len(entry['faqs']), len(entry['sources']), len(entry['intro']),
        len(z['sections']), len(z['faqs']), len(z['sources']), len(entry['answer'].split()), len(entry['relatedProjects']),
        '; '.join(problems) or 'ok'))
    return entry


def main():
    known = load_projects()
    report, entries = [], []
    for path in sorted(glob.glob(os.path.join(SRC, 'GD*.txt'))):
        entries.append(build(path, known, report))
    order = {t: i for i, t in enumerate(TOPICS)}
    entries.sort(key=lambda e: (order.get(e['topic'], 99), -(e['volume'] or 0), e['id']))
    slugs = [e['slug'] for e in entries]
    dupes = sorted({s for s in slugs if slugs.count(s) > 1})
    if dupes:
        report.append('DUPLICATE SLUGS: ' + ', '.join(dupes))
    with open(OUT, 'w', encoding='utf-8') as fh:
        json.dump(entries, fh, ensure_ascii=False, indent=2)
        fh.write('\n')
    report.append('%d guides -> %s (project pages known: %d)' % (len(entries), OUT, len(known)))
    print('\n'.join(report))


if __name__ == '__main__':
    main()
