#!/usr/bin/env bash
# GitHub Trending - Data Collector
# Fetches trending repos + READMEs, saves structured JSON
set -euo pipefail
export LC_ALL=C.UTF-8
export LANG=C.UTF-8

workdir="$(mktemp -d)"
html="$workdir/trending.html"
json_out="$workdir/trending_data.json"
today="$(date -u +%F)"

# --- Step 1: Fetch trending page ---
curl -fsSL --retry 3 --retry-delay 2 'https://github.com/trending' -o "$html"

# --- Step 2 & 3: Parse repos + fetch READMEs ---
python3 - "$html" "$json_out" "$today" <<'PY'
import json, re, sys, base64, time, urllib.request
from pathlib import Path
from html.parser import HTMLParser

html_path, json_path, today = sys.argv[1:4]
text = Path(html_path).read_text(encoding='utf-8', errors='replace')

# --- Parse trending page ---
class Parser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.items=[]
        self.in_article=False
        self.cur=None
        self.capture=None
        self.buf=[]
    def handle_starttag(self, tag, attrs):
        d = dict(attrs)
        if tag=='article' and d.get('class','').startswith('Box-row'):
            self.in_article=True; self.cur={}
        if not self.in_article: return
        if tag=='h2':
            self.capture='h2'; self.buf=[]; self.cur['url']=''
        elif tag=='a' and self.capture=='h2':
            href = d.get('href','')
            if href: self.cur['url'] = 'https://github.com' + href
        elif tag=='p' and self.cur.get('description') is None and 'col-9' in d.get('class',''):
            self.capture='desc'; self.buf=[]
        elif tag=='span' and d.get('itemprop')=='programmingLanguage':
            self.capture='lang'; self.buf=[]
        elif tag=='a' and d.get('href','').endswith('/stargazers'):
            self.capture='stars'; self.buf=[]
    def handle_data(self, data):
        if self.capture: self.buf.append(data)
    def handle_endtag(self, tag):
        if not self.in_article: return
        if self.capture == 'h2':
            if tag == 'a':
                val = ' '.join(''.join(self.buf).split())
                self.cur['name'] = val.replace(' / ', '/').strip()
                self.capture = None; self.buf = []
            return
        if tag in {'p','span','a'} and self.capture:
            val = ' '.join(''.join(self.buf).split())
            if self.capture=='desc': self.cur['description']=val
            elif self.capture=='lang': self.cur['language']=val
            elif self.capture=='stars':
                self.cur['stars']=int(val.replace(',',''))
            self.capture=None; self.buf=[]
        elif tag=='article':
            if self.cur and self.cur.get('url') and 'sponsors/' not in self.cur['url']:
                self.cur['topics'] = []
                self.items.append(self.cur)
            self.in_article=False; self.cur=None; self.capture=None; self.buf=[]

p=Parser(); p.feed(text)

# --- Fetch README for each repo ---
def fetch_readme(owner_repo):
    """Fetch README content via GitHub API. Returns (text, error)."""
    api_url = f"https://api.github.com/repos/{owner_repo}/readme"
    req = urllib.request.Request(api_url)
    req.add_header('Accept', 'application/vnd.github.raw+json')
    try:
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = resp.read()
            return data.decode('utf-8', errors='replace'), None
    except urllib.error.HTTPError as e:
        # Fallback: try regular API with base64
        try:
            fallback_url = f"https://api.github.com/repos/{owner_repo}/readme"
            req2 = urllib.request.Request(fallback_url)
            req2.add_header('Accept', 'application/vnd.github.v3+json')
            with urllib.request.urlopen(req2, timeout=15) as resp2:
                body = json.loads(resp2.read().decode())
                if body.get('encoding') == 'base64' and body.get('content'):
                    return base64.b64decode(body['content']).decode('utf-8', errors='replace'), None
                return '', 'no readme content'
        except Exception as e2:
            return '', str(e2)
    except Exception as e:
        return '', str(e)

repos = []
rank = 0
for x in p.items:
    if len(repos) >= 20: break
    rank += 1
    full_name = x.get('name','').replace(' / ','/').strip()
    url = x.get('url','')
    
    # Extract owner/repo from URL for API calls
    path_match = re.search(r'github\.com/([^/]+/[^/]+)', url)
    owner_repo = path_match.group(1) if path_match else full_name
    
    print(f"  [{rank}/10] Fetching README: {owner_repo} ...", file=sys.stderr)
    readme_text, readme_err = fetch_readme(owner_repo)
    if readme_err:
        print(f"    ⚠ README error: {readme_err}", file=sys.stderr)
    
    repos.append({
        'rank': rank,
        'name': full_name,
        'url': url,
        'language': x.get('language') or 'Não informado',
        'stars': x.get('stars') or 0,
        'description': x.get('description') or 'Sem descrição no Trending.',
        'readme': readme_text,
        'readme_error': readme_err or None,
        'topics': x.get('topics', []),
    })

output = {
    'date': today,
    'source': 'https://github.com/trending',
    'repos': repos,
}

Path(json_path).write_text(json.dumps(output, ensure_ascii=False, indent=2), encoding='utf-8')
print(f"\n✅ Dados salvos: {json_path}", file=sys.stderr)
print(json_path)  # stdout: path to JSON for the agent
PY