                if not description:
                    og_el = page.query_selector('meta[property="og:description"]')
                    if og_el:
                        description = (og_el.get_attribute("content") or "").strip()
                        description = re.sub(r'…$', '', description).strip()
                # Filter out Zhihu generic site description (not question-specific)
                if description and "知乎，中文互联网" in description:
                    description = None

"""Zhihu Hunter — vault scanning, question discovery, and answer drafting.

Workflow:
    1. scan_and_hunt()  →  scans recent vault files, extracts keywords,
                           searches Tavily for matching Zhihu questions.
    2. draft_answer()   →  uses existing RAG + ZhihuGenerator to produce
                           a Zhihu-style answer draft ready for review.
"""

import os
import re
import requests
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import List, Optional

from pydantic import BaseModel, Field

from health.utils.logging_config import setup_logger
from slack_bot.llm.gemini import GeminiLLM
from slack_bot.obsidian.generators import ZhihuGenerator
from slack_bot.obsidian.indexer import ObsidianIndexer
from slack_bot.obsidian.vector_store import ChromaVectorStore

logger = setup_logger(__name__)

# ── Vault directories to watch for new/updated content ────────────────────────
# Paths are relative to vault_path root.
WATCH_DIRS: List[str] = [
    "notes",
    "Article",
    "auto_report/deep-dive",
    "CLAW/deepfact",
    "Clippings",
]

# Article is always included regardless of time; Revised files are prioritized.
_ARTICLE_DIR = "Article"

# Short names users can type to target a specific directory.
DIR_SHORTCUTS: dict[str, str] = {
    "notes":      "notes",
    "article":    "Article",
    "clippings":  "Clippings",
    "claw":       "CLAW/deepfact",
    "deepfact":   "CLAW/deepfact",
    "deep-dive":  "auto_report/deep-dive",
}

_MAX_FILES_PER_DIR   = 4    # most-recent files per directory (keeps dir diversity)
_MAX_FILES_ARTICLE   = 8    # Article gets more slots (higher priority)
_MAX_KEYWORDS        = 10   # keywords extracted from vault scan
_MAX_QUESTIONS_TOTAL = 12   # total Zhihu questions surfaced per hunt
# Must contain /question/<id>; answer/comment subpages are excluded in _find_questions
_MIN_QUESTION_URL_RE = re.compile(r'zhihu\.com/question/\d+')

# ── Data Models ────────────────────────────────────────────────────────────────

class ZhihuQuestion(BaseModel):
    """A Zhihu question candidate surfaced by the hunter."""

    title: str
    url: str
    snippet: Optional[str] = None
    keywords: List[str] = Field(default_factory=list)
    found_at: str = Field(
        default_factory=lambda: datetime.now(tz=timezone.utc).isoformat()
    )
    description: Optional[str] = None   # question body/detail text from Zhihu page
    source_file: Optional[str] = None  # vault-relative path, set by scan_single_file()
    outline: Optional[str] = None       # user-provided answer logic/structure (direct URL mode)

class AnswerDraft(BaseModel):
    """A generated answer draft awaiting human review."""

    question: ZhihuQuestion
    content: str
    generated_at: str = Field(
        default_factory=lambda: datetime.now(tz=timezone.utc).isoformat()
    )
    rag_sources: List[str] = Field(default_factory=list)
    vault_file: Optional[str] = None  # absolute path set after saving to vault

# ── Main Class ─────────────────────────────────────────────────────────────────

class ZhihuHunter:
    """Discovers Zhihu questions from vault content and drafts answers.

    Example:
        hunter = ZhihuHunter(vault_path, vector_store, indexer)
        questions = hunter.scan_and_hunt(since_days=3)
        draft = hunter.draft_answer(questions[0])
    """

    def __init__(
        self,
        vault_path: Path,
        vector_store: ChromaVectorStore,
        indexer: ObsidianIndexer,
    ) -> None:
        """Initialize the hunter.

        Args:
            vault_path: Root path of the Obsidian vault.
            vector_store: ChromaDB vector store for semantic retrieval.
            indexer: ObsidianIndexer for keyword-based search and writing samples.
        """
        self.vault_path = vault_path
        self.vector_store = vector_store
        self.indexer = indexer
        self.llm = GeminiLLM()

    # ── Public API ─────────────────────────────────────────────────────────────

    def scan_and_hunt(
        self,
        since_days: int = 7,
        dirs: Optional[List[str]] = None,
        topic_hint: str = "",
    ) -> List[ZhihuQuestion]:
        """Scan vault files and surface matching Zhihu questions.

        Args:
            since_days: Age threshold for non-Article directories.
            dirs: Restrict scan to these vault dirs (relative paths). None = all WATCH_DIRS.
            topic_hint: Optional focus hint injected into keyword extraction prompt.

        Returns:
            List of ZhihuQuestion candidates (up to _MAX_QUESTIONS_TOTAL).
        """
        scan_desc = f"dirs={dirs or 'all'}" + (f", topic='{topic_hint}'" if topic_hint else "")
        logger.info(f"ZhihuHunter: scanning vault ({scan_desc})")

        keywords = self._scan_vault_for_keywords(since_days, dirs=dirs, topic_hint=topic_hint)
        if not keywords:
            logger.info("No keywords extracted from vault files")
            return []

        logger.info(f"Extracted keywords: {keywords}")
        questions = self._find_questions(keywords)
        logger.info(f"Found {len(questions)} Zhihu questions")
        return questions

    def scan_single_file(self, rel_path: str) -> List[ZhihuQuestion]:
        """Extract keywords from one vault file and find matching Zhihu questions.

        The returned questions carry ``source_file`` set to ``rel_path`` so that
        ``draft_answer()`` will ground the generated answer in that article.

        Args:
            rel_path: Path relative to vault_path (e.g. "Article/8-xxx-Revised.md").

        Returns:
            List of ZhihuQuestion candidates with source_file populated.

        Raises:
            FileNotFoundError: If the file does not exist under the vault root.
        """
        file_path = self.vault_path / rel_path
        if not file_path.exists():
            raise FileNotFoundError(f"文件不存在：{file_path}")

        text = file_path.read_text(encoding="utf-8", errors="ignore")
        text = re.sub(r'^---\n.*?\n---\n', '', text, flags=re.DOTALL).strip()
        logger.info(f"scan_single_file: {rel_path} ({len(text)} chars)")

        keywords = self._extract_keywords_with_llm(text[:5000])
        if not keywords:
            logger.info("No keywords extracted from single file")
            return []

        logger.info(f"Keywords from file: {keywords}")
        questions = self._find_questions(keywords)

        for q in questions:
            q.source_file = rel_path

        return questions

    def scan_glob_files(self, pattern: str) -> List[ZhihuQuestion]:
        """Find vault files matching a glob pattern and hunt Zhihu questions.

        All matched files are combined for keyword extraction, so the questions
        reflect the full set of content.  The most-recently-modified matched file
        is stored as ``source_file`` to ground the answer draft.

        Args:
            pattern: Glob pattern relative to vault_path
                     (e.g. "Article/*openclaw*-Revised.md").

        Returns:
            List of ZhihuQuestion candidates with source_file set.

        Raises:
            FileNotFoundError: If no files match the pattern.
        """
        matched = sorted(
            self.vault_path.glob(pattern),
            key=lambda p: p.stat().st_mtime,
            reverse=True,
        )
        if not matched:
            raise FileNotFoundError(f"没有文件匹配：{pattern}")

        names = [p.name for p in matched]
        logger.info(f"scan_glob_files: '{pattern}' → {len(matched)} files: {names}")

        # Combine content from all matched files (per-file budget so no one file
        # dominates) and extract keywords from the aggregate.
        parts: List[str] = []
        per_file_budget = 1500 if len(matched) == 1 else 800
        for fp in matched:
            try:
                text = fp.read_text(encoding="utf-8", errors="ignore")
                text = re.sub(r'^---\n.*?\n---\n', '', text, flags=re.DOTALL).strip()
                parts.append(f"[{fp.name}]\n{text[:per_file_budget]}")
            except OSError as e:
                logger.debug(f"Could not read {fp}: {e}")

        combined = "\n\n---\n\n".join(parts)
        keywords = self._extract_keywords_with_llm(combined[:8000])
        if not keywords:
            logger.info("No keywords extracted from glob-matched files")
            return []

        logger.info(f"Keywords from glob: {keywords}")
        questions = self._find_questions(keywords)

        # Primary source = most-recently-modified matched file
        primary = str(matched[0].relative_to(self.vault_path))
        for q in questions:
            q.source_file = primary

        return questions

    def hunt_direct_url(self, url: str) -> List[ZhihuQuestion]:
        """Create a question card directly from a Zhihu question URL.

        Used when the user pastes a URL they were invited to answer.
        Fetches the page title and description so the card displays readable info.

        Args:
            url: A zhihu.com/question/<id> URL.

        Returns:
            Single-element list containing the ZhihuQuestion.
        """
        clean_url = re.sub(r'[>\'")\].,;]+$', '', url).lstrip("<").split("?")[0].split("#")[0].rstrip("/")
        title, description = self._fetch_question_info(clean_url)
        logger.info(f"Direct URL hunt: {clean_url!r} → title={title!r}, desc={len(description or '')} chars")
        return [ZhihuQuestion(title=title, url=clean_url, description=description)]

    def _fetch_question_info(self, url: str) -> tuple[str, Optional[str]]:
        """Fetch the question title and description via Playwright.

        Falls back to a URL-derived label if Playwright is unavailable or fails.

        Args:
            url: Cleaned question URL.

        Returns:
            Tuple of (title, description). Description may be None.
        """
        state_file = Path(__file__).resolve().parents[3] / "data" / "zhihu_state.json"
        try:
            from playwright.sync_api import sync_playwright
            from playwright_stealth import Stealth
            import json as _json

            with sync_playwright() as p:
                browser = p.chromium.launch(headless=True)
                context = browser.new_context(
                    user_agent=(
                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                        "AppleWebKit/537.36 (KHTML, like Gecko) "
                        "Chrome/122.0.0.0 Safari/537.36"
                    )
                )
                # Load saved cookies if available
                if state_file.exists():
                    data = _json.loads(state_file.read_text(encoding="utf-8"))
                    cookies = data.get("cookies", [])
                    if cookies:
                        context.add_cookies(cookies)

                page = context.new_page()
                Stealth().apply_stealth_sync(page)
                page.goto(url, wait_until="domcontentloaded", timeout=15000)
                page.wait_for_timeout(2000)

                # Extract title
                title = ""
                title_el = page.query_selector("h1.QuestionHeader-title")
                if title_el:
                    title = title_el.inner_text().strip()
                if not title:
                    title = page.title() or ""
                    title = title.split(" - 知乎")[0].strip()
                # Strip "(XX 条消息)" notification prefix from tab title
                title = re.sub(r'^\(\d+ 条消息\)\s*', '', title)

                # Extract question description
                description = None
                desc_el = page.query_selector("div.QuestionRichText")
                if desc_el:
                    description = desc_el.inner_text().strip()
                    # Clean up truncation artifacts and zero-width chars
                    description = description.replace("\u200b", "").strip()
                    description = re.sub(r'…?\s*显示全部\s*$', '', description).strip()
                if not description:
                    og_el = page.query_selector('meta[property="og:description"]')
                    if og_el:
                        description = (og_el.get_attribute("content") or "").strip()
                        description = re.sub(r'…$', '', description).strip()
                if description:
                    description = description[:2000]
                else:
                    description = None

                browser.close()

                if title and title != "知乎":
                    return title, description
        except Exception as e:
            logger.debug(f"Playwright question fetch failed for {url}: {e}")

        # Fallback: use the question ID
        qid = url.rstrip("/").split("/")[-1]
        return f"知乎问题 #{qid}", None

    def _fetch_recent_web_context(self, query: str) -> str:
        """Search the web for recent articles/news about the query topic.

        Results are injected into the answer prompt so it reads like it
        comes from someone who's actively following the latest developments.

        Args:
            query: Topic or question title to search.

        Returns:
            Formatted string of recent snippets, or empty string on failure.
        """
        api_key = os.environ.get("TAVILY_API_KEY")
        if not api_key:
            return ""
        try:
            from tavily import TavilyClient
            client = TavilyClient(api_key=api_key)
            results = client.search(
                query=query,
                search_depth="basic",
                max_results=4,
                days=90,
                exclude_domains=["zhihu.com"],
            )
            items = results.get("results", [])
            if not items:
                return ""
            parts = [
                f"- {r['title']} ({r['url'].split('/')[2]}): {r.get('content', '')[:250]}"
                for r in items
            ]
            return "\n".join(parts)
        except Exception as e:
            logger.debug(f"Recent web context fetch failed: {e}")
            return ""

    def draft_answer(self, question: ZhihuQuestion, feedback: str = "") -> AnswerDraft:
        """Generate a Zhihu-style answer draft for a question.

        Delegates to the existing ZhihuGenerator (reuses prompt + style guide).
        The RAG context is enriched by both semantic (ChromaDB) and
        keyword-based (ObsidianIndexer) retrieval.

        Args:
            question: ZhihuQuestion to answer.
            feedback: Optional revision instructions from the user (e.g. "更口语化一些").

        Returns:
            AnswerDraft with generated content and source list.
        """
        logger.info(f"Drafting answer for: {question.title}" + (f" [feedback: {feedback[:40]}]" if feedback else ""))

        # Collect RAG sources for transparency
        rag_sources: List[str] = []

        semantic_results = self.vector_store.search_knowledge(question.title, top_k=3)
        for r in semantic_results:
            rag_sources.append(Path(r.source_path).name)

        keyword_results = self.indexer.search(question.title, limit=5)
        for note in keyword_results:
            first_line = note.split("\n")[0].strip("# ").strip()
            if first_line:
                rag_sources.append(first_line)

        # If the question was pinned to a specific source file, load it as the
        # primary article so the answer stays grounded in its viewpoints.
        primary_article = ""
        if question.source_file:
            fp = self.vault_path / question.source_file
            try:
                raw = fp.read_text(encoding="utf-8", errors="ignore")
                raw = re.sub(r'^---\n.*?\n---\n', '', raw, flags=re.DOTALL).strip()
                primary_article = raw[:4000]
                rag_sources.insert(0, Path(question.source_file).name)
                logger.info(f"Loaded primary article: {question.source_file} ({len(primary_article)} chars)")
            except OSError as e:
                logger.warning(f"Could not read source file {question.source_file}: {e}")

        # Delegate to existing ZhihuGenerator for consistent style
        generator = ZhihuGenerator(self.indexer, self.vector_store)
        prompt_input = question.title
        if question.description:
            prompt_input += f"\n\n问题详细描述：{question.description}"
        if question.snippet:
            prompt_input += f" | 背景：{question.snippet}"

        # User-provided outline takes highest priority — structure the answer strictly
        # around it, pulling matching content from the knowledge base for each point.
        if question.outline:
            prompt_input += (
                f"\n\n用户提供的回答思路（必须严格按照这个逻辑框架展开，"
                f"从知识库中找到对应的内容来支撑每个论点，不要自由发挥框架结构）：\n{question.outline}"
            )
            logger.info(f"Injected user outline: {question.outline[:60]!r}")

        if primary_article:
            prompt_input += (
                f"\n\n主要参考文章（请基于此文的观点和论据来回答，"
                f"可直接引用其中的分析和结论）：\n{primary_article}"
            )

        # Inject recent web context so the answer feels up-to-date
        recent_context = self._fetch_recent_web_context(question.title)
        if recent_context:
            prompt_input += (
                f"\n\n近期相关资讯（最近90天内的动态，用来补充最新工具/发布/讨论，"
                f"让回答更有时效感，自然地融入即可，不要刻意罗列）：\n{recent_context}"
            )
            logger.info(f"Injected recent web context ({len(recent_context)} chars)")

        if feedback:
            prompt_input += f"\n\n用户修改意见：{feedback}"

        content, _ = generator.chat(prompt_input, history=[])

        return AnswerDraft(
            question=question,
            content=content,
            rag_sources=list(dict.fromkeys(rag_sources)),  # deduplicate, keep order
        )

    # ── Vault Scanning ─────────────────────────────────────────────────────────

    def _scan_vault_for_keywords(
        self,
        since_days: int,
        dirs: Optional[List[str]] = None,
        topic_hint: str = "",
    ) -> List[str]:
        """Collect vault files and extract topic keywords via LLM.

        Args:
            since_days: Age threshold for non-Article directories.
            dirs: Dirs to scan; None = all WATCH_DIRS.
            topic_hint: Optional focus instruction for keyword extraction.

        Returns:
            Deduplicated list of topic keywords (up to _MAX_KEYWORDS).
        """
        cutoff = datetime.now(tz=timezone.utc) - timedelta(days=since_days)
        files = self._collect_recent_files(cutoff, dirs=dirs)

        if not files:
            return []

        # Article-only scans get a larger per-file budget (higher priority content)
        scan_dirs = dirs or WATCH_DIRS
        per_file_budget = 800 if scan_dirs == [_ARTICLE_DIR] else 500
        combined = self._read_files_summary(files, max_chars_per_file=per_file_budget)
        return self._extract_keywords_with_llm(combined, topic_hint=topic_hint)

    def _collect_recent_files(
        self,
        cutoff: datetime,
        dirs: Optional[List[str]] = None,
    ) -> List[Path]:
        """Walk vault directories and return candidate files.

        Article directory:
          - No time filter (always included regardless of cutoff)
          - Revised files sorted first, then by mtime descending
          - Up to _MAX_FILES_ARTICLE slots
        Other directories:
          - Only files modified after cutoff
          - Sorted by mtime descending
          - Up to _MAX_FILES_PER_DIR slots

        Args:
            cutoff: Freshness threshold for non-Article directories.
            dirs: Directories to scan (relative paths). None = all WATCH_DIRS.

        Returns:
            List of Path objects.
        """
        scan_dirs = dirs if dirs is not None else WATCH_DIRS
        collected: List[Path] = []

        for rel_dir in scan_dirs:
            watch_dir = self.vault_path / rel_dir
            if not watch_dir.is_dir():
                logger.debug(f"Watch dir not found, skipping: {watch_dir}")
                continue

            # (mtime, is_revised, path)
            candidates: List[tuple[float, bool, Path]] = []
            for md_file in watch_dir.rglob("*.md"):
                try:
                    mtime = md_file.stat().st_mtime
                    if rel_dir != _ARTICLE_DIR:
                        file_dt = datetime.fromtimestamp(mtime, tz=timezone.utc)
                        if file_dt < cutoff:
                            continue
                    is_revised = md_file.name.lower().endswith("-revised.md")
                    candidates.append((mtime, is_revised, md_file))
                except OSError:
                    continue

            if rel_dir == _ARTICLE_DIR:
                # Revised files first, then newest; give double slots
                candidates.sort(key=lambda t: (0 if t[1] else 1, -t[0]))
                max_files = _MAX_FILES_ARTICLE
            else:
                candidates.sort(key=lambda t: -t[0])
                max_files = _MAX_FILES_PER_DIR

            collected.extend(p for _, _, p in candidates[:max_files])

        logger.debug(f"Collected {len(collected)} files from vault (dirs={scan_dirs})")
        return collected

    def _read_files_summary(
        self,
        files: List[Path],
        max_chars: int = 8000,
        max_chars_per_file: int = 500,
    ) -> str:
        """Read and concatenate file contents up to a character budget.

        Each file is capped at max_chars_per_file so that no single large file
        exhausts the total budget and crowds out other directories.

        Args:
            files: List of markdown file paths.
            max_chars: Total character budget across all files.
            max_chars_per_file: Per-file cap to ensure directory diversity.

        Returns:
            Concatenated content string with file-name headers.
        """
        parts: List[str] = []
        remaining = max_chars

        for f in files:
            if remaining <= 0:
                break
            try:
                text = f.read_text(encoding="utf-8", errors="ignore")
                # Strip YAML frontmatter
                text = re.sub(r'^---\n.*?\n---\n', '', text, flags=re.DOTALL).strip()
                chunk = text[:min(max_chars_per_file, remaining)]
                parts.append(f"[{f.name}]\n{chunk}")
                remaining -= len(chunk)
            except OSError as e:
                logger.debug(f"Could not read {f}: {e}")

        return "\n\n---\n\n".join(parts)

    def _extract_keywords_with_llm(
        self,
        combined_text: str,
        topic_hint: str = "",
    ) -> List[str]:
        """Use LLM to distill topic keywords from aggregated vault content.

        Args:
            combined_text: Concatenated recent vault content.
            topic_hint: Optional focus instruction (e.g. "agent开发").

        Returns:
            List of keyword strings (up to _MAX_KEYWORDS).
        """
        topic_instruction = (
            f"\n- 重点关注与「{topic_hint}」相关的话题，优先提取这个方向的关键词"
            if topic_hint else ""
        )
        prompt = f"""你是一位知乎内容运营专家。请从以下笔记内容中提炼出适合在知乎搜索问题的关键词。

要求：
- 输出 {_MAX_KEYWORDS} 个以内的关键词，每行一个，无序号无符号
- 关键词要在知乎上有真实讨论热度，即普通中国用户会关心并提问的话题领域
- 避免过于偏门的专有名词（如具体项目代号、小众产品名），要用它所属的上一级概念
  例：「OpenClaw」→「AI智能体安全」；「BadJs」→「微信监控机制」；「Moon Monitor」→「健康监测设备」
- 不要太泛（如"AI""技术""健康"），也不要太窄（如只有一个工具代码名称）
- 优先选择最近讨论最多、最有深度的话题{topic_instruction}
- 只输出关键词列表，不要解释

=== 笔记内容 ===
{combined_text[:5000]}
"""
        try:
            response, _ = self.llm.generate_response(prompt, [])
            keywords = [
                line.strip()
                for line in response.strip().splitlines()
                if line.strip() and not line.strip().startswith("#")
            ]
            return keywords[:_MAX_KEYWORDS]
        except Exception as e:
            logger.error(f"Keyword extraction failed: {e}")
            return []

    # ── Question Discovery ─────────────────────────────────────────────────────

    def _find_questions(self, keywords: List[str]) -> List[ZhihuQuestion]:
        """Search Tavily for Zhihu questions matching each keyword.

        Collects candidates from ALL keywords before truncating so that later
        keywords (which can cover different facets) are not silently dropped.

        Args:
            keywords: List of topic keywords to search.

        Returns:
            Deduplicated ZhihuQuestion list (up to _MAX_QUESTIONS_TOTAL).
        """
        from tavily import TavilyClient

        api_key = os.environ.get("TAVILY_API_KEY")
        if not api_key:
            raise RuntimeError("TAVILY_API_KEY not set")

        client = TavilyClient(api_key=api_key)
        seen_urls: set[str] = set()
        questions: List[ZhihuQuestion] = []

        for kw in keywords:
            try:
                results = self._tavily_search(client, kw)
                for item in results:
                    url = item.get("url", "")
                    if not _MIN_QUESTION_URL_RE.search(url):
                        continue  # skip non-question URLs (e.g. zhihu.com/people/...)
                    # Normalize: extract bare URL to strip invisible chars, then remove params
                    _m = re.search(r'https?://\S+', url)
                    if not _m:
                        continue
                    clean_url = re.sub(r'[>\'")\].,;]+$', '', _m.group(0)).split("?")[0].split("#")[0].rstrip("/")
                    # Skip answer/comment subpages — we want the question itself
                    if re.search(r'/answer/|/comment/', clean_url):
                        continue
                    if clean_url in seen_urls:
                        continue
                    seen_urls.add(clean_url)
                    questions.append(
                        ZhihuQuestion(
                            title=item.get("title", kw),
                            url=clean_url,
                            snippet=item.get("content", "")[:200] or None,
                            keywords=[kw],
                        )
                    )
            except Exception as e:
                logger.warning(f"Tavily search failed for keyword '{kw}': {e}")

        logger.info(f"Found {len(questions)} Zhihu questions (capped at {_MAX_QUESTIONS_TOTAL})")
        return questions[:_MAX_QUESTIONS_TOTAL]

    def _tavily_search(self, client, keyword: str) -> List[dict]:
        """Run a single Tavily search scoped to zhihu.com/question.

        Args:
            client: Authenticated TavilyClient instance.
            keyword: Search keyword.

        Returns:
            List of result dicts from Tavily (url, title, content, score).
        """
        query = f"site:zhihu.com/question {keyword}"
        logger.debug(f"Tavily search: {query}")
        response = client.search(
            query=query,
            search_depth="basic",
            max_results=8,
            include_domains=["zhihu.com"],
            days=365,   # prefer questions with recent activity (higher chance of top rank)
        )
        return response.get("results", [])

+                # Filter out Zhihu generic site description (not question-specific)

+                if description and "知乎，中文互联网" in description:


Bash

source /root/projects/butler/venv/bin/activate && python3 -c "
import os, sys
os.chdir('/root/projects/butler')
sys.path.insert(0, '.')
from slack_bot.zhihu.zhihu_hunter import ZhihuHunter
from slack_bot.obsidian.indexer import ObsidianIndexer
from slack_bot.obsidian.embeddings import get_embedding_provider
from slack_bot.obsidian.vector_store import ChromaVectorStore
from pathlib import Path
vault = os.environ.get('OBSIDIAN_VAULT_PATH', '/root/vault/obsidian_vault/obsidian/obsidian/')
indexer = ObsidianIndexer(vault); indexer.scan_vault()
vs = ChromaVectorStore(embedding_provider=get_embedding_provider())
hunter = ZhihuHunter(vault_path=Path(vault), vector_store=vs, indexer=indexer)
qs = hunter.hunt_direct_url('https://www.zhihu.com/question/2015396999915139338')
print(f'Title: {qs[0].title!r}')
print(f'Desc:  {qs[0].description!r}')
" 2>&1 | grep -E "^(Title|Desc)"

Verify OpenClaw description is now None


Title: '你会使用OpenClaw吗，风险大于获得？'
Desc:  None


所有改动完成。更新 tasks：


Updated task #5 status


Updated task #6 status


/clear


## 修改总结

**5 个改动，涉及 3 个文件：**

### zhihu_hunter.py
1. **ZhihuQuestion 加 `description` 字段** — 存储问题正文
2. **`_fetch_question_title()` → `_fetch_question_info()`** — 同时抓标题和描述
   - 标题：优先用 `h1.QuestionHeader-title`（精确），fallback 到 `page.title()`
   - 清理 `(XX 条消息)` 污染前缀
   - 描述：优先 `div.QuestionRichText`，fallback 到 `og:description`，过滤知乎通用描述
3. **`draft_answer()` 注入问题描述** — `question.description` 追加到 prompt_input

### generators.py — ZhihuGenerator
4. **生成 prompt 瘦身** — 删除 POWELL_REVISE（7.8K chars），替换为 5 行核心规则。prompt 从 ~30K 降到 ~22K
5. **Review prompt 瘦身** — 删除 PRIORITY 3 (zhihu_bot_style) 和 PRIORITY 4 (POWELL_REVISE)，从 ~16K 降到 ~4K。kill list 集中在一个屏幕内

### generators.py — WritingAssistant
6. **同步更新 kill list** — 删除 POWELL_REVISE（PRIORITY 3），新增词汇和句式规则

### Kill list 新增项
- **夸张修饰**: 极其脆弱、瞬间崩断、精疲力竭、真金白银
- **大词空转**: 命门
- **概念包装**: 裸奔、修路/飙车、提款机、独立房间
- **句式**: 过渡词换皮（"第一个要说的是"）、伪分类二分法（"一部分人X另一部分人Y"）、"这不对，它的本质是…"

### 风险评估
- 最可能的问题：review prompt 变短后如果 Gemini 还是不跟 kill list，那是模型能力问题，不是规则覆盖问题。但 prompt 变短后注意力更集中，kill list 命中率应该显著提升
- 生成 prompt 没了 POWELL_REVISE 后，第一稿可能 AI 味更重，但这没关系——review pass 专门负责杀 AI 味，职责分离更清晰

**验证：** 重启 bot 后测试同一个 OpenClaw 问题。


bash -lc 'input=$(cat); for root in "${CLAUDE_PLUGIN_ROOT:-}" "$HOME/.claude/plugins/everything-claude-code" "$HOME/.claude/plugins/everything-claude-code@everything-claude-code" "$HOME/.claude/plugins/marketplace/everything-claude-code"; do if [ -n "$root" ] && [ -f "$root/scripts/hooks/run-with-flags.js" ]; then printf "%s" "$input" | node "$root/scripts/hooks/run-with-flags.js" "session:start" "scripts/hooks/session-start.js" "minimal,standard,strict"; exit $?; fi; done; for parent in "$HOME/.claude/plugins" "$HOME/.claude/plugins/marketplace"; do if [ -d "$parent" ]; then candidate=$(find "$parent" -maxdepth 2 -type f -path "*/scripts/hooks/run-with-flags.js" 2>/dev/null | head -n 1); if [ -n "$candidate" ]; then root=$(dirname "$(dirname "$(dirname "$candidate")")"); printf "%s" "$input" | node "$root/scripts/hooks/run-with-flags.js" "session:start" "scripts/hooks/session-start.js" "minimal,standard,strict"; exit $?; fi; fi; done; echo "[SessionStart] WARNING: could not resolve ECC plugin root; skipping session-start hook" >&2; printf "%s" "$input"; exit 0'


<command-message>superpowers:brainstorm</command-message>
<command-name>/superpowers:brainstorm</command-name>
<command-args>我打算做一个新功能在obsidian bot,取名为mode info。这个功能是把日常工作当中遇到的碎片信息保存在obsidian目录中，初步想法是按不同项目在不同的子目录保存，然后由于碎片或多或少，每周一个文件，新的信息就append文件末尾，每个信息包括至少三个部分：添加日期、原始内容、简要点评其内容。以后遇到项目问题，就可以从这里面，或者指定载入别的笔记内容一起来汇总整个项目的进展和问题。这里面也会有一些问题，在于怎么逐渐地update信息，因为也需要不断有一些交互获得了结果需要再放回目录当中，还就是不同项目目前倾向于互相独立，但也可能有机会同时问多个项目的意义。也看看你有什么建议和意见</command-args>


Tell your human partner that this command is deprecated and will be removed in the next major release. They should ask you to use the "superpowers brainstorming" skill instead.

ARGUMENTS: 我打算做一个新功能在obsidian bot,取名为mode info。这个功能是把日常工作当中遇到的碎片信息保存在obsidian目录中，初步想法是按不同项目在不同的子目录保存，然后由于碎片或多或少，每周一个文件，新的信息就append文件末尾，每个信息包括至少三个部分：添加日期、原始内容、简要点评其内容。以后遇到项目问题，就可以从这里面，或者指定载入别的笔记内容一起来汇总整个项目的进展和问题。这里面也会有一些问题，在于怎么逐渐地update信息，因为也需要不断有一些交互获得了结果需要再放回目录当中，还就是不同项目目前倾向于互相独立，但也可能有机会同时问多个项目的意义。也看看你有什么建议和意见


The user wants to brainstorm a new feature called "mode info" for the Obsidian bot. The `/superpowers:brainstorm` command is deprecated and should use the `superpowers:brainstorming` skill instead. Let me invoke that skill.


提示：`/superpowers:brainstorm` 已弃用，将在下一个主版本移除。请改用 `superpowers:brainstorming`。

我来用正确的 skill 帮你 brainstorm：


Skill

我打算做一个新功能在obsidian bot,取名为mode info。这个功能是把日常工作当中遇到的碎片信息保存在obsidian目录中，初步想法是按不同项目在不同的子目录保存，然后由于碎片或多或少，每周一个文件，新的信息就append文件末尾，每个信息包括至少三个部分：添加日期、原始内容、简要点评其内容。以后遇到项目问题，就可以从这里面，或者指定载入别的笔记内容一起来汇总整个项目的进展和问题。这里面也会有一些问题，在于怎么逐渐地update信息，因为也需要不断有一些交互获得了结果需要再放回目录当中，还就是不同项目目前倾向于互相独立，但也可能有机会同时问多个项目的意义。也看看你有什么建议和意见


bash "${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags-shell.sh" "pre:observe" "skills/continuous-learning-v2/hooks/observe.sh" "standard,strict"


Launching skill: superpowers:brainstorming


bash "${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags-shell.sh" "post:observe" "skills/continuous-learning-v2/hooks/observe.sh" "standard,strict"


Base directory for this skill: /root/.claude/plugins/cache/superpowers-marketplace/superpowers/5.0.2/skills/brainstorming

# Brainstorming Ideas Into Designs

Help turn ideas into fully formed designs and specs through natural collaborative dialogue.

Start by understanding the current project context, then ask questions one at a time to refine the idea. Once you understand what you're building, present the design and get user approval.

<HARD-GATE>
Do NOT invoke any implementation skill, write any code, scaffold any project, or take any implementation action until you have presented a design and the user has approved it. This applies to EVERY project regardless of perceived simplicity.
</HARD-GATE>

## Anti-Pattern: "This Is Too Simple To Need A Design"

Every project goes through this process. A todo list, a single-function utility, a config change — all of them. "Simple" projects are where unexamined assumptions cause the most wasted work. The design can be short (a few sentences for truly simple projects), but you MUST present it and get approval.

## Checklist

You MUST create a task for each of these items and complete them in order:

1. **Explore project context** — check files, docs, recent commits
2. **Offer visual companion** (if topic will involve visual questions) — this is its own message, not combined with a clarifying question. See the Visual Companion section below.
3. **Ask clarifying questions** — one at a time, understand purpose/constraints/success criteria
4. **Propose 2-3 approaches** — with trade-offs and your recommendation
5. **Present design** — in sections scaled to their complexity, get user approval after each section
6. **Write design doc** — save to `docs/superpowers/specs/YYYY-MM-DD-<topic>-design.md` and commit
7. **Spec review loop** — dispatch spec-document-reviewer subagent with precisely crafted review context (never your session history); fix issues and re-dispatch until approved (max 5 iterations, then surface to human)
8. **User reviews written spec** — ask user to review the spec file before proceeding
9. **Transition to implementation** — invoke writing-plans skill to create implementation plan

## Process Flow

```dot
digraph brainstorming {
    "Explore project context" [shape=box];
    "Visual questions ahead?" [shape=diamond];
    "Offer Visual Companion\n(own message, no other content)" [shape=box];
    "Ask clarifying questions" [shape=box];
    "Propose 2-3 approaches" [shape=box];
    "Present design sections" [shape=box];
    "User approves design?" [shape=diamond];
    "Write design doc" [shape=box];
    "Spec review loop" [shape=box];
    "Spec review passed?" [shape=diamond];
    "User reviews spec?" [shape=diamond];
    "Invoke writing-plans skill" [shape=doublecircle];

    "Explore project context" -> "Visual questions ahead?";
    "Visual questions ahead?" -> "Offer Visual Companion\n(own message, no other content)" [label="yes"];
    "Visual questions ahead?" -> "Ask clarifying questions" [label="no"];
    "Offer Visual Companion\n(own message, no other content)" -> "Ask clarifying questions";
    "Ask clarifying questions" -> "Propose 2-3 approaches";
    "Propose 2-3 approaches" -> "Present design sections";
    "Present design sections" -> "User approves design?";
    "User approves design?" -> "Present design sections" [label="no, revise"];
    "User approves design?" -> "Write design doc" [label="yes"];
    "Write design doc" -> "Spec review loop";
    "Spec review loop" -> "Spec review passed?";
    "Spec review passed?" -> "Spec review loop" [label="issues found,\nfix and re-dispatch"];
    "Spec review passed?" -> "User reviews spec?" [label="approved"];
    "User reviews spec?" -> "Write design doc" [label="changes requested"];
    "User reviews spec?" -> "Invoke writing-plans skill" [label="approved"];
}
```

**The terminal state is invoking writing-plans.** Do NOT invoke frontend-design, mcp-builder, or any other implementation skill. The ONLY skill you invoke after brainstorming is writing-plans.

## The Process

**Understanding the idea:**

- Check out the current project state first (files, docs, recent commits)
- Before asking detailed questions, assess scope: if the request describes multiple independent subsystems (e.g., "build a platform with chat, file storage, billing, and analytics"), flag this immediately. Don't spend questions refining details of a project that needs to be decomposed first.
- If the project is too large for a single spec, help the user decompose into sub-projects: what are the independent pieces, how do they relate, what order should they be built? Then brainstorm the first sub-project through the normal design flow. Each sub-project gets its own spec → plan → implementation cycle.
- For appropriately-scoped projects, ask questions one at a time to refine the idea
- Prefer multiple choice questions when possible, but open-ended is fine too
- Only one question per message - if a topic needs more exploration, break it into multiple questions
- Focus on understanding: purpose, constraints, success criteria

**Exploring approaches:**

- Propose 2-3 different approaches with trade-offs
- Present options conversationally with your recommendation and reasoning
- Lead with your recommended option and explain why

**Presenting the design:**

- Once you believe you understand what you're building, present the design
- Scale each section to its complexity: a few sentences if straightforward, up to 200-300 words if nuanced
- Ask after each section whether it looks right so far
- Cover: architecture, components, data flow, error handling, testing
- Be ready to go back and clarify if something doesn't make sense

**Design for isolation and clarity:**

- Break the system into smaller units that each have one clear purpose, communicate through well-defined interfaces, and can be understood and tested independently
- For each unit, you should be able to answer: what does it do, how do you use it, and what does it depend on?
- Can someone understand what a unit does without reading its internals? Can you change the internals without breaking consumers? If not, the boundaries need work.
- Smaller, well-bounded units are also easier for you to work with - you reason better about code you can hold in context at once, and your edits are more reliable when files are focused. When a file grows large, that's often a signal that it's doing too much.

**Working in existing codebases:**

- Explore the current structure before proposing changes. Follow existing patterns.
- Where existing code has problems that affect the work (e.g., a file that's grown too large, unclear boundaries, tangled responsibilities), include targeted improvements as part of the design - the way a good developer improves code they're working in.
- Don't propose unrelated refactoring. Stay focused on what serves the current goal.

## After the Design

**Documentation:**

- Write the validated design (spec) to `docs/superpowers/specs/YYYY-MM-DD-<topic>-design.md`
  - (User preferences for spec location override this default)
- Use elements-of-style:writing-clearly-and-concisely skill if available
- Commit the design document to git

**Spec Review Loop:**
After writing the spec document:

1. Dispatch spec-document-reviewer subagent (see spec-document-reviewer-prompt.md)
2. If Issues Found: fix, re-dispatch, repeat until Approved
3. If loop exceeds 5 iterations, surface to human for guidance

**User Review Gate:**
After the spec review loop passes, ask the user to review the written spec before proceeding:

> "Spec written and committed to `<path>`. Please review it and let me know if you want to make any changes before we start writing out the implementation plan."

Wait for the user's response. If they request changes, make them and re-run the spec review loop. Only proceed once the user approves.

**Implementation:**

- Invoke the writing-plans skill to create a detailed implementation plan
- Do NOT invoke any other skill. writing-plans is the next step.

## Key Principles

- **One question at a time** - Don't overwhelm with multiple questions
- **Multiple choice preferred** - Easier to answer than open-ended when possible
- **YAGNI ruthlessly** - Remove unnecessary features from all designs
- **Explore alternatives** - Always propose 2-3 approaches before settling
- **Incremental validation** - Present design, get approval before moving on
- **Be flexible** - Go back and clarify when something doesn't make sense

## Visual Companion

A browser-based companion for showing mockups, diagrams, and visual options during brainstorming. Available as a tool — not a mode. Accepting the companion means it's available for questions that benefit from visual treatment; it does NOT mean every question goes through the browser.

**Offering the companion:** When you anticipate that upcoming questions will involve visual content (mockups, layouts, diagrams), offer it once for consent:
> "Some of what we're working on might be easier to explain if I can show it to you in a web browser. I can put together mockups, diagrams, comparisons, and other visuals as we go. This feature is still new and can be token-intensive. Want to try it? (Requires opening a local URL)"

**This offer MUST be its own message.** Do not combine it with clarifying questions, context summaries, or any other content. The message should contain ONLY the offer above and nothing else. Wait for the user's response before continuing. If they decline, proceed with text-only brainstorming.

**Per-question decision:** Even after the user accepts, decide FOR EACH QUESTION whether to use the browser or the terminal. The test: **would the user understand this better by seeing it than reading it?**

- **Use the browser** for content that IS visual — mockups, wireframes, layout comparisons, architecture diagrams, side-by-side visual designs
- **Use the terminal** for content that is text — requirements questions, conceptual choices, tradeoff lists, A/B/C/D text options, scope decisions

A question about a UI topic is not automatically a visual question. "What does personality mean in this context?" is a conceptual question — use the terminal. "Which wizard layout works better?" is a visual question — use the browser.

If they agree to the companion, read the detailed guide before proceeding:
`skills/brainstorming/visual-companion.md`

ARGUMENTS: 我打算做一个新功能在obsidian bot,取名为mode info。这个功能是把日常工作当中遇到的碎片信息保存在obsidian目录中，初步想法是按不同项目在不同的子目录保存，然后由于碎片或多或少，每周一个文件，新的信息就append文件末尾，每个信息包括至少三个部分：添加日期、原始内容、简要点评其内容。以后遇到项目问题，就可以从这里面，或者指定载入别的笔记内容一起来汇总整个项目的进展和问题。这里面也会有一些问题，在于怎么逐渐地update信息，因为也需要不断有一些交互获得了结果需要再放回目录当中，还就是不同项目目前倾向于互相独立，但也可能有机会同时问多个项目的意义。也看看你有什么建议和意见


TaskCreate

探索项目上下文

了解现有 Obsidian bot 的 mode 架构、dispatcher 路由、存储模式，为 mode info 设计提供基础


Task #7 created successfully: 探索项目上下文

探索项目上下文


逐个问题澄清 mode info 的需求细节

逐个提问澄清需求


Task #8 created successfully: 逐个提问澄清需求

逐个提问澄清需求


提出 2-3 个实现方案，分析利弊并给出推荐

提出 2-3 个方案并推荐


Task #9 created successfully: 提出 2-3 个方案并推荐

提出 2-3 个方案并推荐


分段呈现完整设计，逐段获取用户确认

分段呈现设计并获取确认


Task #10 created successfully: 分段呈现设计并获取确认

分段呈现设计并获取确认


将确认的设计写入 spec 文件并提交

撰写设计文档并提交


Task #11 created successfully: 撰写设计文档并提交

撰写设计文档并提交


Updated task #7 status


先了解一下现有 bot 的 mode 架构和数据组织方式。


Agent

Explore existing mode architecture

Prompt:
I need to understand the existing Obsidian bot mode architecture in /root/projects/butler for planning a new "mode info" feature. Please find and summarize:

1. How modes are registered and routed in dispatcher.py (the VALID_MODES set, how generators map to modes)
2. How the existing MODE_NOTE works (note_ingester.py) - this is the closest existing feature to "mode info"
3. The vault directory structure under the OBSIDIAN_VAULT_PATH - what subdirectories exist, especially for notes
4. How NoteIngester saves notes (file naming, frontmatter format, directory structure)
5. The Pydantic models used (ExtractedNote, etc.)

Don't read generators.py (I already have it). Focus on: dispatcher.py mode registration, note_ingester.py, and vault structure. Be concise.


I need to understand the existing Obsidian bot mode architecture in /root/projects/butler for planning a new "mode info" feature. Please find and summarize:

1. How modes are registered and routed in dispatcher.py (the VALID_MODES set, how generators map to modes)
2. How the existing MODE_NOTE works (note_ingester.py) - this is the closest existing feature to "mode info"
3. The vault directory structure under the OBSIDIAN_VAULT_PATH - what subdirectories exist, especially for notes
4. How NoteIngester saves notes (file naming, frontmatter format, directory structure)
5. The Pydantic models used (ExtractedNote, etc.)

Don't read generators.py (I already have it). Focus on: dispatcher.py mode registration, note_ingester.py, and vault structure. Be concise.

Prompt:
I need to understand the existing Obsidian bot mode architecture in /root/projects/butler for planning a new "mode info" feature. Please find and summarize:

1. How modes are registered and routed in dispatcher.py (the VALID_MODES set, how generators map to modes)
2. How the existing MODE_NOTE works (note_ingester.py) - this is the closest existing feature to "mode info"
3. The vault directory structure under the OBSIDIAN_VAULT_PATH - what subdirectories exist, especially for notes
4. How NoteIngester saves notes (file naming, frontmatter format, directory structure)
5. The Pydantic models used (ExtractedNote, etc.)

Don't read generators.py (I already have it). Focus on: dispatcher.py mode registration, note_ingester.py, and vault structure. Be concise.


Bash

find /root/vault/obsidian_vault -type d -maxdepth 3 2>/dev/null | head -30

Explore Obsidian vault directory structure


grep -r "OBSIDIAN_VAULT_PATH" /root/projects/butler --include="*.py" | head -5

Find vault path references


     1→import os
     2→from pathlib import Path
     3→from typing import Optional, List, Dict, Any
     4→from slack_sdk import WebClient
     5→
     6→from health.utils.logging_config import setup_logger
     7→from slack_bot.obsidian.indexer import ObsidianIndexer
     8→from slack_bot.obsidian.generators import WritingAssistant, ReplyGenerator, DecisionSupport, SearchAnalyzer, DeAIReviser, ZhihuGenerator
     9→from slack_bot.context.storage import ContextStorage as BaseStorage
    10→
    11→logger = setup_logger(__name__)
    12→
    13→class ObsidianContextStorage(BaseStorage):
    14→    """Specific storage for Obsidian bot to avoid mixing with Health bot."""
    15→    def __init__(self, channel_id: str):
    16→        super().__init__(channel_id)
    17→        from health import config
    18→        self.context_dir = config.DATA_DIR / "obsidian_context"
    19→        self.context_dir.mkdir(parents=True, exist_ok=True)
    20→        self.file_path = self.context_dir / f"{channel_id}.json"
    21→        self.state_file = self.context_dir / f"{channel_id}_state.json"
    22→
    23→    def save_state(self, state: Dict[str, Any]):
    24→        import json
    25→        with open(self.state_file, "w") as f:
    26→            json.dump(state, f)
    27→
    28→    def load_state(self) -> Dict[str, Any]:
    29→        import json
    30→        if not self.state_file.exists():
    31→            return {"mode": "none"}
    32→        try:
    33→            with open(self.state_file, "r") as f:
    34→                return json.load(f)
    35→        except:
    36→            return {"mode": "none"}
    37→
    38→class ObsidianDispatcher:
    39→    """Routes Obsidian Slack Bot messages."""
    40→
    41→    MODE_WRITE = "write"
    42→    MODE_REPLY = "reply"
    43→    MODE_DECIDE = "decide"
    44→    MODE_SEARCH = "search"
    45→    MODE_DEAI = "deai"
    46→    MODE_ZHIHU = "zhihu"
    47→    MODE_NOTE = "note"
    48→    MODE_ZHIHU_HUNTER = "zhihu-hunter"
    49→
    50→    VALID_MODES = {MODE_WRITE, MODE_REPLY, MODE_DECIDE, MODE_SEARCH, MODE_DEAI, MODE_ZHIHU, MODE_NOTE, MODE_ZHIHU_HUNTER}
    51→
    52→    def __init__(self, vault_path: str):
    53→        self.indexer = ObsidianIndexer(vault_path)
    54→        self.indexer.scan_vault()
    55→
    56→        # Initialize RAG vector store and note ingester
    57→        from slack_bot.obsidian.embeddings import get_embedding_provider
    58→        from slack_bot.obsidian.vector_store import ChromaVectorStore
    59→        from slack_bot.obsidian.note_ingester import NoteIngester
    60→
    61→        self.vector_store = ChromaVectorStore(embedding_provider=get_embedding_provider())
    62→        self.note_ingester = NoteIngester(
    63→            vault_path=Path(vault_path),
    64→            vector_store=self.vector_store,
    65→        )
    66→
    67→        self.generators = {
    68→            self.MODE_WRITE: WritingAssistant(self.indexer, self.vector_store),
    69→            self.MODE_REPLY: ReplyGenerator(self.indexer, self.vector_store),
    70→            self.MODE_DECIDE: DecisionSupport(self.indexer),
    71→            self.MODE_SEARCH: SearchAnalyzer(self.indexer),
    72→            self.MODE_DEAI: DeAIReviser(self.indexer),
    73→            self.MODE_ZHIHU: ZhihuGenerator(self.indexer, self.vector_store),
    74→        }
    75→
    76→        token = os.environ.get("OBSIDIAN_SLACK_BOT_TOKEN")
    77→        self.client = WebClient(token=token)
    78→
    79→        # Initialize Zhihu Hunter components
    80→        from slack_bot.zhihu.zhihu_hunter import ZhihuHunter
    81→        from slack_bot.zhihu.zhihu_playwright_engine import ZhihuPlaywrightEngine
    82→        from slack_bot.zhihu.slack_interactive_gateway import SlackInteractiveGateway
    83→
    84→        zhihu_notify_channel = os.environ.get("ZHIHU_NOTIFY_CHANNEL", "")
    85→        self.zhihu_hunter = ZhihuHunter(
    86→            vault_path=Path(vault_path),
    87→            vector_store=self.vector_store,
    88→            indexer=self.indexer,
    89→        )
    90→        _zhihu_engine = ZhihuPlaywrightEngine(
    91→            slack_client=self.client,
    92→            notify_channel=zhihu_notify_channel,
    93→        )
    94→        self.zhihu_gateway = SlackInteractiveGateway(
    95→            slack_client=self.client,
    96→            hunter=self.zhihu_hunter,
    97→            engine=_zhihu_engine,
    98→            notify_channel=zhihu_notify_channel,
    99→        )
   100→        logger.info("ObsidianDispatcher initialized")
   101→
   102→    def dispatch(self, text: str, channel_id: str, user_id: str, response_ts: Optional[str] = None):
   103→        storage = ObsidianContextStorage(channel_id)
   104→        state = storage.load_state()
   105→        current_mode = state.get("mode", "none")
   106→
   107→        cmd = text.strip().lower()
   108→
   109→        # 1. Handle Commands
   110→        if cmd.startswith("mode "):
   111→            new_mode = cmd.split(" ", 1)[1].strip()
   112→            if new_mode in self.VALID_MODES:
   113→                storage.clear()
   114→                storage.save_state({"mode": new_mode})
   115→                self._reply(channel_id, f"✅ Switched to mode: *{new_mode}* (History cleared)", response_ts)
   116→                return
   117→            else:
   118→                self._reply(channel_id, f"❌ Invalid mode. Available: {', '.join(sorted(self.VALID_MODES))}", response_ts)
   119→                return
   120→
   121→        if cmd == "clear":
   122→            storage.clear()
   123→            self._reply(channel_id, "🧹 Context cleared.", response_ts)
   124→            return
   125→
   126→        if cmd == "reload":
   127→            self.indexer.scan_vault()
   128→            self._reply(channel_id, "🔄 Vault reloaded.", response_ts)
   129→            return
   130→
   131→        # 2. Handle Content based on mode
   132→        if current_mode == "none":
   133→            self._reply(channel_id, "⚠ No mode selected. Use `mode write/reply/decide/search/deai/zhihu/note/zhihu-hunter` to start.", response_ts)
   134→            return
   135→
   136→        # MODE_NOTE: ingest text or URL into vault + vector store
   137→        if current_mode == self.MODE_NOTE:
   138→            try:
   139→                result = self.note_ingester.ingest(text)
   140→                self._reply(channel_id, result, response_ts)
   141→            except Exception as e:
   142→                logger.error(f"Note ingestion failed: {e}", exc_info=True)
   143→                self._reply(channel_id, f"🐛 Ingestion error: {str(e)}", response_ts)
   144→            return
   145→
   146→        # MODE_ZHIHU_HUNTER: scan vault for keywords, surface Zhihu question candidates
   147→        #
   148→        # Command format:
   149→        #   开始                               → full scan, all dirs, auto keywords
   150→        #   article                            → scan Article only (Revised first, no time filter)
   151→        #   article agent开发                  → scan Article, focus on "agent开发"
   152→        #   notes agent开发                    → scan notes, focus on "agent开发"
   153→        #   关键词1 关键词2                     → skip scan, search directly with keywords
   154→        #   Article/8-xxx-Revised.md           → single file: grounded in that article
   155→        #   Article/*openclaw*-Revised.md      → glob: combine matching files
   156→        #   https://www.zhihu.com/question/... → direct URL: answer this specific question
   157→        #
   158→        # Dir shortcuts: notes / article / clippings / claw / deepfact / deep-dive
   159→        if current_mode == self.MODE_ZHIHU_HUNTER:
   160→            from slack_bot.zhihu.zhihu_hunter import DIR_SHORTCUTS
   161→            try:
   162→                trigger_words = {"开始", "hunt", "扫描", "start", "go"}
   163→                words = [w for w in text.strip().split() if w.lower() not in trigger_words]
   164→
   165→                if not words:
   166→                    # Bare trigger → full auto scan
   167→                    self._reply(channel_id, "🔍 正在扫描知乎候选问题（所有目录）...", response_ts)
   168→                    questions = self.zhihu_hunter.scan_and_hunt(since_days=7)
   169→
   170→                elif words and 'zhihu.com/question/' in words[0]:
   171→                    # Direct URL paste — user was invited or found a question to answer.
   172→                    # Any text after the URL is treated as an optional answer outline.
   173→                    # Strip trailing Slack/HTML junk chars (>, ", etc.) from the URL.
   174→                    import re as _re
   175→                    url = _re.sub(r'[>\'")\].,;]+$', '', words[0]).lstrip("<")
   176→                    outline = " ".join(words[1:])[:500] if len(words) > 1 else ""
   177→                    hint = f"（已记录回答思路）" if outline else ""
   178→                    self._reply(channel_id, f"🔍 获取问题标题，准备生成草稿{hint}...", response_ts)
   179→                    questions = self.zhihu_hunter.hunt_direct_url(url)
   180→                    if outline and questions:
   181→                        questions[0].outline = outline
   182→                    logger.info(f"ZhihuHunter: direct URL {url!r} → {questions[0].title!r}" +
   183→                                (f" outline={outline[:40]!r}" if outline else ""))
   184→
   185→                elif words and ('*' in words[0] or '?' in words[0]):
   186→                    # Glob pattern → combine all matched files for keyword extraction
   187→                    pattern = words[0]
   188→                    self._reply(
   189→                        channel_id,
   190→                        f"🔍 匹配 `{pattern}` 中的文件，提取观点搜索知乎问题...",
   191→                        response_ts,
   192→                    )
   193→                    questions = self.zhihu_hunter.scan_glob_files(pattern)
   194→                    logger.info(f"ZhihuHunter: glob '{pattern}' → {len(questions)} questions")
   195→
   196→                elif words and ('/' in words[0] or words[0].endswith('.md')):
   197→                    # Single-file mode: extract questions from one specific file,
   198→                    # and pass source_file so answers stay grounded in that article.
   199→                    rel_path = words[0]
   200→                    self._reply(
   201→                        channel_id,
   202→                        f"🔍 从 `{rel_path}` 提取观点，搜索匹配知乎问题...",
   203→                        response_ts,
   204→                    )
   205→                    questions = self.zhihu_hunter.scan_single_file(rel_path)
   206→                    logger.info(f"ZhihuHunter: single-file scan '{rel_path}' → {len(questions)} questions")
   207→
   208→                elif words[0].lower() in DIR_SHORTCUTS:
   209→                    # Dir-scoped scan with optional topic hint
   210→                    rel_dir = DIR_SHORTCUTS[words[0].lower()]
   211→                    topic_hint = " ".join(words[1:])
   212→                    hint_desc = f"，聚焦：{topic_hint}" if topic_hint else ""
   213→                    self._reply(
   214→                        channel_id,
   215→                        f"🔍 扫描 {rel_dir} 目录{hint_desc}，请稍候...",
   216→                        response_ts,
   217→                    )
   218→                    questions = self.zhihu_hunter.scan_and_hunt(
   219→                        since_days=7, dirs=[rel_dir], topic_hint=topic_hint
   220→                    )
   221→
   222→                else:
   223→                    # Direct keywords → skip vault scan entirely
   224→                    self._reply(channel_id, f"🔍 使用指定关键词搜索：{' · '.join(words)}", response_ts)
   225→                    questions = self.zhihu_hunter._find_questions(words)
   226→                    logger.info(f"ZhihuHunter: manual keywords {words} → {len(questions)} questions")
   227→
   228→                self.zhihu_gateway.post_question_cards(questions, channel=channel_id)
   229→            except Exception as e:
   230→                logger.error(f"Zhihu hunt failed: {e}", exc_info=True)
   231→                self._reply(channel_id, f"❌ 扫描失败：{e}", response_ts)
   232→            return
   233→
   234→        # Get history
   235→        history = storage.get_context()
   236→        generator = self.generators[current_mode]
   237→
   238→        try:
   239→            logger.info(f"Generating Obsidian response in mode: {current_mode}")
   240→            # The chat method handles first turn logic and returns updated history
   241→            # But wait, our generators' chat method appends to history internally.
   242→            # In our Slack setup, we want to append the result to ContextStorage.
   243→
   244→            # Note: generator.chat(text, history) returns (response, new_history_list)
   245→            # new_history_list contains [...history, {user: rich_prompt}, {assistant: response}]
   246→            response_text, updated_history = generator.chat(text, history)
   247→
   248→            # Since generator.chat already did the LLM call and returned history,
   249→            # we just need to save the ONLY NEW messages to our storage.
   250→            # However, during FIRST TURN, text is replaced by rich_prompt.
   251→
   252→            # For simplicity: completely overwrite storage with updated_history
   253→            # ContextStorage doesn't support overwrite easily, but we can clear and re-add.
   254→            # Better: Modify ObsidianContextStorage to support set_context.
   255→
   256→            # Actually, let's just use the updated_history and manually add to storage
   257→            # Warning: first turn 'user' message in storage will be the RICH PROMPT
   258→            # This is actually better for context continuity in Slack too.
   259→
   260→            storage.clear()
   261→            for msg in updated_history:
   262→                storage.add_message(msg["role"], msg["content"])
   263→
   264→            self._reply(channel_id, response_text, response_ts)
   265→
   266→        except Exception as e:
   267→            logger.error(f"Obsidian generation failed: {e}", exc_info=True)
   268→            self._reply(channel_id, f"🐛 Error: {str(e)}", response_ts)
   269→
   270→    def _reply(self, channel_id: str, text: str, ts: Optional[str] = None):
   271→        from slack_bot.utils.mrkdwn import SlackFormatter
   272→
   273→        # Slack API limits (official documentation):
   274→        # - chat.postMessage: 40,000 chars
   275→        # - chat.update: 4,000 chars
   276→        # Slack markdown formatting can expand text by 50-100% (bold, links, code blocks, etc.)
   277→        TRUNCATE_WARNING = "\n\n⚠ (Response truncated, full text uploaded as file)"
   278→        FILE_UPLOAD_HINT = "📄 Full response uploaded as file above ☝"
   279→
   280→        # Calculate safe pre-formatting limit with aggressive overhead buffer
   281→        if ts:
   282→            # For updates: 4000 limit - 60% overhead for safety = ~1600 chars
   283→            max_raw_length = 1500
   284→        else:
   285→            # For new messages: 40000 limit - 40% overhead = ~24000 chars
   286→            max_raw_length = 24000
   287→
   288→        original_length = len(text)
   289→        should_upload_file = original_length > max_raw_length
   290→
   291→        if should_upload_file:
   292→            # Upload full text as file first
   293→            self._upload_as_file(channel_id, text, ts)
   294→
   295→            # Then send a truncated preview
   296→            truncate_at = text.rfind('\n\n', 0, 800)
   297→            if truncate_at == -1 or truncate_at < 400:
   298→                truncate_at = text.rfind('\n', 0, 800)
   299→            if truncate_at == -1 or truncate_at < 400:
   300→                truncate_at = 800
   301→
   302→            preview_text = text[:truncate_at] + f"\n\n...\n\n{FILE_UPLOAD_HINT}"
   303→            formatted_text = SlackFormatter.convert(preview_text)
   304→            logger.info(f"Uploaded full response ({original_length} chars) as file, sending preview ({len(preview_text)} chars)")
   305→        else:
   306→            # Text is short enough, just format and send
   307→            formatted_text = SlackFormatter.convert(text)
   308→
   309→            # Final safety check after formatting
   310→            final_limit = 3900 if ts else 39900
   311→            if len(formatted_text) > final_limit:
   312→                logger.warning(f"Formatted text ({len(formatted_text)} chars) exceeds limit after formatting. Uploading as file.")
   313→                self._upload_as_file(channel_id, text, ts)
   314→                # Send simple notification
   315→                formatted_text = FILE_UPLOAD_HINT
   316→
   317→        # Send the message
   318→        try:
   319→            if ts:
   320→                self.client.chat_update(channel=channel_id, ts=ts, text=formatted_text)
   321→            else:
   322→                self.client.chat_postMessage(channel=channel_id, text=formatted_text)
   323→        except Exception as e:
   324→            if "msg_too_long" in str(e):
   325→                logger.error(f"Still got msg_too_long error even after truncation. Text length: {len(formatted_text)}")
   326→                # Ultimate fallback: send error message only
   327→                try:
   328→                    error_msg = "❌ Response formatting error. Please check the uploaded file above."
   329→                    if ts:
   330→                        self.client.chat_update(channel=channel_id, ts=ts, text=error_msg)
   331→                    else:
   332→                        self.client.chat_postMessage(channel=channel_id, text=error_msg)
   333→                except:
   334→                    logger.error("Failed to send even error message", exc_info=True)
   335→            else:
   336→                raise
   337→
   338→    def _upload_as_file(self, channel_id: str, content: str, ts: Optional[str] = None):
   339→        """Fallback: upload long responses as text files."""
   340→        try:
   341→            import tempfile
   342→            from datetime import datetime
   343→
   344→            filename = f"response_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
   345→
   346→            with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as tmp:
   347→                tmp.write(content)
   348→                tmp_path = tmp.name
   349→
   350→            self.client.files_upload_v2(
   351→                channel=channel_id,
   352→                file=tmp_path,
   353→                filename=filename,
   354→                initial_comment="📄 Response too long for Slack message, uploaded as file:",
   355→                thread_ts=ts
   356→            )
   357→
   358→            os.unlink(tmp_path)
   359→            logger.info(f"Uploaded long response as file: {filename}")
   360→        except Exception as e:
   361→            logger.error(f"Failed to upload file: {e}", exc_info=True)
   362→            # Ultimate fallback: send error message
   363→            error_msg = "❌ Response too long to display. Please try a more specific query."
   364→            if ts:
   365→                self.client.chat_update(channel=channel_id, ts=ts, text=error_msg)
   366→            else:
   367→                self.client.chat_postMessage(channel=channel_id, text=error_msg)
   368→
   369→


1→"""Note ingestion engine for the Second Brain RAG system."""
     2→
     3→import json
     4→import os
     5→import re
     6→from datetime import datetime, timezone
     7→from pathlib import Path
     8→from typing import List, Optional
     9→
    10→from pydantic import BaseModel
    11→
    12→from health.utils.logging_config import setup_logger
    13→from slack_bot.llm.gemini import GeminiLLM
    14→from slack_bot.obsidian.vector_store import ChunkMetadata, ChromaVectorStore, split_md_by_headers
    15→
    16→logger = setup_logger(__name__)
    17→
    18→
    19→class ExtractedNote(BaseModel):
    20→    """LLM-extracted structured note from raw input text."""
    21→
    22→    title: str
    23→    summary: str           # 2-3 sentences
    24→    key_points: List[str]  # up to 7 items
    25→    entities: List[str]    # people, concepts, projects; wrapped as [[entity]] in cleaned_body
    26→    cleaned_body: str      # full text with entities wrapped in [[double brackets]]
    27→    source_url: Optional[str] = None
    28→    language: str = "zh"
    29→
    30→
    31→class NoteIngester:
    32→    """Ingests raw text or URLs into the Obsidian vault and vector store."""
    33→
    34→    def __init__(self, vault_path: Path, vector_store: ChromaVectorStore) -> None:
    35→        """Initialize the note ingester.
    36→
    37→        Args:
    38→            vault_path: Root path of the Obsidian vault.
    39→            vector_store: ChromaDB vector store for indexing chunks.
    40→        """
    41→        self.vault_path = vault_path
    42→        self.notes_dir = vault_path / "notes"
    43→        self.notes_dir.mkdir(parents=True, exist_ok=True)
    44→        self.vector_store = vector_store
    45→        self.llm = GeminiLLM()
    46→
    47→    def ingest(self, user_input: str) -> str:
    48→        """Ingest text or a URL into the vault and vector store.
    49→
    50→        Args:
    51→            user_input: Raw text or a URL to ingest.
    52→
    53→        Returns:
    54→            Slack-formatted confirmation message.
    55→        """
    56→        url = self._detect_url(user_input)
    57→        source_url: Optional[str] = None
    58→
    59→        if url:
    60→            logger.info(f"Detected URL: {url}. Fetching content...")
    61→            try:
    62→                raw_text = self._fetch_url(url)
    63→                source_url = url
    64→            except Exception as e:
    65→                logger.warning(f"URL fetch failed ({e})")
    66→                # 无法抓取（微信/付费墙等），提示用户手动粘贴正文
    67→                return (
    68→                    f"⚠ 无法自动抓取该链接内容（{e}）\n\n"
    69→                    f"请将文章正文直接粘贴发送给我，我来帮你保存入库。"
    70→                )
    71→        else:
    72→            raw_text = user_input
    73→
    74→        try:
    75→            note = self._extract_with_llm(raw_text, source_url)
    76→        except Exception as e:
    77→            logger.error(f"LLM extraction failed: {e}")
    78→            return f"❌ 内容提取失败（LLM 解析出错）：{e}\n\n请重试，或检查原始内容是否正常。"
    79→
    80→        file_path = self._write_md_file(note)
    81→        self._chunk_and_index(note, file_path)
    82→
    83→        entity_links = " ".join(f"[[{e}]]" for e in note.entities[:5])
    84→        rel_path = file_path.relative_to(self.vault_path)
    85→        return (
    86→            f"✅ *Note saved:* {note.title}\n"
    87→            f"{note.summary}\n"
    88→            f"_{entity_links}_\n"
    89→            f"📁 `{rel_path}`"
    90→        )
    91→
    92→    def _detect_url(self, text: str) -> Optional[str]:
    93→        """Extract the first HTTP/HTTPS URL from text, handling Slack's <URL> format.
    94→
    95→        Slack wraps URLs as ``<url>`` or ``<url|display text>``. The angle
    96→        brackets and pipe are excluded from the match so they don't get
    97→        appended to the URL.
    98→
    99→        Args:
   100→            text: Input text to scan.
   101→
   102→        Returns:
   103→            First clean URL found, or None.
   104→        """
   105→        # Exclude Slack delimiters: angle brackets and pipe
   106→        match = re.search(r'https?://[^\s<>|]+', text)
   107→        if not match:
   108→            return None
   109→        return match.group(0).rstrip('.,;:')
   110→
   111→    # Domains that block generic scrapers; try Jina first for these
   112→    _JINA_FIRST_DOMAINS = {"mp.weixin.qq.com", "weixin.qq.com"}
   113→    _MIN_CONTENT_LEN = 200
   114→
   115→    def _fetch_url(self, url: str) -> str:
   116→        """Fetch URL content using multiple strategies with automatic fallback.
   117→
   118→        Strategy order:
   119→        - WeChat / known blocked domains: direct js_content parse → Tavily → Playwright
   120→        - All other URLs: Tavily → Jina → Playwright
   121→
   122→        Args:
   123→            url: URL to fetch.
   124→
   125→        Returns:
   126→            Extracted text content (>= _MIN_CONTENT_LEN chars).
   127→
   128→        Raises:
   129→            RuntimeError: If all strategies fail.
   130→        """
   131→        host_match = re.search(r'https?://([^/]+)', url)
   132→        host = host_match.group(1) if host_match else ""
   133→
   134→        if host in self._JINA_FIRST_DOMAINS:
   135→            # WeChat: direct parse first, Playwright as last resort
   136→            strategies = [self._fetch_direct_wechat, self._fetch_tavily, self._fetch_playwright]
   137→        else:
   138→            strategies = [self._fetch_tavily, self._fetch_jina, self._fetch_playwright]
   139→
   140→        last_err: Exception = RuntimeError("No strategies available")
   141→        for strategy in strategies:
   142→            try:
   143→                content = strategy(url)
   144→                if len(content) >= self._MIN_CONTENT_LEN:
   145→                    logger.info(f"Fetched {len(content)} chars via {strategy.__name__}")
   146→                    return content
   147→                logger.debug(f"{strategy.__name__} returned too little content ({len(content)} chars)")
   148→            except Exception as e:
   149→                logger.debug(f"{strategy.__name__} failed: {e}")
   150→                last_err = e
   151→
   152→        raise RuntimeError(f"所有抓取策略均失败: {last_err}")
   153→
   154→    def _fetch_jina(self, url: str) -> str:
   155→        """Fetch via Jina Reader (r.jina.ai) — works well for WeChat and most sites.
   156→
   157→        Args:
   158→            url: Target URL.
   159→
   160→        Returns:
   161→            Markdown text extracted by Jina.
   162→        """
   163→        import httpx
   164→
   165→        jina_url = f"https://r.jina.ai/{url}"
   166→        headers: dict = {"Accept": "text/plain", "X-Return-Format": "markdown"}
   167→        jina_key = os.environ.get("JINA_API_KEY")
   168→        if jina_key:
   169→            headers["Authorization"] = f"Bearer {jina_key}"
   170→
   171→        resp = httpx.get(jina_url, headers=headers, timeout=30, follow_redirects=True)
   172→        resp.raise_for_status()
   173→        return resp.text
   174→
   175→    def _fetch_direct_wechat(self, url: str) -> str:
   176→        """Directly fetch a WeChat public article by parsing js_content from HTML.
   177→
   178→        WeChat public articles (mp.weixin.qq.com) embed the full body in
   179→
. This method extracts title + body without
   180→        needing a headless browser or external proxy.
   181→
   182→        Args:
   183→            url: WeChat article URL.
   184→
   185→        Returns:
   186→            Plain text of title + article body.
   187→        """
   188→        import httpx
   189→
   190→        headers = {
   191→            "User-Agent": (
   192→                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
   193→                "AppleWebKit/537.36 (KHTML, like Gecko) "
   194→                "Chrome/122.0.0.0 Safari/537.36"
   195→            ),
   196→            "Accept-Language": "zh-CN,zh;q=0.9",
   197→        }
   198→        resp = httpx.get(url, headers=headers, timeout=15, follow_redirects=True)
   199→        resp.raise_for_status()
   200→        html = resp.text
   201→
   202→        parts: list[str] = []
   203→
   204→        # Extract title
   205→        title_m = re.search(
   206→            r'class=["\'][^"\']*rich_media_title[^"\']*["\'][^>]*>(.*?)
',
   207→            html, re.DOTALL
   208→        )
   209→        if title_m:
   210→            title_text = re.sub(r'<[^>]+>', '', title_m.group(1)).strip()
   211→            if title_text:
   212→                parts.append(f"# {title_text}")
   213→
   214→        # Extract body: find js_content start, end at first known landmark
   215→        start_m = re.search(r'id=["\']js_content["\']', html)
   216→        if start_m:
   217→            content_start = html.index('>', start_m.end()) + 1
   218→            # Landmarks that mark the end of the article body in WeChat HTML
   219→            end_markers = [
   220→                'id="js_bottom_bar"',
   221→                'id="js_related_posts"',
   222→                'id="js_msg_card"',
   223→                'id="js_sg_bar"',
   224→                'id="js_pc_qr_code"',
   225→                'class="qr_code_pc_outer"',
   226→            ]
   227→            content_end = len(html)
   228→            for marker in end_markers:
   229→                idx = html.find(marker, content_start)
   230→                if 0 < idx < content_end:
   231→                    content_end = idx
   232→            body_html = html[content_start:content_end]
   233→            body = re.sub(r'<[^>]+>', ' ', body_html)
   234→            body = re.sub(r' ', ' ', body)
   235→            body = re.sub(r'&[a-zA-Z]+;', '', body)
   236→            body = re.sub(r'\s+', ' ', body).strip()
   237→            if body:
   238→                parts.append(body)
   239→
   240→        if not parts:
   241→            # Fallback: strip all tags from full HTML
   242→            text = re.sub(r'<[^>]+>', ' ', html)
   243→            return re.sub(r'\s+', ' ', text).strip()
   244→
   245→        return "\n\n".join(parts)
   246→
   247→    def _fetch_tavily(self, url: str) -> str:
   248→        """Fetch via Tavily extract API.
   249→
   250→        Args:
   251→            url: Target URL.
   252→
   253→        Returns:
   254→            Raw content string from Tavily.
   255→
   256→        Raises:
   257→            RuntimeError: If API key missing or no content returned.
   258→        """
   259→        from tavily import TavilyClient
   260→
   261→        api_key = os.environ.get("TAVILY_API_KEY")
   262→        if not api_key:
   263→            raise RuntimeError("TAVILY_API_KEY is not set in environment")
   264→
   265→        client = TavilyClient(api_key=api_key)
   266→        result = client.extract(urls=[url])
   267→        if result and result.get("results"):
   268→            return result["results"][0].get("raw_content", "")
   269→        raise RuntimeError("Tavily 未能提取到内容")
   270→
   271→    def _fetch_playwright(self, url: str) -> str:
   272→        """Fetch via headless Chromium with stealth mode (last resort).
   273→
   274→        Renders the full page in a real browser, bypassing JS-heavy SPAs and
   275→        most anti-bot fingerprinting. Slower (~3-5s) but most comprehensive.
   276→
   277→        Args:
   278→            url: Target URL.
   279→
   280→        Returns:
   281→            Visible body text extracted from the rendered page.
   282→
   283→        Raises:
   284→            ImportError: If playwright or playwright-stealth is not installed.
   285→            RuntimeError: If the page loads but body text is too short.
   286→        """
   287→        try:
   288→            from playwright.sync_api import sync_playwright
   289→        except ImportError:
   290→            raise ImportError(
   291→                "playwright is not installed. Run: pip install playwright && "
   292→                "playwright install chromium"
   293→            )
   294→        try:
   295→            from playwright_stealth import Stealth
   296→        except ImportError:
   297→            raise ImportError(
   298→                "playwright-stealth is not installed. Run: pip install playwright-stealth"
   299→            )
   300→
   301→        with sync_playwright() as p:
   302→            browser = p.chromium.launch(headless=True)
   303→            try:
   304→                page = browser.new_page()
   305→                Stealth().apply_stealth_sync(page)
   306→                page.goto(url, timeout=30000, wait_until="domcontentloaded")
   307→                # Allow JS rendering to settle
   308→                page.wait_for_timeout(2000)
   309→                content = page.inner_text("body").strip()
   310→                logger.info(f"Playwright fetched {len(content)} chars from {url}")
   311→                return content
   312→            finally:
   313→                browser.close()
   314→
   315→    def _extract_with_llm(self, raw_text: str, source_url: Optional[str]) -> ExtractedNote:
   316→        """Use LLM to extract structured note data from raw text.
   317→
   318→        Args:
   319→            raw_text: Raw content to process (truncated to 8000 chars).
   320→            source_url: Optional source URL for context.
   321→
   322→        Returns:
   323→            Validated ExtractedNote model.
   324→
   325→        Raises:
   326→            ValueError: If LLM output cannot be parsed as valid JSON.
   327→        """
   328→        url_context = f"Source URL: {source_url}\n" if source_url else ""
   329→        prompt = f"""You are a knowledge extraction assistant. Extract structured information from the text below.
   330→{url_context}
   331→=== TEXT ===
   332→{raw_text[:8000]}
   333→
   334→=== TASK ===
   335→Return a JSON object with EXACTLY these fields:
   336→- "title": A concise, descriptive title (max 80 chars)
   337→- "summary": 2-3 sentence summary of the core content
   338→- "key_points": Array of up to 7 key points (strings)
   339→- "entities": Array of important people, concepts, projects, or tools mentioned (max 10)
   340→- "cleaned_body": The full text, cleaned and reorganized. Wrap entity names in [[double brackets]] like [[entity name]].
   341→- "language": "zh" if the content is primarily Chinese, "en" otherwise
   342→
   343→CRITICAL:
   344→- Output ONLY valid JSON, no markdown code blocks, no explanation
   345→- Do NOT include tags like #writing_sample or #reply_sample
   346→- Entities should be wrapped as [[entity]] in cleaned_body
   347→"""
   348→        response, _ = self.llm.generate_response(prompt, [])
   349→
   350→        # Strip potential markdown fences
   351→        response = response.strip()
   352→        if response.startswith("```"):
   353→            response = re.sub(r'^```(?:json)?\n?', '', response)
   354→            response = re.sub(r'\n?```$', '', response)
   355→
   356→        data = json.loads(response)
   357→        return ExtractedNote.model_validate(data)
   358→
   359→    def _fallback_note(self, raw_text: str, source_url: Optional[str]) -> ExtractedNote:
   360→        """Create a minimal note when LLM extraction fails.
   361→
   362→        Args:
   363→            raw_text: Raw content.
   364→            source_url: Optional source URL.
   365→
   366→        Returns:
   367→            Basic ExtractedNote with raw text as body.
   368→        """
   369→        title = raw_text[:60].replace('\n', ' ').strip() + "..."
   370→        return ExtractedNote(
   371→            title=title,
   372→            summary=raw_text[:200],
   373→            key_points=[],
   374→            entities=[],
   375→            cleaned_body=raw_text,
   376→            source_url=source_url,
   377→            language="zh",
   378→        )
   379→
   380→    def _write_md_file(self, note: ExtractedNote) -> Path:
   381→        """Write note to vault as a Markdown file with YAML frontmatter.
   382→
   383→        Args:
   384→            note: Extracted note data.
   385→
   386→        Returns:
   387→            Path to the written file.
   388→        """
   389→        date_str = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
   390→        slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', note.title)[:50].strip('-')
   391→        filename = f"{date_str}_{slug}.md"
   392→        file_path = self.notes_dir / filename
   393→
   394→        source_line = f"source_url: \"{note.source_url}\"\n" if note.source_url else ""
   395→        key_points_yaml = "\n".join(f"  - \"{kp}\"" for kp in note.key_points)
   396→        ingested_at = datetime.now(tz=timezone.utc).isoformat()
   397→
   398→        content = (
   399→            f"---\n"
   400→            f"title: \"{note.title}\"\n"
   401→            f"date: {date_str}\n"
   402→            f"tags:\n"
   403→            f"  - auto_ingested\n"
   404→            f"language: {note.language}\n"
   405→            f"{source_line}"
   406→            f"key_points:\n"
   407→            f"{key_points_yaml}\n"
   408→            f"ingested_at: {ingested_at}\n"
   409→            f"---\n\n"
   410→            f"## Summary\n\n"
   411→            f"{note.summary}\n\n"
   412→            f"## Content\n\n"
   413→            f"{note.cleaned_body}\n"
   414→        )
   415→
   416→        file_path.write_text(content, encoding="utf-8")
   417→        logger.info(f"Note written: {file_path}")
   418→        return file_path
   419→
   420→    def _chunk_and_index(self, note: ExtractedNote, file_path: Path) -> None:
   421→        """Split note body into chunks and index them in the vector store.
   422→
   423→        Args:
   424→            note: Extracted note data.
   425→            file_path: Path to the written markdown file.
   426→        """
   427→        body = f"## Summary\n\n{note.summary}\n\n## Content\n\n{note.cleaned_body}"
   428→        chunks = split_md_by_headers(body)
   429→
   430→        if not chunks:
   431→            # Fallback: single chunk from summary + start of body
   432→            chunks = [(note.summary + "\n\n" + note.cleaned_body)[:1000]]
   433→
   434→        ingested_at = datetime.now(tz=timezone.utc).isoformat()
   435→        metadatas = [
   436→            ChunkMetadata(
   437→                source_path=str(file_path),
   438→                chunk_index=i,
   439→                header_hierarchy=(
   440→                    chunk.split('\n')[0][:100] if chunk.startswith('#') else "body"
   441→                ),
   442→                note_title=note.title,
   443→                tags=["auto_ingested"],
   444→                ingested_at=ingested_at,
   445→            )
   446→            for i, chunk in enumerate(chunks)
   447→        ]
   448→
   449→        self.vector_store.add_chunks(chunks, metadatas)
   450→        logger.info(f"Indexed {len(chunks)} chunks for note: {note.title}")
   451→


/root/vault/obsidian_vault
/root/vault/obsidian_vault/obsidian
/root/vault/obsidian_vault/obsidian/obsidian
/root/vault/obsidian_vault/obsidian/obsidian/AppleNotes
/root/vault/obsidian_vault/obsidian/obsidian/presentation-workspace
/root/vault/obsidian_vault/obsidian/obsidian/attachments
/root/vault/obsidian_vault/obsidian/obsidian/Clippings
/root/vault/obsidian_vault/obsidian/obsidian/accounts
/root/vault/obsidian_vault/obsidian/obsidian/Article
/root/vault/obsidian_vault/obsidian/obsidian/obsidian
/root/vault/obsidian_vault/obsidian/obsidian/feelings
/root/vault/obsidian_vault/obsidian/obsidian/daily
/root/vault/obsidian_vault/obsidian/obsidian/必备知识
/root/vault/obsidian_vault/obsidian/obsidian/工作记录
/root/vault/obsidian_vault/obsidian/obsidian/AI
/root/vault/obsidian_vault/obsidian/obsidian/英语学习
/root/vault/obsidian_vault/obsidian/obsidian/私人
/root/vault/obsidian_vault/obsidian/obsidian/md
/root/vault/obsidian_vault/obsidian/obsidian/CLAW
/root/vault/obsidian_vault/obsidian/obsidian/录音记录
/root/vault/obsidian_vault/obsidian/obsidian/auto_report
/root/vault/obsidian_vault/obsidian/obsidian/notes
/root/vault/obsidian_vault/obsidian/obsidian/zhihu
/root/vault/obsidian_vault/obsidian/Documents
/root/vault/obsidian_vault/obsidian/Documents/obsidian
/root/vault/obsidian_vault/obsidian/Documents/copilot


/root/projects/butler/list_samples.py:vault_path = os.getenv("OBSIDIAN_VAULT_PATH")
/root/projects/butler/scripts/obsidian_bot.py:    vault_path = os.environ.get("OBSIDIAN_VAULT_PATH")
/root/projects/butler/scripts/obsidian_bot.py:        logger.error("OBSIDIAN_VAULT_PATH not set in .env")
/root/projects/butler/scripts/obsidian_distiller.py:        self.vault_path = os.getenv("OBSIDIAN_VAULT_PATH")
/root/projects/butler/scripts/obsidian_distiller.py:            raise ValueError("OBSIDIAN_VAULT_PATH not found in .env")


ls -la /root/vault/obsidian_vault/obsidian/obsidian/ 2>/dev/null | head -30

List main vault directories


ls -la /root/vault/obsidian_vault/obsidian/obsidian/notes/ 2>/dev/null | head -20

Check notes subdirectory


Found 2 files
slack_bot/obsidian/dispatcher.py
slack_bot/obsidian/cli.py


total 2728
drwxr-xr-x 22 root root    4096 Mar 14 21:14 .
drwxr-xr-x  4 root root      39 Mar  2 10:00 ..
-rw-r--r--  1 root root       0 Mar  3 19:27 2026-03-03.md
drwxr-xr-x  2 root root     183 Feb 26 21:17 accounts
drwxr-xr-x  2 root root    4096 Feb 26 21:17 AI
-rw-r--r--  1 root root       0 Mar 14 21:14 AI.md
drwxr-xr-x  9 root root    8192 Feb 26 21:17 AppleNotes
drwxr-xr-x  2 root root    4096 Mar  4 17:03 Article
drwxr-xr-x  2 root root    8192 Feb 26 21:16 attachments
drwxr-xr-x  3 root root    4096 Mar 17 09:02 auto_report
drwxr-xr-x  6 root root     184 Mar  4 23:01 CLAW
drwxr-xr-x  2 root root    4096 Mar 12 19:52 Clippings
drwxr-xr-x  2 root root    4096 Mar  3 16:19 daily
-rw-r--r--  1 root root    7642 Mar 15 21:31 DRAFT.md
drwxr-xr-x  2 root root     142 Feb 26 21:10 feelings
-rw-r--r--  1 root root 1328257 Feb 26 21:17 IMG_0883.png
-rw-r--r--  1 root root 1196690 Feb 26 21:18 IMG_0884.png
drwxr-xr-x  2 root root    4096 Feb 26 21:17 md
drwxr-xr-x  2 root root    4096 Mar 14 22:08 notes
drwxr-xr-x  3 root root      23 Feb 26 21:09 obsidian
-rw-r--r--  1 root root  157852 Feb 26 21:10 Pasted image 20260115100609.png
-rw-r--r--  1 root root    7120 Feb 26 21:10 Pasted image 20260204175955.png
drwxr-xr-x  2 root root      30 Feb 26 21:09 presentation-workspace
drwxr-xr-x  2 root root    4096 Mar 17 11:52 zhihu
drwxr-xr-x  2 root root    4096 Mar  3 16:19 工作记录
drwxr-xr-x  2 root root     120 Feb 26 23:54 录音记录
drwxr-xr-x  2 root root    4096 Feb 26 21:18 必备知识
-rw-r--r--  1 root root     151 Feb 26 23:44 文件树.base
-rw-r--r--  1 root root      40 Feb 26 21:18 未命名.base


total 24
drwxr-xr-x  2 root root 4096 Mar 14 22:08 .
drwxr-xr-x 22 root root 4096 Mar 14 21:14 ..
-rw-r--r--  1 root root 3025 Mar  3 13:00 2026-03-03_微软-OpenAI-亚马逊-500亿美元投资下的AI三体博弈.md
-rw-r--r--  1 root root 3401 Mar  3 17:19 2026-03-03_斯坦福等高校发布Agents-of-Chaos-真实环境下的OpenClaw安全漏洞研究.md
-rw-r--r--  1 root root 2608 Mar 14 22:00 2026-03-14_Adopting-Agentic-AI-in-2026--5-Strategic-Steps-for.md
-rw-r--r--  1 root root 2939 Mar 14 22:08 2026-03-14_Agentic-AI--Evolution--Mechanics--and-Strategic-Cy.md


find /root/projects/butler/slack_bot/obsidian -name "*.py" -type f | sort

List all obsidian bot files


     1→import cmd
     2→import os
     3→import sys
     4→from slack_bot.obsidian.indexer import ObsidianIndexer
     5→from slack_bot.obsidian.indexer import ObsidianIndexer
     6→from slack_bot.obsidian.generators import WritingAssistant, ReplyGenerator, DecisionSupport, SearchAnalyzer
     7→from health.utils.logging_config import setup_logger
     8→
     9→logger = setup_logger(__name__)
    10→
    11→class ObsidianShell(cmd.Cmd):
    12→    intro = 'Welcome to the Obsidian Assistant Shell. Type help or ? to list commands.\n'
    13→    prompt = '(obsidian) '
    14→
    15→    # Modes
    16→    MODE_NONE = "none"
    17→    MODE_WRITE = "write"
    18→    MODE_REPLY = "reply"
    19→    MODE_DECIDE = "decide"
    20→    MODE_SEARCH = "search"
    21→
    22→    def __init__(self, vault_path: str):
    23→        super().__init__()
    24→        self.vault_path = vault_path
    25→        self.indexer = ObsidianIndexer(vault_path)
    26→        self.indexer.scan_vault()
    27→
    28→        self.generators = {
    29→            self.MODE_WRITE: WritingAssistant(self.indexer),
    30→            self.MODE_REPLY: ReplyGenerator(self.indexer),
    31→            self.MODE_DECIDE: DecisionSupport(self.indexer),
    32→            self.MODE_SEARCH: SearchAnalyzer(self.indexer)
    33→        }
    34→
    35→        self.current_mode = self.MODE_NONE
    36→        self.history = [] # Stores chat history for current session
    37→        self._set_prompt()
    38→
    39→    def _set_prompt(self):
    40→        if self.current_mode == self.MODE_NONE:
    41→            self.prompt = '(obsidian) '
    42→        else:
    43→            turn_indicator = f" [{len(self.history)//2} turns]" if self.history else ""
    44→            self.prompt = f'(obsidian:{self.current_mode}{turn_indicator}) '
    45→
    46→    def do_mode(self, arg):
    47→        """Switch mode: mode [write|reply|decide|search]"""
    48→        if arg in [self.MODE_WRITE, self.MODE_REPLY, self.MODE_DECIDE, self.MODE_SEARCH]:
    49→            self.current_mode = arg
    50→            self.history = [] # Clear history on mode switch
    51→            self._set_prompt()
    52→            print(f"Switched to mode: {arg} (History cleared)")
    53→        else:
    54→            print(f"Invalid mode. Available modes: {self.MODE_WRITE}, {self.MODE_REPLY}, {self.MODE_DECIDE}, {self.MODE_SEARCH}")
    55→            self.current_mode = self.MODE_NONE
    56→            self.history = []
    57→            self._set_prompt()
    58→
    59→    def do_reload(self, arg):
    60→        """Reloads the vault index."""
    61→        print("Reloading index...")
    62→        self.indexer.scan_vault()
    63→        print("Index reloaded.")
    64→
    65→    def do_clear(self, arg):
    66→        """Clears the current conversation history."""
    67→        self.history = []
    68→        self._set_prompt()
    69→        print("Context cleared.")
    70→
    71→    def do_context(self, arg):
    72→        """Show current context samples."""
    73→        if self.current_mode == self.MODE_NONE:
    74→            print("No mode selected.")
    75→            return
    76→
    77→        print(f"--- Context for {self.current_mode} ---")
    78→        if self.current_mode == self.MODE_WRITE:
    79→            samples = self.indexer.writing_samples
    80→            print(f"Total Writing Samples (#writing_sample): {len(samples)}")
    81→            for s in samples:
    82→                print(f" - {os.path.basename(s)}")
    83→
    84→        elif self.current_mode == self.MODE_REPLY:
    85→            samples = self.indexer.reply_samples
    86→            print(f"Total Reply Samples (#reply_sample): {len(samples)}")
    87→            for s in samples:
    88→                print(f" - {os.path.basename(s)}")
    89→            print(" + REPLY-SAMPLE.md (Local)")
    90→
    91→        elif self.current_mode == self.MODE_DECIDE:
    92→            print("Referencing: decision.md, methodology.md")
    93→
    94→        elif self.current_mode == self.MODE_SEARCH:
    95→            print("Hybrid Search: Obsidian (Local) + DuckDuckGo (Web)")
    96→
    97→    def default(self, line):
    98→        """Handle input based on current mode."""
    99→        if self.current_mode == self.MODE_NONE:
   100→            print("Please select a mode first using `mode [write|reply|decide]`")
   101→            return
   102→
   103→        user_input = line.strip()
   104→        if not user_input:
   105→            return
   106→
   107→        print("\nThinking...\n")
   108→        generator = self.generators[self.current_mode]
   109→
   110→        try:
   111→            # All generators now use chat interface
   112→            response, updated_history = generator.chat(user_input, self.history)
   113→
   114→            self.history = updated_history
   115→            self._set_prompt()
   116→
   117→            print(f"\n{response}\n")
   118→
   119→        except Exception as e:
   120→            logger.error(f"Generation failed: {e}")
   121→            print(f"Error: {e}")
   122→
   123→    def do_quit(self, arg):
   124→        """Quit the shell."""
   125→        return True
   126→
   127→    def do_exit(self, arg):
   128→        """Exit the shell."""
   129→        return True
   130→
   131→    def emptyline(self):
   132→        pass
   133→