     1→"""
     2→Pydantic models for daily health metrics.
     3→
     4→Includes models for steps, heart rate, sleep, stress, body battery, and more.
     5→"""
     6→
     7→from datetime import date as date_type, time as time_type
     8→from typing import Optional, List, Dict, Any
     9→from pydantic import BaseModel, Field
    10→
    11→
    12→class StepsData(BaseModel):
    13→    """Daily step count and related metrics."""
    14→
    15→    date: date_type = Field(..., description="Date of the data")
    16→    total_steps: Optional[int] = Field(default=None, description="Total steps taken")
    17→    total_distance_meters: Optional[float] = Field(
    18→        default=None, description="Total distance in meters"
    19→    )
    20→    calories_burned: Optional[int] = Field(
    21→        default=None, description="Calories burned from steps"
    22→    )
    23→    step_goal: Optional[int] = Field(default=None, description="Daily step goal")
    24→    raw_data: Optional[Dict[str, Any]] = Field(
    25→        default=None, description="Raw API response"
    26→    )
    27→
    28→
    29→class HeartRateData(BaseModel):
    30→    """Daily heart rate metrics."""
    31→
    32→    date: date_type = Field(..., description="Date of the data")
    33→    resting_heart_rate: Optional[int] = Field(
    34→        default=None, description="Resting heart rate in bpm"
    35→    )
    36→    min_heart_rate: Optional[int] = Field(
    37→        default=None, description="Minimum heart rate in bpm"
    38→    )
    39→    max_heart_rate: Optional[int] = Field(
    40→        default=None, description="Maximum heart rate in bpm"
    41→    )
    42→    average_heart_rate: Optional[int] = Field(
    43→        default=None, description="Average heart rate in bpm"
    44→    )
    45→    heart_rate_zones: Optional[Dict[str, int]] = Field(
    46→        default=None, description="Time in each heart rate zone (minutes)"
    47→    )
    48→    raw_data: Optional[Dict[str, Any]] = Field(
    49→        default=None, description="Raw API response"
    50→    )
    51→
    52→
    53→class SleepData(BaseModel):
    54→    """Daily sleep metrics."""
    55→
    56→    date: date_type = Field(..., description="Date of the sleep (night)")
    57→    total_sleep_seconds: Optional[int] = Field(
    58→        default=None, description="Total sleep time in seconds"
    59→    )
    60→    deep_sleep_seconds: Optional[int] = Field(

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


Bash

ls /root/vault/obsidian_vault/obsidian/obsidian/ 2>/dev/null | head -20

Check Obsidian vault structure


List context module structure


compressor.py
__pycache__
storage.py


accounts
AI
AppleNotes
Article
attachments
auto_report
CLAW
Clippings
daily
DRAFT.md
feelings
IMG_0883.png
IMG_0884.png
md
obsidian
Pasted image 20260115100609.png
Pasted image 20260204175955.png
presentation-workspace
工作记录
录音记录


ls /root/vault/obsidian_vault/obsidian/obsidian/notes 2>/dev/null && echo "exists" || echo "not found"

Check if notes subdirectory exists in vault


     1→import json
     2→import time
     3→from pathlib import Path
     4→from typing import List, Dict, Any, Optional
     5→from health.utils.logging_config import setup_logger
     6→from health import config
     7→
     8→logger = setup_logger(__name__)
     9→
    10→CONTEXT_DIR = config.DATA_DIR / "slack_context"
    11→CONTEXT_DIR.mkdir(parents=True, exist_ok=True)
    12→
    13→class ContextStorage:
    14→    """Manages persistent conversation history for Slack channels."""
    15→
    16→    # Maximum messages to keep (simple sliding window)
    17→    MAX_MESSAGES = 40
    18→
    19→    def __init__(self, channel_id: str):
    20→        self.channel_id = channel_id
    21→        self.file_path = CONTEXT_DIR / f"{channel_id}.json"
    22→
    23→    def _load(self) -> List[Dict[str, Any]]:
    24→        if not self.file_path.exists():
    25→            return []
    26→        try:
    27→            with open(self.file_path, "r", encoding="utf-8") as f:
    28→                return json.load(f)
    29→        except Exception as e:
    30→            logger.error(f"Failed to load context for {self.channel_id}: {e}")
    31→            return []
    32→
    33→    def _save(self, context: List[Dict[str, Any]]) -> None:
    34→        try:
    35→            with open(self.file_path, "w", encoding="utf-8") as f:
    36→                json.dump(context, f, ensure_ascii=False, indent=2)
    37→        except Exception as e:
    38→            logger.error(f"Failed to save context for {self.channel_id}: {e}")
    39→
    40→    def add_message(self, role: str, content: str, model: Optional[str] = None) -> None:
    41→        """
    42→        Add a message to the context.
    43→
    44→        Args:
    45→            role: 'user' or 'assistant'
    46→            content: Message content
    47→            model: Model identifier (only for assistant messages, currently always 'gemini')
    48→        """
    49→        context = self._load()
    50→        msg = {
    51→            "role": role,
    52→            "content": content,
    53→            "timestamp": time.time()
    54→        }
    55→        if model:
    56→            msg["model"] = model
    57→
    58→        context.append(msg)
    59→
    60→        # Apply sliding window compression
    61→        if len(context) > self.MAX_MESSAGES:
    62→            context = context[-self.MAX_MESSAGES:]
    63→
    64→        self._save(context)
    65→
    66→    def get_context(self) -> List[Dict[str, Any]]:
    67→        """
    68→        Get conversation context in standard format.
    69→
    70→        Returns messages as [{"role": "user", "content": "..."}, ...]
    71→        """
    72→        full_context = self._load()
    73→
    74→        # Convert to standard format (remove timestamp and model metadata)
    75→        filtered = []
    76→        for msg in full_context:
    77→            filtered.append({
    78→                "role": msg["role"],
    79→                "content": msg["content"]
    80→            })
    81→
    82→        return filtered
    83→
    84→    def clear(self) -> None:
    85→        """Clear all context for this channel."""
    86→        if self.file_path.exists():
    87→            self.file_path.unlink()
    88→        logger.info(f"Cleared context for {self.channel_id}")
    89→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


not found


ls /root/vault/obsidian_vault/obsidian/obsidian/Article/ 2>/dev/null | head -5

Peek at an example vault subdirectory


cat /root/projects/butler/.env 2>/dev/null | grep -v "KEY\|TOKEN\|PASSWORD\|SECRET" | head -30 || echo "no .env or no readable fields"

Check .env for non-secret config values


GEMINI_CONFIG_PATH=.gemini.current.env

 SLACK_REPORT_CHANNEL=D0AA4GBQRKK
GARMIN_EMAIL=350134531@qq.com

OBSIDIAN_VAULT_PATH="/root/vault/obsidian_vault/obsidian/obsidian/"


10-再谈 OpenClaw 与私人服务器：从“宠物”到“牛群”的代码成人礼.md
10-再谈 OpenClaw 与私人服务器：从"宠物"到"牛群"的代码成人礼-Revised.md
1-Claude code的热潮，代表了Unix哲学的复兴-revised.md
20211230毛利提升汇报.md
2023-11-21 友邦保险交流.md


     1→#!/usr/bin/env python3
     2→"""
     3→Weekly health report script.
     4→
     5→Generates a Slack-formatted summary of the previous week's health data
     6→and posts it to the configured Slack channel.
     7→
     8→Usage:
     9→    python scripts/weekly_report.py
    10→
    11→Cron example (every Monday at 11:00 AM):
    12→    0 11 * * 1 cd /root/projects/butler && /root/projects/butler/venv/bin/python /root/projects/butler/scripts/weekly_report.py >> /root/projects/butler/logs/cron.log 2>&1
    13→
    14→Required env vars:
    15→    SLACK_BOT_TOKEN      - Slack bot OAuth token (xoxb-...)
    16→    SLACK_REPORT_CHANNEL - Target channel ID (C0XXXXXXXXX)
    17→"""
    18→
    19→import os
    20→import sys
    21→from datetime import date, timedelta
    22→from pathlib import Path
    23→
    24→# Add project root to path
    25→sys.path.insert(0, str(Path(__file__).parent.parent))
    26→
    27→from health.analytics.weekly_report import WeeklyReportGenerator
    28→from health.utils.env_loader import load_env_with_extras
    29→from health.utils.logging_config import setup_logger
    30→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


     1→"""Weekly health report generator.
     2→
     3→Generates a Slack-formatted weekly health summary covering sleep, steps,
     4→recovery, body battery, and lifestyle data.
     5→"""
     6→
     7→from datetime import date, timedelta
     8→from typing import Any, Optional
     9→
    10→from health.services.manual_log_storage import ManualLogStorage
    11→from health.services.query import HealthDataQuery
    12→from health.utils.logging_config import setup_logger
    13→
    14→logger = setup_logger(__name__)
    15→
    16→
    17→class WeeklyReportGenerator:
    18→    """Generates weekly health reports from stored Garmin and manual log data."""
    19→
    20→    def __init__(self) -> None:
    21→        self.query = HealthDataQuery()
    22→        self.manual_storage = ManualLogStorage()
    23→
    24→    def generate(self, week_start: date) -> str:
    25→        """Generate a Slack-formatted weekly health report.
    26→
    27→        Args:
    28→            week_start: The Monday that starts the report week.
    29→
    30→        Returns:
    31→            Formatted report string ready for Slack posting.
    32→        """
    33→        week_end = week_start + timedelta(days=6)
    34→        prev_week_start = week_start - timedelta(days=7)
    35→        prev_week_end = week_start - timedelta(days=1)
    36→
    37→        logger.info(f"Generating weekly report for {week_start} ~ {week_end}")
    38→
    39→        sleep_stats = self._get_sleep_stats(week_start, week_end)
    40→        steps_stats = self._get_steps_stats(week_start, week_end)

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


I now have a thorough understanding of the codebase. Let me compose the complete implementation plan.


ls /root/projects/butler/slack_bot/obsidian/__pycache__/ 2>/dev/null | head -10

Check obsidian pycache to see compiled modules (confirms what modules exist)


cat /root/projects/butler/health/utils/time_utils.py 2>/dev/null | head -20

Quick look at time_utils for get_current_time_str signature


dispatcher.cpython-311.pyc
generators.cpython-311.pyc
indexer.cpython-311.pyc
__init__.cpython-311.pyc


from datetime import datetime, timedelta, timezone, date

def get_cst_now() -> datetime:
    """Returns the current datetime in UTC+8 (China Standard Time)."""
    tz = timezone(timedelta(hours=8))
    return datetime.now(tz)

def get_cst_today() -> date:
    """Returns today's date in UTC+8 (China Standard Time)."""
    return get_cst_now().date()

def get_current_time_str() -> str:
    """
    Returns the current time in UTC+8 (China Standard Time).
    Format: YYYY-MM-DD HH:MM:SS (CST)
    """
    return get_cst_now().strftime("%Y-%m-%d %H:%M:%S (CST)")


I have now fully explored the codebase. Here is the complete, detailed implementation plan.

---

## Implementation Plan: Second Brain RAG System

### Architecture Overview

The design adds three new layers on top of the existing `obsidian/` package without breaking any existing code paths:

1. **Embedding layer** (`obsidian/embeddings.py`) - pluggable providers
2. **Vector store layer** (`obsidian/vector_store.py`) - ChromaDB wrapper with `search_knowledge()`
3. **Note ingestion layer** (`obsidian/note_ingester.py`) - URL/text -> cleaned .md -> vector DB
4. **CLI indexer** (`scripts/init_vault_indexer.py`) - bulk backfill of existing vault
5. **Generator enrichment** - three existing generators get a RAG pre-fetch injected before their first-turn prompt

The existing `ObsidianIndexer` (keyword search) is kept intact and runs alongside the new vector store. The two are complementary: keyword search remains the fallback for exact-match queries, while semantic search provides concept-level retrieval.

---

### 1. New Directory Structure

```
/root/projects/butler/
├── slack_bot/
│   └── obsidian/
│       ├── __init__.py          (unchanged)
│       ├── cli.py               (unchanged)
│       ├── dispatcher.py        (MODIFIED - add MODE_NOTE, wire NoteIngester)
│       ├── generators.py        (MODIFIED - add RAG injection to Write/Reply/Zhihu)
│       ├── indexer.py           (unchanged)
│       ├── embeddings.py        (NEW - EmbeddingProvider ABC + two backends)
│       ├── vector_store.py      (NEW - ChromaVectorStore + search_knowledge())
│       └── note_ingester.py     (NEW - NoteIngester for mode note)
├── scripts/
│   └── init_vault_indexer.py    (NEW - CLI bulk indexer with click + tqdm)
└── requirements.txt             (MODIFIED - add chromadb, tavily-python, etc.)
```

No new sub-packages are needed. All new files live in the existing `slack_bot/obsidian/` module, keeping imports flat and consistent with the current pattern (`from slack_bot.obsidian.vector_store import ...`).

---

### 2. Data Flow

#### Ingestion path (mode note)

```
Slack: "mode note"
  -> dispatcher routes to NoteIngester.ingest(user_input)
     -> if URL detected: TavilyClient.extract(url) -> raw text + title
     -> else: use raw user_input as text
     -> GeminiLLM.generate_response(clean_prompt, [])
        -> returns: {title, summary, key_points, entities[], cleaned_body}
     -> write .md file to vault/notes/<slug>.md
        (YAML frontmatter with source_url, date, tags: [#auto_ingested])
     -> split .md into chunks by markdown header
     -> EmbeddingProvider.embed(chunks[])
     -> ChromaVectorStore.add_chunks(chunks, metadata)
  -> reply to Slack with: "Saved note: <title>\n<summary>"
```

#### Search / RAG injection path

```
Slack: "mode write" -> "write an article about [topic]"
  -> dispatcher calls WritingAssistant.chat(topic, [])
     -> is_first_turn = True
     -> BEFORE building prompt:
        search_results = vector_store.search_knowledge(topic, top_k=3)
     -> inject results into prompt as "=== SEMANTIC KNOWLEDGE FROM VAULT ==="
        (augments the existing keyword RAG block already there)
     -> rest of prompt building is unchanged
```

#### Bulk indexing path

```
$ python scripts/init_vault_indexer.py --path /root/vault/... [--reset]
  -> click parses args
  -> if --reset: ChromaVectorStore.reset()
  -> glob all .md files under path
  -> tqdm progress bar over files
  -> for each file:
     -> parse YAML frontmatter (python-frontmatter)
     -> split content by markdown headers into chunks
     -> EmbeddingProvider.embed(chunk texts[])
     -> ChromaVectorStore.add_chunks(chunks, metadata)
     -> on error: log + skip
```

---

### 3. Component-by-Component Implementation Plan

---

#### Component A: `slack_bot/obsidian/embeddings.py` (NEW)

This is the pluggable embedding provider. The env var `EMBEDDING_BACKEND` switches between `cloud` and `local`.

**Pydantic models needed:**

```python
from pydantic import BaseModel
from typing import List

class EmbeddingResult(BaseModel):
    """Result from a single embedding call."""
    text: str
    vector: List[float]
    model: str
```

**Key classes and signatures:**

```python
from abc import ABC, abstractmethod
from typing import List
from pathlib import Path
import os
from slack_bot.obsidian.embeddings import EmbeddingResult  # self-import for clarity
from health.utils.logging_config import setup_logger

class EmbeddingProvider(ABC):
    """Abstract base class for embedding backends."""

    @abstractmethod
    def embed(self, texts: List[str]) -> List[List[float]]:
        """Embed a list of texts and return a list of float vectors.

        Args:
            texts: List of strings to embed.

        Returns:
            List of float vectors, one per input text.
        """
        pass

    @abstractmethod
    def get_dimension(self) -> int:
        """Return the dimensionality of this model's output vectors."""
        pass

    @abstractmethod
    def get_model_name(self) -> str:
        """Return the display name of the embedding model."""
        pass

class QwenEmbeddingProvider(EmbeddingProvider):
    """Cloud embedding via Qwen3-Embedding-8B through OpenAI-compatible API."""

    def __init__(self) -> None:
        # Reuses the same OpenAI client pattern as GeminiLLM
        # EMBEDDING_API_KEY, EMBEDDING_BASE_URL, EMBEDDING_MODEL from env

    def embed(self, texts: List[str]) -> List[List[float]]: ...
    def get_dimension(self) -> int: ...   # 4096 for Qwen3-Embedding-8B
    def get_model_name(self) -> str: ...

class SentenceTransformerProvider(EmbeddingProvider):
    """Local embedding via sentence-transformers (all-MiniLM-L6-v2 by default)."""

    def __init__(self) -> None:
        # Lazy-loads model on first embed() call to avoid startup penalty
        # EMBEDDING_LOCAL_MODEL from env, default = "all-MiniLM-L6-v2"

    def embed(self, texts: List[str]) -> List[List[float]]: ...
    def get_dimension(self) -> int: ...   # 384 for MiniLM
    def get_model_name(self) -> str: ...

def get_embedding_provider() -> EmbeddingProvider:
    """Factory function. Reads EMBEDDING_BACKEND env var.

    Returns:
        QwenEmbeddingProvider if EMBEDDING_BACKEND=cloud, else SentenceTransformerProvider.
    """
    backend = os.environ.get("EMBEDDING_BACKEND", "local").lower()
    if backend == "cloud":
        return QwenEmbeddingProvider()
    return SentenceTransformerProvider()
```

**Integration note:** The `QwenEmbeddingProvider` should use `openai.OpenAI(api_key=..., base_url=...)` and call `client.embeddings.create(input=texts, model=model_name)` - exactly the same pattern as `GeminiLLM._generate_with_openai()`. No new HTTP library is needed.

---

#### Component B: `slack_bot/obsidian/vector_store.py` (NEW)

ChromaDB wrapper. This is the only place ChromaDB is imported - all other code goes through this interface.

**Pydantic models needed:**

```python
from pydantic import BaseModel, Field
from typing import List, Optional
from pathlib import Path

class ChunkMetadata(BaseModel):
    """Metadata stored alongside each vector in ChromaDB."""
    source_path: str          # Absolute path to the .md source file
    chunk_index: int          # 0-based position within the file
    header_hierarchy: str     # e.g. "## Section > ### Subsection"
    note_title: Optional[str] = None
    tags: List[str] = Field(default_factory=list)
    ingested_at: str          # ISO datetime string

class SearchResult(BaseModel):
    """A single result returned by search_knowledge()."""
    text: str                 # The chunk text
    source_path: str
    header_hierarchy: str
    distance: float           # Lower = more similar (ChromaDB default)
    note_title: Optional[str] = None
```

**Key class and signatures:**

```python
from pathlib import Path
from typing import List, Optional
import chromadb
from slack_bot.obsidian.embeddings import EmbeddingProvider, get_embedding_provider
from slack_bot.obsidian.vector_store import ChunkMetadata, SearchResult
from health.utils.logging_config import setup_logger
from health import config

# DB stored at: config.DATA_DIR / "vector_db"
# Collection name: "obsidian_notes"

class ChromaVectorStore:
    """ChromaDB-backed vector store for Obsidian note chunks.

    Args:
        embedding_provider: The embedding backend to use.
        db_path: Path to persist the ChromaDB database.
    """

    def __init__(
        self,
        embedding_provider: Optional[EmbeddingProvider] = None,
        db_path: Optional[Path] = None,
    ) -> None: ...

    def add_chunks(
        self,
        texts: List[str],
        metadatas: List[ChunkMetadata],
    ) -> None:
        """Embed texts and upsert into ChromaDB.

        Args:
            texts: The chunk text content to embed.
            metadatas: Matching metadata for each chunk.
        """
        ...

    def search_knowledge(
        self,
        query: str,
        top_k: int = 3,
    ) -> List[SearchResult]:
        """Semantic search over indexed note chunks.

        Args:
            query: Natural language query string.
            top_k: Number of top results to return.

        Returns:
            List of SearchResult objects ordered by relevance.
        """
        ...

    def reset(self) -> None:
        """Delete and recreate the collection (used by --reset flag in CLI)."""
        ...

    def get_stats(self) -> dict:
        """Return collection statistics (count, etc.) for diagnostics."""
        ...
```

**ChromaDB document IDs** should be `sha256(source_path + str(chunk_index))[:16]` - this makes re-indexing idempotent (upsert will update existing chunks rather than duplicating them).

**Important:** ChromaDB's `EmbeddingFunction` interface is NOT used here. We embed manually via `EmbeddingProvider.embed()` and pass raw vectors to ChromaDB's `collection.upsert(embeddings=...)`. This keeps the embedding backend fully decoupled from ChromaDB.

---

#### Component C: `slack_bot/obsidian/note_ingester.py` (NEW)

Handles the `mode note` command. Takes text or URL, calls LLM to clean/structure it, writes an .md file, and indexes the note.

**Pydantic model needed:**

```python
from pydantic import BaseModel, Field
from typing import List, Optional

class ExtractedNote(BaseModel):
    """LLM-extracted structured content from raw input."""
    title: str
    summary: str                   # 2-3 sentence summary
    key_points: List[str]          # Bullet list, max 7 items
    entities: List[str]            # Named entities to become [[wiki links]]
    cleaned_body: str              # Full prose, entities wrapped in [[...]]
    source_url: Optional[str] = None
    language: str = "zh"           # "zh" or "en" - detected by LLM
```

**Key class and signatures:**

```python
from pathlib import Path
from typing import Optional
import re
import json
from datetime import datetime

from slack_bot.llm.gemini import GeminiLLM
from slack_bot.obsidian.vector_store import ChromaVectorStore
from slack_bot.obsidian.note_ingester import ExtractedNote
from health.utils.logging_config import setup_logger
from health.utils.time_utils import get_cst_today

NOTE_EXTRACTION_PROMPT = """..."""  # defined as module-level constant

class NoteIngester:
    """Ingests plaintext or URL content into the Obsidian vault and vector DB.

    Args:
        vault_path: Path to the Obsidian vault root.
        vector_store: Initialized ChromaVectorStore instance.
    """

    def __init__(
        self,
        vault_path: Path,
        vector_store: ChromaVectorStore,
    ) -> None:
        self.notes_dir = vault_path / "notes"
        self.vector_store = vector_store
        self.llm = GeminiLLM()  # Uses default system prompt; overridden by per-call prompt

    def ingest(self, user_input: str) -> str:
        """Main entry point. Detects URL vs text, runs pipeline, returns summary reply.

        Args:
            user_input: Raw Slack message text (URL or free text).

        Returns:
            Slack-formatted confirmation with title and summary.
        """
        ...

    def _fetch_url(self, url: str) -> str:
        """Fetch URL content using Tavily extract API.

        Args:
            url: The URL to fetch.

        Returns:
            Extracted text content from the URL.
        """
        ...

    def _extract_with_llm(self, raw_text: str, source_url: Optional[str]) -> ExtractedNote:
        """Call LLM to clean, structure, and extract entities from raw text.

        Args:
            raw_text: Raw input text.
            source_url: Original URL if from web fetch.

        Returns:
            Validated ExtractedNote model.
        """
        ...

    def _write_md_file(self, note: ExtractedNote) -> Path:
        """Write the note to vault/notes/<slug>.md with YAML frontmatter.

        Args:
            note: The extracted note data.

        Returns:
            Path to the written file.
        """
        ...

    def _chunk_and_index(self, note: ExtractedNote, file_path: Path) -> None:
        """Split note by markdown headers and index chunks in vector DB.

        Args:
            note: The extracted note.
            file_path: Absolute path of the saved .md file.
        """
        ...

    @staticmethod
    def _detect_url(text: str) -> Optional[str]:
        """Return URL if text starts with a URL pattern, else None."""
        ...

    @staticmethod
    def _slugify(title: str) -> str:
        """Convert title to filesystem-safe slug."""
        ...
```

**LLM extraction prompt strategy:** The `_extract_with_llm` method sends a single prompt asking for JSON output conforming to `ExtractedNote`. The response is parsed with `json.loads()` and then `ExtractedNote.model_validate()`. This is Pydantic v2 pattern consistent with `health/models/daily_metrics.py`.

**Chunking strategy** (`_chunk_and_index`): Split the `cleaned_body` by markdown H2/H3 headers using `re.split(r'\n(?=#{2,3} )', body)`. Each chunk includes its header as the first line. The frontmatter is treated as chunk 0 with header_hierarchy = "frontmatter". Minimum chunk size: 50 characters (skip trivially short sections).

---

#### Component D: `scripts/init_vault_indexer.py` (NEW)

Standalone CLI script. Follows the `scripts/weekly_report.py` pattern of adding `sys.path.insert(0, ...)` before project imports.

**Key structure and signatures:**

```python
#!/usr/bin/env python3
"""CLI tool to bulk-index an Obsidian vault into the vector database."""

import sys
import re
from pathlib import Path
from typing import Optional
import click
import tqdm
import frontmatter  # python-frontmatter

sys.path.insert(0, str(Path(__file__).parent.parent))

from health.utils.env_loader import load_env_with_extras
from health.utils.logging_config import setup_logger
from slack_bot.obsidian.embeddings import get_embedding_provider
from slack_bot.obsidian.vector_store import ChromaVectorStore, ChunkMetadata

load_env_with_extras()
logger = setup_logger(__name__)

def parse_md_file(file_path: Path) -> list[dict]:
    """Parse a markdown file into chunks with metadata.

    Args:
        file_path: Path to the .md file.

    Returns:
        List of dicts with keys: text, header_hierarchy, chunk_index.
    """
    ...

@click.command()
@click.option(
    "--path",
    required=True,
    type=click.Path(exists=True),
    help="File or directory to index",
)
@click.option(
    "--reset",
    is_flag=True,
    default=False,
    help="Clear the vector DB before indexing",
)
def main(path: str, reset: bool) -> None:
    """Bulk-index Obsidian markdown files into the vector database."""
    ...
```

**Error handling strategy:** Each file is wrapped in a `try/except`. On error, `logger.warning(f"Skipping {file}: {e}")` and the loop continues. API errors (from `EmbeddingProvider.embed()`) should be retried up to 3 times with a 2-second exponential backoff, matching the pattern in `GeminiLLM._retry_with_backoff()`. After all retries exhausted, log and skip.

**Frontmatter parsing:** Use `python-frontmatter` (`import frontmatter; post = frontmatter.load(path)`). `post.metadata` gives the YAML dict; `post.content` gives the body. Extract `tags` and `title` from metadata if present.

**Chunking in init_vault_indexer** uses the same header-split logic as `NoteIngester._chunk_and_index()`. To avoid duplication, extract this into a shared private function `_split_by_headers(body: str) -> List[str]` defined in `vector_store.py` or as a utility. The simplest approach: put `split_md_by_headers(body: str) -> List[str]` as a module-level function in `vector_store.py`, imported by both `NoteIngester` and `init_vault_indexer.py`.

---

#### Component E: Modifications to `generators.py`

Three generators need RAG injection: `WritingAssistant`, `ReplyGenerator`, `ZhihuGenerator`.

The `BaseGenerator.__init__` signature changes to accept an optional `vector_store`:

```python
class BaseGenerator:
    def __init__(
        self,
        indexer: ObsidianIndexer,
        vector_store: Optional[ChromaVectorStore] = None,
    ) -> None:
        ...
        self.vector_store = vector_store
```

A private helper is added to `BaseGenerator`:

```python
def _semantic_search(self, query: str, top_k: int = 3) -> str:
    """Run semantic search and format results as a context block.

    Args:
        query: The topic or question to search for.
        top_k: Number of results.

    Returns:
        Formatted string block for insertion into prompts, or empty string
        if vector_store is None or returns no results.
    """
    if self.vector_store is None:
        return ""
    results = self.vector_store.search_knowledge(query, top_k=top_k)
    if not results:
        return ""
    lines = ["=== SEMANTIC KNOWLEDGE FROM VAULT ==="]
    for r in results:
        lines.append(f"[Source: {Path(r.source_path).name} / {r.header_hierarchy}]")
        lines.append(r.text)
        lines.append("")
    return "\n".join(lines)
```

In each affected generator's `chat()` method, inside the `if is_first_turn:` block, one line is added before the `prompt = f"""..."""` string construction:

```python
# Inside WritingAssistant.chat(), is_first_turn block:
semantic_context = self._semantic_search(topic, top_k=3)

# Then in the prompt f-string, after the existing RAG block:
=== SEMANTIC KNOWLEDGE FROM VAULT ===
{semantic_context}
```

This is a purely additive change. If `vector_store` is `None` (not yet set up), `_semantic_search()` returns `""` and the prompt is identical to today's behavior. No existing tests break.

---

#### Component F: Modifications to `dispatcher.py`

Two changes:

1. **Add `MODE_NOTE`** constant and wire `NoteIngester`:

```python
MODE_NOTE = "note"

def __init__(self, vault_path: str):
    ...
    # After existing generator init:
    from slack_bot.obsidian.vector_store import ChromaVectorStore
    from slack_bot.obsidian.embeddings import get_embedding_provider
    from slack_bot.obsidian.note_ingester import NoteIngester

    self.vector_store = ChromaVectorStore(
        embedding_provider=get_embedding_provider()
    )
    self.note_ingester = NoteIngester(
        vault_path=Path(self.indexer.vault_path),
        vector_store=self.vector_store,
    )

    # Pass vector_store to generators that support RAG injection:
    self.generators = {
        self.MODE_WRITE: WritingAssistant(self.indexer, self.vector_store),
        self.MODE_REPLY: ReplyGenerator(self.indexer, self.vector_store),
        self.MODE_DECIDE: DecisionSupport(self.indexer),       # no injection needed
        self.MODE_SEARCH: SearchAnalyzer(self.indexer),        # no injection needed
        self.MODE_DEAI: DeAIReviser(self.indexer),             # no injection needed
        self.MODE_ZHIHU: ZhihuGenerator(self.indexer, self.vector_store),
    }
```

2. **Route `mode note`** in `dispatch()`:

`MODE_NOTE` is NOT added to `self.generators` because it has a different signature (it doesn't use `chat()`). Instead, handle it explicitly in `dispatch()`:

```python
# In dispatch(), after the other mode checks:
if current_mode == self.MODE_NOTE:
    try:
        result = self.note_ingester.ingest(text)
        self._reply(channel_id, result, response_ts)
    except Exception as e:
        logger.error(f"Note ingestion failed: {e}", exc_info=True)
        self._reply(channel_id, f"Error ingesting note: {e}", response_ts)
    return
```

The `mode note` command itself is handled by the existing `mode ` prefix handler - it saves `"note"` as the state, which then routes to the `NoteIngester` path. The available modes list in the error message also needs `"note"` appended.

---

### 4. requirements.txt Additions

```
# Vector DB (second brain RAG)
chromadb>=0.5.0

# URL content extraction
tavily-python>=0.3.0

# YAML frontmatter parsing for .md files
python-frontmatter>=1.0.0

# Progress bar for CLI indexer
tqdm>=4.66.0

# Local embedding model (only loaded if EMBEDDING_BACKEND=local)
sentence-transformers>=3.0.0

# Markdown header splitting (stdlib re is sufficient; no extra dep needed)
```

**Note on `sentence-transformers`:** It pulls in `torch` as a dependency, which is ~2GB. If local embedding is not needed, set `EMBEDDING_BACKEND=cloud` in `.env` and `sentence-transformers` can be omitted from the install. Consider adding a comment in `requirements.txt` to make this optional, and use a lazy import inside `SentenceTransformerProvider.__init__()` with a helpful error if not installed.

---

### 5. Environment Variables

Add to `.env` (alongside the existing `OBSIDIAN_VAULT_PATH`):

```bash
# --- Second Brain RAG ---

# Embedding backend: "local" (default) or "cloud"
EMBEDDING_BACKEND=local

# Cloud embedding (required only if EMBEDDING_BACKEND=cloud)
EMBEDDING_API_KEY=your_openrouter_or_custom_key
EMBEDDING_BASE_URL=https://openrouter.ai/api
EMBEDDING_MODEL=qwen/qwen3-embedding-8b

# Local embedding model name (optional, defaults to all-MiniLM-L6-v2)
# EMBEDDING_LOCAL_MODEL=all-MiniLM-L6-v2

# Tavily API key (required for URL ingestion in mode note)
TAVILY_API_KEY=your_tavily_api_key
```

---

### 6. Implementation Sequencing

The components should be implemented in this order, as each step is independently testable:

**Step 1** - `embeddings.py` (no deps on new code, immediately testable with `python -c "from slack_bot.obsidian.embeddings import get_embedding_provider; p = get_embedding_provider(); print(p.embed(['hello']))"`)

**Step 2** - `vector_store.py` with `split_md_by_headers()` utility (depends on Step 1)

**Step 3** - `scripts/init_vault_indexer.py` (depends on Steps 1-2; lets you index the vault before touching the bot)

**Step 4** - `note_ingester.py` (depends on Steps 1-2)

**Step 5** - Modify `generators.py` - add `vector_store` param to `BaseGenerator`, add `_semantic_search()`, inject into 3 generators. This is the change with the highest risk: test with `EMBEDDING_BACKEND=local` and confirm prompt length stays within token limits.

**Step 6** - Modify `dispatcher.py` - add `MODE_NOTE`, wire `NoteIngester` and `ChromaVectorStore`, pass `vector_store` to generators.

---

### 7. Key Design Decisions and Trade-offs

**Why ChromaDB over sqlite-vec?**
ChromaDB is a higher-level abstraction with a Python-native API, persistent storage by default, and no need for a compilation step. sqlite-vec requires building a native extension and has a lower-level API. For this use case (single-user, < 100k chunks), ChromaDB's overhead is negligible and the developer experience is significantly better.

**Why keep keyword search alongside vector search?**
The existing `ObsidianIndexer.search()` is fast (in-memory), handles exact Chinese token matches well (the existing tokenization in `indexer.py` is already tuned for this), and requires no API calls. The semantic search adds concept-level retrieval that keyword search misses. Both populate different sections of the prompt. This is the right choice given the vault contains both structured notes with exact terms and conceptual essays.

**Why not embed in `BaseGenerator.__init__` directly?**
Embedding initialization involves either network calls (cloud) or model loading (local). `BaseGenerator` is currently instantiated at bot startup in `dispatcher.py`. Lazy-loading the `ChromaVectorStore` in `dispatcher.py.__init__()` and passing it as an argument keeps startup failures isolated and allows the bot to start even if the vector DB is unavailable.

**Token budget concern for RAG injection:**
The three injected generators already produce large prompts (writing samples, style guides, methodology files). Adding 3 semantic search results (avg ~300 tokens each = ~900 tokens) is acceptable for a 128k context model like Gemini Flash. The existing keyword RAG (`rag_text`) already does this. The semantic results should be injected as an additional section, not replacing the keyword section, since they retrieve different things.

**YAML frontmatter in auto-generated notes:**
The `NoteIngester._write_md_file()` should produce frontmatter that is compatible with the existing `ObsidianIndexer.scan_vault()`. Since `scan_vault()` reads the full file content and checks for `#writing_sample` / `#reply_sample` tags, the auto-ingested notes must NOT accidentally contain those tags. The frontmatter should use `tags: [auto_ingested]` (no hashtag, YAML list format) and the LLM extraction prompt must explicitly instruct not to add writing/reply sample tags.

---

### Critical Files for Implementation

- `/root/projects/butler/slack_bot/obsidian/generators.py` - Core logic to modify: add `vector_store` param to `BaseGenerator`, add `_semantic_search()` helper, inject semantic context into `WritingAssistant`, `ReplyGenerator`, and `ZhihuGenerator` first-turn prompts
- `/root/projects/butler/slack_bot/obsidian/dispatcher.py` - Wiring point: must instantiate `ChromaVectorStore` and `NoteIngester`, pass them to generators, add `MODE_NOTE` routing; this is the integration hub for all new components
- `/root/projects/butler/slack_bot/llm/gemini.py` - Pattern to follow exactly: the `QwenEmbeddingProvider` in `embeddings.py` must replicate its OpenRouter client init pattern (`default_headers`, `base_url + "/v1"`) to call cloud embedding APIs correctly
- `/root/projects/butler/requirements.txt` - Must be updated before any new code runs; `chromadb` in particular has deep transitive dependencies that may require a fresh `pip install`
- `/root/projects/butler/scripts/weekly_report.py` - Pattern to follow for `init_vault_indexer.py`: the `sys.path.insert`, `load_env_with_extras()`, and click CLI structure should be copied verbatim as the scaffolding template