Now I understand the root cause of Reddit returning 0 results:

**Root cause: `start_published_date` filter causes 0 results because Exa doesn't have `published_date` metadata for Reddit posts.**

All Reddit results have `published: None`. When we pass `start_published_date`, Exa must be filtering out all posts without published_date metadata, resulting in 0 results.

Without the date filter, we get 13 quality results from very relevant subreddits (r/AI_Agents, r/ClaudeAI, r/ClaudeCode, r/n8n, etc.).

For Reddit, I need to drop the date filter entirely, remove the strict subreddit whitelist, and instead weave target subreddits directly into the search queries while casting a wider net across multiple search variations. For Hacker News, adding a `created_at_i` time filter to pull only the last 24-48 hours would trim the results from 441 down to around 10 per run.

GitHub's collection is already solid at 122 items, but the ValueFilter is being too restrictive. I should update it to explicitly pass GitHub repos and tool announcements—things like frameworks, libraries, or projects that directly enable AI automation and workflow optimization.

Now I'm checking the `max_items_per_run` cap and how the interleaving works. With HN at 424 items and GitHub at 90, the ValueFilter might be filtering out more GitHub items than expected, so I need to revisit the earlier report to understand what's actually being collected.

Looking back at the report, it shows only 3 processed items total from HN and 0 from GitHub, which doesn't match the volume we should be seeing after interleaving. Either the ValueFilter is rejecting too many GitHub items, or there's an issue with how items are being processed. Let me check the database to see what actually happened on March 4th.


Bash

source venv/bin/activate && python -c "
from ai_usecases_explorer.storage.database import Database
from ai_usecases_explorer.settings import Settings

s = Settings()
db = Database(db_path=s.db_path)
# Get recent items to see what was stored on March 4
items = db.get_recent(days=2)
print(f'Items in DB from last 2 days: {len(items)}')
for item in items[:10]:
    print(f'  [{item.source_platform}] {item.title[:60]} (collected: {item.collected_at.date()})')
" 2>&1

Check DB for recent March 4 entries


Error: Exit code 1
Traceback (most recent call last):
  File "<string>", line 8, in <module>
TypeError: Database.get_recent() got an unexpected keyword argument 'days'

Traceback (most recent call last):
  File "<string>", line 8, in <module>
TypeError: Database.get_recent() got an unexpected keyword argument 'days'


     1→"""SQLite storage for UseCase records."""
     2→import json
     3→import sqlite3
     4→from pathlib import Path
     5→
     6→from ai_usecases_explorer.models.usecase import Novelty, ScenarioType, UseCase
     7→
     8→_CREATE_TABLE = """
     9→CREATE TABLE IF NOT EXISTS usecases (
    10→    id TEXT PRIMARY KEY,
    11→    title TEXT NOT NULL,
    12→    summary TEXT NOT NULL,
    13→    source_url TEXT NOT NULL,
    14→    source_platform TEXT NOT NULL,
    15→    author TEXT NOT NULL,
    16→    published_at TEXT NOT NULL,
    17→    collected_at TEXT NOT NULL,
    18→    scenario_type TEXT NOT NULL,
    19→    tools_used TEXT NOT NULL,
    20→    has_local_data INTEGER NOT NULL,
    21→    has_workflow_opt INTEGER NOT NULL,
    22→    value_score REAL NOT NULL,
    23→    is_real_value INTEGER NOT NULL,
    24→    novelty TEXT NOT NULL,
    25→    similar_to_ids TEXT NOT NULL,
    26→    raw_content TEXT NOT NULL
    27→)
    28→"""
    29→
    30→
    31→class Database:
    32→    """Thin SQLite wrapper for UseCase persistence."""
    33→
    34→    def __init__(self, db_path: Path) -> None:
    35→        self._path = db_path
    36→        db_path.parent.mkdir(parents=True, exist_ok=True)
    37→        self._conn = sqlite3.connect(str(db_path))
    38→        self._conn.row_factory = sqlite3.Row
    39→        self._conn.execute(_CREATE_TABLE)
    40→        self._conn.commit()
    41→
    42→    def save(self, uc: UseCase) -> str:
    43→        """Insert or replace a UseCase. Returns its id."""
    44→        self._conn.execute(
    45→            """
    46→            INSERT OR REPLACE INTO usecases VALUES (
    47→                :id, :title, :summary, :source_url, :source_platform,
    48→                :author, :published_at, :collected_at, :scenario_type,
    49→                :tools_used, :has_local_data, :has_workflow_opt,
    50→                :value_score, :is_real_value, :novelty, :similar_to_ids,
    51→                :raw_content
    52→            )
    53→            """,
    54→            {
    55→                "id": uc.id,
    56→                "title": uc.title,
    57→                "summary": uc.summary,
    58→                "source_url": uc.source_url,
    59→                "source_platform": uc.source_platform,
    60→                "author": uc.author,
    61→                "published_at": uc.published_at.isoformat(),
    62→                "collected_at": uc.collected_at.isoformat(),
    63→                "scenario_type": uc.scenario_type.value,
    64→                "tools_used": json.dumps(uc.tools_used, ensure_ascii=False),
    65→                "has_local_data": int(uc.has_local_data),
    66→                "has_workflow_opt": int(uc.has_workflow_opt),
    67→                "value_score": uc.value_score,
    68→                "is_real_value": int(uc.is_real_value),
    69→                "novelty": uc.novelty.value,
    70→                "similar_to_ids": json.dumps(uc.similar_to_ids),
    71→                "raw_content": uc.raw_content,
    72→            },
    73→        )
    74→        self._conn.commit()
    75→        return uc.id
    76→
    77→    def exists(self, id: str) -> bool:
    78→        """Return True if a UseCase with this id is already stored."""
    79→        row = self._conn.execute(
    80→            "SELECT 1 FROM usecases WHERE id = ?", (id,)
    81→        ).fetchone()
    82→        return row is not None
    83→
    84→    def get_by_scenario(
    85→        self, scenario_type: ScenarioType, limit: int = 100
    86→    ) -> list[UseCase]:
    87→        """Return up to `limit` UseCases matching the given scenario type."""
    88→        rows = self._conn.execute(
    89→            "SELECT * FROM usecases WHERE scenario_type = ? ORDER BY collected_at DESC LIMIT ?",
    90→            (scenario_type.value, limit),
    91→        ).fetchall()
    92→        return [self._row_to_usecase(r) for r in rows]
    93→
    94→    def get_recent(self, limit: int = 200) -> list[UseCase]:
    95→        """Return the most recently collected UseCases."""
    96→        rows = self._conn.execute(
    97→            "SELECT * FROM usecases ORDER BY collected_at DESC LIMIT ?",
    98→            (limit,),
    99→        ).fetchall()
   100→        return [self._row_to_usecase(r) for r in rows]
   101→
   102→    @staticmethod
   103→    def _row_to_usecase(row: sqlite3.Row) -> UseCase:
   104→        from datetime import datetime
   105→
   106→        return UseCase(
   107→            id=row["id"],
   108→            title=row["title"],
   109→            summary=row["summary"],
   110→            source_url=row["source_url"],
   111→            source_platform=row["source_platform"],
   112→            author=row["author"],
   113→            published_at=datetime.fromisoformat(row["published_at"]),
   114→            collected_at=datetime.fromisoformat(row["collected_at"]),
   115→            scenario_type=ScenarioType(row["scenario_type"]),
   116→            tools_used=json.loads(row["tools_used"]),
   117→            has_local_data=bool(row["has_local_data"]),
   118→            has_workflow_opt=bool(row["has_workflow_opt"]),
   119→            value_score=row["value_score"],
   120→            is_real_value=bool(row["is_real_value"]),
   121→            novelty=Novelty(row["novelty"]),
   122→            similar_to_ids=json.loads(row["similar_to_ids"]),
   123→            raw_content=row["raw_content"],
   124→        )
   125→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""SQLite storage for UseCase records."""
import json
import sqlite3
from pathlib import Path

from ai_usecases_explorer.models.usecase import Novelty, ScenarioType, UseCase

_CREATE_TABLE = """
CREATE TABLE IF NOT EXISTS usecases (
    id TEXT PRIMARY KEY,
    title TEXT NOT NULL,
    summary TEXT NOT NULL,
    source_url TEXT NOT NULL,
    source_platform TEXT NOT NULL,
    author TEXT NOT NULL,
    published_at TEXT NOT NULL,
    collected_at TEXT NOT NULL,
    scenario_type TEXT NOT NULL,
    tools_used TEXT NOT NULL,
    has_local_data INTEGER NOT NULL,
    has_workflow_opt INTEGER NOT NULL,
    value_score REAL NOT NULL,
    is_real_value INTEGER NOT NULL,
    novelty TEXT NOT NULL,
    similar_to_ids TEXT NOT NULL,
    raw_content TEXT NOT NULL
)
"""

class Database:
    """Thin SQLite wrapper for UseCase persistence."""

    def __init__(self, db_path: Path) -> None:
        self._path = db_path
        db_path.parent.mkdir(parents=True, exist_ok=True)
        self._conn = sqlite3.connect(str(db_path))
        self._conn.row_factory = sqlite3.Row
        self._conn.execute(_CREATE_TABLE)
        self._conn.commit()

    def save(self, uc: UseCase) -> str:
        """Insert or replace a UseCase. Returns its id."""
        self._conn.execute(
            """
            INSERT OR REPLACE INTO usecases VALUES (
                :id, :title, :summary, :source_url, :source_platform,
                :author, :published_at, :collected_at, :scenario_type,
                :tools_used, :has_local_data, :has_workflow_opt,
                :value_score, :is_real_value, :novelty, :similar_to_ids,
                :raw_content
            )
            """,
            {
                "id": uc.id,
                "title": uc.title,
                "summary": uc.summary,
                "source_url": uc.source_url,
                "source_platform": uc.source_platform,
                "author": uc.author,
                "published_at": uc.published_at.isoformat(),
                "collected_at": uc.collected_at.isoformat(),
                "scenario_type": uc.scenario_type.value,
                "tools_used": json.dumps(uc.tools_used, ensure_ascii=False),
                "has_local_data": int(uc.has_local_data),
                "has_workflow_opt": int(uc.has_workflow_opt),
                "value_score": uc.value_score,
                "is_real_value": int(uc.is_real_value),
                "novelty": uc.novelty.value,
                "similar_to_ids": json.dumps(uc.similar_to_ids),
                "raw_content": uc.raw_content,
            },
        )
        self._conn.commit()
        return uc.id

    def exists(self, id: str) -> bool:
        """Return True if a UseCase with this id is already stored."""
        row = self._conn.execute(
            "SELECT 1 FROM usecases WHERE id = ?", (id,)
        ).fetchone()
        return row is not None

    def get_by_scenario(
        self, scenario_type: ScenarioType, limit: int = 100
    ) -> list[UseCase]:
        """Return up to `limit` UseCases matching the given scenario type."""
        rows = self._conn.execute(
            "SELECT * FROM usecases WHERE scenario_type = ? ORDER BY collected_at DESC LIMIT ?",
            (scenario_type.value, limit),
        ).fetchall()
        return [self._row_to_usecase(r) for r in rows]

    def get_recent(self, limit: int = 200) -> list[UseCase]:
        """Return the most recently collected UseCases."""
        rows = self._conn.execute(
            "SELECT * FROM usecases ORDER BY collected_at DESC LIMIT ?",
            (limit,),
        ).fetchall()
        return [self._row_to_usecase(r) for r in rows]

    @staticmethod
    def _row_to_usecase(row: sqlite3.Row) -> UseCase:
        from datetime import datetime

        return UseCase(
            id=row["id"],
            title=row["title"],
            summary=row["summary"],
            source_url=row["source_url"],
            source_platform=row["source_platform"],
            author=row["author"],
            published_at=datetime.fromisoformat(row["published_at"]),
            collected_at=datetime.fromisoformat(row["collected_at"]),
            scenario_type=ScenarioType(row["scenario_type"]),
            tools_used=json.loads(row["tools_used"]),
            has_local_data=bool(row["has_local_data"]),
            has_workflow_opt=bool(row["has_workflow_opt"]),
            value_score=row["value_score"],
            is_real_value=bool(row["is_real_value"]),
            novelty=Novelty(row["novelty"]),
            similar_to_ids=json.loads(row["similar_to_ids"]),
            raw_content=row["raw_content"],
        )


source venv/bin/activate && python -c "
from ai_usecases_explorer.storage.database import Database
from ai_usecases_explorer.settings import Settings

s = Settings()
db = Database(db_path=s.db_path)
items = db.get_recent(limit=20)
print(f'Most recent DB items: {len(items)}')
for item in items[:10]:
    print(f'  [{item.source_platform}] {item.title[:60]} (collected: {item.collected_at.date()})')
" 2>&1

Check DB recent entries to see what was stored


Most recent DB items: 20
  [hackernews] Scaling long-running autonomous coding (collected: 2026-03-03)
  [hackernews] Show HN: Now I Get It – Translate scientific papers into int (collected: 2026-03-03)
  [hackernews] Claude Code Remote Control (collected: 2026-03-03)
  [github] farion1231/cc-switch (collected: 2026-03-02)
  [github] flipped-aurora/gin-vue-admin (collected: 2026-03-02)
  [github] github/github-mcp-server (collected: 2026-03-02)
  [github] CopilotKit/CopilotKit (collected: 2026-03-02)
  [github] jeecgboot/JeecgBoot (collected: 2026-03-02)
  [reddit] Some Programmers Use AI (LLMs) Quite Differently : r/singula (collected: 2026-03-02)
  [github] RunanywhereAI/runanywhere-sdks (collected: 2026-03-01)


Now I understand what happened:

1. On March 3rd (collected_at 2026-03-03): 3 HN items were stored
2. On March 2nd: GitHub + Reddit items were stored
3. March 4th report shows HN(3), Reddit(0), GitHub(0)

So on March 4th, the 3 collected items (all HN) that passed the filter were NOT stored yet (they had collected_at: 2026-03-03, meaning they were stored on March 3rd's run).

Wait, that's confusing. Let me re-read: collected_at: 2026-03-03 means collected on March 3, but the cron runs at 7am.

So the March 4th report at 7am shows 3 HN items that were processed that day—these items have already been stored, and the `collected_at` timestamp reflects when they were actually processed.

But there's a discrepancy: the report shows `HN(3)` for March 4th, yet the database shows those same 3 items with `collected_at: 2026-03-03`, meaning they were stored on March 3rd, not during today's run. I'm realizing that the report date and the `collected_at` timestamp are tracking different things—the report date is when the report was generated, while `collected_at` is when the items were actually collected and stored.

Looking at the database more carefully, the items collected on March 3rd include both the 3 HN items and the GitHub items from March 2nd, which suggests there's a timezone issue at play. The cron runs at 7am, and depending on the server's timezone offset from UTC, March 4th at 7am could actually be March 3rd in UTC.

Rather than getting lost in the timezone details, I should focus on identifying the actual root causes. The main issue I'm seeing is with Reddit—the `start_published_date` filter is eliminating all results. xa doesn't know Reddit post dates (published_date = None for all Reddit results). Fix: remove `start_published_date` for Reddit.

2. **HN: No date filter means collecting ALL-TIME data**, not just today's posts. This gives 441 items total but most are old. The `search_by_date` endpoint sorts by date desc but without a time filter. Fix: add `created_at_i > {24h_ago_unix}` to numericFilters.

3. **GitHub: ValueFilter too strict for repos**. Tool repos like CherryHQ/cherry-studio (40K stars) get rejected because the filter prompt is designed for user-story posts. Actually in my test, cherry-studio was rejected but openclaw and chatgpt-on-wechat passed. Let me think about this more.

4. **Reddit: post-filter too strict**. The URL check that requires posts to come from our target subreddits is filtering out valid content, so I need to either remove it or make it more permissive. I should also broaden the search queries to better capture the AI community.

For the fixes, I'm adding a date filter to HN to pull only the last 24 hours of data. For Reddit, I'll drop the date requirement and relax the URL validation. And for the ValueFilter, I want to add a criterion that recognizes repositories and tools that enable practical AI integration into workflows.

The REJECT criteria also needs refinement—calling any tool description a "marketing demo" is too harsh. A GitHub repo with a tool description shouldn't automatically get rejected.

I notice the console output already shows raw collection counts before filtering, but the Obsidian report only shows post-filter numbers. I could update the report to include both raw and filtered counts for better visibility.

Now I'm prioritizing what to actually fix: the Reddit date filter is broken and returning nothing, the GitHub ValueFilter is too strict and also returning nothing, HN needs proper time filtering for recent items, and the report should show the raw collection counts alongside filtered results. Let me check the tests to see what needs updating.


이제 근본 원인을 다 파악했습니다. 테스트 코드도 확인하겠습니다.


     1→"""Tests for HackerNews, Reddit, and GitHub collectors."""
     2→from datetime import datetime
     3→from unittest.mock import MagicMock, patch
     4→
     5→from pytest_httpx import HTTPXMock
     6→
     7→from ai_usecases_explorer.collectors.github import GitHubCollector
     8→from ai_usecases_explorer.collectors.hackernews import HackerNewsCollector
     9→from ai_usecases_explorer.collectors.reddit import RedditCollector
    10→
    11→# ---------------------------------------------------------------------------
    12→# HackerNews Collector
    13→# ---------------------------------------------------------------------------
    14→
    15→HN_RESPONSE = {
    16→    "hits": [
    17→        {
    18→            "objectID": "12345",
    19→            "title": "Show HN: I built a Claude Code workflow for local DB queries",
    20→            "url": "https://example.com/claude-db",
    21→            "author": "hnuser1",
    22→            "points": 45,
    23→            "created_at": "2026-02-26T10:00:00.000Z",
    24→            "story_text": "I connected Claude Code directly to my local Postgres instance...",
    25→            "num_comments": 12,
    26→        },
    27→        {
    28→            "objectID": "12346",
    29→            "title": "Low points post",
    30→            "url": "https://example.com/low",
    31→            "author": "hnuser2",
    32→            "points": 3,
    33→            "created_at": "2026-02-26T09:00:00.000Z",
    34→            "story_text": "Not much here.",
    35→            "num_comments": 1,
    36→        },
    37→    ],
    38→    "nbHits": 2,
    39→}
    40→
    41→HN_RESPONSE_NO_URL = {
    42→    "hits": [
    43→        {
    44→            "objectID": "99999",
    45→            "title": "Ask HN: Best LLM workflow tools?",
    46→            "url": None,
    47→            "author": "asker",
    48→            "points": 50,
    49→            "created_at": "2026-02-26T08:00:00.000Z",
    50→            "story_text": "I want to automate my dev workflow...",
    51→            "num_comments": 5,
    52→        }
    53→    ],
    54→    "nbHits": 1,
    55→}
    56→
    57→
    58→HN_EMPTY = {"hits": [], "nbHits": 0}
    59→_HN_KEYWORD_COUNT = 12  # must match len(_KEYWORDS) in hackernews.py
    60→
    61→
    62→def _mock_hn(httpx_mock: HTTPXMock, first_response: dict) -> None:
    63→    """Add first_response for the first keyword, empty for the rest."""
    64→    httpx_mock.add_response(json=first_response)
    65→    for _ in range(_HN_KEYWORD_COUNT - 1):
    66→        httpx_mock.add_response(json=HN_EMPTY)
    67→
    68→
    69→class TestHackerNewsCollector:
    70→    def test_fetch_returns_list_of_raw_items(self, httpx_mock: HTTPXMock) -> None:
    71→        _mock_hn(httpx_mock, HN_RESPONSE)
    72→        collector = HackerNewsCollector(min_points=10)
    73→        items = collector.fetch()
    74→        assert isinstance(items, list)
    75→
    76→    def test_filters_by_min_points(self, httpx_mock: HTTPXMock) -> None:
    77→        _mock_hn(httpx_mock, HN_RESPONSE)
    78→        collector = HackerNewsCollector(min_points=10)
    79→        items = collector.fetch()
    80→        assert len(items) == 1
    81→        assert items[0].author == "hnuser1"
    82→
    83→    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
    84→        _mock_hn(httpx_mock, HN_RESPONSE)
    85→        collector = HackerNewsCollector(min_points=10)
    86→        items = collector.fetch()
    87→        item = items[0]
    88→        assert item.title == "Show HN: I built a Claude Code workflow for local DB queries"
    89→        assert item.source_url == "https://example.com/claude-db"
    90→        assert item.author == "hnuser1"
    91→        assert item.source_platform == "hackernews"
    92→        assert isinstance(item.published_at, datetime)
    93→
    94→    def test_fallback_url_for_ask_hn(self, httpx_mock: HTTPXMock) -> None:
    95→        _mock_hn(httpx_mock, HN_RESPONSE_NO_URL)
    96→        collector = HackerNewsCollector(min_points=10)
    97→        items = collector.fetch()
    98→        assert len(items) == 1
    99→        assert "news.ycombinator.com" in items[0].source_url
   100→
   101→    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
   102→        for _ in range(_HN_KEYWORD_COUNT):
   103→            httpx_mock.add_response(json=HN_EMPTY)
   104→        collector = HackerNewsCollector(min_points=10)
   105→        items = collector.fetch()
   106→        assert items == []
   107→
   108→    def test_raw_content_includes_title_and_text(self, httpx_mock: HTTPXMock) -> None:
   109→        _mock_hn(httpx_mock, HN_RESPONSE)
   110→        collector = HackerNewsCollector(min_points=10)
   111→        items = collector.fetch()
   112→        assert "Show HN" in items[0].raw_content
   113→        assert "Postgres" in items[0].raw_content
   114→
   115→    def test_deduplicates_across_queries(self, httpx_mock: HTTPXMock) -> None:
   116→        # Same item appears in multiple keyword results — returned only once
   117→        for _ in range(_HN_KEYWORD_COUNT):
   118→            httpx_mock.add_response(json=HN_RESPONSE)
   119→        collector = HackerNewsCollector(min_points=10)
   120→        items = collector.fetch()
   121→        ids = [i.source_url for i in items]
   122→        assert len(ids) == len(set(ids))
   123→
   124→
   125→# ---------------------------------------------------------------------------
   126→# Reddit Collector (Exa.ai neural search, subreddit-based)
   127→# ---------------------------------------------------------------------------
   128→
   129→
   130→def _make_exa_result(
   131→    url: str = "https://reddit.com/r/ClaudeCode/comments/abc123/title/",
   132→    title: str = "How I automated my daily reports with Claude",
   133→    text: str = "I built a Python script that calls Claude every morning...",
   134→    published_date: str = "2026-02-26T10:00:00Z",
   135→    author: str = "redditor1",
   136→) -> MagicMock:
   137→    result = MagicMock()
   138→    result.url = url
   139→    result.title = title
   140→    result.text = text
   141→    result.published_date = published_date
   142→    result.author = author
   143→    return result
   144→
   145→
   146→def _make_exa_response(results: list[MagicMock]) -> MagicMock:
   147→    response = MagicMock()
   148→    response.results = results
   149→    return response
   150→
   151→
   152→class TestRedditCollector:
   153→    def test_returns_empty_when_no_api_key(self) -> None:
   154→        collector = RedditCollector(subreddits=["ClaudeCode"], api_key="")
   155→        items = collector.fetch()
   156→        assert items == []
   157→
   158→    def test_fetches_and_returns_items(self) -> None:
   159→        mock_response = _make_exa_response([_make_exa_result()])
   160→        mock_exa = MagicMock()
   161→        mock_exa.search_and_contents.return_value = mock_response
   162→
   163→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   164→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   165→            items = collector.fetch()
   166→
   167→        assert len(items) == 1
   168→        assert items[0].title == "How I automated my daily reports with Claude"
   169→        assert items[0].source_platform == "reddit"
   170→        assert "reddit.com" in items[0].source_url
   171→
   172→    def test_item_has_required_fields(self) -> None:
   173→        mock_response = _make_exa_response([_make_exa_result()])
   174→        mock_exa = MagicMock()
   175→        mock_exa.search_and_contents.return_value = mock_response
   176→
   177→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   178→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   179→            items = collector.fetch()
   180→
   181→        item = items[0]
   182→        assert item.title == "How I automated my daily reports with Claude"
   183→        assert item.source_platform == "reddit"
   184→        assert item.author == "redditor1"
   185→        assert isinstance(item.published_at, datetime)
   186→
   187→    def test_deduplicates_within_subreddit(self) -> None:
   188→        # Same URL appearing twice in results should be deduplicated
   189→        same_result = _make_exa_result()
   190→        mock_exa = MagicMock()
   191→        mock_exa.search_and_contents.return_value = _make_exa_response([same_result, same_result])
   192→
   193→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   194→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   195→            items = collector.fetch()
   196→
   197→        urls = [i.source_url for i in items]
   198→        assert len(urls) == len(set(urls))
   199→
   200→    def test_multiple_subreddits_aggregate_results(self) -> None:
   201→        result_a = _make_exa_result(url="https://reddit.com/r/vibecoding/comments/aaa/")
   202→        result_b = _make_exa_result(url="https://reddit.com/r/singularity/comments/bbb/")
   203→        mock_exa = MagicMock()
   204→        mock_exa.search_and_contents.side_effect = [
   205→            _make_exa_response([result_a]),
   206→            _make_exa_response([result_b]),
   207→        ]
   208→
   209→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   210→            collector = RedditCollector(subreddits=["vibecoding", "singularity"], api_key="test-key")
   211→            items = collector.fetch()
   212→
   213→        assert len(items) == 2
   214→
   215→    def test_filters_out_wrong_subreddit_urls(self) -> None:
   216→        # Exa returns URL from wrong subreddit — post-filter must reject it
   217→        wrong_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
   218→        mock_exa = MagicMock()
   219→        mock_exa.search_and_contents.return_value = _make_exa_response([wrong_result])
   220→
   221→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   222→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   223→            items = collector.fetch()
   224→
   225→        assert items == []
   226→
   227→    def test_empty_results_returns_empty_list(self) -> None:
   228→        mock_exa = MagicMock()
   229→        mock_exa.search_and_contents.return_value = _make_exa_response([])
   230→
   231→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   232→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   233→            items = collector.fetch()
   234→
   235→        assert items == []
   236→
   237→    def test_gracefully_handles_exa_error(self) -> None:
   238→        mock_exa = MagicMock()
   239→        mock_exa.search_and_contents.side_effect = RuntimeError("network error")
   240→
   241→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   242→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   243→            items = collector.fetch()
   244→
   245→        assert items == []
   246→
   247→    def test_raw_content_includes_title_and_text(self) -> None:
   248→        mock_response = _make_exa_response([_make_exa_result()])
   249→        mock_exa = MagicMock()
   250→        mock_exa.search_and_contents.return_value = mock_response
   251→
   252→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   253→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   254→            items = collector.fetch()
   255→
   256→        assert "automated" in items[0].raw_content
   257→        assert "Python" in items[0].raw_content
   258→
   259→    def test_passes_subreddit_domain_to_exa(self) -> None:
   260→        mock_exa = MagicMock()
   261→        mock_exa.search_and_contents.return_value = _make_exa_response([])
   262→
   263→        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
   264→            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
   265→            collector.fetch()
   266→
   267→        call_kwargs = mock_exa.search_and_contents.call_args
   268→        assert call_kwargs.kwargs["include_domains"] == ["reddit.com/r/ClaudeCode"]
   269→        assert "start_published_date" in call_kwargs.kwargs
   270→        assert call_kwargs.kwargs["text"] is True
   271→
   272→
   273→# ---------------------------------------------------------------------------
   274→# GitHub Collector
   275→# ---------------------------------------------------------------------------
   276→
   277→GITHUB_SEARCH_RESPONSE = {
   278→    "total_count": 2,
   279→    "incomplete_results": False,
   280→    "items": [
   281→        {
   282→            "id": 111,
   283→            "name": "claude-local-rag",
   284→            "full_name": "user1/claude-local-rag",
   285→            "html_url": "https://github.com/user1/claude-local-rag",
   286→            "description": "RAG pipeline connecting Claude to local PDF files",
   287→            "topics": ["claude", "rag", "llm", "python"],
   288→            "stargazers_count": 42,
   289→            "language": "Python",
   290→            "pushed_at": "2026-02-26T12:00:00Z",
   291→            "created_at": "2026-02-25T08:00:00Z",
   292→            "owner": {"login": "user1"},
   293→        },
   294→        {
   295→            "id": 222,
   296→            "name": "low-star-repo",
   297→            "full_name": "user2/low-star-repo",
   298→            "html_url": "https://github.com/user2/low-star-repo",
   299→            "description": "Just a test",
   300→            "topics": ["llm"],
   301→            "stargazers_count": 2,
   302→            "language": "JavaScript",
   303→            "pushed_at": "2026-02-26T10:00:00Z",
   304→            "created_at": "2026-02-26T09:00:00Z",
   305→            "owner": {"login": "user2"},
   306→        },
   307→    ],
   308→}
   309→
   310→
   311→GITHUB_EMPTY = {"total_count": 0, "incomplete_results": False, "items": []}
   312→_GITHUB_TOPIC_COUNT = 8  # must match len(_TOPICS) in github.py
   313→
   314→
   315→def _mock_github(httpx_mock: HTTPXMock, first_response: dict) -> None:
   316→    """First topic returns real data, rest return empty."""
   317→    httpx_mock.add_response(json=first_response)
   318→    for _ in range(_GITHUB_TOPIC_COUNT - 1):
   319→        httpx_mock.add_response(json=GITHUB_EMPTY)
   320→
   321→
   322→class TestGitHubCollector:
   323→    def test_fetch_returns_list(self, httpx_mock: HTTPXMock) -> None:
   324→        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
   325→        collector = GitHubCollector(min_stars=10)
   326→        items = collector.fetch()
   327→        assert isinstance(items, list)
   328→
   329→    def test_filters_by_min_stars(self, httpx_mock: HTTPXMock) -> None:
   330→        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
   331→        collector = GitHubCollector(min_stars=10)
   332→        items = collector.fetch()
   333→        assert len(items) == 1
   334→        assert items[0].author == "user1"
   335→
   336→    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
   337→        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
   338→        collector = GitHubCollector(min_stars=10)
   339→        items = collector.fetch()
   340→        item = items[0]
   341→        assert item.title == "user1/claude-local-rag"
   342→        assert item.source_url == "https://github.com/user1/claude-local-rag"
   343→        assert item.source_platform == "github"
   344→        assert isinstance(item.published_at, datetime)
   345→
   346→    def test_raw_content_includes_description_and_topics(self, httpx_mock: HTTPXMock) -> None:
   347→        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
   348→        collector = GitHubCollector(min_stars=10)
   349→        items = collector.fetch()
   350→        assert "RAG pipeline" in items[0].raw_content
   351→        assert "claude" in items[0].raw_content
   352→
   353→    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
   354→        for _ in range(_GITHUB_TOPIC_COUNT):
   355→            httpx_mock.add_response(json=GITHUB_EMPTY)
   356→        collector = GitHubCollector(min_stars=10)
   357→        items = collector.fetch()
   358→        assert items == []
   359→
   360→    def test_deduplicates_across_topics(self, httpx_mock: HTTPXMock) -> None:
   361→        # Same repo appears under multiple topic queries — returned only once
   362→        for _ in range(_GITHUB_TOPIC_COUNT):
   363→            httpx_mock.add_response(json=GITHUB_SEARCH_RESPONSE)
   364→        collector = GitHubCollector(min_stars=10)
   365→        items = collector.fetch()
   366→        urls = [i.source_url for i in items]
   367→        assert len(urls) == len(set(urls))
   368→
   369→    def test_uses_auth_header_when_token_provided(self, httpx_mock: HTTPXMock) -> None:
   370→        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
   371→        collector = GitHubCollector(min_stars=10, token="ghp_test123")
   372→        items = collector.fetch()
   373→        assert isinstance(items, list)
   374→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""Tests for HackerNews, Reddit, and GitHub collectors."""
from datetime import datetime
from unittest.mock import MagicMock, patch

from pytest_httpx import HTTPXMock

from ai_usecases_explorer.collectors.github import GitHubCollector
from ai_usecases_explorer.collectors.hackernews import HackerNewsCollector
from ai_usecases_explorer.collectors.reddit import RedditCollector

# ---------------------------------------------------------------------------
# HackerNews Collector
# ---------------------------------------------------------------------------

HN_RESPONSE = {
    "hits": [
        {
            "objectID": "12345",
            "title": "Show HN: I built a Claude Code workflow for local DB queries",
            "url": "https://example.com/claude-db",
            "author": "hnuser1",
            "points": 45,
            "created_at": "2026-02-26T10:00:00.000Z",
            "story_text": "I connected Claude Code directly to my local Postgres instance...",
            "num_comments": 12,
        },
        {
            "objectID": "12346",
            "title": "Low points post",
            "url": "https://example.com/low",
            "author": "hnuser2",
            "points": 3,
            "created_at": "2026-02-26T09:00:00.000Z",
            "story_text": "Not much here.",
            "num_comments": 1,
        },
    ],
    "nbHits": 2,
}

HN_RESPONSE_NO_URL = {
    "hits": [
        {
            "objectID": "99999",
            "title": "Ask HN: Best LLM workflow tools?",
            "url": None,
            "author": "asker",
            "points": 50,
            "created_at": "2026-02-26T08:00:00.000Z",
            "story_text": "I want to automate my dev workflow...",
            "num_comments": 5,
        }
    ],
    "nbHits": 1,
}

HN_EMPTY = {"hits": [], "nbHits": 0}
_HN_KEYWORD_COUNT = 12  # must match len(_KEYWORDS) in hackernews.py

def _mock_hn(httpx_mock: HTTPXMock, first_response: dict) -> None:
    """Add first_response for the first keyword, empty for the rest."""
    httpx_mock.add_response(json=first_response)
    for _ in range(_HN_KEYWORD_COUNT - 1):
        httpx_mock.add_response(json=HN_EMPTY)

class TestHackerNewsCollector:
    def test_fetch_returns_list_of_raw_items(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert isinstance(items, list)

    def test_filters_by_min_points(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert len(items) == 1
        assert items[0].author == "hnuser1"

    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        item = items[0]
        assert item.title == "Show HN: I built a Claude Code workflow for local DB queries"
        assert item.source_url == "https://example.com/claude-db"
        assert item.author == "hnuser1"
        assert item.source_platform == "hackernews"
        assert isinstance(item.published_at, datetime)

    def test_fallback_url_for_ask_hn(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE_NO_URL)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert len(items) == 1
        assert "news.ycombinator.com" in items[0].source_url

    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
        for _ in range(_HN_KEYWORD_COUNT):
            httpx_mock.add_response(json=HN_EMPTY)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert items == []

    def test_raw_content_includes_title_and_text(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert "Show HN" in items[0].raw_content
        assert "Postgres" in items[0].raw_content

    def test_deduplicates_across_queries(self, httpx_mock: HTTPXMock) -> None:
        # Same item appears in multiple keyword results — returned only once
        for _ in range(_HN_KEYWORD_COUNT):
            httpx_mock.add_response(json=HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        ids = [i.source_url for i in items]
        assert len(ids) == len(set(ids))

# ---------------------------------------------------------------------------
# Reddit Collector (Exa.ai neural search, subreddit-based)
# ---------------------------------------------------------------------------

def _make_exa_result(
    url: str = "https://reddit.com/r/ClaudeCode/comments/abc123/title/",
    title: str = "How I automated my daily reports with Claude",
    text: str = "I built a Python script that calls Claude every morning...",
    published_date: str = "2026-02-26T10:00:00Z",
    author: str = "redditor1",
) -> MagicMock:
    result = MagicMock()
    result.url = url
    result.title = title
    result.text = text
    result.published_date = published_date
    result.author = author
    return result

def _make_exa_response(results: list[MagicMock]) -> MagicMock:
    response = MagicMock()
    response.results = results
    return response

class TestRedditCollector:
    def test_returns_empty_when_no_api_key(self) -> None:
        collector = RedditCollector(subreddits=["ClaudeCode"], api_key="")
        items = collector.fetch()
        assert items == []

    def test_fetches_and_returns_items(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 1
        assert items[0].title == "How I automated my daily reports with Claude"
        assert items[0].source_platform == "reddit"
        assert "reddit.com" in items[0].source_url

    def test_item_has_required_fields(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        item = items[0]
        assert item.title == "How I automated my daily reports with Claude"
        assert item.source_platform == "reddit"
        assert item.author == "redditor1"
        assert isinstance(item.published_at, datetime)

    def test_deduplicates_within_subreddit(self) -> None:
        # Same URL appearing twice in results should be deduplicated
        same_result = _make_exa_result()
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([same_result, same_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        urls = [i.source_url for i in items]
        assert len(urls) == len(set(urls))

    def test_multiple_subreddits_aggregate_results(self) -> None:
        result_a = _make_exa_result(url="https://reddit.com/r/vibecoding/comments/aaa/")
        result_b = _make_exa_result(url="https://reddit.com/r/singularity/comments/bbb/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.side_effect = [
            _make_exa_response([result_a]),
            _make_exa_response([result_b]),
        ]

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["vibecoding", "singularity"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 2

    def test_filters_out_wrong_subreddit_urls(self) -> None:
        # Exa returns URL from wrong subreddit — post-filter must reject it
        wrong_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([wrong_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_empty_results_returns_empty_list(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_gracefully_handles_exa_error(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.side_effect = RuntimeError("network error")

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_raw_content_includes_title_and_text(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert "automated" in items[0].raw_content
        assert "Python" in items[0].raw_content

    def test_passes_subreddit_domain_to_exa(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            collector.fetch()

        call_kwargs = mock_exa.search_and_contents.call_args
        assert call_kwargs.kwargs["include_domains"] == ["reddit.com/r/ClaudeCode"]
        assert "start_published_date" in call_kwargs.kwargs
        assert call_kwargs.kwargs["text"] is True

# ---------------------------------------------------------------------------
# GitHub Collector
# ---------------------------------------------------------------------------

GITHUB_SEARCH_RESPONSE = {
    "total_count": 2,
    "incomplete_results": False,
    "items": [
        {
            "id": 111,
            "name": "claude-local-rag",
            "full_name": "user1/claude-local-rag",
            "html_url": "https://github.com/user1/claude-local-rag",
            "description": "RAG pipeline connecting Claude to local PDF files",
            "topics": ["claude", "rag", "llm", "python"],
            "stargazers_count": 42,
            "language": "Python",
            "pushed_at": "2026-02-26T12:00:00Z",
            "created_at": "2026-02-25T08:00:00Z",
            "owner": {"login": "user1"},
        },
        {
            "id": 222,
            "name": "low-star-repo",
            "full_name": "user2/low-star-repo",
            "html_url": "https://github.com/user2/low-star-repo",
            "description": "Just a test",
            "topics": ["llm"],
            "stargazers_count": 2,
            "language": "JavaScript",
            "pushed_at": "2026-02-26T10:00:00Z",
            "created_at": "2026-02-26T09:00:00Z",
            "owner": {"login": "user2"},
        },
    ],
}

GITHUB_EMPTY = {"total_count": 0, "incomplete_results": False, "items": []}
_GITHUB_TOPIC_COUNT = 8  # must match len(_TOPICS) in github.py

def _mock_github(httpx_mock: HTTPXMock, first_response: dict) -> None:
    """First topic returns real data, rest return empty."""
    httpx_mock.add_response(json=first_response)
    for _ in range(_GITHUB_TOPIC_COUNT - 1):
        httpx_mock.add_response(json=GITHUB_EMPTY)

class TestGitHubCollector:
    def test_fetch_returns_list(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert isinstance(items, list)

    def test_filters_by_min_stars(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert len(items) == 1
        assert items[0].author == "user1"

    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        item = items[0]
        assert item.title == "user1/claude-local-rag"
        assert item.source_url == "https://github.com/user1/claude-local-rag"
        assert item.source_platform == "github"
        assert isinstance(item.published_at, datetime)

    def test_raw_content_includes_description_and_topics(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert "RAG pipeline" in items[0].raw_content
        assert "claude" in items[0].raw_content

    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
        for _ in range(_GITHUB_TOPIC_COUNT):
            httpx_mock.add_response(json=GITHUB_EMPTY)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert items == []

    def test_deduplicates_across_topics(self, httpx_mock: HTTPXMock) -> None:
        # Same repo appears under multiple topic queries — returned only once
        for _ in range(_GITHUB_TOPIC_COUNT):
            httpx_mock.add_response(json=GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        urls = [i.source_url for i in items]
        assert len(urls) == len(set(urls))

    def test_uses_auth_header_when_token_provided(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10, token="ghp_test123")
        items = collector.fetch()
        assert isinstance(items, list)


Now I have a complete picture of the root causes and what tests need to be updated. Let me plan the fixes:

**Root Cause Summary:**

1. **Reddit BROKEN**: `start_published_date` filter causes 0 results because Exa doesn't have published_date metadata for Reddit posts (all returned as `None`). The strict post-filter (`if "/r/{sub}/" not in url`) then rejects anything that slipped through, making things worse. The `include_domains=[f"reddit.com/r/{sub}"]` path-based filter returns 0 results.

2. **HN historical data**: HN collector fetches ALL-TIME posts (no time filter), giving 441 items total.

Most of these are old posts that may not be in the database, with only about 10 items from the last 24 hours actually matching the time-based filtering.

3. **GitHub ValueFilter too strict**: The filter prompt was designed for user-story posts ("I did X with AI"), not tool/repo announcements. GitHub repos like CherryHQ/cherry-studio get rejected as "marketing demos".

**Fixes:**

### Fix 1: `reddit.py` - Remove date filter, broaden collection
- Use `include_domains=['reddit.com']` only (no path)
- Remove `start_published_date` - Exa has no reliable date for Reddit
- Use multiple queries targeting AI communities, not specific subreddits
- Remove the strict URL post-filter (or make it check for `reddit.com` only)

### Fix 2: `hackernews.py` - Add 24h time filter
- Add `created_at_i>{unix_ts}` to `numericFilters` to get only recent posts

### Fix 3: `filter.py` - Add GitHub/tool-specific PASS criterion
- Add criterion: "A tool, library, framework, or project that directly enables real AI use cases, automation, or workflow integration"

### Fix 4: `obsidian.py` - Show raw collection counts in report
- Add a new section showing raw counts vs filtered counts for better visibility

Now I need to update the tests to reflect these changes, particularly checking that the subreddit domain filtering works as expected.

The real issue with Reddit is that Exa returns posts without published_date metadata, and the date filter eliminates everything. I need to remove the date constraint for Reddit specifically while still ensuring we're getting recent content. Looking at the test results, removing the date filter gave us 13 results, which suggests Exa might return older posts without that constraint. Let me implement these fixes now. I'll expand the Reddit queries to cast a wider net—adding searches around Claude AI agents, AI productivity for solopreneurs, and vibe coding approaches. For Hacker News, keeping that 24-hour filter makes sense since it naturally surfaces about 10 relevant posts daily, which fits the report cadence. Now I need to refine the filter logic in filter.py, particularly the PASS criteria to better identify tool repos and projects that enable AI automation workflows.

I'm adding a third PASS criterion specifically for open-source tools, frameworks, and repositories built for AI automation and workflow integration, while tightening the REJECT criteria to exclude marketing fluff that lacks actual implementation. With those criteria in place, I can now implement the code changes: updating hackernews.py with a 24-hour timestamp filter, redesigning reddit.py to use multiple targeted queries without the strict URL filtering, and updating filter.py with the expanded PASS logic.

I need to update the test suite to reflect these behavioral changes, particularly the Reddit collector tests that currently verify the buggy behavior. I'll follow the TDD approach by writing the corrected tests first that expose what the new behavior should be, then implement the fixes to make them pass. I'm seeing that topic-based queries with `include_domains=['reddit.com']` actually pull relevant results from multiple subreddits naturally, so I don't need to hardcode specific subreddit paths anymore. The key insight is that my queries should be crafted around the topics we care about—AI tools, agents, automation—and let the search surface results from whatever communities discuss those topics. Defining search queries...

I'm realizing the subreddit-specific filtering isn't working reliably with domain filters, so I should shift to a broader strategy: use multiple topical queries without date restrictions and let Reddit results surface naturally. The subreddits list becomes optional context rather than a hard filter. I'll construct queries around different AI themes—workflow automation, Claude-specific development, AI advancement—and accept any reddit.com URLs that match.

Building the RedditCollector class...

I'm realizing the interface change removes the `subreddits` parameter, but the Settings class still passes `reddit_subreddits` to it. I could keep the parameter for backward compatibility, but since I'm using predefined queries instead, I should probably just remove it from the constructor and update the Settings class accordingly. Actually, let me stick with the simpler approach and drop the parameter entirely.

Wait, that's a breaking change to Settings and tests. The cleanest solution is to keep the `subreddits` parameter but repurpose it—instead of using each subreddit as a domain filter, I'll incorporate the subreddit names as keywords in the search query alongside `include_domains=['reddit.com']`. This way Exa understands we're looking for relevant posts about those communities.

I tested this approach with a broad query like "openclaw vibecoding ClaudeCode singularity AI workflow" combined with the domain filter, and it's working better than path-based filtering. Let me verify this is the right direction.

My implementation plan: rename the default subreddits to topic-based queries, remove the date filter, set domains to just `['reddit.com']`, drop the strict URL post-filter, and keep the `subreddits` parameter to build targeted queries that mention each subreddit name as context.

Building the search logic... I'll keep the `subreddits` parameter for backward compatibility but change how it's used—each subreddit gets incorporated into a query like `"AI workflow use case tool {sub}"`, and I'll add a few general broad queries on top of that. I'm removing the date filter and strict URL validation, relying instead on `include_domains=['reddit.com']` to keep results focused. Now I need to update the tests since they're checking for domain filtering that no longer applies in the same way.

Building the query construction logic...

Actually, I'm reconsidering the approach—using general queries without subreddit filtering seems more reliable based on what I've tested. I'm thinking through a few broad search queries that cast a wider net across relevant communities: one targeting AI agents and workflow automation, another focused on Claude and coding, and a third around AI and automation themes. This way I'm not relying on exact subreddit matches, which haven't been working well.

I'm setting up a queries list with these semantic search terms instead of explicit subreddit filtering. But now I'm realizing there's a problem with the constructor—the Settings class is passing subreddits to RedditCollector, but I'm changing the signature to just take an API key.

I could keep the subreddits parameter for backward compatibility but not use it, or better yet, use the subreddit names themselves to build more targeted queries that hint at those communities.

Actually, I'm overcomplicating this. The simplest approach is to keep the subreddits parameter to maintain compatibility with existing tests and settings, but replace the query logic with a hardcoded list of relevant AI topics instead of dynamically building queries from subreddit names. I can note in the docstring that the old subreddit-based behavior has been replaced, and if we need configurable queries later, we can add that then.

Let me just implement the straightforward fix: keep the subreddits parameter for backward compatibility, but switch to using a predefined set of queries instead of querying Exa per subreddit.

Updating the tests...

Actually, I realize the core issue is simpler than I thought—I just need to remove the `start_published_date` parameter, switch to `include_domains=['reddit.com']` without path filtering, and update the URL validation to accept any reddit.com domain. I can keep the per-subreddit query loop but use a broader search query that works across all subreddits, then adjust the test mocks to handle the new query structure. I need to update the tests to match these changes—removing the date filter check and deleting the subreddit URL filter test since we're no longer enforcing strict URL matching. With 4 subreddits querying 10 results each, I should get around 20-30 unique results after deduplication, which is reasonable. Since Exa doesn't support date filtering for Reddit content, I'll rely on their ranking algorithm to surface the most relevant recent posts naturally.

I'm reconsidering whether adding subreddit names to the query actually helps—Exa's semantic search might not benefit from keyword matching like that. Instead, I'll stick with the approach that worked in testing: broad queries focused on the topic with `include_domains=['reddit.com']`, making multiple queries with different angles to capture varied perspectives across the subreddits.

Defining the collector class... I'm settling on keeping the subreddit-based structure to maintain backward compatibility with the existing test mocks—each subreddit will trigger one API call. I'll update the query parameters to include the domain filter, remove the date constraint, drop the URL post-filtering, and append the subreddit name for context. Beyond that, I'm adding a set of general queries that run independently of specific subreddits to broaden the coverage. Let me stop deliberating and implement this.

Writing the implementation... Building the Reddit collector...

I'm realizing the extra queries approach would break the existing tests since they mock exactly N Exa calls for N subreddits. I should simplify this to just fix the core issues per subreddit—correcting the domain format, removing the date filtering that doesn't work with Reddit, and dropping the URL post-filter—while keeping the test structure intact. I can always add the extra queries later if needed.

For the test updates, I need to verify the new domain parameter is just `["reddit.com"]` without the date parameter, and flip the subreddit filtering test to confirm we're accepting results from any Reddit subreddit rather than rejecting them. Then I'll move to the HackerNews changes, starting with adding the time import and calculating a timestamp.

For the filter and Obsidian updates, I'm realizing the reporter needs to show raw collection counts before filtering, but the current interface only has access to the filtered results. I could either modify the reporter interface to accept raw counts or calculate them separately—for now, I'll update the report to display the raw counts alongside the filtered ones.

Now I'm ready to write the actual code changes. For the tests, I need to update the assertions to match the new filtering logic, particularly around subreddit domain handling and HN keyword counts. Let me also finalize the PASS criteria for filter.py—the key is identifying when users are integrating AI with their personal data or automating workflows...

Writing the code changes...

Actually, I'm realizing the filter criteria needs refinement. Tool announcements and open-source projects should pass through, but the current rejection logic is too broad. I should adjust it to specifically target commercial product announcements without implementation details, while allowing genuine tool repos and "Show HN" style posts to pass.

Let me refine the criteria: add a pass criterion for software tools and frameworks designed for AI workflows, clarify that GitHub repos should be evaluated on their technical merit rather than personal testimonials, and make the marketing rejection more precise so it only catches commercial products lacking substance.

Now I need to write out the actual filter prompts, starting with the HN date filtering approach.

Adding a 24-hour timestamp filter to HN will dramatically reduce the result set from 441 items down to around 10, which is much more manageable. The existing tests mock the API response with hardcoded data, so they won't be affected by the timestamp parameter change. There's a risk of getting zero results on slower days, but that's preferable to wading through hundreds of irrelevant posts.

Time to implement the changes across the codebase.

Now looking at the Obsidian report format—it currently shows HN(3) which implies only 3 items were collected, when really it means 3 items passed the filter. After these improvements, I expect to see HN collecting around 10 items with 3-5 passing, Reddit pulling 20-30 items with 5-10 passing, and GitHub getting 90+ items with roughly 10 making it through the filter. The report should clarify this distinction between collected and filtered results.

To properly show both counts, I'd need to either pass raw collection numbers to the reporter or add them to the Scout class, but that's a bigger refactor. For now I'll stick with fixing the core collection issues and let users see the raw counts from console output.

Before I start writing the fixes, I should also consider whether to adjust GitHub's `min_stars` threshold and `lookback_days` parameters.

Looking at the current settings, `min_stars=10` is reasonable for quality filtering, and `lookback_days=2` captures recent repositories well. Even though topics like "openclaw" have massive star counts, keeping `min_stars=10` maintains good coverage without being too permissive. The user's concern is about coverage anyway, not filtering for only the most popular repos.

Now I need to update the ValueFilter for GitHub by adding a PASS criterion that properly handles tool repositories. The current criteria focus on users integrating AI with personal data or automating workflows, but I should refine this to better capture relevant tool repos.

Updating the criteria to include a third PASS condition for tools, libraries, and frameworks built specifically for AI agents and workflow automation, while also making the REJECT criteria more precise—distinguishing between pure Q&A that just replaces ChatGPT, vague hype posts without technical depth, and marketing content lacking real implementation details.

Now I'm updating the collector and filter logic to properly catch GitHub repositories and apply these refined criteria.


이제 근본 원인을 완전히 파악했습니다. 수정 작업을 시작하겠습니다.

**근본 원인 요약:**

1. **Reddit**: Exa.ai는 Reddit 게시물에 `published_date` 메타데이터가 없음(모두 `None`) → `start_published_date` 필터가 결과를 완전 차단. 또한 `include_domains=['reddit.com/r/ClaudeCode']` 경로 기반 필터는 아예 지원 안 됨 (항상 0건)
2. **HN**: 날짜 필터 없이 ALL-TIME 데이터 수집 → 441건 수집되지만 대부분 오래된 글. 실제 24시간 필터 적용 시 10건만 해당
3. **GitHub**: 122건 수집되지만 ValueFilter가 repo 발표를 "마케팅 데모"로 과도하게 거부


TaskCreate

Fix Reddit Exa date filter bug

Remove start_published_date and fix include_domains from path-based to domain-only. Update tests.


Task #1 created successfully: Fix Reddit Exa date filter bug

Fix Reddit Exa date filter bug


Add created_at_i Unix timestamp filter to HN collector to only get last 24h posts instead of all-time.

Add HN 24h date filter


Task #2 created successfully: Add HN 24h date filter

Add HN 24h date filter


Add 3rd PASS criterion for tool repos and soften REJECT criteria so legitimate GitHub repos are not incorrectly rejected as marketing demos.

Fix ValueFilter too strict for GitHub repos


Task #3 created successfully: Fix ValueFilter too strict for GitHub repos


Updated task #1 status


먼저 TDD에 따라 테스트를 먼저 수정합니다:


Edit

    def test_filters_out_wrong_subreddit_urls(self) -> None:
        # Exa returns URL from wrong subreddit — post-filter must reject it
        wrong_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([wrong_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_accepts_results_from_any_reddit_subreddit(self) -> None:
        # Exa may return URLs from subreddits not in the target list — accept them
        # (path-based domain filtering is unsupported by Exa; we accept any reddit.com URL)
        other_sub_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([other_sub_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 1


The file /root/projects/ai_usecases_explorer/tests/test_collectors.py has been updated successfully.

    def test_accepts_results_from_any_reddit_subreddit(self) -> None:
        # Exa may return URLs from subreddits not in the target list — accept them
        # (path-based domain filtering is unsupported by Exa; we accept any reddit.com URL)
        other_sub_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([other_sub_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 1

"""Tests for HackerNews, Reddit, and GitHub collectors."""
from datetime import datetime
from unittest.mock import MagicMock, patch

from pytest_httpx import HTTPXMock

from ai_usecases_explorer.collectors.github import GitHubCollector
from ai_usecases_explorer.collectors.hackernews import HackerNewsCollector
from ai_usecases_explorer.collectors.reddit import RedditCollector

# ---------------------------------------------------------------------------
# HackerNews Collector
# ---------------------------------------------------------------------------

HN_RESPONSE = {
    "hits": [
        {
            "objectID": "12345",
            "title": "Show HN: I built a Claude Code workflow for local DB queries",
            "url": "https://example.com/claude-db",
            "author": "hnuser1",
            "points": 45,
            "created_at": "2026-02-26T10:00:00.000Z",
            "story_text": "I connected Claude Code directly to my local Postgres instance...",
            "num_comments": 12,
        },
        {
            "objectID": "12346",
            "title": "Low points post",
            "url": "https://example.com/low",
            "author": "hnuser2",
            "points": 3,
            "created_at": "2026-02-26T09:00:00.000Z",
            "story_text": "Not much here.",
            "num_comments": 1,
        },
    ],
    "nbHits": 2,
}

HN_RESPONSE_NO_URL = {
    "hits": [
        {
            "objectID": "99999",
            "title": "Ask HN: Best LLM workflow tools?",
            "url": None,
            "author": "asker",
            "points": 50,
            "created_at": "2026-02-26T08:00:00.000Z",
            "story_text": "I want to automate my dev workflow...",
            "num_comments": 5,
        }
    ],
    "nbHits": 1,
}

HN_EMPTY = {"hits": [], "nbHits": 0}
_HN_KEYWORD_COUNT = 12  # must match len(_KEYWORDS) in hackernews.py

def _mock_hn(httpx_mock: HTTPXMock, first_response: dict) -> None:
    """Add first_response for the first keyword, empty for the rest."""
    httpx_mock.add_response(json=first_response)
    for _ in range(_HN_KEYWORD_COUNT - 1):
        httpx_mock.add_response(json=HN_EMPTY)

class TestHackerNewsCollector:
    def test_fetch_returns_list_of_raw_items(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert isinstance(items, list)

    def test_filters_by_min_points(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert len(items) == 1
        assert items[0].author == "hnuser1"

    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        item = items[0]
        assert item.title == "Show HN: I built a Claude Code workflow for local DB queries"
        assert item.source_url == "https://example.com/claude-db"
        assert item.author == "hnuser1"
        assert item.source_platform == "hackernews"
        assert isinstance(item.published_at, datetime)

    def test_fallback_url_for_ask_hn(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE_NO_URL)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert len(items) == 1
        assert "news.ycombinator.com" in items[0].source_url

    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
        for _ in range(_HN_KEYWORD_COUNT):
            httpx_mock.add_response(json=HN_EMPTY)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert items == []

    def test_raw_content_includes_title_and_text(self, httpx_mock: HTTPXMock) -> None:
        _mock_hn(httpx_mock, HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        assert "Show HN" in items[0].raw_content
        assert "Postgres" in items[0].raw_content

    def test_deduplicates_across_queries(self, httpx_mock: HTTPXMock) -> None:
        # Same item appears in multiple keyword results — returned only once
        for _ in range(_HN_KEYWORD_COUNT):
            httpx_mock.add_response(json=HN_RESPONSE)
        collector = HackerNewsCollector(min_points=10)
        items = collector.fetch()
        ids = [i.source_url for i in items]
        assert len(ids) == len(set(ids))

# ---------------------------------------------------------------------------
# Reddit Collector (Exa.ai neural search, subreddit-based)
# ---------------------------------------------------------------------------

def _make_exa_result(
    url: str = "https://reddit.com/r/ClaudeCode/comments/abc123/title/",
    title: str = "How I automated my daily reports with Claude",
    text: str = "I built a Python script that calls Claude every morning...",
    published_date: str = "2026-02-26T10:00:00Z",
    author: str = "redditor1",
) -> MagicMock:
    result = MagicMock()
    result.url = url
    result.title = title
    result.text = text
    result.published_date = published_date
    result.author = author
    return result

def _make_exa_response(results: list[MagicMock]) -> MagicMock:
    response = MagicMock()
    response.results = results
    return response

class TestRedditCollector:
    def test_returns_empty_when_no_api_key(self) -> None:
        collector = RedditCollector(subreddits=["ClaudeCode"], api_key="")
        items = collector.fetch()
        assert items == []

    def test_fetches_and_returns_items(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 1
        assert items[0].title == "How I automated my daily reports with Claude"
        assert items[0].source_platform == "reddit"
        assert "reddit.com" in items[0].source_url

    def test_item_has_required_fields(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        item = items[0]
        assert item.title == "How I automated my daily reports with Claude"
        assert item.source_platform == "reddit"
        assert item.author == "redditor1"
        assert isinstance(item.published_at, datetime)

    def test_deduplicates_within_subreddit(self) -> None:
        # Same URL appearing twice in results should be deduplicated
        same_result = _make_exa_result()
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([same_result, same_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        urls = [i.source_url for i in items]
        assert len(urls) == len(set(urls))

    def test_multiple_subreddits_aggregate_results(self) -> None:
        result_a = _make_exa_result(url="https://reddit.com/r/vibecoding/comments/aaa/")
        result_b = _make_exa_result(url="https://reddit.com/r/singularity/comments/bbb/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.side_effect = [
            _make_exa_response([result_a]),
            _make_exa_response([result_b]),
        ]

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["vibecoding", "singularity"], api_key="test-key")
            items = collector.fetch()

        assert len(items) == 2

    def test_filters_out_wrong_subreddit_urls(self) -> None:
        # Exa returns URL from wrong subreddit — post-filter must reject it
        wrong_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([wrong_result])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_empty_results_returns_empty_list(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_gracefully_handles_exa_error(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.side_effect = RuntimeError("network error")

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert items == []

    def test_raw_content_includes_title_and_text(self) -> None:
        mock_response = _make_exa_response([_make_exa_result()])
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = mock_response

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            items = collector.fetch()

        assert "automated" in items[0].raw_content
        assert "Python" in items[0].raw_content

    def test_passes_subreddit_domain_to_exa(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            collector.fetch()

        call_kwargs = mock_exa.search_and_contents.call_args
        assert call_kwargs.kwargs["include_domains"] == ["reddit.com/r/ClaudeCode"]
        assert "start_published_date" in call_kwargs.kwargs
        assert call_kwargs.kwargs["text"] is True

# ---------------------------------------------------------------------------
# GitHub Collector
# ---------------------------------------------------------------------------

GITHUB_SEARCH_RESPONSE = {
    "total_count": 2,
    "incomplete_results": False,
    "items": [
        {
            "id": 111,
            "name": "claude-local-rag",
            "full_name": "user1/claude-local-rag",
            "html_url": "https://github.com/user1/claude-local-rag",
            "description": "RAG pipeline connecting Claude to local PDF files",
            "topics": ["claude", "rag", "llm", "python"],
            "stargazers_count": 42,
            "language": "Python",
            "pushed_at": "2026-02-26T12:00:00Z",
            "created_at": "2026-02-25T08:00:00Z",
            "owner": {"login": "user1"},
        },
        {
            "id": 222,
            "name": "low-star-repo",
            "full_name": "user2/low-star-repo",
            "html_url": "https://github.com/user2/low-star-repo",
            "description": "Just a test",
            "topics": ["llm"],
            "stargazers_count": 2,
            "language": "JavaScript",
            "pushed_at": "2026-02-26T10:00:00Z",
            "created_at": "2026-02-26T09:00:00Z",
            "owner": {"login": "user2"},
        },
    ],
}

GITHUB_EMPTY = {"total_count": 0, "incomplete_results": False, "items": []}
_GITHUB_TOPIC_COUNT = 8  # must match len(_TOPICS) in github.py

def _mock_github(httpx_mock: HTTPXMock, first_response: dict) -> None:
    """First topic returns real data, rest return empty."""
    httpx_mock.add_response(json=first_response)
    for _ in range(_GITHUB_TOPIC_COUNT - 1):
        httpx_mock.add_response(json=GITHUB_EMPTY)

class TestGitHubCollector:
    def test_fetch_returns_list(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert isinstance(items, list)

    def test_filters_by_min_stars(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert len(items) == 1
        assert items[0].author == "user1"

    def test_item_has_required_fields(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        item = items[0]
        assert item.title == "user1/claude-local-rag"
        assert item.source_url == "https://github.com/user1/claude-local-rag"
        assert item.source_platform == "github"
        assert isinstance(item.published_at, datetime)

    def test_raw_content_includes_description_and_topics(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert "RAG pipeline" in items[0].raw_content
        assert "claude" in items[0].raw_content

    def test_empty_response_returns_empty_list(self, httpx_mock: HTTPXMock) -> None:
        for _ in range(_GITHUB_TOPIC_COUNT):
            httpx_mock.add_response(json=GITHUB_EMPTY)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        assert items == []

    def test_deduplicates_across_topics(self, httpx_mock: HTTPXMock) -> None:
        # Same repo appears under multiple topic queries — returned only once
        for _ in range(_GITHUB_TOPIC_COUNT):
            httpx_mock.add_response(json=GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10)
        items = collector.fetch()
        urls = [i.source_url for i in items]
        assert len(urls) == len(set(urls))

    def test_uses_auth_header_when_token_provided(self, httpx_mock: HTTPXMock) -> None:
        _mock_github(httpx_mock, GITHUB_SEARCH_RESPONSE)
        collector = GitHubCollector(min_stars=10, token="ghp_test123")
        items = collector.fetch()
        assert isinstance(items, list)

-    def test_filters_out_wrong_subreddit_urls(self) -> None:

-        # Exa returns URL from wrong subreddit — post-filter must reject it

-        wrong_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")

+    def test_accepts_results_from_any_reddit_subreddit(self) -> None:

+        # Exa may return URLs from subreddits not in the target list — accept them

+        # (path-based domain filtering is unsupported by Exa; we accept any reddit.com URL)

+        other_sub_result = _make_exa_result(url="https://reddit.com/r/LocalLLaMA/comments/xyz/")

-        mock_exa.search_and_contents.return_value = _make_exa_response([wrong_result])

+        mock_exa.search_and_contents.return_value = _make_exa_response([other_sub_result])


    def test_passes_subreddit_domain_to_exa(self) -> None:
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            collector.fetch()

        call_kwargs = mock_exa.search_and_contents.call_args
        assert call_kwargs.kwargs["include_domains"] == ["reddit.com/r/ClaudeCode"]
        assert "start_published_date" in call_kwargs.kwargs
        assert call_kwargs.kwargs["text"] is True

    def test_passes_reddit_domain_to_exa(self) -> None:
        # include_domains must be ["reddit.com"] — Exa does not support path-based filtering.
        # start_published_date must NOT be set — Exa has no published_date for Reddit posts.
        mock_exa = MagicMock()
        mock_exa.search_and_contents.return_value = _make_exa_response([])

        with patch("ai_usecases_explorer.collectors.reddit.Exa", return_value=mock_exa):
            collector = RedditCollector(subreddits=["ClaudeCode"], api_key="test-key")
            collector.fetch()

        call_kwargs = mock_exa.search_and_contents.call_args
        assert call_kwargs.kwargs["include_domains"] == ["reddit.com"]
        assert "start_published_date" not in call_kwargs.kwargs
        assert call_kwargs.kwargs["text"] is True