Taxonomy-enhanced session search with SKOS concept layer

┌──────────────────────────────────────────────────────────────────┐
│  Qdrant Collections                                              │
│                                                                  │
│  taxonomy_concepts         session_transcripts                   │
│  ├─ vector: 768-dim        ├─ vector: 768-dim                   │
│  ├─ prefLabel              ├─ text (chunk content)               │
│  ├─ altLabels[]            ├─ concept_ids[]  ←── taxonomy link   │
│  ├─ broader[]              ├─ source (pi|claude|codex)           │
│  ├─ narrower[]             ├─ sessionId                          │
│  ├─ related[]              ├─ timestamp_start / timestamp_end    │
│  ├─ exactMatch{}           ├─ turn_roles[] (user|assistant|tool) │
│  ├─ closeMatch{}           ├─ files_read[]                       │
│  ├─ scopeNote              ├─ files_modified[]                   │
│  ├─ definition             ├─ vault_notes[]                      │
│  ├─ conceptScheme          ├─ slog_tool_refs[]                   │
│  └─ vault_note_path        ├─ codebase_paths[]                   │
│                            ├─ adr_refs[]                         │
│  memory_observations       ├─ chunk_index                        │
│  (ADR-0021, enriched)      ├─ total_chunks                       │
│  ├─ concept_ids[]  ←─┐     ├─ context_prefix                    │
│  └─ (existing schema)│     └─ ttl_expires_at (codex only)        │
│                      │                                           │
│                      └──── shared concept_ids enable cross-query │
└──────────────────────────────────────────────────────────────────┘
 
┌──────────────────────────────────────────────────────────────────┐
│  Vault (human layer)                                             │
│                                                                  │
│  Resources/taxonomy/                                             │
│  ├─ _index.md              (concept scheme overview)             │
│  ├─ agent-infrastructure.md                                      │
│  │   frontmatter: { prefLabel, altLabels, narrower, related }    │
│  │   body: scope note, definition, links to projects             │
│  ├─ memory-system.md                                             │
│  ├─ video-pipeline.md                                            │
│  ├─ joelclaw.md                                                  │
│  └─ ...                                                          │
│                                                                  │
│  Wikilinks = broader/narrower/related graph                      │
│  Frontmatter = machine-readable SKOS fields                      │
│  Sync: Vault notes → Qdrant taxonomy_concepts (Inngest function) │
└──────────────────────────────────────────────────────────────────┘

User/Agent query: "how did we fix the worker crash?"
│
├─ 1. Embed query (nomic-embed-text-v1.5, 768-dim)
│
├─ 2. Taxonomy expansion
│     ├─ Search taxonomy_concepts by vector similarity
│     │   → finds: jc:system-bus-worker (score 0.82)
│     ├─ Traverse broader[]: jc:agent-infrastructure
│     ├─ Traverse related[]: jc:inngest, jc:launchd, jc:docker
│     ├─ Collect altLabels: ["worker", "event bus", "system-bus"]
│     └─ Build expanded concept set: [jc:system-bus-worker, jc:inngest, ...]
│
├─ 3. Hybrid retrieval
│     ├─ Dense: vector similarity on session_transcripts + memory_observations
│     ├─ Sparse: BM25 on altLabels + query terms (future: Qdrant sparse vectors)
│     ├─ Filter: concept_ids overlap with expanded concept set (payload filter)
│     └─ Combine via Reciprocal Rank Fusion (RRF)
│
├─ 4. Re-rank top-k results
│     ├─ Cross-encoder or LLM-based re-ranking
│     └─ Score by: relevance + recency + source priority
│
└─ 5. Return ranked chunks with concept context
      ├─ Each result includes: text, source, timestamp, concept_ids, files
      └─ Reflect tool synthesizes answer from top chunks

interface TaxonomyConcept {
  // Identity
  id: string;                    // e.g., "jc:system-bus-worker"
  prefLabel: string;             // "system-bus worker"
  altLabels: string[];           // ["worker", "event bus worker", "system-bus"]
  hiddenLabels: string[];        // typos, abbreviations: ["sb-worker", "sysbus"]
  
  // Hierarchy (SKOS semantic relations)
  broader: string[];             // ["jc:agent-infrastructure"]
  narrower: string[];            // ["jc:inngest-functions", "jc:launchd-plist"]
  related: string[];             // ["jc:inngest", "jc:docker", "jc:bun"]
  
  // Cross-system mappings (SKOS mapping properties)
  exactMatch: {
    slog_tool?: string;          // "system-bus-worker"
    vault_project?: string;      // "Projects/07-event-bus/"
    codebase_path?: string;      // "packages/system-bus/"
    skill?: string;              // "inngest"
  };
  closeMatch: {
    vault_notes?: string[];      // related Vault notes
    adr_refs?: string[];         // ["ADR-0021", "ADR-0022"]
  };
  
  // Documentation (SKOS documentation properties)
  scopeNote: string;             // brief description of concept scope
  definition?: string;           // formal definition
  
  // Metadata
  conceptScheme: string;         // "jc:system" | "jc:tools" | "jc:projects"
  vault_note_path?: string;      // "Resources/taxonomy/system-bus-worker.md"
  created: string;               // ISO 8601
  modified: string;              // ISO 8601
  source: "mined" | "curated";  // how the concept was created
}

{"type":"user","message":"fix the worker crash","timestamp":"2026-02-15T10:30:00Z"}
{"type":"assistant","message":"Let me check the logs...","timestamp":"2026-02-15T10:30:05Z"}
{"type":"tool_use","tool":"bash","input":"docker logs ...","timestamp":"2026-02-15T10:30:06Z"}
{"type":"tool_result","output":"Error: Cannot find module '@qdrant/js-client-rest'","timestamp":"2026-02-15T10:30:07Z"}
{"type":"assistant","message":"The worker crashed because...","timestamp":"2026-02-15T10:30:10Z"}

[SESSION: {source} | {date} | {session_name}]
[TOPIC: {auto-detected or taxonomy concept labels}]
[FILES: {files_read + files_modified, truncated to top 5}]

                    ┌─────────────────────────────────┐
                    │  Inngest Functions               │
                    │                                  │
  File watcher or   │  search/session.index.requested  │
  manual trigger ──→│    ① Find unindexed sessions     │
                    │    ② For each: parse JSONL        │
                    │    ③ Chunk (adaptive recursive)   │
                    │    ④ Extract metadata per chunk   │
                    │    ⑤ Tag with concept_ids         │
                    │       (match against taxonomy)    │
                    │    ⑥ Add context prefix           │
                    │    ⑦ Embed (nomic-embed-text)     │
                    │    ⑧ Upsert to Qdrant             │
                    │    ⑨ Mark session as indexed       │
                    │       (Redis: indexed:{hash})     │
                    │                                  │
  Vault change or   │  search/taxonomy.sync.requested  │
  manual trigger ──→│    ① Read Resources/taxonomy/*.md │
                    │    ② Parse frontmatter → SKOS     │
                    │    ③ Embed prefLabel + definition  │
                    │    ④ Upsert to taxonomy_concepts  │
                    │    ⑤ Re-tag affected chunks        │
                    │       (concept label changes)     │
                    │                                  │
  Daily cron ──────→│  search/session.ttl.cleanup       │
                    │    ① Find points where             │
                    │       ttl_expires_at < now         │
                    │    ② Delete expired points         │
                    │    ③ Log cleanup stats             │
                    └─────────────────────────────────┘

// session_transcripts
{
  vectors: { size: 768, distance: "Cosine" },
  optimizers_config: {
    indexing_threshold: 20000,  // delay indexing until 20k points (batch-friendly)
  },
  // Payload indexes for filtered search
  payload_indexes: [
    { field: "source", type: "keyword" },
    { field: "sessionId", type: "keyword" },
    { field: "concept_ids", type: "keyword" },  // array of concept IDs
    { field: "timestamp_start", type: "integer" },
    { field: "files_read", type: "keyword" },
    { field: "files_modified", type: "keyword" },
    { field: "ttl_expires_at", type: "integer" },
  ],
}
 
// taxonomy_concepts
{
  vectors: { size: 768, distance: "Cosine" },
  payload_indexes: [
    { field: "prefLabel", type: "keyword" },
    { field: "altLabels", type: "keyword" },
    { field: "conceptScheme", type: "keyword" },
    { field: "broader", type: "keyword" },
    { field: "narrower", type: "keyword" },
    { field: "related", type: "keyword" },
    { field: "exactMatch.slog_tool", type: "keyword" },
    { field: "exactMatch.vault_project", type: "keyword" },
  ],
}

---
type: taxonomy-concept
concept_id: "jc:system-bus-worker"
prefLabel: "system-bus worker"
altLabels:
  - worker
  - event bus worker
  - system-bus
hiddenLabels:
  - sb-worker
  - sysbus
broader:
  - "[[agent-infrastructure]]"
narrower:
  - "[[inngest-functions]]"
  - "[[launchd-plist]]"
related:
  - "[[inngest]]"
  - "[[docker]]"
  - "[[bun]]"
exactMatch:
  slog_tool: system-bus-worker
  vault_project: "Projects/07-event-bus/"
  codebase_path: "packages/system-bus/"
conceptScheme: jc:system
tags:
  - taxonomy
---
 
# System Bus Worker
 
The Inngest worker process that registers and executes durable functions for the event bus. Runs as a launchd service on the Mac Mini, serving the `/api/inngest` endpoint.
 
## Scope
 
Covers the worker process itself, its start script (`start.sh`), the launchd plist (`com.joel.system-bus-worker.plist`), and the serve entrypoint (`src/serve.ts`). Does NOT cover individual Inngest functions (those have their own concepts) or the Inngest server (see [[inngest]]).
 
## See Also
 
- [[Projects/07-event-bus/index|Event Bus Project]]
- [[ADR-0022|Webhook to System Event Pipeline]]