search method

Future<KnowledgeSearchResponse> search(String query, { KnowledgeRetrievalMode mode = KnowledgeRetrievalMode.bm25, KnowledgeSearchPolicy? policy, int limit = 5, int? contextLimit, int candidateLimit = 50, })

Implementation

Future<KnowledgeSearchResponse> search(
  String query, {
  KnowledgeRetrievalMode mode = KnowledgeRetrievalMode.bm25,
  KnowledgeSearchPolicy? policy,
  int limit = 5,
  int? contextLimit,
  int candidateLimit = 50,
}) async {
  final snapshot = _snapshot;
  if (snapshot == null || _synchronizing) {
    throw StateError('Synchronize the index before searching.');
  }
  if (candidateLimit <= 0) {
    throw ArgumentError.value(candidateLimit, 'candidateLimit');
  }
  final budget = contextLimit ?? limit;
  if (query.trim().isEmpty || limit <= 0 || budget <= 0) {
    return KnowledgeSearchResponse(matches: [], context: [], notices: []);
  }
  final effective = policy ?? KnowledgeSearchPolicy();
  final conceptPaths = snapshot.conceptPaths;
  for (final path in [
    ...effective.governingSources.keys,
    ...effective.governingSources.values,
  ]) {
    if (!conceptPaths.contains(path)) {
      throw ArgumentError('Unknown governing concept: $path');
    }
  }
  _searches++;
  try {
    final eligible = {
      for (final chunk in snapshot.chunks)
        if (effective.allows(snapshot, chunk.sourcePath)) chunk.id: chunk,
    };
    final options = SearchOptions(
      filePaths: eligible.values
          .map((chunk) => chunk.sourcePath)
          .toSet()
          .toList(),
    );
    final lexical = mode == KnowledgeRetrievalMode.dense || eligible.isEmpty
        ? <SearchResult>[]
        : (_lexical ??= BM25LexicalIndex.fromChunks(
                snapshot.chunks.map(
                  (chunk) => chunk.copyWith(
                    id: chunk.id,
                    content: snapshot.textFor(
                      chunk,
                      includeContext: includeContext,
                    ),
                  ),
                ),
              ))
              .search(query, limit: snapshot.chunks.length, options: options)
              .where((hit) => eligible.containsKey(hit.chunk.id))
              .map(
                (hit) => SearchResult(
                  chunk: eligible[hit.chunk.id]!,
                  embedding: null,
                  similarity: hit.similarity,
                ),
              )
              .toList();
    final dense = <SearchResult>[];
    if (mode != KnowledgeRetrievalMode.bm25) {
      if (embedder == null) {
        throw StateError('Semantic search requires an embedder.');
      }
      if (eligible.isNotEmpty) {
        final key = query.trim();
        final vector =
            _queryCache.remove(key) ??
            await embedder!.generateQueryVector(key);
        _queryCache[key] = vector;
        if (_queryCache.length > 100) {
          _queryCache.remove(_queryCache.keys.first);
        }
        final stored = await store.getEmbeddingsForChunks(
          eligible.keys.toSet(),
          source: embedder!.sourceName,
          modelName: embeddingModelName!,
        );
        if (stored.length != eligible.length) {
          throw StateError('Vectors changed; synchronize the index again.');
        }
        for (final embedding in stored) {
          dense.add(
            SearchResult(
              chunk: eligible[embedding.chunkId]!,
              embedding: embedding,
              similarity: cosineSimilarity(vector, embedding.vector),
            ),
          );
        }
        dense.sort((a, b) {
          final score = b.similarity.compareTo(a.similarity);
          if (score != 0) return score;
          final path = a.chunk.sourcePath.compareTo(b.chunk.sourcePath);
          return path == 0 ? a.chunk.id.compareTo(b.chunk.id) : path;
        });
      }
    }
    final ranked = switch (mode) {
      KnowledgeRetrievalMode.bm25 => lexical,
      KnowledgeRetrievalMode.dense => dense,
      KnowledgeRetrievalMode.hybrid => ReciprocalRankFusion.fuse([
        lexical
            .take(candidateLimit < limit ? limit : candidateLimit)
            .toList(),
        dense.take(candidateLimit < limit ? limit : candidateLimit).toList(),
      ], limit: eligible.length),
    };
    return contextForMatches(
      ranked,
      policy: effective,
      limit: limit,
      contextLimit: budget,
    );
  } finally {
    _searches--;
  }
}