KnowledgeSnapshot.fromSources constructor

KnowledgeSnapshot.fromSources(Map<String, String> sources, { required String bundleId, Iterable<String> assets = const [], int maxChunkLength = 1000, Iterable<OkfLinkField> linkFields = const [], })

Parses source text through OKF; producer-defined metadata is retained. bundleId must be stable across updates and unique within a shared store.

Implementation

factory KnowledgeSnapshot.fromSources(
  Map<String, String> sources, {
  required String bundleId,
  Iterable<String> assets = const [],
  int maxChunkLength = 1000,
  Iterable<OkfLinkField> linkFields = const [],
}) {
  if (bundleId.trim().isEmpty) {
    throw ArgumentError.value(bundleId, 'bundleId');
  }
  final parsed = _parseBundle(sources, assets);
  final bundle = parsed.bundle;
  final metadata = parsed.metadata;
  final chunker = MarkdownChunker(maxChunkLength: maxChunkLength);
  final chunks = <Chunk>[];
  final contextTexts = <String, String>{};
  for (final entry in bundle.concepts.entries) {
    final path = entry.key.documentPath;
    if (parsed.excludedPaths.contains(path)) continue;
    final document = entry.value;
    final frontmatter = parsed.frontmatter[path]!;
    final source = sources[path]!
        .replaceAll('\r\n', '\n')
        .replaceAll('\r', '\n');
    // OKF's body is an exact normalized suffix, including its optional blank
    // separator handling. Derive offsets without another frontmatter parser.
    final prefix = source.substring(0, source.length - document.body.length);
    final offset = '\n'.allMatches(prefix).length;
    final bodyChunks = chunker.chunkContent(
      document.body,
      ChunkMetadata(sourcePath: path, contentType: 'markdown'),
    );
    final headings = <({int level, String text})>[];
    // Headings and footnote definitions are document apparatus: headings
    // become context for the passages under them, and OKF resolves per-claim
    // attribution through `sources` rather than footnote prose (OKF 0.2
    // §5.1). A body made only of apparatus still has to be searchable.
    final onlyApparatus = bodyChunks.every(
      (chunk) => chunk.type == 'heading' || chunk.type == 'footnote',
    );
    for (final raw in bodyChunks) {
      if (raw.type == 'heading') {
        final level = raw.metadata['headingLevel']! as int;
        headings.removeWhere((heading) => heading.level >= level);
        headings.add((level: level, text: raw.content));
        if (!onlyApparatus) continue;
      } else if (raw.type == 'footnote' && !onlyApparatus) {
        continue;
      }
      // Identity uses body-relative ranges so adding frontmatter does not
      // invalidate unchanged body embeddings. Citation ranges remain absolute.
      final id = sha256
          .convert(utf8.encode(jsonEncode([bundleId, raw.id])))
          .toString();
      final chunk = raw.copyWith(
        id: id,
        lineStart: raw.lineStart + offset,
        lineEnd: raw.lineEnd + offset,
        metadata: {
          ...raw.metadata,
          'okf': {
            'bundleId': bundleId,
            'frontmatter': frontmatter,
            'headingPath': headings.map((heading) => heading.text).toList(),
          },
        },
      );
      chunks.add(chunk);
      final title = document.title ?? entry.key.value.split('/').last;
      final description = frontmatter['description'];
      // Bound optional summary context independently of model token fitting.
      // Omit oversized descriptions rather than cutting a Unicode sequence or
      // embedding an incomplete claim. The original field remains metadata.
      final summary = description is String ? description.trim() : '';
      contextTexts[id] = [
        if (title.trim().isNotEmpty) 'Title: $title',
        if (summary.isNotEmpty && summary.runes.length <= 240)
          'Description: $summary',
        if (headings.isNotEmpty)
          'Section: ${headings.map((h) => h.text).join(' > ')}',
        if (raw.metadata['tableHeader'] case final String header)
          'Table columns: $header',
        if (raw.metadata['codeLanguage'] case final String language)
          'Code language: $language',
        'Passage:',
        raw.content,
      ].join('\n\n');
    }
  }
  return KnowledgeSnapshot._(
    bundleId,
    List.unmodifiable(chunks),
    OkfGraph.fromBundle(bundle),
    List.unmodifiable(okfFieldEdges(bundle, linkFields)),
    Map.unmodifiable(metadata),
    Map.unmodifiable(contextTexts),
    Map.unmodifiable(sources),
    List.unmodifiable(assets),
  );
}