KnowledgeSnapshot.fromSources constructor

KnowledgeSnapshot.fromSources(Map<String, String> sources, { required String bundleId, Iterable<String> assets = const [], int maxChunkLength = 1000, })

Parses source text through OKF; producer-defined metadata is retained. bundleId must be stable across updates and unique within a shared store.

Implementation

factory KnowledgeSnapshot.fromSources(
  Map<String, String> sources, {
  required String bundleId,
  Iterable<String> assets = const [],
  int maxChunkLength = 1000,
}) {
  if (bundleId.trim().isEmpty) {
    throw ArgumentError.value(bundleId, 'bundleId');
  }
  final documents = <String, OkfDocument>{};
  final indexes = <String, String>{};
  final logs = <String, String>{};
  for (final entry in sources.entries) {
    switch (p.posix.basename(entry.key)) {
      case 'index.md':
        indexes[entry.key] = entry.value;
      case 'log.md':
        logs[entry.key] = entry.value;
      default:
        documents[entry.key] = OkfDocument.parse(
          entry.value,
          sourcePath: entry.key,
        );
    }
  }
  final bundle = OkfBundle.fromDocuments(
    documents,
    indexes: indexes,
    logs: logs,
    assets: assets,
  );
  final chunker = MarkdownChunker(maxChunkLength: maxChunkLength);
  final chunks = <Chunk>[];
  final metadata = <String, OkfMetadata>{};
  final contextTexts = <String, String>{};
  for (final entry in bundle.concepts.entries) {
    final path = entry.key.documentPath;
    final document = entry.value;
    final frontmatter = freezeMetadataMap(
      document.frontmatter,
      name: 'frontmatter',
    );
    metadata[path] = OkfMetadata.fromFrontmatter(frontmatter);
    final source = sources[path]!
        .replaceAll('\r\n', '\n')
        .replaceAll('\r', '\n');
    // OKF's body is an exact normalized suffix, including its optional blank
    // separator handling. Derive offsets without another frontmatter parser.
    final prefix = source.substring(0, source.length - document.body.length);
    final offset = '\n'.allMatches(prefix).length;
    final bodyChunks = chunker.chunkContent(
      document.body,
      ChunkMetadata(sourcePath: path, contentType: 'markdown'),
    );
    final headings = <({int level, String text})>[];
    // Headings and footnote definitions are document apparatus: headings
    // become context for the passages under them, and OKF resolves per-claim
    // attribution through `sources` rather than footnote prose (OKF 0.2
    // §5.1). A body made only of apparatus still has to be searchable.
    final onlyApparatus = bodyChunks.every(
      (chunk) => chunk.type == 'heading' || chunk.type == 'footnote',
    );
    for (final raw in bodyChunks) {
      if (raw.type == 'heading') {
        final level = raw.metadata['headingLevel']! as int;
        headings.removeWhere((heading) => heading.level >= level);
        headings.add((level: level, text: raw.content));
        if (!onlyApparatus) continue;
      } else if (raw.type == 'footnote' && !onlyApparatus) {
        continue;
      }
      // Identity uses body-relative ranges so adding frontmatter does not
      // invalidate unchanged body embeddings. Citation ranges remain absolute.
      final id = sha256
          .convert(utf8.encode(jsonEncode([bundleId, raw.id])))
          .toString();
      final chunk = raw.copyWith(
        id: id,
        lineStart: raw.lineStart + offset,
        lineEnd: raw.lineEnd + offset,
        metadata: {
          'okf': {
            'bundleId': bundleId,
            'frontmatter': frontmatter,
            'headingPath': headings.map((heading) => heading.text).toList(),
          },
        },
      );
      chunks.add(chunk);
      final context = <String>{
        document.title ?? entry.key.value.split('/').last,
        ...headings.map((heading) => heading.text),
      }.where((text) => text.trim().isNotEmpty && text != raw.content);
      contextTexts[id] = [...context, raw.content].join('\n\n');
    }
  }
  return KnowledgeSnapshot._(
    bundleId,
    List.unmodifiable(chunks),
    OkfGraph.fromBundle(bundle),
    Map.unmodifiable(metadata),
    Map.unmodifiable(contextTexts),
    Map.unmodifiable(sources),
    List.unmodifiable(assets),
  );
}