initialize static method

Future<RagEngine> initialize({ required RagConfig config, void onProgress(String status)?, })

Initialize RagEngine with all dependencies.

This method handles:

1. Copying tokenizer asset to documents directory
2. Initializing the tokenizer
3. Loading the ONNX embedding model
4. Initializing the RAG database

config - Configuration containing asset paths and options. onProgress - Optional callback for initialization status updates.

Example:

final rag = await RagEngine.initialize(
  config: RagConfig.fromAssets(
    tokenizerAsset: 'assets/tokenizer.json',
    modelAsset: 'assets/model.onnx',
  ),
  onProgress: (status) => setState(() => _status = status),
);

Implementation

static Future<RagEngine> initialize({
  required RagConfig config,
  void Function(String status)? onProgress,
}) async {
  // 0. Auto-initialize Rust library (safe to call multiple times)
  await _ensureRustInitialized();

  // 1. Get app documents directory
  final dir = await getApplicationDocumentsDirectory();
  final dbPath = "${dir.path}/${config.databaseName ?? 'rag.sqlite'}";
  final tokenizerPath =
      config.preparedTokenizerPath ?? "${dir.path}/tokenizer.json";
  final modelPath =
      config.preparedModelPath ??
      "${dir.path}/${config.modelAsset.split('/').last}";

  // 2. Copy and initialize tokenizer
  onProgress?.call('Initializing tokenizer...');
  if (config.preparedTokenizerPath == null) {
    await _copyAssetToFile(config.tokenizerAsset, tokenizerPath);
  }
  await initTokenizer(tokenizerPath: tokenizerPath);
  final vocabSize = getVocabSize();

  // 3. Prepare ONNX embedding model (Copy logic)
  onProgress?.call('Preparing embedding model...');
  // Copy model asset to file (optimized for memory)
  if (config.preparedModelPath == null) {
    await _copyAssetToFile(config.modelAsset, modelPath);
  }

  final normalizedMaxChunkChars = normalizeMaxChunkChars(
    config.maxChunkChars,
    context: 'RagEngine.initialize',
  );
  final normalizedOverlapChars = normalizeOverlapChars(
    config.overlapChars,
    context: 'RagEngine.initialize',
  );
  warnThreadConfigConflict(
    threadLevel: config.threadLevel,
    embeddingIntraOpNumThreads: config.embeddingIntraOpNumThreads,
    context: 'RagEngine.initialize',
  );

  // Default to half the cores if not specified to prevent full CPU usage
  // Calculate threads based on configuration
  int threads;
  final totalCores = Platform.numberOfProcessors;

  if (config.threadLevel != null) {
    // 1. Thread Level (Percentage based)
    switch (config.threadLevel!) {
      case ThreadUseLevel.low:
        threads = (totalCores * 0.2).ceil();
        break;
      case ThreadUseLevel.medium:
        threads = (totalCores * 0.4).ceil();
        break;
      case ThreadUseLevel.high:
        threads = (totalCores * 0.8).ceil();
        break;
    }
  } else if (config.embeddingIntraOpNumThreads != null) {
    // 2. Manual Count
    threads = config.embeddingIntraOpNumThreads!;
  } else {
    // 3. Priority: Default (50% safe fallback)
    threads = (totalCores > 1 ? (totalCores / 2).ceil() : 1);
  }

  // Ensure at least 1 thread
  if (threads < 1) threads = 1;

  debugPrint(
    '[RagEngine] Configured ONNX embedding threads: $threads (Total Cores: $totalCores)',
  );

  // Init EmbeddingService on a background worker isolate.
  // Thread config is passed as a raw int; the worker creates
  // OrtSessionOptions internally (native objects can't cross isolate
  // boundaries).
  onProgress?.call('Loading embedding model...');
  var embeddingWorkerInitialized = false;
  try {
    await EmbeddingService.init(
      modelPath: modelPath,
      intraOpNumThreads: threads,
    );
    embeddingWorkerInitialized = true;

    // 4. Probe the actual model output dimension and configure Rust before
    // any database or MMAP write can occur. The host explicitly supplies the
    // variance profile; the filename and dimension are never used to infer it.
    onProgress?.call('Validating VABQ profile...');
    final probe = await EmbeddingService.embed(_kFingerprintProbeText);
    final expectedDimension = config.expectedEmbeddingDimension;
    if (expectedDimension != null && probe.length != expectedDimension) {
      RagModelPackManifest.validateExpectedEmbeddingDimension(
        expectedDimension: expectedDimension,
        actualDimension: probe.length,
      );
    }
    await vabq_config.configureVabqProfile(
      profile: config.vabqProfile == VabqProfile.none
          ? null
          : vabqProfileWireName(config.vabqProfile),
      embeddingDimension: probe.length,
    );

    // 5. Initialize database connection pool
    onProgress?.call('Initializing connection pool...');
    await initDbPool(dbPath: dbPath, maxSize: 4);

    final currentFingerprint = computeEmbeddingFingerprint(
      modelBasename: embeddingModelBasename(modelPath),
      dim: probe.length,
      quant: embeddingQuantizationFingerprintAxis(config.vabqProfile),
    );

    // 6. Initialize the database and resolve the embedding fingerprint before
    // any deferred BM25/HNSW warmup can contend for the same SQLite database.
    onProgress?.call('Initializing database...');
    final ragService = SourceRagService(
      dbPath: dbPath,
      modelPath: modelPath,
      maxChunkChars: normalizedMaxChunkChars,
      overlapChars: normalizedOverlapChars,
    );
    RagEmbeddingFingerprintLock? initialLock;
    await ragService.initForEngine(
      deferIndexWarmup: config.deferIndexWarmup,
      afterDatabaseInitialized: () async {
        // The fingerprint tables are created by SourceRagService.init, so this
        // must remain inside its pre-warmup database-ready boundary.
        onProgress?.call('Validating embedding fingerprint...');
        initialLock = await _resolveFingerprintGate(currentFingerprint);
      },
    );

    // 7. Surface an existing embedding mismatch after the database and index
    // initialization sequence has been established.
    final resolvedInitialLock = initialLock;
    if (resolvedInitialLock != null) {
      debugPrint(
        '[RagEngine] Embedding fingerprint mismatch detected: '
        'stored="${resolvedInitialLock.stored}", current="$currentFingerprint", '
        'remaining=${resolvedInitialLock.remainingChunks}, '
        'resume=${resolvedInitialLock.resumeInProgress}. '
        'Search/ingest will be locked until reembedAll() or clearAndRestart().',
      );
    }

    onProgress?.call('Ready!');
    return RagEngine._(
      ragService: ragService,
      dbPath: dbPath,
      vocabSize: vocabSize,
      currentEmbeddingFingerprint: currentFingerprint,
      initialLock: resolvedInitialLock,
      deferIndexWarmup: config.deferIndexWarmup,
    );
  } catch (_) {
    if (embeddingWorkerInitialized) {
      await EmbeddingService.disposeAsync();
    }
    rethrow;
  }
}