extract static method

Future<AiExtractionResult> extract(String directory, String outputDirectory, { required AiProvider provider, required Iterable<String> languages, String sourceLanguage = 'en', String? model, String? apiKey, bool dryRun = false, bool acceptAi = false, int maxRequests = 100, bool includeGenerated = false, Iterable<String> exclude = const [], Iterable<String> functions = const [], Iterable<String> getters = const [], String? sourceArb, AiClient? client, })

Implementation

static Future<AiExtractionResult> extract(
  String directory,
  String outputDirectory, {
  required AiProvider provider,
  required Iterable<String> languages,
  String sourceLanguage = 'en',
  String? model,
  String? apiKey,
  bool dryRun = false,
  bool acceptAi = false,
  int maxRequests = 100,
  bool includeGenerated = false,
  Iterable<String> exclude = const [],
  Iterable<String> functions = const [],
  Iterable<String> getters = const [],
  String? sourceArb,
  AiClient? client,
}) async {
  if (!dryRun && !acceptAi) {
    throw ArgumentError(
      'Read the AI disclaimer and pass --accept-ai to permit provider requests and API charges. Use --dry-run to preview locally.',
    );
  }
  if (maxRequests < 1) throw ArgumentError('maxRequests must be positive.');
  if (client != null && client.provider != provider)
    throw ArgumentError(
      'AI client provider does not match the selected provider.',
    );
  final sourceLocale = _locale(sourceLanguage);
  final targets = languages.map(_locale).toSet().toList()..sort();
  if (targets.isEmpty || targets.contains(sourceLocale)) {
    throw ArgumentError(
      'Supply at least one target language different from the source language.',
    );
  }
  final folder = p.normalize(p.absolute(outputDirectory));
  final files = {
    for (final locale in [sourceLocale, ...targets])
      locale: File(p.join(folder, '$locale.json')),
  };
  final catalogs = <String, Map<String, dynamic>>{};
  final original = <String, String?>{};
  for (final entry in files.entries) {
    final file = entry.value;
    CatalogIO.validateOutput(file);
    if (sourceArb != null &&
        file.existsSync() &&
        FileSystemEntity.identicalSync(file.path, sourceArb)) {
      throw ArgumentError('Output must differ from the source ARB catalog.');
    }
    original[entry.key] = file.existsSync() ? file.readAsStringSync() : null;
    catalogs[entry.key] = file.existsSync()
        ? CatalogIO.readDictionary(file)
        : <String, dynamic>{};
  }
  final sourceFile = files[sourceLocale]!;
  ExtractionResult scan(bool allStrings) =>
      FlutterStringExtractor.extractStrings(
        directory,
        sourceFile.path,
        allStrings: allStrings,
        dryRun: true,
        replace: true,
        includeGenerated: includeGenerated,
        exclude: exclude,
        functions: functions,
        getters: getters,
        sourceArb: sourceArb,
      );
  final discovered = scan(true);
  final marked = scan(false).translations;
  final warnings = [...discovered.warnings];
  final key = apiKey ?? Platform.environment[provider.environmentKey];
  final credentials = [
    if (key != null && key.isNotEmpty) key,
    for (final provider in AiProvider.values)
      if (Platform.environment[provider.environmentKey]?.isNotEmpty == true)
        Platform.environment[provider.environmentKey]!,
  ];
  final candidates = <String, String>{};
  var sensitive = 0;
  var oversized = 0;
  for (final entry in discovered.translations.entries) {
    final value = entry.value as String;
    final existing = _lookup(catalogs[sourceLocale]!, entry.key);
    if (_sensitive(entry.key, credentials) ||
        _sensitive(value, credentials) ||
        (existing is String && _sensitive(existing, credentials))) {
      sensitive++;
      continue;
    }
    if (value.trim().isEmpty) continue;
    if (value.length > 2000 ||
        (existing is String && existing.length > 2000)) {
      oversized++;
      continue;
    }
    // A named Intl/ARB message already owns this text: avoid a second raw key.
    if (!marked.containsKey(entry.key) && marked.values.contains(value))
      continue;
    candidates[entry.key] = value;
  }
  if (sensitive > 0)
    warnings.add(
      'Skipped $sensitive possible credential strings; filtering cannot guarantee that all sensitive text is detected.',
    );
  if (oversized > 0)
    warnings.add(
      'Skipped $oversized messages over the 2,000-character AI limit.',
    );
  var requests = 0;
  var translated = 0;
  final selected = <String, String>{};
  final active = dryRun
      ? null
      : client ??
            AiClient(provider: provider, apiKey: key ?? '', model: model);
  Future<Map<String, dynamic>> ask(
    String instructions,
    Map<String, dynamic> input,
  ) async {
    if (requests >= maxRequests)
      throw AiException(
        'Reached the $maxRequests request limit. No catalogs were changed. Narrow the scan or increase --max-ai-requests. Earlier requests may have incurred charges.',
      );
    requests++;
    return active!.complete(instructions, input);
  }

  try {
    if (dryRun) {
      selected.addAll(candidates);
    } else {
      final unmarked = <MapEntry<String, String>>[];
      for (final entry in candidates.entries) {
        if (marked.containsKey(entry.key) ||
            CatalogIO.containsKey(catalogs[sourceLocale]!, entry.key)) {
          selected[entry.key] = entry.value;
        } else {
          unmarked.add(entry);
        }
      }
      for (final batch in _batches(unmarked)) {
        final messages = _numbered(batch);
        final answer = await ask(_classify, {
          'task': 'classify',
          'source_language': sourceLocale,
          'messages': messages,
        });
        final ids = answer['selected_ids'];
        if (answer.length != 1 ||
            ids is! List ||
            ids.any((id) => id is! String || !messages.containsKey(id)) ||
            ids.toSet().length != ids.length) {
          throw AiException(
            'Invalid discovery IDs returned by the provider.',
          );
        }
        for (final id in ids.cast<String>()) {
          final entry = batch[int.parse(id)];
          selected[entry.key] = entry.value;
        }
      }
      for (final entry in selected.entries.toList()) {
        final existing = _lookup(catalogs[sourceLocale]!, entry.key);
        if (existing is String && existing.trim().isNotEmpty) {
          selected[entry.key] = existing;
        } else if (!CatalogIO.containsKey(
          catalogs[sourceLocale]!,
          entry.key,
        )) {
          CatalogIO.insert(catalogs[sourceLocale]!, entry.key, entry.value);
        } else {
          selected.remove(entry.key);
          warnings.add(
            'Skipped a source entry with a non-string or empty existing value; review the source catalog.',
          );
        }
      }
      final complex = selected.values.where(_complexIcu).length;
      if (complex > 0)
        warnings.add(
          'Skipped AI translation of $complex ICU plural/select messages. Source messages are retained; use an ICU-aware translation workflow for these entries.',
        );
      for (final locale in targets) {
        final missing = selected.entries
            .where(
              (e) =>
                  !CatalogIO.containsKey(catalogs[locale]!, e.key) &&
                  !_complexIcu(e.value),
            )
            .toList();
        for (final batch in _batches(missing)) {
          final messages = _numbered(batch);
          final answer = await ask(_translate, {
            'task': 'translate',
            'source_language': sourceLocale,
            'target_language': locale,
            'messages': messages,
          });
          final values = answer['translations'];
          if (answer.length != 1 ||
              values is! Map<String, dynamic> ||
              values.length != messages.length ||
              values.keys.any((id) => !messages.containsKey(id))) {
            throw AiException(
              'Provider translation IDs did not match the requested messages.',
            );
          }
          for (var i = 0; i < batch.length; i++) {
            final value = values['$i'];
            if (value is! String ||
                value.trim().isEmpty ||
                value.length > 12000 ||
                _sensitive(value, credentials) ||
                jsonEncode(_placeholders(value)) !=
                    jsonEncode(_placeholders(batch[i].value))) {
              throw AiException(
                'Provider returned an invalid translation or changed placeholders. No catalogs were changed.',
              );
            }
            CatalogIO.insert(catalogs[locale]!, batch[i].key, value);
            translated++;
          }
        }
      }
      // Finish all remote work and detect intervening edits before any write.
      for (final entry in files.entries) {
        CatalogIO.validateOutput(entry.value);
        final now = entry.value.existsSync()
            ? entry.value.readAsStringSync()
            : null;
        if (now != original[entry.key])
          throw FileSystemException(
            'A catalog changed during AI processing; rerun to preserve the edit',
            entry.value.path,
          );
      }
      for (final entry in files.entries) {
        final dictionary = catalogs[entry.key]!;
        final sorted = {
          for (final key in dictionary.keys.toList()..sort())
            key: dictionary[key],
        };
        final json =
            '${const JsonEncoder.withIndent('  ').convert(sorted)}\n';
        if (json != original[entry.key])
          CatalogIO.writeOutput(entry.value, json);
      }
    }
    return AiExtractionResult(
      messages: Map.unmodifiable(selected),
      filesScanned: discovered.filesScanned,
      translatedCount: translated,
      requestCount: requests,
      warnings: List.unmodifiable(warnings),
      sourceLanguage: sourceLocale,
      languages: List.unmodifiable(targets),
      outputFiles: List.unmodifiable(files.values.map((file) => file.path)),
    );
  } finally {
    if (client == null) active?.close();
  }
}