validateProject method

Future<ValidationEntry> validateProject(String projectPath)

Run all analyzers on a single project and produce a ValidationEntry

Implementation

Future<ValidationEntry> validateProject(String projectPath) async {
  final expectedFindings = await loadExpectedFindings(projectPath);

  // Extract project name from expected_findings.json
  final content = await fileSystem.readAsStringAsync(
    fileSystem.join(projectPath, 'expected_findings.json'),
  );
  final json = jsonDecode(content) as Map<String, dynamic>;
  final projectName =
      json['projectName'] as String? ?? projectPath.split('/').last;

  // Run all analyzers
  final context = AnalyzerContext(
    projectPath: projectPath,
    fileSystem: fileSystem,
  );
  final results = await analyzerService.runAll(context);

  // Collect all actual findings
  final actualFindings = <DiagnosticIssue>[];
  for (final result in results) {
    for (final issue in result.issues) {
      actualFindings.add(issue);
    }
  }

  // Match expected vs actual
  final matchedExpected = <int>{};
  final truePositives = <ExpectedFinding>[];
  final falsePositives = <DiagnosticIssue>[];

  for (final actual in actualFindings) {
    var matched = false;
    for (var i = 0; i < expectedFindings.length; i++) {
      final expected = expectedFindings[i];
      if (expected.shouldBeFound &&
          expected.code == actual.code &&
          expected.analyzerName == _analyzerNameForCode(actual.code)) {
        matchedExpected.add(i);
        matched = true;
        break;
      }
    }
    if (matched) {
      truePositives.add(expectedFindings.elementAt(matchedExpected.last));
    } else {
      falsePositives.add(actual);
    }
  }

  // False negatives: expected to be found but weren't
  final falseNegatives = <ExpectedFinding>[];
  for (var i = 0; i < expectedFindings.length; i++) {
    if (expectedFindings[i].shouldBeFound && !matchedExpected.contains(i)) {
      falseNegatives.add(expectedFindings[i]);
    }
  }

  // Unmatched not-should-be-found = correct true negatives (not explicitly tracked)
  // Total checks = all expected findings
  final totalChecks = expectedFindings.length;
  // correct = truePositives + trueNegatives
  // trueNegatives = expected where shouldBeFound:false AND not in falsePositives
  final shouldBeFalseCount = expectedFindings
      .where((e) => !e.shouldBeFound)
      .length;
  final trueNegatives = (shouldBeFalseCount - falsePositives.length).clamp(
    0,
    shouldBeFalseCount,
  );
  final totalCorrect = truePositives.length + trueNegatives;
  final accuracy = totalChecks > 0 ? totalCorrect / totalChecks : 1.0;
  final precision = (truePositives.length + falsePositives.length) > 0
      ? truePositives.length / (truePositives.length + falsePositives.length)
      : 1.0;
  final recall = (truePositives.length + falseNegatives.length) > 0
      ? truePositives.length / (truePositives.length + falseNegatives.length)
      : 1.0;

  return ValidationEntry(
    projectName: projectName,
    projectPath: projectPath,
    totalChecks: totalChecks,
    truePositives: truePositives,
    falseNegatives: falseNegatives,
    falsePositives: falsePositives,
    accuracy: accuracy,
    precision: precision,
    recall: recall,
  );
}