extract static method

ExtractionResult extract(Uint8List bytes, { BanglaOcrHook? ocrHook, String password = '', })

Extracts bytes.

Never throws: an unreadable document comes back as an empty result with BanglaTextEncoding.none rather than an exception, because the documents most worth extracting are also the most likely to be damaged. password opens a document that needs one. Most protected PDFs do not: they carry an owner password and an empty user password, so they are decrypted without it.

Implementation

static ExtractionResult extract(
  Uint8List bytes, {
  BanglaOcrHook? ocrHook,
  String password = '',
}) {
  final reader = PdfReader.open(bytes, password: password);
  if (reader == null) {
    return const ExtractionResult(
      text: '',
      pages: <ExtractedPage>[],
      encodingDetected: BanglaTextEncoding.none,
      confidence: 0,
    );
  }

  final pages = <ExtractedPage>[];
  var sawUnicode = false;
  var sawBijoy = false;
  var certainty = 0.0;
  var counted = 0;

  final pageDicts = reader.pages();
  for (var i = 0; i < pageDicts.length; i++) {
    final page = pageDicts[i];
    final fonts = _fontsOf(reader, page);
    final content = walkContentStream(reader.contentOf(page), fonts);

    final buffer = StringBuffer();
    var pageUnicode = false;
    var pageBijoy = false;
    var mapped = 0;
    var inferred = 0;
    var unmapped = 0;

    double? lastY;
    var lastObject = -1;
    for (final run in content.runs) {
      if (run.text.isEmpty) continue;
      if (lastY != null && (lastY - run.y).abs() > run.fontSize * 0.4) {
        // A change in device-space Y is a new line.
        buffer.write('\n');
      } else if (lastY != null &&
          run.objectIndex != lastObject &&
          _needsSpaceBetween(buffer.toString(), run.text)) {
        // Same line, new text object: a word-by-word producer drew a space
        // by repositioning rather than by emitting one.
        buffer.write(' ');
      }
      lastY = run.y;
      lastObject = run.objectIndex;

      final font = run.font;
      // Only a run's own font can say whether its bytes are Bijoy: the byte
      // 'e' is English "e" in one font and ব in another.
      final treatAsBijoy =
          font != null && font.isBijoy && bijoyConfidence(run.text) > 0;
      if (treatAsBijoy) {
        buffer.write(bijoyToUnicode(run.text));
        pageBijoy = true;
        mapped++;
      } else {
        buffer.write(run.text);
        if (RegExp('[ঀ-৿]').hasMatch(run.text)) pageUnicode = true;
        // Whitespace says nothing about how well a document was read. Word
        // draws thousands of lone spaces in a mapping-free WinAnsi font;
        // counted, they would bury a perfectly read page.
        if (run.text.trim().isEmpty) continue;
        // Text read back through the font is inference, whether the document
        // gave no mapping or one that had to be ignored: better than
        // nothing, not as good as being told. A simple font's codes are
        // defined by its encoding even without a CMap; only a CID font can
        // say nothing at all.
        if (font != null &&
            font.textMappingUntrusted &&
            font.reverseMap != null) {
          inferred++;
        } else if (font == null ||
            (font.toUnicode.isEmpty && font.codesAreGlyphIds)) {
          unmapped++;
        } else {
          mapped++;
        }
      }
    }

    var text = buffer.toString().trim();
    final encoding = _classify(pageUnicode, pageBijoy, text);

    var pageResult = ExtractedPage(
      number: i + 1,
      text: text,
      encodingDetected: encoding,
      hasImages: content.imageCount > 0,
    );

    // A page with an image and no words is a scan even if something is
    // written on it: Word stamps a page number on an inserted scan, and that
    // alone must not keep the page from OCR.
    final wordless = !RegExp(r'\p{L}', unicode: true).hasMatch(text);
    if (ocrHook != null &&
        (text.isEmpty || (pageResult.hasImages && wordless))) {
      final recovered = ocrHook(pageResult);
      if (recovered != null && recovered.isNotEmpty) {
        text = recovered;
        pageResult = ExtractedPage(
          number: i + 1,
          text: text,
          encodingDetected: BanglaTextEncoding.unicode,
          hasImages: pageResult.hasImages,
        );
      }
    }

    pages.add(pageResult);
    if (pageUnicode) sawUnicode = true;
    if (pageBijoy) sawBijoy = true;

    if (text.isNotEmpty) {
      counted++;
      certainty += content.actualTextUsed
          ? 1.0
          : (mapped + inferred + unmapped == 0
              ? 0.0
              : (mapped + inferred * _inferredWeight) /
                  (mapped + inferred + unmapped) *
                  (pageBijoy ? 0.85 : 1.0));
    }
  }

  final joined =
      pages.map((p) => p.text).where((t) => t.isNotEmpty).join('\n\n');

  return ExtractionResult(
    text: joined,
    pages: pages,
    encodingDetected: _classify(sawUnicode, sawBijoy, joined),
    confidence: counted == 0 ? 0.0 : (certainty / counted).clamp(0.0, 1.0),
    isEncrypted: reader.isEncrypted,
    isLocked: reader.isLocked,
  );
}