uploadBytes method

Future<void> uploadBytes(Uint8List data, String fileName, String parentUuid, { int maxConcurrentChunks = kDefaultUploadConcurrency, dynamic onProgress(int bytesUploaded, int totalBytes)?, })

Upload raw bytes from memory (needed for Web platform where File I/O is not available).

Implementation

Future<void> uploadBytes(
  Uint8List data,
  String fileName,
  String parentUuid, {
  int maxConcurrentChunks = kDefaultUploadConcurrency,
  Function(int bytesUploaded, int totalBytes)? onProgress,
}) async {
  api.log(
      'Starting memory upload for $fileName (${formatSize(data.length)})');

  final size = data.length;
  final uuid = crypto.uuid();
  final mk = masterKeys.last;
  if (mk.isEmpty) throw Exception('No master keys available');

  final fileKeyStr = crypto.randomString(32);
  final fileKeyBytes = Uint8List.fromList(utf8.encode(fileKeyStr));

  if (size == 0) {
    final metaJson = json.encode({
      'name': fileName,
      'size': 0,
      'mime': 'application/octet-stream',
      'key': fileKeyStr,
      'hash': '',
      'lastModified': DateTime.now().millisecondsSinceEpoch,
    });

    final nameEncrypted =
        await crypto.encryptMetadata002(fileName, fileKeyStr);
    final sizeEncrypted = await crypto.encryptMetadata002('0', fileKeyStr);
    final mimeEncrypted = await crypto.encryptMetadata002(
        'application/octet-stream', fileKeyStr);
    final metadataEncrypted = await crypto.encryptMetadata002(metaJson, mk);
    final nameHashed = await crypto.hashFileName(fileName, masterKeys, email);

    await api.post('/v3/upload/empty', {
      'uuid': uuid,
      'name': nameEncrypted,
      'nameHashed': nameHashed,
      'size': sizeEncrypted,
      'parent': parentUuid,
      'mime': mimeEncrypted,
      'metadata': metadataEncrypted,
      'version': 2,
    });

    if (onProgress != null) onProgress(0, 0);
    cache.invalidate(parentUuid);
    return;
  }

  final uploadKey = crypto.randomString(32);
  final rm = crypto.randomString(32);
  const chunkSz = 1048576;
  final ingest = 'https://ingest.filen.io';
  final totalChunks = (size / chunkSz).ceil();

  final digestSink = DigestSink();
  final byteSink = crypto_pkg.sha512.startChunkedConversion(digestSink);

  // POST one already-encrypted chunk with up to 3 retries; throws if all fail.
  Future<void> postChunk(int idx, Uint8List enc) async {
    final hashHex =
        HEX.encode(crypto_pkg.sha512.convert(enc).bytes).toLowerCase();
    final url = Uri.parse(
        '$ingest/v3/upload?uuid=$uuid&index=$idx&parent=$parentUuid&uploadKey=$uploadKey&hash=$hashHex');
    var retry = 0;
    while (true) {
      try {
        final r = await api.client.post(url, body: enc, headers: {
          'Authorization': 'Bearer ${api.apiKey}'
        }).timeout(Duration(seconds: 45));
        if (r.statusCode != 200) {
          throw Exception('Status ${r.statusCode}: ${r.body}');
        }
        return;
      } catch (e) {
        retry++;
        api.log('Chunk $idx failed (Attempt $retry): $e');
        if (retry >= 3) rethrow;
        await Future.delayed(Duration(seconds: 1));
      }
    }
  }

  // Tiny files keep the simple sequential path; larger ones overlap chunks
  // bounded by both the semaphore (count) and MemoryGate (bytes).
  final useConcurrency =
      maxConcurrentChunks > 1 && totalChunks > kSequentialChunkThreshold;
  var doneBytes = 0;

  if (useConcurrency) {
    final sem = ChunkSemaphore(maxConcurrentChunks);
    final inflight = <Future<void>>[];
    Object? firstError;

    var offset = 0;
    var index = 0;
    while (offset < size) {
      final end = min(size, offset + chunkSz);
      final chunkBytes = data.sublist(offset, end);
      byteSink.add(chunkBytes); // in-order plaintext hash
      offset = end;
      final myIdx = index++;
      if (firstError != null) break;

      await sem.acquire();
      if (firstError != null) {
        sem.release();
        break;
      }
      final enc = await crypto.encryptData(chunkBytes, fileKeyBytes);
      final budget = chunkBytes.length + enc.length;
      await memoryGate.acquire(budget);

      inflight.add(() async {
        try {
          await postChunk(myIdx, enc);
          doneBytes += chunkBytes.length;
          if (onProgress != null) onProgress(doneBytes, size);
        } catch (e) {
          firstError ??= e;
        } finally {
          memoryGate.release(budget);
          sem.release();
        }
      }());
    }

    await Future.wait(inflight);
    if (firstError != null) throw firstError!;
  } else {
    var offset = 0;
    var index = 0;
    while (offset < size) {
      final end = min(size, offset + chunkSz);
      final chunkBytes = data.sublist(offset, end);
      byteSink.add(chunkBytes); // in-order plaintext hash
      offset = end;
      final enc = await crypto.encryptData(chunkBytes, fileKeyBytes);
      await postChunk(index++, enc);
      doneBytes += chunkBytes.length;
      if (onProgress != null) onProgress(doneBytes, size);
    }
  }

  byteSink.close();
  final totalHash = HEX.encode(digestSink.value?.bytes ?? []).toLowerCase();

  final metaJsonWithHash = json.encode({
    'name': fileName,
    'size': size,
    'mime': 'application/octet-stream',
    'key': fileKeyStr,
    'hash': totalHash,
    'lastModified': DateTime.now().millisecondsSinceEpoch,
  });

  final nameEncrypted = await crypto.encryptMetadata002(fileName, fileKeyStr);
  final sizeEncrypted =
      await crypto.encryptMetadata002(size.toString(), fileKeyStr);
  final mimeEncrypted =
      await crypto.encryptMetadata002('application/octet-stream', fileKeyStr);
  final metadataEncryptedWithHash =
      await crypto.encryptMetadata002(metaJsonWithHash, mk);
  final nameHashed = await crypto.hashFileName(fileName, masterKeys, email);

  await api.post('/v3/upload/done', {
    'uuid': uuid,
    'name': nameEncrypted,
    'nameHashed': nameHashed,
    'size': sizeEncrypted,
    'chunks': totalChunks,
    'mime': mimeEncrypted,
    'rm': rm,
    'metadata': metadataEncryptedWithHash,
    'version': 2,
    'uploadKey': uploadKey,
  });

  cache.invalidate(parentUuid);
}