dsbuild 0.1.0-alpha.5
dsbuild: ^0.1.0-alpha.5 copied to clipboard
Dataset preparation for LM training. Usable as a standalone tool and a library. Process conversations using StreamTransformers configured with simple YAML. Supports multithreading.
import 'dart:io';
import 'package:dsbuild/dsbuild.dart';
import 'package:dsbuild/progress.dart';
import 'package:yaml/yaml.dart';
void main(List<String> args) async {
// Load a descriptor. You can also build a descriptor programmatically.
String descriptorPath = args.isEmpty ? 'dataset.yaml' : args[0];
DsBuild dsBuild = DsBuild(
DatasetDescriptor.fromYaml(
loadYaml(await File(descriptorPath).readAsString())),
ProgressBloc(ProgressState()));
// Listen to progress events
DateTime lastProgressOutput = DateTime.timestamp();
dsBuild.progress.stream.listen((state) {
if (DateTime.timestamp().difference(lastProgressOutput).inSeconds > 5) {
lastProgressOutput = DateTime.timestamp();
print("Progress:\n"
"Messages processed: ${state.messagesProcessed} / ${state.messagesTotal}\n"
"Conversations processed: ${state.conversationsProcessed} / ${state.conversationsTotal}");
}
});
// Register additional transformers.
// The builtin transformers can also be replaced.
//dsBuild.registry.registerPreprocessor(name, (config) => Preprocessor())
// verifyDescriptor is a convenience function to output a simple list of error strings for any missing transformers.
List<String> errors = dsBuild.verifyDescriptor();
if (errors.isNotEmpty) {
for (String validationError in dsBuild.verifyDescriptor()) {
print(validationError);
}
exit(1);
} else {
print("Descriptor valid.");
}
// fetchRequirements can be used to automatically download input files with a provided source uri.
await dsBuild.fetchRequirements().forEach((input) async {
print("${input.source} retrieved.");
});
// Now could also be a good time to verify the integrity of any downloaded files.
// This may be included in the API in the future, as it is an expected feature of any downstream application.
// Build the transformation pipeline. This is the result of transforming and concatenating all inputs.
// Custom pipelines can be built using the registry and the information available in the descriptor.
for (PassDescriptor pass in dsBuild.repository.descriptor.passes) {
Stream<List<Conversation>> conversations =
dsBuild.buildPipeline(pass.steps);
await conversations.drain();
}
// Final progress stream events are expected to be pushed by the build script.
// This is more useful if you intend to trigger additional build steps.
dsBuild.progress.add(const BuildComplete());
await dsBuild.progress.close();
print("Done!");
}