scrape method

Future<Map<String, Object>> scrape({
  1. required Uri url,
  2. required Map<String, List<Config>> configMap,
  3. int configIndex = 0,
  4. Uri? proxyUrl,
  5. bool debug = false,
  6. Document? html,
  7. Map<String, String>? cookies,
  8. Map<String, String>? headers,
  9. String? userAgent,
  10. bool concurrentParsing = false,
})

Implementation

Future<Map<String, Object>> scrape({
  required Uri url,
  required Map<String, List<Config>> configMap,
  int configIndex = 0,
  Uri? proxyUrl,
  bool debug = false,
  Document? html,
  Map<String, String>? cookies,
  Map<String, String>? headers,
  String? userAgent,
  bool concurrentParsing = false,
}) async {
  /// Fetch config and target
  Config? config = getConfig(
    url,
    configs: configMap,
    configIndex: configIndex,
  );
  UrlTarget? urlTarget;
  if (config != null) {
    urlTarget = fetchTarget(config.urlTargets, url);
  }
  if (config == null || urlTarget == null) {
    throw WebScraperError('Unsupported URL');
  }

  /// Scrape the URL
  Scraper scraping = Scraper();
  Data scrapedData = await scraping.scrape(
    url: url,
    html: html,
    debug: debug,
    config: config,
    cookies: cookies,
    headers: headers,
    userAgent: userAgent,
    proxyUrl: proxyUrl,
  );

  /// Parse HTML
  WebParser webParser = WebParser();
  Map<String, Object> parsedData = await webParser.parse(
    scrapedData: scrapedData,
    config: config,
    proxyUrl: proxyUrl,
    cookies: cookies,
    debug: debug,
    concurrentParsing: concurrentParsing,
  );

  return parsedData;
}