cmoncrawl.processor.pipeline.downloader.AsyncDownloader

class cmoncrawl.processor.pipeline.downloader.AsyncDownloader(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 10, encoding: str = 'latin-1')

__init__(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 10, encoding: str = 'latin-1')

Methods

__init__([base_url, digest_verification, ...])
aclose(exc_type, exc_val[, exc_tb])
aopen()
download(domain_record)
unwrap(response, domain_record)