data-prep-toolkit>=0.2.1
duckdb>=0.10.1
fasttext==0.9.2
langcodes==3.3.0
huggingface-hub<1.0.0,>=0.21.4
numpy==1.26.4
mmh3>=4.1.0
xxhash==3.4.1
tqdm==4.66.3
scipy==1.12.0
mmh3>=4.1.0
xxhash==3.4.1
sentence-transformers>=3.0.1
trafilatura==1.12.0
transformers==4.38.2
