data-prep-toolkit>=0.2.1
#filter
duckdb>=0.10.1
#langid
fasttext==0.9.2
langcodes==3.3.0
huggingface-hub >= 0.21.4, <1.0.0
numpy==1.26.4
#fdedup
mmh3>=4.1.0
xxhash==3.4.1
tqdm==4.66.3
scipy==1.12.0
# ededup
mmh3>=4.1.0,
xxhash==3.4.1
#text_encoder
sentence-transformers>=3.0.1
#html2parquet
trafilatura==1.12.0
#tokenization
transformers==4.38.2

#ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
#data-prep-toolkit-transforms 0.2.2.dev0 requires duckdb==0.10.1, but you have duckdb 1.1.0 which is incompatible.
#data-prep-toolkit-transforms 0.2.2.dev0 requires sentence-transformers==3.0.1, but you have sentence-transformers 3.1.1 which is incompatible.






