modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_documents#

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_documents(input_dir, index_start, index_end, output_dir, dataset_name)[source]#

previous

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.generate_hashes

next

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_features