modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_documents

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_documents(input_dir, index_start, index_end, output_dir, dataset_name)

previous

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.generate_hashes

next

modelzoo.transformers.data_processing.slimpajama.dedup.to_hash.get_features