modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.split_files#

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.split_files(input_dir, n_proc)[source]#

previous

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.lsh

next

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicates_dict