modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.lsh#

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.lsh(out_file, doc_queue, idx)[source]#

previous

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.get_hashes

next

modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs.split_files