modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train.deduplicate_train_holdout_sets#

modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train.deduplicate_train_holdout_sets(train_path, holdout_path, deduped_train_path, chunk_id)[source]#

previous

modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train

next

modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components