modelzoo.transformers.data_processing.slimpajama.utils.utils.write_lmd_dataset

modelzoo.transformers.data_processing.slimpajama.utils.utils.write_lmd_dataset(fh, lines, indices=None, return_total_written=False)

previous

modelzoo.transformers.data_processing.slimpajama.utils.utils.utf8len

next

modelzoo.transformers.data_processing.tokenizers