cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.check_fim_special_tokens

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.check_fim_special_tokens(params, tokenizer)

previous

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.add_summarization_vsl_args

next

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.chunk