cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_text#

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_text(text, sp_model)[source]#

previous

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_dataset

next

cerebras.modelzoo.data_preparation.nlp.t5.test_convert_t5_dataset