cerebras.modelzoo.data_preparation.data_preprocessing.utils.validate_tokens

cerebras.modelzoo.data_preparation.data_preprocessing.utils.validate_tokens(tokens, min_len=2)

previous

cerebras.modelzoo.data_preparation.data_preprocessing.utils.update_params

next

cerebras.modelzoo.data_preparation.data_preprocessing.utils.wikitext_detokenizer