cerebras.modelzoo.data_preparation.data_preprocessing.utils.append_eos_to_multiple_semantic_regions#

cerebras.modelzoo.data_preparation.data_preprocessing.utils.append_eos_to_multiple_semantic_regions(formatted_data, data_ranges, eos_token, image_token, is_chat_data)[source]#

previous

cerebras.modelzoo.data_preparation.data_preprocessing.utils

next

cerebras.modelzoo.data_preparation.data_preprocessing.utils.check_fim_special_tokens