cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.TokenFlowDataProcessor#

class cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.TokenFlowDataProcessor(filepath, data_params)[source]#

Bases: object

Methods

get_datadict

get_stats

get_top_tokens

load_data

previous

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.launch_tokenflow.load_params

next

cerebras.modelzoo.data_preparation.data_preprocessing.tokenflow.tokenizer