modelzoo.transformers.data_processing.tokenizers.BPETokenizer.BPETokenizer

class modelzoo.transformers.data_processing.tokenizers.BPETokenizer.BPETokenizer

Bases: object

Methods

add_token
bpe
decode
encode
get_token_id

__init__(vocab_file, encoder_file, errors='replace', special_tokens=None)

previous

modelzoo.transformers.data_processing.tokenizers.BPETokenizer.get_pairs

next

modelzoo.transformers.data_processing.tokenizers.HFTokenizer