modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.VerificationArgs#

class modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.VerificationArgs[source]#

Bases: object

VerificationArgs(processes: int, files_per_record: int, max_seq_length: int, tokenizer_obj: object, eos_id: int, pad_id: int, use_vsl: bool)

Methods

Attributes

`processes`
`files_per_record`
`max_seq_length`
`tokenizer_obj`
`eos_id`
`pad_id`
`use_vsl`

__init__(processes: int, files_per_record: int, max_seq_length: int, tokenizer_obj: object, eos_id: int, pad_id: int, use_vsl: bool) → None#

previous

modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.Reader

next

modelzoo.transformers.data_processing.scripts.hdf5_shuffling