modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.Reader#

class modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.Reader[source]#

Bases: object

Methods

read_gz

read_json

read_jsongz

read_jsonl

read_jsonl_tar

read_jsonl_zst

read_parquet

read_txt

stream_data

__init__(in_path, tokenizable_columns)[source]#

previous

modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.DatasetStats

next

modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.VerificationArgs