Index

_

__add__() (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures method) (modelzoo.transformers.data_processing.utils.maskedLmInstance method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures method) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures method) __call__() (modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling.CSDataCollatorForLanguageModeling method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.PositionalEncoding method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.TransformerModel method) (modelzoo.vision.pytorch.dit.input.transforms.LabelDropout method) (modelzoo.vision.pytorch.dit.input.transforms.NoiseGenerator method) (modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D.DownEncoderBlock2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Downsample2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.ResnetBlock2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Upsample2D method) (modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D.UpDecoderBlock2D method) (modelzoo.vision.pytorch.dit.pipeline.DiffusionPipeline method) (modelzoo.vision.pytorch.input.classification.mixup.RandomCutmix method) (modelzoo.vision.pytorch.input.classification.mixup.RandomMixup method) (modelzoo.vision.pytorch.input.classification.sampler.RepeatedAugSampler method) __init__() (cerebras_pytorch.utils.data.DataExecutor method) (cerebras_pytorch.utils.data.DataLoader method) (cerebras_pytorch.utils.data.RestartableDataLoader method) (cerebras_pytorch.utils.data.SyntheticDataset method) (cerebras_pytorch.utils.tensorboard.SummaryReader method) (cerebras_pytorch.utils.tensorboard.SummaryWriter method) (modelzoo.transformers.data_processing.bert.bertsum_data_processor.BertData method) (modelzoo.transformers.data_processing.bert.bertsum_data_processor.JsonConverter method) (modelzoo.transformers.data_processing.bert.bertsum_data_processor.RougeBasedLabelsFormatter method) (modelzoo.transformers.data_processing.bert.bertsum_data_processor.Tokenizer method) (modelzoo.transformers.data_processing.bert.dynamic_processor.PreprocessInstance method) (modelzoo.transformers.data_processing.bert.mlm_only_processor.MLMOnlyInstance method) (modelzoo.transformers.data_processing.bert.ner_data_processor.InputExample method) (modelzoo.transformers.data_processing.bert.sentence_pair_processor.SentencePairInstance method) (modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling.CSDataCollatorForLanguageModeling method) (modelzoo.transformers.data_processing.qa.qa_utils.SquadExample method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator.SummarizationTokenGenerator method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.DatasetStats method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.Reader method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils.VerificationArgs method) (modelzoo.transformers.data_processing.scripts.pubmed.preprocess.Downloader.Downloader method) (modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextFormatting.TextFormatting method) (modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding.NLTKSegmenter method) (modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding.Sharding method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method) (modelzoo.transformers.data_processing.tokenizers.BPETokenizer.BPETokenizer method) (modelzoo.transformers.data_processing.tokenizers.HFTokenizer.HFTokenizer method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.BaseTokenizer method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.FullTokenizer method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.WordPieceTokenizer method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertCSVFormatter method) (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.PositionalEncoding method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.TransformerModel method) (modelzoo.transformers.utils.attention_utils.FixedSparseAttentionBuilder method) (modelzoo.transformers.utils.attention_utils.SparseAttentionBuilder method) (modelzoo.transformers.utils.attention_utils.StridedSparseAttentionBuilder method) (modelzoo.vision.pytorch.dit.input.transforms.LabelDropout method) (modelzoo.vision.pytorch.dit.input.transforms.NoiseGenerator method) (modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D.DownEncoderBlock2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Downsample2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.ResnetBlock2D method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Upsample2D method) (modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D.UpDecoderBlock2D method) (modelzoo.vision.pytorch.dit.layers.vae.utils.DiagonalGaussianDistribution method) (modelzoo.vision.pytorch.dit.pipeline.DiffusionPipeline method) (modelzoo.vision.pytorch.dit.sample_generator.SampleGenerator method) (modelzoo.vision.pytorch.dit.samplers.DDIMSampler.DDIMSampler method) (modelzoo.vision.pytorch.dit.samplers.DDIMSampler.DDIMSamplerOutput method) (modelzoo.vision.pytorch.dit.samplers.DDPMSampler.DDPMSampler method) (modelzoo.vision.pytorch.dit.samplers.DDPMSampler.DDPMSamplerOutput method) (modelzoo.vision.pytorch.input.classification.mixup.RandomCutmix method) (modelzoo.vision.pytorch.input.classification.mixup.RandomMixup method) (modelzoo.vision.pytorch.input.classification.sampler.RepeatedAugSampler method)

__mul__() (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures method) (modelzoo.transformers.data_processing.utils.maskedLmInstance method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures method) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures method) __new__() (modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling.CSDataCollatorForLanguageModeling static method) (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures static method) (modelzoo.transformers.data_processing.utils.maskedLmInstance static method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures static method) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures static method) (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.PositionalEncoding static method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.TransformerModel static method) (modelzoo.vision.pytorch.dit.input.transforms.LabelDropout static method) (modelzoo.vision.pytorch.dit.input.transforms.NoiseGenerator static method) (modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D.DownEncoderBlock2D static method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Downsample2D static method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.ResnetBlock2D static method) (modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D.Upsample2D static method) (modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D.UpDecoderBlock2D static method) (modelzoo.vision.pytorch.dit.pipeline.DiffusionPipeline static method) (modelzoo.vision.pytorch.input.classification.mixup.RandomCutmix static method) (modelzoo.vision.pytorch.input.classification.mixup.RandomMixup static method) (modelzoo.vision.pytorch.input.classification.sampler.RepeatedAugSampler static method)

A

accumulate_and_yield() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) add() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) add_common_args() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) add_common_parser_args() (in module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils) add_csx_arguments() (in module modelzoo.common.run_utils.cli_parser) add_general_arguments() (in module modelzoo.common.run_utils.cli_parser) add_gpu_arguments() (in module modelzoo.common.run_utils.cli_parser) add_lm_args() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) add_mlm_nsp_specific_args() (in module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils) add_mlm_only_specific_args() (in module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils) add_tensor() (cerebras_pytorch.utils.tensorboard.SummaryWriter method) add_token() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method)

aggregate_state_dict() (cerebras_pytorch.utils.data.RestartableDataLoader method) already_shuffled() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method) append_to_hdf5() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) assemble_disallowlist() (in module modelzoo.common.run_utils.cli_parser)

B

BaseTokenizer (class in modelzoo.transformers.data_processing.tokenizers.Tokenization) batch_metrics (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute) batch_specs (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) BatchMetrics (class in cerebras_pytorch.utils.benchmark.utils.dataloader) benchmark_dataloader() (in module cerebras_pytorch.utils.benchmark) BertCSVFormatter (class in modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils)

BertData (class in modelzoo.transformers.data_processing.bert.bertsum_data_processor) BertInputFeatures (class in modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils) BlockType (class in modelzoo.vision.pytorch.dit.utils) BPETokenizer (class in modelzoo.transformers.data_processing.tokenizers.BPETokenizer) build_inputs() (modelzoo.vision.pytorch.dit.pipeline.DiffusionPipeline method) build_vocab() (in module modelzoo.transformers.pytorch.bert.input.utils) bytes_to_unicode() (in module modelzoo.transformers.data_processing.tokenizers.BPETokenizer)

C

calculate_total_chunks() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) calculate_total_size() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) cerebras_pytorch.utils.CSConfig (built-in class) cerebras_pytorch.utils.data.DataLoaderCheckpoint (built-in class) check_and_create_output_dirs() (in module modelzoo.common.input.utils) check_fim_special_tokens() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) check_output() (in module modelzoo.transformers.data_processing.bert.bertsum_data_processor) check_tokenizer_scheme() (in module modelzoo.transformers.data_processing.qa.qa_utils) check_unused_model_params() (in module modelzoo.transformers.pytorch.bert.utils) check_unused_params() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) chunk() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) ChunkDataPreprocessor (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor) clean() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.filter) clear() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) cls_ids (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures attribute) collect_stats() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) concatenate_documents() (in module modelzoo.transformers.pytorch.t5.input.utils) construct_denoising_objective() (in module modelzoo.transformers.pytorch.t5.input.utils) construct_graph() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components) convert_dataset_to_HDF5() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.convert_dataset_to_HDF5) convert_examples_to_features() (in module modelzoo.transformers.data_processing.qa.qa_utils) convert_examples_to_features_and_write() (in module modelzoo.transformers.data_processing.qa.qa_utils) (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner) convert_ids_to_tokens() (modelzoo.transformers.data_processing.tokenizers.Tokenization.FullTokenizer method) convert_single_example() (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner) convert_str_to_int_list() (in module modelzoo.transformers.data_processing.utils) convert_to_bert_format_files() (in module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.write_csv_data) convert_to_json_files() (in module modelzoo.transformers.data_processing.bert.bertsum_data_processor) convert_to_unicode() (in module modelzoo.transformers.data_processing.utils) (in module modelzoo.transformers.pytorch.bert.input.utils) convert_tokens_to_ids() (modelzoo.transformers.data_processing.tokenizers.Tokenization.FullTokenizer method) copy() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) count() (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures method) (modelzoo.transformers.data_processing.utils.maskedLmInstance method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures method) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures method) count_lines() (in module modelzoo.common.model_utils.count_lines)

count_total_documents() (in module modelzoo.transformers.data_processing.utils) create_arg_parser() (in module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils) create_csv() (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only_static_masking) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_static_masking) create_dataset() (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) create_diffusion_model() (modelzoo.vision.pytorch.dit.sample_generator.SampleGenerator method) create_features_auto_lm() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) create_features_auto_lm_vsl() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) create_features_summarization() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) create_features_summarization_vsl() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) create_fixed_sparse_attention_mask() (in module modelzoo.transformers.pytorch.gpt2.sparse_mask) create_h5() (in module modelzoo.transformers.pytorch.bert.input.scripts.create_hdf5_files) create_h5_mp() (in module modelzoo.transformers.pytorch.bert.input.scripts.create_hdf5_files) create_masked_lm_features() (in module modelzoo.transformers.data_processing.bert.mlm_only_processor) create_masked_lm_predictions() (in module modelzoo.transformers.data_processing.utils) (in module modelzoo.transformers.pytorch.bert.input.utils) create_npz_from_sample_folder() (in module modelzoo.vision.pytorch.dit.sample_generator) create_parser() (in module modelzoo.transformers.data_processing.bert.bertsum_data_processor) (in module modelzoo.transformers.data_processing.bert.ner_data_processor) create_pipeline() (modelzoo.vision.pytorch.dit.sample_generator.SampleGenerator method) create_preprocessing_params_with_defaults() (in module modelzoo.vision.pytorch.input.classification.utils) create_sampler() (modelzoo.vision.pytorch.dit.sample_generator.SampleGenerator method) create_transform() (in module modelzoo.vision.pytorch.input.transforms) create_transformer_input_features() (in module modelzoo.transformers.pytorch.t5.input.utils) CSDataCollatorForLanguageModeling (class in modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling) CSoftPackage (class in modelzoo) CurationCorpusPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor) cycle_documents() (in module modelzoo.transformers.data_processing.slimpajama.utils.utils)

D

data_generator() (in module modelzoo.transformers.data_processing.bert.dynamic_processor) (in module modelzoo.transformers.data_processing.bert.mlm_only_processor) (in module modelzoo.transformers.data_processing.bert.sentence_pair_processor) DataExecutor (class in cerebras_pytorch.utils.data) DataFrame (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) DataLoader (class in cerebras_pytorch.utils.data) dataloader_build_time (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) Dataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) DatasetStats (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) DDIMSampler (class in modelzoo.vision.pytorch.dit.samplers.DDIMSampler) DDIMSamplerOutput (class in modelzoo.vision.pytorch.dit.samplers.DDIMSampler) DDPMSampler (class in modelzoo.vision.pytorch.dit.samplers.DDPMSampler) DDPMSamplerOutput (class in modelzoo.vision.pytorch.dit.samplers.DDPMSampler) deaggregate_state_dict() (cerebras_pytorch.utils.data.RestartableDataLoader method) debug_or_download_individual_file() (in module modelzoo.transformers.data_processing.scripts.pile.download) deduplicate_train_holdout_sets() (in module modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train) devices() (modelzoo.common.run_utils.utils.DeviceType class method) DeviceType (class in modelzoo.common.run_utils.utils) DiagonalGaussianDistribution (class in modelzoo.vision.pytorch.dit.layers.vae.utils) DiffusionPipeline (class in modelzoo.vision.pytorch.dit.pipeline) dir_path() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method)

disable_dataloader_checkpointing() (cerebras_pytorch.utils.data.DataLoader method) discard_params() (in module modelzoo.common.run_utils.cli_parser) display_images() (in module modelzoo.vision.pytorch.dit.display_images) doc_span_index (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) documents() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method) DownEncoderBlock2D (class in modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D) download_pile() (in module modelzoo.transformers.data_processing.scripts.pile.download) download_tokenizer_files() (in module modelzoo.transformers.data_processing.scripts.pile.download) Downloader (class in modelzoo.transformers.data_processing.scripts.pubmed.preprocess.Downloader) Downsample2D (class in modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D) dump_args() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) dump_result() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.utils) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils)

E

encode() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator.SummarizationTokenGenerator method) encode_leftover_prefix() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) end_position (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute)

end_time_ns (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute) (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) epoch_metrics (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) epoch_step (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute) EpochMetrics (class in cerebras_pytorch.utils.benchmark.utils.dataloader) example_index (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute)

F

file_read_generator() (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) file_split_process_dataset() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) filter_dataset() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.filter) fim() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) FIMDataPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors) FIMTokenGenerator (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator)

find_connected_components() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components) find_last_paragraph_or_sentence_end() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) FixedSparseAttentionBuilder (class in modelzoo.transformers.utils.attention_utils) flat_map() (in module modelzoo.transformers.pytorch.t5.input.utils) format_fim() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) forward() (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.PositionalEncoding method) (modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer.TransformerModel method) (modelzoo.vision.pytorch.dit.input.transforms.NoiseGenerator method) (modelzoo.vision.pytorch.dit.pipeline.DiffusionPipeline method) (modelzoo.vision.pytorch.input.classification.mixup.RandomCutmix method) (modelzoo.vision.pytorch.input.classification.mixup.RandomMixup method) fromkeys() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) FullTokenizer (class in modelzoo.transformers.data_processing.tokenizers.Tokenization)

G

generate_connected_components_mp() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components) generate_duplicates() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicates_dict) generate_hashes() (in module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash) generate_pairs() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs) generate_samples() (in module modelzoo.transformers.data_processing.slimpajama.split_dataset) generate_square_subsequent_mask() (in module modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer) get() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) get_all_args() (in module modelzoo.common.run_utils.cli_parser) get_beta_schedule() (in module modelzoo.vision.pytorch.dit.layers.schedulers) get_compression_factor() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor) get_data_size() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) get_default_inis() (in module modelzoo.vision.pytorch.utils.run_utils) get_documents() (in module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash) get_features() (in module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash) get_files() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) get_files_in_metadata() (in module modelzoo.transformers.data_processing.utils) get_hashes() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs) get_label_id_map() (in module modelzoo.transformers.data_processing.utils) get_meta_data() (in module modelzoo.transformers.pytorch.bert.input.utils) get_named_beta_schedule() (in module modelzoo.vision.pytorch.dit.layers.schedulers) get_output_dir() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) get_output_type_shapes() (in module modelzoo.transformers.data_processing.utils) get_pairs() (in module modelzoo.transformers.data_processing.tokenizers.BPETokenizer) get_params() (in module modelzoo.common.run_utils.cli_parser) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) get_params_file() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) get_params_from_args() (in module modelzoo.common.run_utils.cli_parser) (in module modelzoo.common.run_utils.cli_pytorch) get_parser() (in module modelzoo.common.run_utils.cli_parser) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils)

get_parser_args() (in module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils) (in module modelzoo.vision.pytorch.dit.display_images) get_raw_sequence_lengths() (in module modelzoo.transformers.pytorch.t5.input.utils) get_sampler() (in module modelzoo.vision.pytorch.dit.samplers.get_sampler) get_short_documents() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.filter) get_token_id() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator.SummarizationTokenGenerator method) get_tokenizer_fns() (in module modelzoo.transformers.data_processing.qa.write_csv_qa) get_tokenizer_vocab() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) get_tokens_and_labels() (in module modelzoo.transformers.data_processing.bert.ner_data_processor) get_urls_for_tokenizer_files() (in module modelzoo.transformers.data_processing.scripts.pile.download) get_urls_from_split() (in module modelzoo.transformers.data_processing.scripts.pile.download) get_verification_args() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) get_vocab() (in module modelzoo.transformers.data_processing.utils) get_vocab_size() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) get_vocab_words() (modelzoo.transformers.data_processing.tokenizers.Tokenization.FullTokenizer method) get_whole_word_span() (in module modelzoo.transformers.pytorch.bert.input.utils) global_rate (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute) (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) global_sample_rate (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics property) global_step (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute)

H

handle_bos_token_default() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) handle_jsonl() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) handle_metadata_files() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method)

has_valid_extension() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) HDF5BasePreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor) HFTokenizer (class in modelzoo.transformers.data_processing.tokenizers.HFTokenizer) HuggingFace_BookCorpus() (in module modelzoo.transformers.data_processing.huggingface.HuggingFace_BookCorpus) HuggingFace_Eli5() (in module modelzoo.transformers.data_processing.huggingface.HuggingFace_Eli5)

I

index (modelzoo.transformers.data_processing.utils.maskedLmInstance attribute) index() (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures method) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures method) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures method) initialize_gpt2tokenizer() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) initialize_huggingfacetokenizer() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) initialize_miscellaneous_attributes() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) initialize_neoxtokenizer() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) initialize_tokenizer() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) input_ids (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute)

input_mask (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) input_token_ids (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures attribute) InputExample (class in modelzoo.transformers.data_processing.bert.ner_data_processor) InputFeatures (class in modelzoo.transformers.data_processing.qa.qa_utils) (class in modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner) is_impossible (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) is_partial (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) items() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) iteration_time (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute) iterator_creation (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute)

J

JsonConverter (class in modelzoo.transformers.data_processing.bert.bertsum_data_processor)

K

keys() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method)

L

label (modelzoo.transformers.data_processing.utils.maskedLmInstance attribute) LabelDropout (class in modelzoo.vision.pytorch.dit.input.transforms) labels (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures attribute) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures attribute) listdir_or_file() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils)

LMDataPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors) LMDataTokenGenerator (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator) load_state_dict() (cerebras_pytorch.utils.data.DataLoader method) (cerebras_pytorch.utils.data.RestartableDataLoader method) local_rate (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute) lsh() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs)

M

main() (in module modelzoo.fc_mnist.pytorch.run) (in module modelzoo.transformers.data_processing.huggingface.HF_converter_example_BookCorpus) (in module modelzoo.transformers.data_processing.huggingface.HF_converter_example_Eli5) (in module modelzoo.transformers.data_processing.qa.write_csv_qa) (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.create_hdf5_dataset) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.create_hdf5_dataset) (in module modelzoo.transformers.data_processing.scripts.pile.download) (in module modelzoo.transformers.data_processing.slimpajama.main) (in module modelzoo.transformers.data_processing.slimpajama.split_dataset) (in module modelzoo.transformers.pytorch.bert.fine_tuning.classifier.run) (in module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.run) (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.run) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only_static_masking) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_static_masking) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_hdf5_files) (in module modelzoo.transformers.pytorch.bert.run) (in module modelzoo.transformers.pytorch.bloom.run) (in module modelzoo.transformers.pytorch.btlm.run) (in module modelzoo.transformers.pytorch.falcon.run) (in module modelzoo.transformers.pytorch.gpt2.run) (in module modelzoo.transformers.pytorch.gpt3.run) (in module modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining.data_processing.split_trc_dataset) (in module modelzoo.transformers.pytorch.gptj.run) (in module modelzoo.transformers.pytorch.layers_api_demo.run) (in module modelzoo.transformers.pytorch.llama.run) (in module modelzoo.transformers.pytorch.mistral.run) (in module modelzoo.transformers.pytorch.mpt.run) (in module modelzoo.transformers.pytorch.santacoder.run) (in module modelzoo.transformers.pytorch.starcoder.run) (in module modelzoo.transformers.pytorch.t5.run) (in module modelzoo.transformers.pytorch.transformer.input.data_processing.create_meta) (in module modelzoo.transformers.pytorch.transformer.input.data_processing.split_files) (in module modelzoo.transformers.pytorch.transformer.run) (in module modelzoo.transformers.pytorch.vision_transformer.run) (in module modelzoo.vision.pytorch.dit.run) maskedLmInstance (class in modelzoo.transformers.data_processing.utils) Metrics (class in cerebras_pytorch.utils.benchmark.utils.dataloader) MLMOnlyInstance (class in modelzoo.transformers.data_processing.bert.mlm_only_processor) modelzoo module modelzoo.common module modelzoo.common.input module modelzoo.common.input.utils module modelzoo.common.model_utils module modelzoo.common.model_utils.count_lines module modelzoo.common.run_utils module modelzoo.common.run_utils.cli_parser module modelzoo.common.run_utils.cli_pytorch module modelzoo.common.run_utils.utils module modelzoo.fc_mnist module modelzoo.fc_mnist.pytorch module modelzoo.fc_mnist.pytorch.prepare_data module modelzoo.fc_mnist.pytorch.run module modelzoo.fc_mnist.pytorch.utils module modelzoo.transformers module modelzoo.transformers.data_processing module modelzoo.transformers.data_processing.bert module modelzoo.transformers.data_processing.bert.bertsum_data_processor module modelzoo.transformers.data_processing.bert.dynamic_processor module modelzoo.transformers.data_processing.bert.mlm_only_processor module modelzoo.transformers.data_processing.bert.ner_data_processor module modelzoo.transformers.data_processing.bert.sentence_pair_processor module modelzoo.transformers.data_processing.huggingface module modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling module modelzoo.transformers.data_processing.huggingface.HF_converter_example_BookCorpus module modelzoo.transformers.data_processing.huggingface.HF_converter_example_Eli5 module modelzoo.transformers.data_processing.huggingface.HuggingFace_BookCorpus module modelzoo.transformers.data_processing.huggingface.HuggingFace_Eli5 module modelzoo.transformers.data_processing.qa module modelzoo.transformers.data_processing.qa.qa_utils module modelzoo.transformers.data_processing.qa.write_csv_qa module modelzoo.transformers.data_processing.scripts module modelzoo.transformers.data_processing.scripts.chunk_preprocessing module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.create_hdf5_dataset module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.utils module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.convert_dataset_to_HDF5 module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.create_hdf5_dataset module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils module modelzoo.transformers.data_processing.scripts.hdf5_shuffling module modelzoo.transformers.data_processing.scripts.hdf5_shuffling.h5_dataset_shuffle module modelzoo.transformers.data_processing.scripts.pile module modelzoo.transformers.data_processing.scripts.pile.download module modelzoo.transformers.data_processing.scripts.pubmed module modelzoo.transformers.data_processing.scripts.pubmed.preprocess module modelzoo.transformers.data_processing.scripts.pubmed.preprocess.Downloader module modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextFormatting module modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding module modelzoo.transformers.data_processing.slimpajama module modelzoo.transformers.data_processing.slimpajama.dedup module modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train module modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicates_dict module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash module modelzoo.transformers.data_processing.slimpajama.main module modelzoo.transformers.data_processing.slimpajama.preprocessing module modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets module modelzoo.transformers.data_processing.slimpajama.preprocessing.filter module modelzoo.transformers.data_processing.slimpajama.preprocessing.normalize_text module modelzoo.transformers.data_processing.slimpajama.preprocessing.shuffle_holdout module modelzoo.transformers.data_processing.slimpajama.split_dataset module modelzoo.transformers.data_processing.slimpajama.utils module modelzoo.transformers.data_processing.slimpajama.utils.utils module modelzoo.transformers.data_processing.tokenizers module modelzoo.transformers.data_processing.tokenizers.BPETokenizer module modelzoo.transformers.data_processing.tokenizers.HFTokenizer module modelzoo.transformers.data_processing.tokenizers.Tokenization module modelzoo.transformers.data_processing.utils module modelzoo.transformers.pytorch module modelzoo.transformers.pytorch.bert module modelzoo.transformers.pytorch.bert.fine_tuning module modelzoo.transformers.pytorch.bert.fine_tuning.classifier module modelzoo.transformers.pytorch.bert.fine_tuning.classifier.input module modelzoo.transformers.pytorch.bert.fine_tuning.classifier.run module modelzoo.transformers.pytorch.bert.fine_tuning.classifier.utils module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.write_csv_data module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.run module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.utils module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.run module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.utils module modelzoo.transformers.pytorch.bert.input module modelzoo.transformers.pytorch.bert.input.scripts module modelzoo.transformers.pytorch.bert.input.scripts.create_csv module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only_static_masking module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_static_masking module modelzoo.transformers.pytorch.bert.input.scripts.create_hdf5_files module modelzoo.transformers.pytorch.bert.input.scripts.parser_utils module modelzoo.transformers.pytorch.bert.input.utils module modelzoo.transformers.pytorch.bert.run module modelzoo.transformers.pytorch.bert.utils module modelzoo.transformers.pytorch.bloom module modelzoo.transformers.pytorch.bloom.run module modelzoo.transformers.pytorch.btlm module modelzoo.transformers.pytorch.btlm.run module modelzoo.transformers.pytorch.falcon module modelzoo.transformers.pytorch.falcon.run module modelzoo.transformers.pytorch.gpt2 module modelzoo.transformers.pytorch.gpt2.input module modelzoo.transformers.pytorch.gpt2.input.scripts module modelzoo.transformers.pytorch.gpt2.input.scripts.data_processor_utils module modelzoo.transformers.pytorch.gpt2.run module modelzoo.transformers.pytorch.gpt2.scripts module modelzoo.transformers.pytorch.gpt2.sparse_mask module modelzoo.transformers.pytorch.gpt2.utils module modelzoo.transformers.pytorch.gpt3 module modelzoo.transformers.pytorch.gpt3.run module modelzoo.transformers.pytorch.gptj module modelzoo.transformers.pytorch.gptj.fine_tuning module modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining module modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining.data_processing module modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining.data_processing.split_trc_dataset module modelzoo.transformers.pytorch.gptj.run module modelzoo.transformers.pytorch.gptj.utils module modelzoo.transformers.pytorch.layers_api_demo module modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer module modelzoo.transformers.pytorch.layers_api_demo.run module modelzoo.transformers.pytorch.llama module modelzoo.transformers.pytorch.llama.run module modelzoo.transformers.pytorch.mistral module modelzoo.transformers.pytorch.mistral.run module

modelzoo.transformers.pytorch.mpt module modelzoo.transformers.pytorch.mpt.run module modelzoo.transformers.pytorch.santacoder module modelzoo.transformers.pytorch.santacoder.run module modelzoo.transformers.pytorch.starcoder module modelzoo.transformers.pytorch.starcoder.run module modelzoo.transformers.pytorch.t5 module modelzoo.transformers.pytorch.t5.input module modelzoo.transformers.pytorch.t5.input.data_processing module modelzoo.transformers.pytorch.t5.input.data_processing.preprocess_t5_dataset module modelzoo.transformers.pytorch.t5.input.data_processing.test_convert_t5_dataset module modelzoo.transformers.pytorch.t5.input.utils module modelzoo.transformers.pytorch.t5.run module modelzoo.transformers.pytorch.t5.utils module modelzoo.transformers.pytorch.transformer module modelzoo.transformers.pytorch.transformer.input module modelzoo.transformers.pytorch.transformer.input.data_processing module modelzoo.transformers.pytorch.transformer.input.data_processing.create_meta module modelzoo.transformers.pytorch.transformer.input.data_processing.split_files module modelzoo.transformers.pytorch.transformer.run module modelzoo.transformers.pytorch.transformer.utils module modelzoo.transformers.pytorch.vision_transformer module modelzoo.transformers.pytorch.vision_transformer.run module modelzoo.transformers.pytorch.vision_transformer.utils module modelzoo.transformers.utils module modelzoo.transformers.utils.attention_utils module modelzoo.vision module modelzoo.vision.pytorch module modelzoo.vision.pytorch.dit module modelzoo.vision.pytorch.dit.checkpoint_converter module modelzoo.vision.pytorch.dit.display_images module modelzoo.vision.pytorch.dit.input module modelzoo.vision.pytorch.dit.input.scripts module modelzoo.vision.pytorch.dit.input.transforms module modelzoo.vision.pytorch.dit.layers module modelzoo.vision.pytorch.dit.layers.schedulers module modelzoo.vision.pytorch.dit.layers.vae module modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D module modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D module modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D module modelzoo.vision.pytorch.dit.layers.vae.utils module modelzoo.vision.pytorch.dit.pipeline module modelzoo.vision.pytorch.dit.run module modelzoo.vision.pytorch.dit.sample_generator module modelzoo.vision.pytorch.dit.samplers module modelzoo.vision.pytorch.dit.samplers.DDIMSampler module modelzoo.vision.pytorch.dit.samplers.DDPMSampler module modelzoo.vision.pytorch.dit.samplers.get_sampler module modelzoo.vision.pytorch.dit.samplers.sampler_utils module modelzoo.vision.pytorch.dit.samplers.SamplerBase module modelzoo.vision.pytorch.dit.utils module modelzoo.vision.pytorch.input module modelzoo.vision.pytorch.input.classification module modelzoo.vision.pytorch.input.classification.mixup module modelzoo.vision.pytorch.input.classification.sampler module modelzoo.vision.pytorch.input.classification.utils module modelzoo.vision.pytorch.input.transforms module modelzoo.vision.pytorch.utils module modelzoo.vision.pytorch.utils.run_utils module module modelzoo modelzoo.common modelzoo.common.input modelzoo.common.input.utils modelzoo.common.model_utils modelzoo.common.model_utils.count_lines modelzoo.common.run_utils modelzoo.common.run_utils.cli_parser modelzoo.common.run_utils.cli_pytorch modelzoo.common.run_utils.utils modelzoo.fc_mnist modelzoo.fc_mnist.pytorch modelzoo.fc_mnist.pytorch.prepare_data modelzoo.fc_mnist.pytorch.run modelzoo.fc_mnist.pytorch.utils modelzoo.transformers modelzoo.transformers.data_processing modelzoo.transformers.data_processing.bert modelzoo.transformers.data_processing.bert.bertsum_data_processor modelzoo.transformers.data_processing.bert.dynamic_processor modelzoo.transformers.data_processing.bert.mlm_only_processor modelzoo.transformers.data_processing.bert.ner_data_processor modelzoo.transformers.data_processing.bert.sentence_pair_processor modelzoo.transformers.data_processing.huggingface modelzoo.transformers.data_processing.huggingface.CSDataCollatorForLanguageModeling modelzoo.transformers.data_processing.huggingface.HF_converter_example_BookCorpus modelzoo.transformers.data_processing.huggingface.HF_converter_example_Eli5 modelzoo.transformers.data_processing.huggingface.HuggingFace_BookCorpus modelzoo.transformers.data_processing.huggingface.HuggingFace_Eli5 modelzoo.transformers.data_processing.qa modelzoo.transformers.data_processing.qa.qa_utils modelzoo.transformers.data_processing.qa.write_csv_qa modelzoo.transformers.data_processing.scripts modelzoo.transformers.data_processing.scripts.chunk_preprocessing modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor modelzoo.transformers.data_processing.scripts.chunk_preprocessing.create_hdf5_dataset modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator modelzoo.transformers.data_processing.scripts.chunk_preprocessing.utils modelzoo.transformers.data_processing.scripts.hdf5_preprocessing modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.convert_dataset_to_HDF5 modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.create_hdf5_dataset modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils modelzoo.transformers.data_processing.scripts.hdf5_shuffling modelzoo.transformers.data_processing.scripts.hdf5_shuffling.h5_dataset_shuffle modelzoo.transformers.data_processing.scripts.pile modelzoo.transformers.data_processing.scripts.pile.download modelzoo.transformers.data_processing.scripts.pubmed modelzoo.transformers.data_processing.scripts.pubmed.preprocess modelzoo.transformers.data_processing.scripts.pubmed.preprocess.Downloader modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextFormatting modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding modelzoo.transformers.data_processing.slimpajama modelzoo.transformers.data_processing.slimpajama.dedup modelzoo.transformers.data_processing.slimpajama.dedup.dedup_train modelzoo.transformers.data_processing.slimpajama.dedup.generate_connected_components modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicates_dict modelzoo.transformers.data_processing.slimpajama.dedup.to_hash modelzoo.transformers.data_processing.slimpajama.main modelzoo.transformers.data_processing.slimpajama.preprocessing modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets modelzoo.transformers.data_processing.slimpajama.preprocessing.filter modelzoo.transformers.data_processing.slimpajama.preprocessing.normalize_text modelzoo.transformers.data_processing.slimpajama.preprocessing.shuffle_holdout modelzoo.transformers.data_processing.slimpajama.split_dataset modelzoo.transformers.data_processing.slimpajama.utils modelzoo.transformers.data_processing.slimpajama.utils.utils modelzoo.transformers.data_processing.tokenizers modelzoo.transformers.data_processing.tokenizers.BPETokenizer modelzoo.transformers.data_processing.tokenizers.HFTokenizer modelzoo.transformers.data_processing.tokenizers.Tokenization modelzoo.transformers.data_processing.utils modelzoo.transformers.pytorch modelzoo.transformers.pytorch.bert modelzoo.transformers.pytorch.bert.fine_tuning modelzoo.transformers.pytorch.bert.fine_tuning.classifier modelzoo.transformers.pytorch.bert.fine_tuning.classifier.input modelzoo.transformers.pytorch.bert.fine_tuning.classifier.run modelzoo.transformers.pytorch.bert.fine_tuning.classifier.utils modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.write_csv_data modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.run modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.utils modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.run modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.utils modelzoo.transformers.pytorch.bert.input modelzoo.transformers.pytorch.bert.input.scripts modelzoo.transformers.pytorch.bert.input.scripts.create_csv modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only_static_masking modelzoo.transformers.pytorch.bert.input.scripts.create_csv_static_masking modelzoo.transformers.pytorch.bert.input.scripts.create_hdf5_files modelzoo.transformers.pytorch.bert.input.scripts.parser_utils modelzoo.transformers.pytorch.bert.input.utils modelzoo.transformers.pytorch.bert.run modelzoo.transformers.pytorch.bert.utils modelzoo.transformers.pytorch.bloom modelzoo.transformers.pytorch.bloom.run modelzoo.transformers.pytorch.btlm modelzoo.transformers.pytorch.btlm.run modelzoo.transformers.pytorch.falcon modelzoo.transformers.pytorch.falcon.run modelzoo.transformers.pytorch.gpt2 modelzoo.transformers.pytorch.gpt2.input modelzoo.transformers.pytorch.gpt2.input.scripts modelzoo.transformers.pytorch.gpt2.input.scripts.data_processor_utils modelzoo.transformers.pytorch.gpt2.run modelzoo.transformers.pytorch.gpt2.scripts modelzoo.transformers.pytorch.gpt2.sparse_mask modelzoo.transformers.pytorch.gpt2.utils modelzoo.transformers.pytorch.gpt3 modelzoo.transformers.pytorch.gpt3.run modelzoo.transformers.pytorch.gptj modelzoo.transformers.pytorch.gptj.fine_tuning modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining.data_processing modelzoo.transformers.pytorch.gptj.fine_tuning.continuous_pretraining.data_processing.split_trc_dataset modelzoo.transformers.pytorch.gptj.run modelzoo.transformers.pytorch.gptj.utils modelzoo.transformers.pytorch.layers_api_demo modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer modelzoo.transformers.pytorch.layers_api_demo.run modelzoo.transformers.pytorch.llama modelzoo.transformers.pytorch.llama.run modelzoo.transformers.pytorch.mistral modelzoo.transformers.pytorch.mistral.run modelzoo.transformers.pytorch.mpt modelzoo.transformers.pytorch.mpt.run modelzoo.transformers.pytorch.santacoder modelzoo.transformers.pytorch.santacoder.run modelzoo.transformers.pytorch.starcoder modelzoo.transformers.pytorch.starcoder.run modelzoo.transformers.pytorch.t5 modelzoo.transformers.pytorch.t5.input modelzoo.transformers.pytorch.t5.input.data_processing modelzoo.transformers.pytorch.t5.input.data_processing.preprocess_t5_dataset modelzoo.transformers.pytorch.t5.input.data_processing.test_convert_t5_dataset modelzoo.transformers.pytorch.t5.input.utils modelzoo.transformers.pytorch.t5.run modelzoo.transformers.pytorch.t5.utils modelzoo.transformers.pytorch.transformer modelzoo.transformers.pytorch.transformer.input modelzoo.transformers.pytorch.transformer.input.data_processing modelzoo.transformers.pytorch.transformer.input.data_processing.create_meta modelzoo.transformers.pytorch.transformer.input.data_processing.split_files modelzoo.transformers.pytorch.transformer.run modelzoo.transformers.pytorch.transformer.utils modelzoo.transformers.pytorch.vision_transformer modelzoo.transformers.pytorch.vision_transformer.run modelzoo.transformers.pytorch.vision_transformer.utils modelzoo.transformers.utils modelzoo.transformers.utils.attention_utils modelzoo.vision modelzoo.vision.pytorch modelzoo.vision.pytorch.dit modelzoo.vision.pytorch.dit.checkpoint_converter modelzoo.vision.pytorch.dit.display_images modelzoo.vision.pytorch.dit.input modelzoo.vision.pytorch.dit.input.scripts modelzoo.vision.pytorch.dit.input.transforms modelzoo.vision.pytorch.dit.layers modelzoo.vision.pytorch.dit.layers.schedulers modelzoo.vision.pytorch.dit.layers.vae modelzoo.vision.pytorch.dit.layers.vae.DownEncoderBlock2D modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D modelzoo.vision.pytorch.dit.layers.vae.utils modelzoo.vision.pytorch.dit.pipeline modelzoo.vision.pytorch.dit.run modelzoo.vision.pytorch.dit.sample_generator modelzoo.vision.pytorch.dit.samplers modelzoo.vision.pytorch.dit.samplers.DDIMSampler modelzoo.vision.pytorch.dit.samplers.DDPMSampler modelzoo.vision.pytorch.dit.samplers.get_sampler modelzoo.vision.pytorch.dit.samplers.sampler_utils modelzoo.vision.pytorch.dit.samplers.SamplerBase modelzoo.vision.pytorch.dit.utils modelzoo.vision.pytorch.input modelzoo.vision.pytorch.input.classification modelzoo.vision.pytorch.input.classification.mixup modelzoo.vision.pytorch.input.classification.sampler modelzoo.vision.pytorch.input.classification.utils modelzoo.vision.pytorch.input.transforms modelzoo.vision.pytorch.utils modelzoo.vision.pytorch.utils.run_utils

N

name() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method)

NERProcessor (class in modelzoo.transformers.data_processing.bert.ner_data_processor) NLGPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor) NLTKSegmenter (class in modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding) noise_token_span_to_unique_sentinel() (in module modelzoo.transformers.pytorch.t5.input.utils) NoiseGenerator (class in modelzoo.vision.pytorch.dit.input.transforms) normalize_text() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.normalize_text) num_docs() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method)

O

output_results() (in module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash)

P

pad_helper() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) pad_input_sequence() (in module modelzoo.transformers.data_processing.utils) pad_instance_to_max_seq_length() (in module modelzoo.transformers.data_processing.utils) pad_t5_input_features() (in module modelzoo.transformers.pytorch.t5.input.utils) parse_args() (in module modelzoo.transformers.data_processing.qa.write_csv_qa) (in module modelzoo.transformers.data_processing.scripts.pile.download) (in module modelzoo.transformers.data_processing.scripts.pubmed.preprocess.Downloader) (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.normalize_text) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_mlm_only_static_masking) (in module modelzoo.transformers.pytorch.bert.input.scripts.create_csv_static_masking) parse_text() (in module modelzoo.transformers.pytorch.bert.input.utils) (in module modelzoo.transformers.pytorch.t5.input.utils) pass_1_shuffle() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.shuffle_holdout) pass_2_shuffle_holdout() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.shuffle_holdout) pop() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) popitem() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) PositionalEncoding (class in modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer) post_process_params() (in module modelzoo.common.run_utils.cli_parser) preprocessing_generator() (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method)

PreprocessInstance (class in modelzoo.transformers.data_processing.bert.dynamic_processor) previous_timestep() (modelzoo.vision.pytorch.dit.samplers.DDIMSampler.DDIMSampler method) (modelzoo.vision.pytorch.dit.samplers.DDPMSampler.DDPMSampler method) (modelzoo.vision.pytorch.dit.samplers.SamplerBase.SamplerBase method) process_chunks() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) process_dataset() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) process_dataset_params() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) process_files() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) process_params() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) process_processing_params() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) process_setup_params() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) profile_activities (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute)

R

randn_tensor() (in module modelzoo.vision.pytorch.dit.layers.vae.utils) random_spans_noise_mask() (in module modelzoo.transformers.pytorch.t5.input.utils) RandomCutmix (class in modelzoo.vision.pytorch.input.classification.mixup) RandomMixup (class in modelzoo.vision.pytorch.input.classification.mixup) read_checkpoint() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) read_jsongz() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) read_jsonl() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) read_jsonl_tar() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) read_jsonl_zst() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) read_params_file() (in module modelzoo.common.run_utils.cli_parser) read_parquet() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) read_scalar() (cerebras_pytorch.utils.tensorboard.SummaryReader method) read_squad_examples() (in module modelzoo.transformers.data_processing.qa.qa_utils) read_tensor() (cerebras_pytorch.utils.tensorboard.SummaryReader method) read_txt() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) Reader (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) reader_process() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method)

recreate_dataset() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.normalize_text) RedPajamaArXivDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaBooksDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaC4Dataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaCommonCrawlDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaGithubDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaReplication (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaStackExchangeDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) RedPajamaWikipediaDataset (class in modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) redpj_datasets() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets) reload() (cerebras_pytorch.utils.tensorboard.SummaryReader method) RepeatedAugSampler (class in modelzoo.vision.pytorch.input.classification.sampler) rescale_zero_terminal_snr() (in module modelzoo.vision.pytorch.dit.samplers.sampler_utils) ResnetBlock2D (class in modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D) RestartableDataLoader (class in cerebras_pytorch.utils.data) rm_if_exists() (in module modelzoo.transformers.data_processing.slimpajama.utils.utils) RougeBasedLabelsFormatter (class in modelzoo.transformers.data_processing.bert.bertsum_data_processor) run() (modelzoo.vision.pytorch.dit.sample_generator.SampleGenerator method)

S

SampleGenerator (class in modelzoo.vision.pytorch.dit.sample_generator) SamplerBase (class in modelzoo.vision.pytorch.dit.samplers.SamplerBase) sampling_time_ns (cerebras_pytorch.utils.benchmark.utils.dataloader.BatchMetrics attribute) save_params() (in module modelzoo.common.input.utils) save_to_hdf5() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) scalar_names() (cerebras_pytorch.utils.tensorboard.SummaryReader method) seed_runs() (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) segment_ids (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) (modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.input.utils.BertInputFeatures attribute) select_random_chunk() (in module modelzoo.transformers.pytorch.t5.input.utils) SentencePairInstance (class in modelzoo.transformers.data_processing.bert.sentence_pair_processor) serialize_state_dict() (cerebras_pytorch.utils.data.DataLoader method) set_attention_params() (in module modelzoo.transformers.pytorch.gpt2.utils) set_defaults() (in module modelzoo.fc_mnist.pytorch.utils) (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) (in module modelzoo.transformers.pytorch.bert.fine_tuning.classifier.utils) (in module modelzoo.transformers.pytorch.bert.fine_tuning.extractive_summarization.utils) (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.utils) (in module modelzoo.transformers.pytorch.bert.utils) (in module modelzoo.transformers.pytorch.gpt2.utils) (in module modelzoo.transformers.pytorch.gptj.utils) (in module modelzoo.transformers.pytorch.t5.utils) (in module modelzoo.transformers.pytorch.transformer.utils) (in module modelzoo.transformers.pytorch.vision_transformer.utils) (in module modelzoo.vision.pytorch.dit.utils) set_doc_idx() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) set_sampling_timesteps() (in module modelzoo.vision.pytorch.dit.samplers.sampler_utils) set_shuffle_seed() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) set_timesteps() (modelzoo.vision.pytorch.dit.samplers.DDIMSampler.DDIMSampler method) (modelzoo.vision.pytorch.dit.samplers.DDPMSampler.DDPMSampler method) (modelzoo.vision.pytorch.dit.samplers.SamplerBase.SamplerBase method) set_vit_model_defaults() (in module modelzoo.transformers.pytorch.vision_transformer.utils) setdefault() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) setup_output_directory() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) sha256str() (in module modelzoo.transformers.data_processing.slimpajama.utils.utils) shard_and_shuffle_data() (in module modelzoo.transformers.pytorch.bert.input.utils)

Sharding (class in modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding) short_documents_path() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method) shuffle() (in module modelzoo.transformers.pytorch.t5.input.utils) shuffle_dataset() (in module modelzoo.transformers.data_processing.scripts.hdf5_shuffling.h5_dataset_shuffle) size() (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaArXivDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaBooksDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaC4Dataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaCommonCrawlDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaGithubDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaReplication method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaStackExchangeDataset method) (modelzoo.transformers.data_processing.slimpajama.preprocessing.datasets.RedPajamaWikipediaDataset method) space_timesteps() (in module modelzoo.vision.pytorch.dit.samplers.sampler_utils) SparseAttentionBuilder (class in modelzoo.transformers.utils.attention_utils) split_entry_by_paragraph_or_sentence() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader) split_files() (in module modelzoo.transformers.data_processing.slimpajama.dedup.generate_duplicate_pairs) split_list() (in module modelzoo.transformers.data_processing.utils) split_sequences() (in module modelzoo.transformers.pytorch.t5.input.utils) split_text_and_tokenize() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) SquadExample (class in modelzoo.transformers.data_processing.qa.qa_utils) start_position (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) start_time_ns (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute) (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) state_dict() (cerebras_pytorch.utils.data.DataLoader method) (cerebras_pytorch.utils.data.RestartableDataLoader method) step() (modelzoo.vision.pytorch.dit.samplers.DDIMSampler.DDIMSampler method) (modelzoo.vision.pytorch.dit.samplers.DDPMSampler.DDPMSampler method) (modelzoo.vision.pytorch.dit.samplers.SamplerBase.SamplerBase method) stream_data() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.Reader method) StridedSparseAttentionBuilder (class in modelzoo.transformers.utils.attention_utils) SummarizationPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors) SummarizationTokenGenerator (class in modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator) SummaryReader (class in cerebras_pytorch.utils.tensorboard) SummaryWriter (class in cerebras_pytorch.utils.tensorboard) SyntheticDataset (class in cerebras_pytorch.utils.data)

T

task_split_process_dataset() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) tensor_names() (cerebras_pytorch.utils.tensorboard.SummaryReader method) text_to_tokenized_documents() (in module modelzoo.transformers.data_processing.utils) TextFormatting (class in modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextFormatting) threshold_sample() (in module modelzoo.vision.pytorch.dit.samplers.sampler_utils) to_minhash() (in module modelzoo.transformers.data_processing.slimpajama.dedup.to_hash) token_is_max_context (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) token_to_orig_map (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) tokenize() (in module modelzoo.transformers.data_processing.bert.bertsum_data_processor) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.data_reader.DataFrame method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.BaseTokenizer method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.FullTokenizer method) (modelzoo.transformers.data_processing.tokenizers.Tokenization.WordPieceTokenizer method) tokenize_dataset() (in module modelzoo.transformers.pytorch.t5.input.data_processing.preprocess_t5_dataset) tokenize_text() (in module modelzoo.transformers.pytorch.t5.input.data_processing.preprocess_t5_dataset) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.summarization_data_token_generator.SummarizationTokenGenerator method)

tokenize_text_auto_lm() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.fim_data_token_generator.FIMTokenGenerator method) (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.lm_data_token_generator.LMDataTokenGenerator method) Tokenizer (class in modelzoo.transformers.data_processing.bert.bertsum_data_processor) tokenizer_process() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method) tokens (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) (modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner.InputFeatures attribute) total_steps (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics attribute) (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics property) total_time (cerebras_pytorch.utils.benchmark.utils.dataloader.EpochMetrics property) (cerebras_pytorch.utils.benchmark.utils.dataloader.Metrics attribute) training_data_generator() (in module modelzoo.transformers.pytorch.gpt2.input.scripts.data_processor_utils) TransformerModel (class in modelzoo.transformers.pytorch.layers_api_demo.pytorch_transformer) truncate_helper() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) truncate_or_pad_helper() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils)

U

unique_id (modelzoo.transformers.data_processing.qa.qa_utils.InputFeatures attribute) update() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) update_debug_args_with_mem_limits() (in module modelzoo.common.input.utils) update_defaults() (in module modelzoo.common.run_utils.cli_parser) update_params() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) update_params_from_args() (in module modelzoo.common.run_utils.cli_parser) update_params_from_args_and_env() (in module modelzoo.common.run_utils.cli_parser)

update_params_from_file() (in module modelzoo.common.run_utils.cli_parser) update_parser() (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner) update_progress() (in module modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor) update_runconfig_debug_args_path() (in module modelzoo.vision.pytorch.utils.run_utils) UpDecoderBlock2D (class in modelzoo.vision.pytorch.dit.layers.vae.UpDecoderBlock2D) Upsample2D (class in modelzoo.vision.pytorch.dit.layers.vae.ResNetBlock2D) utf8len() (in module modelzoo.transformers.data_processing.slimpajama.utils.utils)

V

validate_tokens() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) values() (modelzoo.transformers.pytorch.bert.input.utils.Vocab method) VerificationArgs (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) verify_saved_hdf5_files() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils)

verify_saved_hdf5_files_mp() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) Vocab (class in modelzoo.transformers.pytorch.bert.input.utils) VSLLMDataPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors) VSLSummarizationPreprocessor (class in modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors)

W

whitespace_tokenize() (in module modelzoo.transformers.data_processing.utils) wikitext_detokenizer() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.utils) WordPieceTokenizer (class in modelzoo.transformers.data_processing.tokenizers.Tokenization) write_csv_files() (in module modelzoo.transformers.data_processing.qa.write_csv_qa) (in module modelzoo.transformers.pytorch.bert.fine_tuning.token_classifier.input.write_csv_ner) write_docs() (in module modelzoo.transformers.data_processing.slimpajama.preprocessing.shuffle_holdout) write_hdf5_file() (in module modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.convert_dataset_to_HDF5) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method)

write_hdf5_files() (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_base_preprocessor.HDF5BasePreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_curation_corpus_preprocessor.CurationCorpusPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.FIMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.LMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.SummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLLMDataPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_dataset_preprocessors.VSLSummarizationPreprocessor method) (modelzoo.transformers.data_processing.scripts.hdf5_preprocessing.hdf5_nlg_preprocessor.NLGPreprocessor method) write_label_map_files() (in module modelzoo.transformers.data_processing.bert.ner_data_processor) write_lmd_dataset() (in module modelzoo.transformers.data_processing.slimpajama.utils.utils) write_samples() (in module modelzoo.transformers.data_processing.slimpajama.split_dataset) writer_process() (modelzoo.transformers.data_processing.scripts.chunk_preprocessing.chunk_data_preprocessor.ChunkDataPreprocessor method)