modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding.Sharding#

class modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding.Sharding[source]#

Bases: object

Methods

distribute_articles_over_shards

get_sentences_per_shard

init_output_files

load_articles

segment_articles_into_sentences

write_shards_to_disk

write_single_shard

__init__(input_files, output_name_prefix, n_training_shards, n_test_shards, fraction_test_set)[source]#

previous

modelzoo.transformers.data_processing.scripts.pubmed.preprocess.TextSharding.NLTKSegmenter

next

modelzoo.transformers.data_processing.slimpajama