Using External Datasets with HuggingFace Data Loader

PaulYoung
Mon, 25 Apr 2022 18:33:46 +0000

Full error message below:

ValueError Traceback (most recent call last)
/[mypath]/nextClassification.ipynb Cell 29’ in <cell line: 1>()
----> 1 train_results = trainer.train()
2 trainer.save_model()
3 trainer.log_metrics(“train”, train_results.metrics)

File ~/.local/lib/python3.8/site-packages/transformers/trainer.py:1396, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
1393 self.control = self.callback_handler.on_epoch_begin(args, self.state, self.control)
1395 step = -1
→ 1396 for step, inputs in enumerate(epoch_iterator):
1397
1398 # Skip past any already trained steps if resuming training
1399 if steps_trained_in_current_epoch > 0:
1400 steps_trained_in_current_epoch -= 1

File ~/.local/lib/python3.8/site-packages/torch/utils/data/dataloader.py:530, in _BaseDataLoaderIter.next(self)
528 if self._sampler_iter is None:
529 self._reset()
→ 530 data = self._next_data()
531 self._num_yielded += 1
532 if self._dataset_kind == _DatasetKind.Iterable and
533 self._IterableDataset_len_called is not None and
534 self._num_yielded > self._IterableDataset_len_called:

File ~/.local/lib/python3.8/site-packages/torch/utils/data/dataloader.py:570, in _SingleProcessDataLoaderIter._next_data(self)
568 def _next_data(self):
569 index = self._next_index() # may raise StopIteration
→ 570 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
571 if self._pin_memory:
572 data = _utils.pin_memory.pin_memory(data)

File ~/.local/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py:49, in _MapDatasetFetcher.fetch(self, possibly_batched_index)
47 def fetch(self, possibly_batched_index):
48 if self.auto_collation:
—> 49 data = [self.dataset[idx] for idx in possibly_batched_index]
50 else:
51 data = self.dataset[possibly_batched_index]

File ~/.local/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py:49, in (.0)
47 def fetch(self, possibly_batched_index):
48 if self.auto_collation:
—> 49 data = [self.dataset[idx] for idx in possibly_batched_index]
50 else:
51 data = self.dataset[possibly_batched_index]

File ~/.local/lib/python3.8/site-packages/datasets/arrow_dataset.py:1764, in Dataset.getitem(self, key)
1762 def getitem(self, key): # noqa: F811
1763 “”“Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools).”“”
→ 1764 return self._getitem(
1765 key,
1766 )

File ~/.local/lib/python3.8/site-packages/datasets/arrow_dataset.py:1749, in Dataset._getitem(self, key, decoded, **kwargs)
1747 formatter = get_formatter(format_type, features=self.features, decoded=decoded, **format_kwargs)
1748 pa_subtable = query_table(self._data, key, indices=self._indices if self._indices is not None else None)
→ 1749 formatted_output = format_table(
1750 pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns
1751 )
1752 return formatted_output

File ~/.local/lib/python3.8/site-packages/datasets/formatting/formatting.py:532, in format_table(table, key, formatter, format_columns, output_all_columns)
530 python_formatter = PythonFormatter(features=None)
531 if format_columns is None:
→ 532 return formatter(pa_table, query_type=query_type)
533 elif query_type == “column”:
534 if key in format_columns:

File ~/.local/lib/python3.8/site-packages/datasets/formatting/formatting.py:281, in Formatter.call(self, pa_table, query_type)
279 def call(self, pa_table: pa.Table, query_type: str) → Union[RowFormat, ColumnFormat, BatchFormat]:
280 if query_type == “row”:
→ 281 return self.format_row(pa_table)
282 elif query_type == “column”:
283 return self.format_column(pa_table)

File ~/.local/lib/python3.8/site-packages/datasets/formatting/formatting.py:387, in CustomFormatter.format_row(self, pa_table)
386 def format_row(self, pa_table: pa.Table) → dict:
→ 387 formatted_batch = self.format_batch(pa_table)
388 try:
389 return _unnest(formatted_batch)

File ~/.local/lib/python3.8/site-packages/datasets/formatting/formatting.py:418, in CustomFormatter.format_batch(self, pa_table)
416 if self.decoded:
417 batch = self.python_features_decoder.decode_batch(batch)
→ 418 return self.transform(batch)

[mypath]/nextClassification.ipynb Cell 17’ in transform(example_batch)
1 def transform(example_batch):
2 # Take a list of PIL images and turn them to pixel values
----> 3 inputs = feature_extractor([x for x in example_batch[‘image’]], return_tensors=‘pt’)
5 # Don’t forget to include the labels!
6 inputs[‘labels’] = example_batch[‘labels’]

File ~/.local/lib/python3.8/site-packages/transformers/models/vit/feature_extraction_vit.py:143, in ViTFeatureExtractor.call(self, images, return_tensors, **kwargs)
141 images = [self.resize(image=image, size=self.size, resample=self.resample) for image in images]
142 if self.do_normalize:
→ 143 images = [self.normalize(image=image, mean=self.image_mean, std=self.image_std) for image in images]
145 # return as BatchFeature
146 data = {“pixel_values”: images}

File ~/.local/lib/python3.8/site-packages/transformers/models/vit/feature_extraction_vit.py:143, in (.0)
141 images = [self.resize(image=image, size=self.size, resample=self.resample) for image in images]
142 if self.do_normalize:
→ 143 images = [self.normalize(image=image, mean=self.image_mean, std=self.image_std) for image in images]
145 # return as BatchFeature
146 data = {“pixel_values”: images}

File ~/.local/lib/python3.8/site-packages/transformers/image_utils.py:186, in ImageFeatureExtractionMixin.normalize(self, image, mean, std)
184 return (image - mean[:, None, None]) / std[:, None, None]
185 else:
→ 186 return (image - mean) / std

ValueError: operands could not be broadcast together with shapes (224,224) (3,)