Remove Text2TextGenerationEvaluator, SummarizationEvaluator, TranslationEvaluator

docs/source/base_evaluator.mdx

@@ -8,9 +8,6 @@ Currently supported tasks are:
 - `"question-answering"`: will use the [`QuestionAnsweringEvaluator`].
 - `"image-classification"`: will use the [`ImageClassificationEvaluator`].
 - `"text-generation"`: will use the [`TextGenerationEvaluator`].
-- `"text2text-generation"`: will use the [`Text2TextGenerationEvaluator`].
-- `"summarization"`: will use the [`SummarizationEvaluator`].
-- `"translation"`: will use the [`TranslationEvaluator`].
 - `"automatic-speech-recognition"`: will use the [`AutomaticSpeechRecognitionEvaluator`].
 - `"audio-classification"`: will use the [`AudioClassificationEvaluator`].
 

docs/source/package_reference/evaluator_classes.mdx

@@ -37,21 +37,6 @@ The base class for all evaluator classes:
 [[autodoc]] evaluate.TextGenerationEvaluator
     - compute
 
-### Text2TextGenerationEvaluator
-
-[[autodoc]] evaluate.Text2TextGenerationEvaluator
-    - compute
-
-### SummarizationEvaluator
-
-[[autodoc]] evaluate.SummarizationEvaluator
-    - compute
-
-### TranslationEvaluator
-
-[[autodoc]] evaluate.TranslationEvaluator
-    - compute
-
 ### AutomaticSpeechRecognitionEvaluator
 
 [[autodoc]] evaluate.AutomaticSpeechRecognitionEvaluator
@@ -60,4 +45,4 @@ The base class for all evaluator classes:
 ### AudioClassificationEvaluator
 
 [[autodoc]] evaluate.AudioClassificationEvaluator
-    - compute
+    - compute

src/evaluate/__init__.py

@@ -33,12 +33,9 @@
     Evaluator,
     ImageClassificationEvaluator,
     QuestionAnsweringEvaluator,
-    SummarizationEvaluator,
-    Text2TextGenerationEvaluator,
     TextClassificationEvaluator,
     TextGenerationEvaluator,
     TokenClassificationEvaluator,
-    TranslationEvaluator,
     evaluator,
 )
 from .hub import push_to_hub

src/evaluate/config.py

@@ -153,23 +153,20 @@
 
 HF_HUB_ALLOWED_TASKS = [
     "image-classification",
-    "translation",
     "image-segmentation",
     "fill-mask",
     "automatic-speech-recognition",
     "token-classification",
     "sentence-similarity",
     "audio-classification",
     "question-answering",
-    "summarization",
     "zero-shot-classification",
     "table-to-text",
     "feature-extraction",
     "other",
     "multiple-choice",
     "text-classification",
     "text-to-image",
-    "text2text-generation",
     "zero-shot-image-classification",
     "tabular-classification",
     "tabular-regression",

src/evaluate/evaluator/__init__.py

@@ -29,7 +29,6 @@
 from .base import Evaluator
 from .image_classification import ImageClassificationEvaluator
 from .question_answering import QuestionAnsweringEvaluator
-from .text2text_generation import SummarizationEvaluator, Text2TextGenerationEvaluator, TranslationEvaluator
 from .text_classification import TextClassificationEvaluator
 from .text_generation import TextGenerationEvaluator
 from .token_classification import TokenClassificationEvaluator
@@ -56,18 +55,6 @@
         "implementation": TextGenerationEvaluator,
         "default_metric_name": "word_count",
     },
-    "text2text-generation": {
-        "implementation": Text2TextGenerationEvaluator,
-        "default_metric_name": "bleu",
-    },
-    "summarization": {
-        "implementation": SummarizationEvaluator,
-        "default_metric_name": "rouge",
-    },
-    "translation": {
-        "implementation": TranslationEvaluator,
-        "default_metric_name": "bleu",
-    },
     "automatic-speech-recognition": {
         "implementation": AutomaticSpeechRecognitionEvaluator,
         "default_metric_name": "wer",
@@ -101,6 +88,8 @@ def check_task(task: str) -> Dict:
     Returns:
         task_defaults: `dict`, contains the implementasion class of a give Evaluator and the default metric name.
     """
+    if task in ["text2text-generation", "summarization", "translation"]:
+        raise KeyError(f"Task {task} is no longer supported, instead use \"text_generation\".")
     if task in TASK_ALIASES:
         task = TASK_ALIASES[task]
     if not check_pipeline_task(task):

src/evaluate/evaluator/base.py

@@ -471,7 +471,7 @@ def prepare_pipeline(
                 pipe = model_or_pipeline
             if tokenizer is not None and feature_extractor is not None:
                 logger.warning("Ignoring the value of the preprocessor argument (`tokenizer` or `feature_extractor`).")
-        if (pipe.task != self.task) and not (self.task == "translation" and pipe.task.startswith("translation")):
+        if (pipe.task != self.task):
             raise ValueError(
                 f"Incompatible `model_or_pipeline`. Please specify `model_or_pipeline` compatible with the `{self.task}` task."
             )