bigbench_discriminative = set("""abstract_narrative_understanding
anachronisms
analogical_similarity
analytic_entailment
arithmetic
authorship_verification
bbq_lite_json
causal_judgment
cause_and_effect
checkmate_in_one
cifar10_classification
code_line_description
color
common_morpheme
conceptual_combinations
contextual_parametric_knowledge_conflicts
crash_blossom
crass_ai
cryobiology_spanish
cs_algorithms
dark_humor_detection
date_understanding
disambiguation_qa
discourse_marker_prediction
dyck_languages
elementary_math_qa
emoji_movie
emojis_emotion_prediction
empirical_judgments
english_proverbs
english_russian_proverbs
entailed_polarity
entailed_polarity_hindi
epistemic_reasoning
evaluating_information_essentiality
fact_checker
fantasy_reasoning
figure_of_speech_detection
formal_fallacies_syllogisms_negation
general_knowledge
geometric_shapes
goal_step_wikihow
gre_reading_comprehension
hhh_alignment
hindu_knowledge
hinglish_toxicity
human_organs_senses
hyperbaton
identify_math_theorems
identify_odd_metaphor
implicatures
implicit_relations
indic_cause_and_effect
intent_recognition
international_phonetic_alphabet_nli
intersect_geometry
irony_identification
kannada
key_value_maps
known_unknowns
language_identification
logic_grid_puzzle
logical_args
logical_deduction
logical_fallacy_detection
logical_sequence
mathematical_induction
medical_questions_russian
metaphor_boolean
metaphor_understanding
misconceptions
misconceptions_russian
mnist_ascii
moral_permissibility
movie_dialog_same_or_different
movie_recommendation
navigate
nonsense_words_grammar
novel_concepts
odd_one_out
parsinlu_qa
penguins_in_a_table
persian_idioms
phrase_relatedness
physical_intuition
physics
play_dialog_same_or_different
presuppositions_as_nli
question_selection
real_or_fake_text
reasoning_about_colored_objects
riddle_sense
ruin_names
salient_translation_error_detection
sentence_ambiguity
similarities_abstraction
simple_arithmetic_json_multiple_choice
simple_ethical_questions
snarks
social_iqa
social_support
sports_understanding
strange_stories
strategyqa
suicide_risk
swahili_english_proverbs
swedish_to_german_proverbs
symbol_interpretation
temporal_sequences
timedial
tracking_shuffled_objects
understanding_fables
undo_permutation
unit_interpretation
vitaminc_fact_verification
what_is_the_tao
which_wiki_edit
winowhy""".split('\n')) - {'simple_arithmetic_json_multiple_choice'}

bigbench_non_english = set("""common_morpheme
cryobiology_spanish
gem
gender_inclusive_sentences_german
kanji_ascii
kannada
language_identification
linguistic_mappings
medical_questions_russian
misconceptions_russian
multiemo
persian_idioms
polish_sequence_labeling
swahili_english_proverbs
swedish_to_german_proverbs
what_is_the_tao
which_wiki_edit""".split('\n')) | {"parsinlu_qa","hinglish_toxicity","indic_cause_and_effect","entailed_polarity_hindi","english_russian_proverbs"}

bbl=set('''auto_debugging
bbq_lite_json
code_line_description
conceptual_combinations
conlang_translation
emoji_movie
formal_fallacies_syllogisms_negation
hindu_knowledge
known_unknowns
language_identification
linguistics_puzzles
logic_grid_puzzle
logical_deduction
misconceptions_russian
novel_concepts
operators
parsinlu_reading_comprehension
play_dialog_same_or_different
repeat_copy_logic
strange_stories
strategyqa
symbol_interpretation
vitaminc_fact_verification
winowhy'''.split('\n'))

bigbench_discriminative_english = bigbench_discriminative - bigbench_non_english