en_answer_mapping = {"true": "yes", "false": "no", True: "yes", False: "no"} def process_docs(dataset): def remove_question_mark(text): text = text.strip() if text.endswith("?") or text.endswith("؟"): text = text[:-1] text = text.strip() return text def _helper(doc): doc["question"] = remove_question_mark(doc["question"]) doc["target"] = en_answer_mapping[doc["answer"]] return doc return dataset.map(_helper)