lm-evaluation-harness/lm_eval/tasks/bbh/fewshot/navigate.yaml

dataset_name: navigate
description: 'Given a series of navigation instructions, determine whether one would
  end up back at the starting point.


  '
doc_to_text: 'Q: {{input}}

  A:'
fewshot_config:
  sampler: first_n
  samples:
  - input: 'If you follow these instructions, do you return to the starting point?
      Turn left. Turn around. Turn left. Take 7 steps. Take 2 steps. Take 4 steps.
      Take 8 steps.

      Options:

      - Yes

      - No'
    target: 'No'
  - input: 'If you follow these instructions, do you return to the starting point?
      Turn around. Take 1 step. Take 6 steps. Turn around. Take 6 steps. Take 9 steps.
      Take 1 step.

      Options:

      - Yes

      - No'
    target: 'No'
  - input: 'If you follow these instructions, do you return to the starting point?
      Always face forward. Take 2 steps right. Take 9 steps left. Take 7 steps right.

      Options:

      - Yes

      - No'
    target: 'Yes'
include: _fewshot_template_yaml
task: bbh_fewshot_navigate