11B_evaluation.yaml

# Config for running the InferenceRecipe in generate.py to generate output from an LLM
#
# This config assumes that you've run the following command before launching:
#   tune download meta-llama/Llama-3.2-11B-Vision-Instruct --output-dir /tmp/Llama-3.2-11B-Vision-Instruct --ignore-patterns "original/consolidated*"
#
# It also assumes that you've downloaded the EleutherAI Eval Harness (v0.4.5 or higher):
#   pip install lm_eval
#
# To launch, run the following command from root torchtune directory:
#    tune run eleuther_eval --config llama3_2_vision/11B_evaluation

output_dir: ./ # Not needed

# Model arguments
model:
  _component_: torchtune.models.llama3_2_vision.llama3_2_vision_11b

# Transform arguments
tokenizer:
  _component_: torchtune.models.llama3_2_vision.llama3_2_vision_transform
  path: /tmp/Llama-3.2-11B-Vision-Instruct/original/tokenizer.model
  max_seq_len: 8192 # Limit the size of our inputs

# Checkpointer
checkpointer:
  _component_: torchtune.training.FullModelHFCheckpointer
  checkpoint_dir: /tmp/Llama-3.2-11B-Vision-Instruct/
  checkpoint_files:
    filename_format: model-{}-of-{}.safetensors
    max_filename: "00005"
  output_dir: ${output_dir}
  model_type: LLAMA3_VISION

# Environment
device: cuda
dtype: bf16
seed: 1234 # It is not recommended to change this seed, b/c it matches EleutherAI's default seed
log_level: INFO

# EleutherAI specific eval args
# Llama3.2 vision reports on MMMU Val using chain-of-thought reasoning
# and image concatenation. This is not currently supported in the EletherAI
# Eval Harness so results may not match the paper OOTB
tasks: ["mmmu_val_science"] # Defaulting to science as a good subset
limit: null
batch_size: 1
enable_kv_cache: True
max_seq_length: 8192

# Quantization specific args
# Quantization is not supported in this specific config
quantizer: null