launch_experiment.sh

#!/bin/bash

print_help() {
    echo "Usage: ./launch_experiment.sh [OPTIONS]"
    echo ""
    echo "Options:"
    echo "  1    Run Llama-3.1-8B experiments"
    echo "  2    Run Mistral-7B experiments"
    echo "  3    Run Llama-3.1-70B experiments"
    echo " -h, --help    Show this help message"
    echo ""
    echo "Example:"
    echo "  ./launch_experiment.sh 1    # Run Llama-3.1-8B experiments"
}

ANS_PATH=data/eli5_eval_bm25_top100_reranked_oracle_answers_llama31_70B_42.json

run_pipeline() {
    local model=$1
    echo "Running pipeline for $model"
    echo "Using answers from $ANS_PATH"

    # Run prompt-based
    python run_pipeline.py \
    --eli5_ans_path $ANS_PATH \
    --attribution_system prompt_based \
    --yaml_config ${model}.yaml \
    --debug_num_samples 100

    # Run sliding window variations
    for suffix in "" "_best" "_smoothing" "_thresholding"; do
        python run_pipeline.py \
        --eli5_ans_path $ANS_PATH \
        --attribution_system sliding_window \
        --yaml_config ${model}${suffix}.yaml \
        --debug_num_samples 100
    done

    # Run gradient-based variations
    for suffix in "" "_best" "_smoothing" "_thresholding"; do
        python run_pipeline.py \
        --eli5_ans_path $ANS_PATH \
        --attribution_system gradient_based \
        --yaml_config ${model}${suffix}.yaml \
        --debug_num_samples 100
    done
}

run_metrics() {
    local model=$1
    echo "Running metrics for $model"

    # Evaluate prompt-based
    python run_eval_char_binary.py \
    --pred_path attribution_results/${model}/prompt_based/eli5.json

    # Evaluate sliding window variations
    for suffix in "" "_best" "_smoothing" "_thresholding"; do
        python run_eval_char_binary.py \
        --pred_path attribution_results/${model}${suffix}/sliding_window/eli5.json
    done

    # Evaluate gradient-based variations
    for suffix in "" "_best" "_smoothing" "_thresholding"; do
        python run_eval_char_binary.py \
        --pred_path attribution_results/${model}${suffix}/gradient_based/eli5.json
    done
}

case "$1" in
    "1")
        run_pipeline "llama31_8B"
        run_metrics "llama31_8B"
        ;;
    "2")
        run_pipeline "mistral_7B"
        run_metrics "mistral_7B"
        ;;
    "3")
        run_pipeline "llama31_70B"
        run_metrics "llama31_70B"
        ;;
    "-h"|"--help"|"")
        print_help
        ;;
    *)
        echo "Invalid option. Use -h or --help to see available options."
        exit 1
        ;;
esac