SaylorTwift HF Staff
run_eval.sh: test split only + Jobs-proxy path (missing train dir killed the env server)
bc532e5 verified | # Standalone eval job (1 GPU): base model vs a LoRA adapter on held-out test tasks. | |
| # | |
| # Expected env: | |
| # HF_TOKEN (secret) hub downloads + sandbox Jobs | |
| # ADAPTER_REPO (env) Hub repo with adapter_config.json at root (or empty for base-only) | |
| # N_TASKS (env) test tasks per model (default 50) | |
| # OUT_REPO (env) where to upload eval_results.json (default: ADAPTER_REPO) | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| echo "=== [setup] ===" | |
| apt-get update -qq && apt-get install -y -qq --no-install-recommends git ca-certificates | |
| python3 -m pip install --quiet \ | |
| 'openenv[harbor]' \ | |
| 'harbor[hf-sandbox]>=0.24.0' \ | |
| math-verify tomli-w \ | |
| aiohttp | |
| git clone --depth 1 https://github.com/huggingface/OpenEnv.git /opt/OpenEnv | |
| SITE=$(python3 -c "import site; print(site.getsitepackages()[0])") | |
| echo "/opt/OpenEnv/envs" > "$SITE/openenv_examples.pth" | |
| echo "=== [setup] pinned test tasks ===" | |
| python3 prepare_tasks.py --output /app/prepared --splits test | |
| # (seam patch happens below, before the env server starts) | |
| echo "=== [start] vLLM (base + adapter as LoRA 'trained') ===" | |
| LORA_ARGS=() | |
| if [ -n "${ADAPTER_REPO:-}" ]; then | |
| LORA_ARGS=(--enable-lora --max-lora-rank 32 --lora-modules "trained=${ADAPTER_REPO}") | |
| fi | |
| CUDA_VISIBLE_DEVICES=0 \ | |
| vllm serve "${BASE_MODEL:-LiquidAI/LFM2.5-2.6B}" \ | |
| --port 8001 \ | |
| --max-model-len 32768 \ | |
| --return-tokens-as-token-ids \ | |
| --logprobs-mode processed_logprobs \ | |
| --enable-auto-tool-choice \ | |
| --tool-call-parser lfm2 \ | |
| "${LORA_ARGS[@]}" & | |
| VLLM_PID=$! | |
| echo "=== [patch] opencode seam: session id as x-api-key (Jobs proxy strips Authorization) ===" | |
| python3 - <<'EOF' | |
| import openenv.harbor.seams as seams | |
| path = seams.__file__ | |
| src = open(path).read() | |
| old = '"apiKey": session,\n "timeout": 600_000,' | |
| new = ('"apiKey": session,\n' | |
| ' "headers": {"x-api-key": session},\n' | |
| ' "timeout": 600_000,') | |
| assert src.count(old) == 1, f"opencode seam not found in {path}" | |
| open(path, "w").write(src.replace(old, new)) | |
| print("patched", path) | |
| EOF | |
| echo "=== [start] env server (test split only; tunnel-free via Jobs proxy) ===" | |
| export SPACE_HOST="${JOB_ID}--8000.hf.jobs" | |
| PREPARED=/app/prepared \ | |
| OPENENV_DATASETS=/app/prepared/datasets/test \ | |
| bash serve_env.sh & | |
| ENV_PID=$! | |
| python3 - <<'EOF' | |
| import time, urllib.request | |
| for name, url in [("vllm", "http://127.0.0.1:8001/v1/models"), | |
| ("env", "http://127.0.0.1:8000/health")]: | |
| for i in range(360): | |
| try: | |
| urllib.request.urlopen(url, timeout=5) | |
| print(name, "ready") | |
| break | |
| except Exception: | |
| time.sleep(5) | |
| else: | |
| raise SystemExit(f"{name} server never became ready") | |
| EOF | |
| echo "=== [eval] N_TASKS=${N_TASKS:-50} ===" | |
| ADAPTER_NAME="${ADAPTER_REPO:+trained}" python3 eval_adapter.py | |
| echo "=== [upload] results ===" | |
| python3 - <<'EOF' | |
| import os | |
| from huggingface_hub import HfApi | |
| repo = os.environ.get("OUT_REPO") or os.environ.get("ADAPTER_REPO") | |
| if repo: | |
| api = HfApi() | |
| api.upload_file(path_or_fileobj="/app/eval_results.json", | |
| path_in_repo=f"eval/{os.environ.get('EVAL_TAG', 'eval')}_results.json", | |
| repo_id=repo) | |
| print("uploaded to", repo) | |
| EOF | |
| kill $VLLM_PID $ENV_PID 2>/dev/null || true | |
| echo "=== done ===" | |