group: Miscellaneous
depends_on: 
  - image-build
steps:
- label: V1 Spec Decode
  device: h200_35gb
  key: v1-spec-decode
  timeout_in_minutes: 30
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/inputs/
    - vllm/model_executor/
    - vllm/platforms/
    - vllm/sampling_params.py
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1/spec_decode
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    # TODO: create another `optional` test group for slow tests
    - pytest -v -s -m 'not slow_test' v1/spec_decode

- label: V1 Sample + Logits
  key: v1-sample-logits
  timeout_in_minutes: 30
  device: h200_18gb
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/engine/
    - vllm/inputs/
    - vllm/logger.py
    - vllm/model_executor/
    - vllm/platforms/
    - vllm/sampling_params.py
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1/sample
    - tests/v1/logits_processors
    - tests/v1/test_oracle.py
    - tests/v1/test_request.py
    - tests/v1/test_outputs.py
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s v1/sample
    - pytest -v -s v1/logits_processors
    - pytest -v -s v1/test_oracle.py
    - pytest -v -s v1/test_request.py
    - pytest -v -s v1/test_outputs.py
  mirror:
    amd:
      device: mi325_1
      depends_on:
      - image-build-amd

- label: V1 Core + KV + Metrics
  key: v1-core-kv-metrics
  timeout_in_minutes: 30
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/engine/
    - vllm/entrypoints/pooling/
    - vllm/inputs/
    - vllm/lora/
    - vllm/model_executor/
    - vllm/multimodal/
    - vllm/outputs.py
    - vllm/platforms/
    - vllm/pooling_params.py
    - vllm/profiler/
    - vllm/sampling_params.py
    - vllm/tokenizers/
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1/core
    - tests/v1/executor
    - tests/v1/kv_offload
    - tests/v1/simple_kv_offload
    - tests/v1/worker
    - tests/v1/kv_connector/unit
    - tests/v1/metrics
    - tests/entrypoints/openai/correctness/test_lmeval.py
  commands:
    - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    # split the test to avoid interference
    - pytest -v -s -m 'not cpu_test' v1/core
    - pytest -v -s v1/executor
    - pytest -v -s v1/kv_offload
    - pytest -v -s v1/simple_kv_offload
    - pytest -v -s v1/worker
    - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
    - pytest -v -s -m 'not cpu_test' v1/metrics
    # Integration test for streaming correctness (requires special branch).
    - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
    - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine

- label: V1 Others (CPU)
  key: v1-others-cpu
  depends_on:
    - image-build-cpu
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/engine/
    - vllm/inputs/
    - vllm/lora/
    - vllm/multimodal/
    - vllm/outputs.py
    - vllm/platforms/
    - vllm/pooling_params.py
    - vllm/profiler/
    - vllm/sampling_params.py
    - vllm/tokenizers/
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1
  device: cpu-small
  commands:
    # split the test to avoid interference
    - pytest -v -s -m 'cpu_test' v1/core
    - pytest -v -s v1/structured_output
    - pytest -v -s v1/test_serial_utils.py
    - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
    - pytest -v -s -m 'cpu_test' v1/metrics

- label: Extract Hidden States Integration
  key: extract-hidden-states-integration
  timeout_in_minutes: 20
  device: h200_18gb
  source_file_dependencies:
    - vllm/v1/spec_decode/extract_hidden_states.py
    - vllm/model_executor/models/extract_hidden_states.py
    - vllm/transformers_utils/configs/extract_hidden_states.py
    - tests/v1/kv_connector/extract_hidden_states_integration
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s v1/kv_connector/extract_hidden_states_integration

- label: Regression
  key: regression
  timeout_in_minutes: 20
  device: h200_18gb
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/test_regression
  commands:
  - pip install modelscope
  - pytest -v -s test_regression.py
  working_dir: "/vllm-workspace/tests" # optional

- label: Examples
  device: h200_35gb
  key: examples
  timeout_in_minutes: 45
  working_dir: "/vllm-workspace/examples"
  source_file_dependencies:
  - vllm/entrypoints
  - vllm/multimodal
  - examples/
  commands:
    - pip install tensorizer # for tensorizer test
     # for basic
    - python3 basic/offline_inference/chat.py
    - python3 basic/offline_inference/generate.py --model facebook/opt-125m
    - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
    - python3 basic/offline_inference/classify.py
    - python3 basic/offline_inference/embed.py
    - python3 basic/offline_inference/score.py
    # for multi-modal models
    - python3 generate/multimodal/audio_language_offline.py --seed 0
    - python3 generate/multimodal/vision_language_offline.py --seed 0
    - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
    - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
     # for pooling models
    - python3 pooling/embed/vision_embedding_offline.py --seed 0
    # for features demo
    - python3 features/automatic_prefix_caching/prefix_caching_offline.py
    - python3 deployment/llm_engine_example.py
    - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
    # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

- label: Metrics, Tracing (2 GPUs)
  key: metrics-tracing-2-gpus
  timeout_in_minutes: 20
  num_devices: 2
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tracing/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/v1/tracing
  commands:
  - "pip install \
      'opentelemetry-sdk>=1.26.0' \
      'opentelemetry-api>=1.26.0' \
      'opentelemetry-exporter-otlp>=1.26.0' \
      'opentelemetry-semantic-conventions-ai>=0.4.1'"
  - pytest -v -s v1/tracing

- label: Python-only Installation
  key: python-only-installation
  depends_on: ~
  optional: true
  timeout_in_minutes: 20
  device: h200_18gb
  source_file_dependencies:
  - tests/standalone_tests/python_only_compile.sh
  - setup.py
  commands:
  - bash standalone_tests/python_only_compile.sh

- label: Async Engine, Inputs, Utils, Worker
  device: h200_35gb
  key: async-engine-inputs-utils-worker
  timeout_in_minutes: 50
  source_file_dependencies:
  - vllm/assets/
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tokenizers/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/detokenizer
  - tests/multimodal
  - tests/utils_
  commands:
  - pytest -v -s detokenizer
  - pytest -v -s -m 'not cpu_test' multimodal
  - pytest -v -s utils_

- label: Async Engine, Inputs, Utils, Worker, Config (CPU)
  key: async-engine-inputs-utils-worker-config-cpu
  depends_on:
  - image-build-cpu
  timeout_in_minutes: 30
  source_file_dependencies:
  - vllm/assets/
  - vllm/config/
  - vllm/engine/arg_utils.py
  - vllm/entrypoints/chat_utils.py
  - vllm/entrypoints/mcp/
  - vllm/entrypoints/openai/chat_completion/protocol.py
  - vllm/entrypoints/openai/engine/protocol.py
  - vllm/envs.py
  - vllm/exceptions.py
  - vllm/inputs/
  - vllm/model_executor/layers/quantization/quark/
  - vllm/multimodal/
  - vllm/outputs.py
  - vllm/platforms/
  - vllm/pooling_params.py
  - vllm/ray/
  - vllm/reasoning/
  - vllm/renderers/
  - vllm/sampling_params.py
  - vllm/tokenizers/
  - vllm/tool_parsers/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/test_inputs.py
  - tests/test_outputs.py
  - tests/test_pooling_params.py
  - tests/test_ray_env.py
  - tests/multimodal
  - tests/renderers
  - tests/standalone_tests/lazy_imports.py
  - tests/tokenizers_
  - tests/reasoning
  - tests/tool_parsers
  - tests/parser
  - tests/transformers_utils
  - tests/config
  device: cpu-small
  commands:
  - python3 standalone_tests/lazy_imports.py
  - pytest -v -s test_inputs.py
  - pytest -v -s test_outputs.py
  - pytest -v -s test_pooling_params.py
  - pytest -v -s test_ray_env.py
  - pytest -v -s -m 'cpu_test' multimodal
  - pytest -v -s renderers
  - pytest -v -s tokenizers_
  - pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
  - pytest -v -s tool_parsers
  - pytest -v -s parser
  - pytest -v -s transformers_utils
  - pytest -v -s config

- label: Batch Invariance (A100)
  key: batch-invariance-a100
  timeout_in_minutes: 30
  device: a100
  source_file_dependencies:
    - vllm/v1/attention
    - vllm/model_executor/layers
    - tests/v1/determinism/
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pip install pytest-timeout pytest-forked
    - pytest -v -s v1/determinism/test_batch_invariance.py
    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]

- label: Batch Invariance (H100)
  key: batch-invariance-h100
  timeout_in_minutes: 30
  device: h100
  source_file_dependencies:
    - vllm/v1/attention
    - vllm/model_executor/layers
    - tests/v1/determinism/
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pip install pytest-timeout pytest-forked
    - pytest -v -s v1/determinism/test_batch_invariance.py
    - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]

- label: Batch Invariance (B200)
  key: batch-invariance-b200
  timeout_in_minutes: 30
  device: b200-k8s
  source_file_dependencies:
    - vllm/v1/attention
    - vllm/model_executor/layers
    - tests/v1/determinism/
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pip install pytest-timeout pytest-forked
    - pytest -v -s v1/determinism/test_batch_invariance.py
    - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
    - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
    - pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
  
- label: Acceptance Length Test (Large Models) # optional
  device: h200_35gb
  key: acceptance-length-test-large-models
  timeout_in_minutes: 25
  gpu: h100
  optional: true
  num_gpus: 1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - vllm/v1/spec_decode/
  - vllm/model_executor/models/mlp_speculator.py
  - tests/v1/spec_decode/test_acceptance_length.py
  commands:
    - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
    - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
