Stage
Infra failed
11 of 12 stages
Running for
74h 28m
committed Oct 16, 21:08 UTC
Round score
—
awaiting round
p99 latency
—
ceiling 2000 / 50 ms
Round 7: infra failed · no score
11/12 stages
Intake & gates
6/6Patch hash recorded; awaiting fetch.
{
"hotkey": "5Dr4a5Lj8BgWJWyRwjXGvybCBq4XRvh5pNcBqXsy5pfQzWCy",
"payment_tx": 25,
"commit_block": 9298521,
"payment_block": 9298518,
"patch_fingerprint": "sha256:d46711c144314a94bcdecdc3d049cd34e30eefb90b9ddba0ce05cfacc7e0b6d4"
}A worker claimed the submission and started the pipeline.
{
"job_id": 276
}Patch artifact retrieved from object storage.
{
"size": 473385,
"patch_hash": "sha256:ef919b75dd7c98e7d505a7d4ac50e3b99ae96e40934b3d5f643ccba9781e32ac"
}Integrity and path constraints checked.
{
"identity": "ok",
"patch_hash": "sha256:ef919b75dd7c98e7d505a7d4ac50e3b99ae96e40934b3d5f643ccba9781e32ac"
}Patch applied onto the baseline checkout.
{
"ok": true
}Public API / surface compatibility held.
{
"files": [
"python/sglang/kernels/jit/csrc/elementwise/l2_prefetch.cuh",
"python/sglang/kernels/jit/csrc/elementwise/set_mla_kv_concat_q.cuh",
"python/sglang/kernels/jit/csrc/kimi_k3/attn_res/fused_tma.cuh",
"python/sglang/kernels/jit/csrc/kimi_k3/comm/ar_fusion.cuh",
"python/sglang/kernels/jit/csrc/moe/k3_decode_moe.cuh",
"python/sglang/kernels/jit/csrc/moe/route_quant_fused.cuh",
"python/sglang/kernels/jit/csrc/moe/route_radix.cuh",
"python/sglang/kernels/ops/attention/attn_res.py",
"python/sglang/kernels/ops/attention/dcp_kernels.py",
"python/sglang/kernels/ops/attention/kda_decode_mtp.py",
"python/sglang/kernels/ops/attention/set_mla_kv_concat_q.py",
"python/sglang/kernels/ops/communication/all_reduce_residual.py",
"python/sglang/kernels/ops/gemm/cutedsl_bf16_gemm.py",
"python/sglang/kernels/ops/gemm/fp8_decode_shadow.py",
"python/sglang/kernels/ops/gemm/kimi_k3.py",
"python/sglang/kernels/ops/layernorm/dual_rmsnorm.py",
"python/sglang/kernels/ops/layernorm/norm.py",
"python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py",
"python/sglang/kernels/ops/memory/l2_prefetch.py",
"python/sglang/kernels/ops/moe/k3_decode_moe.py",
"python/sglang/kernels/ops/moe/moe_route_quant_fused.py",
"python/sglang/kernels/ops/speculative/dspark/commit_track_indices.py",
"python/sglang/kernels/ops/speculative/dspark/dspark_accept.py",
"python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py",
"python/sglang/kernels/ops/speculative/dspark/fused_accept_sampling.py",
"python/sglang/srt/arg_groups/cuda_graph_hook.py",
"python/sglang/srt/arg_groups/fields/spec.py",
"python/sglang/srt/arg_groups/memory_hook.py",
"python/sglang/srt/arg_groups/model_overrides/kimi_k3.py",
"python/sglang/srt/arg_groups/speculative_hook.py",
"python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py",
"python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py",
"python/sglang/srt/entrypoints/openai/serving_chat.py",
"python/sglang/srt/environ.py",
"python/sglang/srt/layers/attention/cutedsl_mla_backend.py",
"python/sglang/srt/layers/attention/linear/kda_backend.py",
"python/sglang/srt/layers/attention/trtllm_mha_backend.py",
"python/sglang/srt/layers/attention/trtllm_mla_backend.py",
"python/sglang/srt/layers/attn_residual.py",
"python/sglang/srt/layers/communication/k3_ar_fusion.py",
"python/sglang/srt/layers/dcp/comm.py",
"python/sglang/srt/layers/layernorm.py",
"python/sglang/srt/layers/moe/k3_moe_premeta.py",
"python/sglang/srt/layers/moe/k3_moe_prune.py",
"python/sglang/srt/layers/moe/k3_prims_ts.py",
"python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py",
"python/sglang/srt/layers/moe/moe_tactic_pins.py",
"python/sglang/srt/layers/moe/route_quant_handoff.py",
"python/sglang/srt/layers/moe/verify_route_alias.py",
"python/sglang/srt/layers/quantization/unquant.py",
"python/sglang/srt/managers/io_struct.py",
"python/sglang/srt/managers/schedule_batch.py",
"python/sglang/srt/managers/scheduler.py",
"python/sglang/srt/managers/scheduler_components/batch_result_processor.py",
"python/sglang/srt/managers/scheduler_components/internal_state.py",
"python/sglang/srt/managers/tokenizer_manager.py",
"python/sglang/srt/managers/utils.py",
"python/sglang/srt/mem_cache/kv_cache_dtype.py",
"python/sglang/srt/model_executor/forward_batch_info.py",
"python/sglang/srt/model_executor/runner/base_runner.py",
"python/sglang/srt/model_executor/runner/flashinfer_autotune.py",
"python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py",
"python/sglang/srt/models/deepseek_v2.py",
"python/sglang/srt/models/dspark.py",
"python/sglang/srt/models/kimi_k3.py",
"python/sglang/srt/speculative/dflash_worker_v2.py",
"python/sglang/srt/speculative/dspark_components/dspark_config.py",
"python/sglang/srt/speculative/dspark_components/dspark_draft.py",
"python/sglang/srt/speculative/dspark_components/dspark_draft_sampler.py",
"python/sglang/srt/speculative/dspark_components/dspark_planner.py",
"python/sglang/srt/speculative/dspark_components/dspark_repeat_guard.py",
"python/sglang/srt/speculative/dspark_components/dspark_verify.py",
"python/sglang/srt/speculative/dspark_components/dspark_verify_depth.py",
"python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py",
"python/sglang/srt/speculative/spec_info.py",
"python/sglang/srt/utils/prompt_ids_cache.py"
]
}Build
3/3Hermetic engine image build in progress.
No structured detail recorded for this step.
Candidate engine image pushed to the registry.
{
"image_ref": "ghcr.io/pareton-ai/pareton-engine@sha256:1520861fa228f1b1dce96565839a0a5a384d48ad7959863e82df360ec46d70e3",
"image_digest": "sha256:1520861fa228f1b1dce96565839a0a5a384d48ad7959863e82df360ec46d70e3"
}Candidate engine image built and digest pinned.
{
"mock": false,
"build_log": "/var/log/pareton/builds/6a66b76b-41ec-44c2-91f4-9b0f84f30331/build.log",
"image_ref": "ghcr.io/pareton-ai/pareton-engine@sha256:1520861fa228f1b1dce96565839a0a5a384d48ad7959863e82df360ec46d70e3",
"image_tag": "ghcr.io/pareton-ai/pareton-engine:ef919b75dd7c98e7d505a7d4ac50e3b99ae96e40934b3d5f643ccba9781e32ac"
}Round
2/3Waiting to be seated in a round.
No structured detail recorded for this step.
Seated in a round; waiting to run or running.
{
"ordinal": 5,
"round_id": "50b1fd89-06c0-4e89-8c0b-1a0a72be2f61"
}Other events
The image never ran because of infrastructure; one requeue.
{
"reason": "concurrency warmup failed: candidate-3/rep0/hf-003: completions HTTP 500 from http://172.18.0.2:8000/v1/completions: Internal Server Error",
"ordinal": 6,
"round_id": "282b5225-0d06-4d59-8e4c-f89df904cbf2"
}The image never ran because of infrastructure; one requeue.
{
"reason": "concurrency warmup failed: candidate-1/rep0/hf-028: completions HTTP 500 from http://172.18.0.2:8000/v1/completions: Internal Server Error",
"ordinal": 7,
"round_id": "9557e279-a534-418f-b766-dd27b5aad65e"
}