{
 "vllm": "0.32.0",
 "target": "unsloth/Meta-Llama-3.1-8B-Instruct",
 "gpu": "NVIDIA H100 80GB HBM3",
 "configs": [
  {
   "config": "off",
   "speculative_config": null,
   "startup_s": 232.2,
   "kv_pool_tokens": 451056,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 9.81,
     "output_tokens": 1519,
     "output_tok_s": 154.8,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 155.9,
     "ttft_ms_p50": 15.7,
     "spec": {}
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 3.38,
     "output_tokens": 3997,
     "output_tok_s": 1183.5,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 251,
     "request_decode_tok_s_p50": 152.0,
     "ttft_ms_p50": 30.8,
     "spec": {}
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 3.74,
     "output_tokens": 16285,
     "output_tok_s": 4354.7,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 143.9,
     "ttft_ms_p50": 83.8,
     "spec": {}
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 4.85,
     "output_tokens": 64727,
     "output_tok_s": 13350.4,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 115.2,
     "ttft_ms_p50": 208.7,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 11.98,
     "output_tokens": 1845,
     "output_tok_s": 154.1,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 155.1,
     "ttft_ms_p50": 19.9,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 4.3,
     "output_tokens": 4855,
     "output_tok_s": 1128.6,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 148.1,
     "ttft_ms_p50": 45.8,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 5.04,
     "output_tokens": 19358,
     "output_tok_s": 3837.6,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 129.6,
     "ttft_ms_p50": 84.4,
     "spec": {}
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 8.13,
     "output_tokens": 77470,
     "output_tok_s": 9533.5,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 87.2,
     "ttft_ms_p50": 506.2,
     "spec": {}
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 11.66,
     "output_tokens": 1536,
     "output_tok_s": 131.7,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 154.7,
     "ttft_ms_p50": 14.2,
     "spec": {}
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 8.44,
     "output_tokens": 16119,
     "output_tok_s": 1910.2,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 139.7,
     "ttft_ms_p50": 1688.0,
     "spec": {}
    }
   ],
   "server_log": [
    "(EngineCore pid=100) INFO 11-11 05:04:19 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='unsloth/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_rev"
   ]
  },
  {
   "config": "ngram",
   "speculative_config": {
    "method": "ngram",
    "num_speculative_tokens": 5,
    "prompt_lookup_min": 2,
    "prompt_lookup_max": 5
   },
   "startup_s": 82.0,
   "kv_pool_tokens": 450240,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 11.11,
     "output_tokens": 1519,
     "output_tok_s": 136.7,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 138.5,
     "ttft_ms_p50": 13.6,
     "spec": {
      "drafts": 1424,
      "draft_tokens": 7120,
      "accepted_tokens": 89,
      "mean_acceptance_length": 1.062,
      "draft_acceptance_rate": 0.0125,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0344,
       0.0169,
       0.0077,
       0.0021,
       0.0014
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 4.06,
     "output_tokens": 3990,
     "output_tok_s": 983.2,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 249,
     "request_decode_tok_s_p50": 126.5,
     "ttft_ms_p50": 22.7,
     "spec": {
      "drafts": 3761,
      "draft_tokens": 18805,
      "accepted_tokens": 213,
      "mean_acceptance_length": 1.057,
      "draft_acceptance_rate": 0.0113,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0332,
       0.0128,
       0.0064,
       0.0029,
       0.0013
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 5.04,
     "output_tokens": 16222,
     "output_tok_s": 3217.8,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 107.7,
     "ttft_ms_p50": 44.4,
     "spec": {
      "drafts": 14819,
      "draft_tokens": 74095,
      "accepted_tokens": 1348,
      "mean_acceptance_length": 1.091,
      "draft_acceptance_rate": 0.0182,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0527,
       0.0199,
       0.0099,
       0.0053,
       0.0032
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 12.01,
     "output_tokens": 64732,
     "output_tok_s": 5390.4,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 44.3,
     "ttft_ms_p50": 93.5,
     "spec": {
      "drafts": 59509,
      "draft_tokens": 297545,
      "accepted_tokens": 5006,
      "mean_acceptance_length": 1.084,
      "draft_acceptance_rate": 0.0168,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0472,
       0.019,
       0.0093,
       0.0052,
       0.0034
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 3.37,
     "output_tokens": 1845,
     "output_tok_s": 548.3,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 571.4,
     "ttft_ms_p50": 18.2,
     "spec": {
      "drafts": 411,
      "draft_tokens": 2055,
      "accepted_tokens": 1428,
      "mean_acceptance_length": 4.474,
      "draft_acceptance_rate": 0.6949,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7299,
       0.6959,
       0.691,
       0.6788,
       0.6788
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.33,
     "output_tokens": 4854,
     "output_tok_s": 3645.8,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 553.8,
     "ttft_ms_p50": 37.2,
     "spec": {
      "drafts": 999,
      "draft_tokens": 4995,
      "accepted_tokens": 3839,
      "mean_acceptance_length": 4.843,
      "draft_acceptance_rate": 0.7686,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.8138,
       0.7708,
       0.7658,
       0.7467,
       0.7457
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 1.92,
     "output_tokens": 19359,
     "output_tok_s": 10079.6,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 404.6,
     "ttft_ms_p50": 85.9,
     "spec": {
      "drafts": 4006,
      "draft_tokens": 20030,
      "accepted_tokens": 15289,
      "mean_acceptance_length": 4.817,
      "draft_acceptance_rate": 0.7633,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.8105,
       0.7646,
       0.7594,
       0.7411,
       0.7409
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 5.39,
     "output_tokens": 77470,
     "output_tok_s": 14368.1,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 142.0,
     "ttft_ms_p50": 235.0,
     "spec": {
      "drafts": 15951,
      "draft_tokens": 79755,
      "accepted_tokens": 61263,
      "mean_acceptance_length": 4.841,
      "draft_acceptance_rate": 0.7681,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.8139,
       0.7699,
       0.7649,
       0.7462,
       0.7458
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 12.64,
     "output_tokens": 1495,
     "output_tok_s": 118.3,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 135.6,
     "ttft_ms_p50": 13.5,
     "spec": {
      "drafts": 1397,
      "draft_tokens": 6985,
      "accepted_tokens": 92,
      "mean_acceptance_length": 1.066,
      "draft_acceptance_rate": 0.0132,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.0387,
       0.0136,
       0.0064,
       0.0036,
       0.0036
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 10.81,
     "output_tokens": 16097,
     "output_tok_s": 1488.6,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 97.0,
     "ttft_ms_p50": 40.5,
     "spec": {
      "drafts": 15095,
      "draft_tokens": 75475,
      "accepted_tokens": 946,
      "mean_acceptance_length": 1.063,
      "draft_acceptance_rate": 0.0125,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.039,
       0.0146,
       0.0056,
       0.0025,
       0.0009
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=461) WARNING 11-11 05:09:59 [vllm.py:1605] Async scheduling not supported with ngram-based speculative decoding and will be disabled.",
    "(EngineCore pid=485) INFO 11-11 05:10:09 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=SpeculativeConfig(method='ngram', model=None, num_spec_tokens=5), tokenizer='unsloth/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_i",
    "(APIServer pid=461) INFO 11-11 05:11:19 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.10, Accepted throughput: 7.94 tokens/s, Drafted throughput: 385.99 tokens/s, Accepted: 92 tokens, Drafted: 4470 tokens, Per-position acceptance rate: 0.048, 0.023, 0.012, 0.010, 0.009, Avg Draft ",
    "(APIServer pid=461) INFO 11-11 05:11:29 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.05, Accepted throughput: 20.40 tokens/s, Drafted throughput: 2006.23 tokens/s, Accepted: 204 tokens, Drafted: 20065 tokens, Per-position acceptance rate: 0.028, 0.012, 0.006, 0.003, 0.002, Avg Dr",
    "(APIServer pid=461) INFO 11-11 05:11:39 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.08, Accepted throughput: 180.21 tokens/s, Drafted throughput: 10901.06 tokens/s, Accepted: 1803 tokens, Drafted: 109065 tokens, Per-position acceptance rate: 0.051, 0.017, 0.008, 0.004, 0.003, Av",
    "(APIServer pid=461) INFO 11-11 05:11:49 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.09, Accepted throughput: 444.24 tokens/s, Drafted throughput: 25581.84 tokens/s, Accepted: 4443 tokens, Drafted: 255855 tokens, Per-position acceptance rate: 0.047, 0.020, 0.010, 0.006, 0.004, Av",
    "(APIServer pid=461) INFO 11-11 05:11:59 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.73, Accepted throughput: 542.53 tokens/s, Drafted throughput: 1565.29 tokens/s, Accepted: 5426 tokens, Drafted: 15655 tokens, Per-position acceptance rate: 0.385, 0.350, 0.340, 0.330, 0.329, Avg ",
    "(APIServer pid=461) INFO 11-11 05:12:09 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 4.84, Accepted throughput: 7653.37 tokens/s, Drafted throughput: 9975.62 tokens/s, Accepted: 76552 tokens, Drafted: 99780 tokens, Per-position acceptance rate: 0.813, 0.769, 0.764, 0.745, 0.745, Av",
    "(APIServer pid=461) INFO 11-11 05:12:19 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.06, Accepted throughput: 6.50 tokens/s, Drafted throughput: 521.48 tokens/s, Accepted: 65 tokens, Drafted: 5215 tokens, Per-position acceptance rate: 0.038, 0.011, 0.006, 0.004, 0.004, Avg Draft ",
    "(APIServer pid=461) INFO 11-11 05:12:29 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.06, Accepted throughput: 58.69 tokens/s, Drafted throughput: 4619.01 tokens/s, Accepted: 587 tokens, Drafted: 46195 tokens, Per-position acceptance rate: 0.040, 0.015, 0.006, 0.002, 0.001, Avg Dr"
   ]
  },
  {
   "config": "suffix",
   "speculative_config": {
    "method": "suffix",
    "num_speculative_tokens": 16
   },
   "startup_s": 80.0,
   "kv_pool_tokens": 436144,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 10.73,
     "output_tokens": 1481,
     "output_tok_s": 138.0,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 139.0,
     "ttft_ms_p50": 13.1,
     "spec": {
      "drafts": 821,
      "draft_tokens": 1180,
      "accepted_tokens": 228,
      "mean_acceptance_length": 1.278,
      "draft_acceptance_rate": 0.1932,
      "draft_tokens_per_draft": 1.437,
      "accepted_through_position": [
       0.1912,
       0.0475,
       0.0171,
       0.0073,
       0.0061,
       0.0024,
       0.0012,
       0.0012,
       0.0012,
       0.0012,
       0.0012,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     },
     "answers_identical_to_off": 2
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 3.72,
     "output_tokens": 3958,
     "output_tok_s": 1065.3,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 248,
     "request_decode_tok_s_p50": 137.5,
     "ttft_ms_p50": 20.2,
     "spec": {
      "drafts": 2557,
      "draft_tokens": 4073,
      "accepted_tokens": 823,
      "mean_acceptance_length": 1.322,
      "draft_acceptance_rate": 0.2021,
      "draft_tokens_per_draft": 1.593,
      "accepted_through_position": [
       0.2358,
       0.0563,
       0.0164,
       0.0059,
       0.0039,
       0.0027,
       0.0008,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 4.26,
     "output_tokens": 16234,
     "output_tok_s": 3813.3,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 130.0,
     "ttft_ms_p50": 37.1,
     "spec": {
      "drafts": 10422,
      "draft_tokens": 20051,
      "accepted_tokens": 4595,
      "mean_acceptance_length": 1.441,
      "draft_acceptance_rate": 0.2292,
      "draft_tokens_per_draft": 1.924,
      "accepted_through_position": [
       0.2867,
       0.0907,
       0.0284,
       0.0137,
       0.0089,
       0.0041,
       0.0032,
       0.0014,
       0.0012,
       0.001,
       0.0009,
       0.0007,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 7.67,
     "output_tokens": 64714,
     "output_tok_s": 8433.4,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 73.3,
     "ttft_ms_p50": 90.6,
     "spec": {
      "drafts": 40507,
      "draft_tokens": 91612,
      "accepted_tokens": 21868,
      "mean_acceptance_length": 1.54,
      "draft_acceptance_rate": 0.2387,
      "draft_tokens_per_draft": 2.262,
      "accepted_through_position": [
       0.3236,
       0.1179,
       0.0448,
       0.0201,
       0.0117,
       0.0066,
       0.0047,
       0.0028,
       0.0023,
       0.002,
       0.0019,
       0.0013,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 1.96,
     "output_tokens": 1845,
     "output_tok_s": 941.9,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 1105.3,
     "ttft_ms_p50": 19.2,
     "spec": {
      "drafts": 198,
      "draft_tokens": 1833,
      "accepted_tokens": 1635,
      "mean_acceptance_length": 9.258,
      "draft_acceptance_rate": 0.892,
      "draft_tokens_per_draft": 9.258,
      "accepted_through_position": [
       0.899,
       0.8333,
       0.7929,
       0.7424,
       0.702,
       0.6818,
       0.6566,
       0.6061,
       0.6061,
       0.5859,
       0.5859,
       0.5657,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 0.89,
     "output_tokens": 4855,
     "output_tok_s": 5432.7,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 821.0,
     "ttft_ms_p50": 57.6,
     "spec": {
      "drafts": 519,
      "draft_tokens": 4891,
      "accepted_tokens": 4383,
      "mean_acceptance_length": 9.445,
      "draft_acceptance_rate": 0.8961,
      "draft_tokens_per_draft": 9.424,
      "accepted_through_position": [
       0.9403,
       0.8709,
       0.842,
       0.7881,
       0.7514,
       0.7225,
       0.7033,
       0.6089,
       0.5954,
       0.5491,
       0.5453,
       0.5279,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 1.67,
     "output_tokens": 19359,
     "output_tok_s": 11561.8,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 482.7,
     "ttft_ms_p50": 121.9,
     "spec": {
      "drafts": 1919,
      "draft_tokens": 19696,
      "accepted_tokens": 17702,
      "mean_acceptance_length": 10.225,
      "draft_acceptance_rate": 0.8988,
      "draft_tokens_per_draft": 10.264,
      "accepted_through_position": [
       0.9802,
       0.9422,
       0.9109,
       0.8718,
       0.8322,
       0.7952,
       0.7827,
       0.7249,
       0.6826,
       0.5956,
       0.5597,
       0.5466,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 5.17,
     "output_tokens": 77470,
     "output_tok_s": 14992.1,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 147.4,
     "ttft_ms_p50": 404.0,
     "spec": {
      "drafts": 7241,
      "draft_tokens": 77857,
      "accepted_tokens": 71589,
      "mean_acceptance_length": 10.887,
      "draft_acceptance_rate": 0.9195,
      "draft_tokens_per_draft": 10.752,
      "accepted_through_position": [
       0.9798,
       0.9685,
       0.9652,
       0.9159,
       0.8964,
       0.8366,
       0.8178,
       0.7865,
       0.7629,
       0.7031,
       0.6579,
       0.5959,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 8.71,
     "output_tokens": 1441,
     "output_tok_s": 165.5,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 246,
     "request_decode_tok_s_p50": 172.0,
     "ttft_ms_p50": 13.4,
     "spec": {
      "drafts": 946,
      "draft_tokens": 2118,
      "accepted_tokens": 459,
      "mean_acceptance_length": 1.485,
      "draft_acceptance_rate": 0.2167,
      "draft_tokens_per_draft": 2.239,
      "accepted_through_position": [
       0.3245,
       0.1184,
       0.0296,
       0.0106,
       0.0021,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 4.69,
     "output_tokens": 16137,
     "output_tok_s": 3441.1,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 119.5,
     "ttft_ms_p50": 38.7,
     "spec": {
      "drafts": 10718,
      "draft_tokens": 24110,
      "accepted_tokens": 4912,
      "mean_acceptance_length": 1.458,
      "draft_acceptance_rate": 0.2037,
      "draft_tokens_per_draft": 2.249,
      "accepted_through_position": [
       0.3029,
       0.0972,
       0.0338,
       0.0123,
       0.0059,
       0.0031,
       0.0021,
       0.0007,
       0.0002,
       0.0001,
       0.0001,
       0.0,
       0.0,
       0.0,
       0.0,
       0.0
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=695) WARNING 11-11 05:12:53 [vllm.py:1605] Async scheduling not supported with suffix-based speculative decoding and will be disabled.",
    "(APIServer pid=695) WARNING 11-11 05:12:53 [vllm.py:732] Model Runner V2 does not yet support speculative method 'suffix'; using the V1 model runner instead.",
    "(EngineCore pid=718) INFO 11-11 05:13:03 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=SpeculativeConfig(method='suffix', model=None, num_spec_tokens=16), tokenizer='unsloth/Meta-Llama-3.1-8B-Instruct', skip_tokenizer",
    "(EngineCore pid=718) WARNING 11-11 05:13:04 [vllm.py:732] Model Runner V2 does not yet support speculative method 'suffix'; using the V1 model runner instead.",
    "(EngineCore pid=718) WARNING 11-11 05:13:06 [__init__.py:211] min_p and logit_bias parameters won't work with speculative decoding.",
    "(EngineCore pid=718) INFO 11-11 05:13:11 [gpu_model_runner.py:5071] Loading drafter model...",
    "(APIServer pid=695) INFO 11-11 05:14:10 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.37, Accepted throughput: 21.69 tokens/s, Drafted throughput: 89.31 tokens/s, Accepted: 248 tokens, Drafted: 1021 tokens, Per-position acceptance rate: 0.199, 0.063, 0.029, 0.016, 0.013, 0.009, 0.",
    "(APIServer pid=695) INFO 11-11 05:14:20 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.38, Accepted throughput: 190.06 tokens/s, Drafted throughput: 850.42 tokens/s, Accepted: 1901 tokens, Drafted: 8506 tokens, Per-position acceptance rate: 0.261, 0.071, 0.023, 0.011, 0.007, 0.003,",
    "(APIServer pid=695) INFO 11-11 05:14:30 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.55, Accepted throughput: 2011.56 tokens/s, Drafted throughput: 8099.22 tokens/s, Accepted: 20128 tokens, Drafted: 81042 tokens, Per-position acceptance rate: 0.325, 0.120, 0.046, 0.022, 0.013, 0.",
    "(APIServer pid=695) INFO 11-11 05:14:40 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.88, Accepted throughput: 1131.50 tokens/s, Drafted throughput: 3316.59 tokens/s, Accepted: 11316 tokens, Drafted: 33169 tokens, Per-position acceptance rate: 0.329, 0.135, 0.073, 0.053, 0.046, 0.",
    "(APIServer pid=695) INFO 11-11 05:14:50 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 10.72, Accepted throughput: 8929.61 tokens/s, Drafted throughput: 9759.73 tokens/s, Accepted: 89305 tokens, Drafted: 97607 tokens, Per-position acceptance rate: 0.978, 0.961, 0.951, 0.904, 0.881, 0",
    "(APIServer pid=695) INFO 11-11 05:15:00 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.48, Accepted throughput: 44.50 tokens/s, Drafted throughput: 206.39 tokens/s, Accepted: 445 tokens, Drafted: 2064 tokens, Per-position acceptance rate: 0.324, 0.119, 0.028, 0.010, 0.002, 0.000, 0"
   ]
  },
  {
   "config": "eagle3 k=3",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 3
   },
   "startup_s": 116.1,
   "kv_pool_tokens": 409584,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 4.85,
     "output_tokens": 1519,
     "output_tok_s": 313.5,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 319.6,
     "ttft_ms_p50": 18.5,
     "spec": {
      "drafts": 620,
      "draft_tokens": 1860,
      "accepted_tokens": 901,
      "mean_acceptance_length": 2.453,
      "draft_acceptance_rate": 0.4844,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.7129,
       0.4581,
       0.2823
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.79,
     "output_tokens": 4009,
     "output_tok_s": 2236.7,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 252,
     "request_decode_tok_s_p50": 315.8,
     "ttft_ms_p50": 39.4,
     "spec": {
      "drafts": 1625,
      "draft_tokens": 4875,
      "accepted_tokens": 2392,
      "mean_acceptance_length": 2.472,
      "draft_acceptance_rate": 0.4907,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6929,
       0.4775,
       0.3015
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 2.12,
     "output_tokens": 16237,
     "output_tok_s": 7664.8,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 274.0,
     "ttft_ms_p50": 67.5,
     "spec": {
      "drafts": 6717,
      "draft_tokens": 20151,
      "accepted_tokens": 9539,
      "mean_acceptance_length": 2.42,
      "draft_acceptance_rate": 0.4734,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6869,
       0.452,
       0.2812
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 4.14,
     "output_tokens": 64711,
     "output_tok_s": 15644.7,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 142.8,
     "ttft_ms_p50": 188.0,
     "spec": {
      "drafts": 26338,
      "draft_tokens": 79014,
      "accepted_tokens": 38407,
      "mean_acceptance_length": 2.458,
      "draft_acceptance_rate": 0.4861,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6993,
       0.4684,
       0.2906
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 4.09,
     "output_tokens": 1845,
     "output_tok_s": 451.3,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 461.5,
     "ttft_ms_p50": 20.9,
     "spec": {
      "drafts": 515,
      "draft_tokens": 1545,
      "accepted_tokens": 1340,
      "mean_acceptance_length": 3.602,
      "draft_acceptance_rate": 0.8673,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9379,
       0.8621,
       0.8019
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.55,
     "output_tokens": 4855,
     "output_tok_s": 3138.3,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 440.4,
     "ttft_ms_p50": 44.6,
     "spec": {
      "drafts": 1339,
      "draft_tokens": 4017,
      "accepted_tokens": 3536,
      "mean_acceptance_length": 3.641,
      "draft_acceptance_rate": 0.8803,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9425,
       0.879,
       0.8193
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 2.12,
     "output_tokens": 19359,
     "output_tok_s": 9141.4,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 342.8,
     "ttft_ms_p50": 120.3,
     "spec": {
      "drafts": 5317,
      "draft_tokens": 15951,
      "accepted_tokens": 14124,
      "mean_acceptance_length": 3.656,
      "draft_acceptance_rate": 0.8855,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9434,
       0.8868,
       0.8262
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 5.28,
     "output_tokens": 77470,
     "output_tok_s": 14682.6,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 145.0,
     "ttft_ms_p50": 423.9,
     "spec": {
      "drafts": 21244,
      "draft_tokens": 63732,
      "accepted_tokens": 56558,
      "mean_acceptance_length": 3.662,
      "draft_acceptance_rate": 0.8874,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.9445,
       0.8866,
       0.8312
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 5.23,
     "output_tokens": 1471,
     "output_tok_s": 281.2,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 300.4,
     "ttft_ms_p50": 19.3,
     "spec": {
      "drafts": 632,
      "draft_tokens": 1896,
      "accepted_tokens": 845,
      "mean_acceptance_length": 2.337,
      "draft_acceptance_rate": 0.4457,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6725,
       0.4256,
       0.2389
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 9.56,
     "output_tokens": 16140,
     "output_tok_s": 1688.4,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 239.4,
     "ttft_ms_p50": 1598.8,
     "spec": {
      "drafts": 7225,
      "draft_tokens": 21675,
      "accepted_tokens": 8925,
      "mean_acceptance_length": 2.235,
      "draft_acceptance_rate": 0.4118,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.6253,
       0.3864,
       0.2235
      ]
     }
    }
   ],
   "server_log": [
    "(EngineCore pid=1037) INFO 11-11 05:16:27 [backends.py:1104] [eagle_head] Using cache directory: /root/.cache/vllm/torch_compile_cache/b9767b8f30/rank_0_0/eagle_head for vLLM's torch.compile",
    "(EngineCore pid=1037) INFO 11-11 05:16:27 [backends.py:1166] [eagle_head] Dynamo bytecode transform time: 0.56 s",
    "(EngineCore pid=1037) INFO 11-11 05:16:29 [backends.py:398] [eagle_head] Compiling a graph for compile range (1, 8192) takes 2.77 s",
    "(EngineCore pid=1037) INFO 11-11 05:16:30 [decorators.py:743] [eagle_head] saved AOT compiled function to /root/.cache/vllm/torch_compile_cache/torch_aot_compile/f5991b09c9b01cd08f744a363a59a856f88cee1c337f78fadc7830f87c8a5c66/rank_0_0/model",
    "(EngineCore pid=1037) INFO 11-11 05:16:30 [monitor.py:62] [eagle_head] torch.compile took 3.52 s in total",
    "(EngineCore pid=1037) INFO 11-11 05:16:30 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(APIServer pid=1001) INFO 11-11 05:17:16 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.49, Accepted throughput: 86.07 tokens/s, Drafted throughput: 173.01 tokens/s, Accepted: 997 tokens, Drafted: 2004 tokens, Per-position acceptance rate: 0.726, 0.472, 0.295, Avg Draft acceptance ",
    "(APIServer pid=1001) INFO 11-11 05:17:26 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.46, Accepted throughput: 3408.93 tokens/s, Drafted throughput: 6988.60 tokens/s, Accepted: 34102 tokens, Drafted: 69912 tokens, Per-position acceptance rate: 0.699, 0.470, 0.295, Avg Draft accep",
    "(APIServer pid=1001) INFO 11-11 05:17:36 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.58, Accepted throughput: 2042.60 tokens/s, Drafted throughput: 3889.20 tokens/s, Accepted: 20426 tokens, Drafted: 38892 tokens, Per-position acceptance rate: 0.722, 0.509, 0.344, Avg Draft accep",
    "(APIServer pid=1001) INFO 11-11 05:17:46 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.67, Accepted throughput: 5966.46 tokens/s, Drafted throughput: 6709.35 tokens/s, Accepted: 59714 tokens, Drafted: 67149 tokens, Per-position acceptance rate: 0.944, 0.889, 0.834, Avg Draft accep",
    "(APIServer pid=1001) INFO 11-11 05:17:56 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.46, Accepted throughput: 1249.77 tokens/s, Drafted throughput: 1522.64 tokens/s, Accepted: 12499 tokens, Drafted: 15228 tokens, Per-position acceptance rate: 0.910, 0.816, 0.736, Avg Draft accep",
    "(APIServer pid=1001) INFO 11-11 05:18:06 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.24, Accepted throughput: 892.42 tokens/s, Drafted throughput: 2167.30 tokens/s, Accepted: 8925 tokens, Drafted: 21675 tokens, Per-position acceptance rate: 0.625, 0.386, 0.224, Avg Draft accepta"
   ]
  },
  {
   "config": "eagle3 k=5",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 5
   },
   "startup_s": 94.1,
   "kv_pool_tokens": 409488,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 4.89,
     "output_tokens": 1519,
     "output_tok_s": 310.8,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 317.8,
     "ttft_ms_p50": 19.1,
     "spec": {
      "drafts": 571,
      "draft_tokens": 2855,
      "accepted_tokens": 956,
      "mean_acceptance_length": 2.674,
      "draft_acceptance_rate": 0.3349,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7058,
       0.4571,
       0.2855,
       0.1541,
       0.0718
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.81,
     "output_tokens": 4021,
     "output_tok_s": 2224.6,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 252,
     "request_decode_tok_s_p50": 319.4,
     "ttft_ms_p50": 40.4,
     "spec": {
      "drafts": 1480,
      "draft_tokens": 7400,
      "accepted_tokens": 2557,
      "mean_acceptance_length": 2.728,
      "draft_acceptance_rate": 0.3455,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.6959,
       0.4757,
       0.2993,
       0.1615,
       0.0953
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 2.21,
     "output_tokens": 16232,
     "output_tok_s": 7350.9,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 266.9,
     "ttft_ms_p50": 56.7,
     "spec": {
      "drafts": 5989,
      "draft_tokens": 29945,
      "accepted_tokens": 10285,
      "mean_acceptance_length": 2.717,
      "draft_acceptance_rate": 0.3435,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.6934,
       0.462,
       0.2902,
       0.1732,
       0.0985
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 5.26,
     "output_tokens": 64721,
     "output_tok_s": 12306.4,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 109.8,
     "ttft_ms_p50": 155.6,
     "spec": {
      "drafts": 23980,
      "draft_tokens": 119900,
      "accepted_tokens": 40913,
      "mean_acceptance_length": 2.706,
      "draft_acceptance_rate": 0.3412,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.6974,
       0.4651,
       0.287,
       0.1658,
       0.0908
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 3.35,
     "output_tokens": 1845,
     "output_tok_s": 550.1,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 574.7,
     "ttft_ms_p50": 21.2,
     "spec": {
      "drafts": 383,
      "draft_tokens": 1915,
      "accepted_tokens": 1474,
      "mean_acceptance_length": 4.849,
      "draft_acceptance_rate": 0.7697,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9295,
       0.8616,
       0.7911,
       0.6789,
       0.5875
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.29,
     "output_tokens": 4854,
     "output_tok_s": 3777.0,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 545.0,
     "ttft_ms_p50": 52.8,
     "spec": {
      "drafts": 977,
      "draft_tokens": 4885,
      "accepted_tokens": 3909,
      "mean_acceptance_length": 5.001,
      "draft_acceptance_rate": 0.8002,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9509,
       0.8772,
       0.8229,
       0.7175,
       0.6325
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 1.9,
     "output_tokens": 19358,
     "output_tok_s": 10211.3,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 401.9,
     "ttft_ms_p50": 135.3,
     "spec": {
      "drafts": 3896,
      "draft_tokens": 19480,
      "accepted_tokens": 15590,
      "mean_acceptance_length": 5.002,
      "draft_acceptance_rate": 0.8003,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9497,
       0.8789,
       0.8288,
       0.7254,
       0.6188
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 5.34,
     "output_tokens": 77470,
     "output_tok_s": 14501.6,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 140.9,
     "ttft_ms_p50": 361.8,
     "spec": {
      "drafts": 15532,
      "draft_tokens": 77660,
      "accepted_tokens": 62450,
      "mean_acceptance_length": 5.021,
      "draft_acceptance_rate": 0.8041,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9547,
       0.8839,
       0.8334,
       0.7239,
       0.6249
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 6.67,
     "output_tokens": 1454,
     "output_tok_s": 217.9,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 243,
     "request_decode_tok_s_p50": 297.0,
     "ttft_ms_p50": 19.9,
     "spec": {
      "drafts": 579,
      "draft_tokens": 2895,
      "accepted_tokens": 883,
      "mean_acceptance_length": 2.525,
      "draft_acceptance_rate": 0.305,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.677,
       0.4387,
       0.2297,
       0.1123,
       0.0674
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 7.89,
     "output_tokens": 16194,
     "output_tok_s": 2051.4,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 230.5,
     "ttft_ms_p50": 1586.7,
     "spec": {
      "drafts": 6643,
      "draft_tokens": 33215,
      "accepted_tokens": 9595,
      "mean_acceptance_length": 2.444,
      "draft_acceptance_rate": 0.2889,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.6434,
       0.3938,
       0.2241,
       0.1194,
       0.0637
      ]
     }
    }
   ],
   "server_log": [
    "(EngineCore pid=1331) INFO 11-11 05:19:07 [monitor.py:40] [eagle_head] Starting torch.compile",
    "(EngineCore pid=1331) INFO 11-11 05:19:08 [backends.py:1104] [eagle_head] Using cache directory: /root/.cache/vllm/torch_compile_cache/b31a018eac/rank_0_0/eagle_head for vLLM's torch.compile",
    "(EngineCore pid=1331) INFO 11-11 05:19:08 [backends.py:1166] [eagle_head] Dynamo bytecode transform time: 0.55 s",
    "(EngineCore pid=1331) INFO 11-11 05:19:10 [backends.py:398] [eagle_head] Compiling a graph for compile range (1, 8192) takes 2.80 s",
    "(EngineCore pid=1331) INFO 11-11 05:19:11 [decorators.py:743] [eagle_head] saved AOT compiled function to /root/.cache/vllm/torch_compile_cache/torch_aot_compile/22c66b1b3e2caf8359f7438a5d460a0686ae03a5ed625775df23a99e3f3bec31/rank_0_0/model",
    "(EngineCore pid=1331) INFO 11-11 05:19:11 [monitor.py:62] [eagle_head] torch.compile took 3.53 s in total",
    "(EngineCore pid=1331) INFO 11-11 05:19:11 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(APIServer pid=1307) INFO 11-11 05:19:55 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.73, Accepted throughput: 91.94 tokens/s, Drafted throughput: 266.34 tokens/s, Accepted: 1058 tokens, Drafted: 3065 tokens, Per-position acceptance rate: 0.715, 0.473, 0.297, 0.162, 0.080, Avg Dr",
    "(APIServer pid=1307) INFO 11-11 05:20:05 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.73, Accepted throughput: 3356.42 tokens/s, Drafted throughput: 9709.55 tokens/s, Accepted: 33564 tokens, Drafted: 97095 tokens, Per-position acceptance rate: 0.700, 0.470, 0.293, 0.170, 0.096, A",
    "(APIServer pid=1307) INFO 11-11 05:20:15 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.88, Accepted throughput: 2482.46 tokens/s, Drafted throughput: 6598.36 tokens/s, Accepted: 24827 tokens, Drafted: 65990 tokens, Per-position acceptance rate: 0.714, 0.495, 0.327, 0.210, 0.135, A",
    "(APIServer pid=1307) INFO 11-11 05:20:25 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 5.03, Accepted throughput: 7040.95 tokens/s, Drafted throughput: 8743.35 tokens/s, Accepted: 70451 tokens, Drafted: 87485 tokens, Per-position acceptance rate: 0.955, 0.883, 0.830, 0.732, 0.627, A",
    "(APIServer pid=1307) INFO 11-11 05:20:35 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 4.41, Accepted throughput: 921.79 tokens/s, Drafted throughput: 1350.84 tokens/s, Accepted: 9219 tokens, Drafted: 13510 tokens, Per-position acceptance rate: 0.888, 0.787, 0.717, 0.537, 0.483, Avg"
   ]
  },
  {
   "config": "draft 1B k=5",
   "speculative_config": {
    "method": "draft_model",
    "model": "unsloth/Llama-3.2-1B-Instruct",
    "num_speculative_tokens": 5
   },
   "startup_s": 112.1,
   "kv_pool_tokens": 345328,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 14.82,
     "output_tokens": 1519,
     "output_tok_s": 102.5,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 108.5,
     "ttft_ms_p50": 69.0,
     "spec": {
      "drafts": 486,
      "draft_tokens": 2430,
      "accepted_tokens": 1048,
      "mean_acceptance_length": 3.156,
      "draft_acceptance_rate": 0.4313,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7366,
       0.5288,
       0.3848,
       0.2922,
       0.214
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 5.63,
     "output_tokens": 3996,
     "output_tok_s": 709.5,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 249,
     "request_decode_tok_s_p50": 105.1,
     "ttft_ms_p50": 138.7,
     "spec": {
      "drafts": 1261,
      "draft_tokens": 6305,
      "accepted_tokens": 2754,
      "mean_acceptance_length": 3.184,
      "draft_acceptance_rate": 0.4368,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7343,
       0.5329,
       0.3941,
       0.295,
       0.2276
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 6.17,
     "output_tokens": 16256,
     "output_tok_s": 2636.7,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 94.0,
     "ttft_ms_p50": 159.5,
     "spec": {
      "drafts": 4982,
      "draft_tokens": 24910,
      "accepted_tokens": 11317,
      "mean_acceptance_length": 3.272,
      "draft_acceptance_rate": 0.4543,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7344,
       0.5548,
       0.4231,
       0.3175,
       0.2417
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 9.02,
     "output_tokens": 64708,
     "output_tok_s": 7174.1,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 68.5,
     "ttft_ms_p50": 252.6,
     "spec": {
      "drafts": 19768,
      "draft_tokens": 98840,
      "accepted_tokens": 45221,
      "mean_acceptance_length": 3.288,
      "draft_acceptance_rate": 0.4575,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.7427,
       0.5584,
       0.4224,
       0.3193,
       0.2448
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 1,
     "requests": 6,
     "seconds": 9.69,
     "output_tokens": 1845,
     "output_tok_s": 190.4,
     "prompt_tokens_p50": 373,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 197.5,
     "ttft_ms_p50": 71.0,
     "spec": {
      "drafts": 311,
      "draft_tokens": 1555,
      "accepted_tokens": 1547,
      "mean_acceptance_length": 5.974,
      "draft_acceptance_rate": 0.9949,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       1.0,
       1.0,
       0.9936,
       0.9904,
       0.9904
      ]
     },
     "answers_identical_to_off": 6
    },
    {
     "workload": "code",
     "concurrency": 8,
     "requests": 16,
     "seconds": 3.6,
     "output_tokens": 4855,
     "output_tok_s": 1348.9,
     "prompt_tokens_p50": 366,
     "completion_tokens_p50": 306,
     "request_decode_tok_s_p50": 185.8,
     "ttft_ms_p50": 132.5,
     "spec": {
      "drafts": 817,
      "draft_tokens": 4085,
      "accepted_tokens": 4070,
      "mean_acceptance_length": 5.982,
      "draft_acceptance_rate": 0.9963,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9988,
       0.9988,
       0.9963,
       0.9939,
       0.9939
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 32,
     "requests": 64,
     "seconds": 4.01,
     "output_tokens": 19359,
     "output_tok_s": 4828.7,
     "prompt_tokens_p50": 368,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 179.9,
     "ttft_ms_p50": 268.3,
     "spec": {
      "drafts": 3256,
      "draft_tokens": 16280,
      "accepted_tokens": 16215,
      "mean_acceptance_length": 5.98,
      "draft_acceptance_rate": 0.996,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       1.0,
       0.9997,
       0.996,
       0.9929,
       0.9914
      ]
     }
    },
    {
     "workload": "code",
     "concurrency": 128,
     "requests": 256,
     "seconds": 6.79,
     "output_tokens": 77470,
     "output_tok_s": 11403.4,
     "prompt_tokens_p50": 367,
     "completion_tokens_p50": 304,
     "request_decode_tok_s_p50": 116.2,
     "ttft_ms_p50": 679.6,
     "spec": {
      "drafts": 13045,
      "draft_tokens": 65225,
      "accepted_tokens": 64930,
      "mean_acceptance_length": 5.977,
      "draft_acceptance_rate": 0.9955,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.9995,
       0.9987,
       0.9953,
       0.9926,
       0.9913
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 1,
     "requests": 6,
     "seconds": 16.73,
     "output_tokens": 1487,
     "output_tok_s": 88.9,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 248,
     "request_decode_tok_s_p50": 104.1,
     "ttft_ms_p50": 69.5,
     "spec": {
      "drafts": 511,
      "draft_tokens": 2555,
      "accepted_tokens": 982,
      "mean_acceptance_length": 2.922,
      "draft_acceptance_rate": 0.3843,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.6751,
       0.4834,
       0.3523,
       0.2329,
       0.1781
      ]
     }
    },
    {
     "workload": "chat_t1",
     "concurrency": 32,
     "requests": 64,
     "seconds": 12.26,
     "output_tokens": 16125,
     "output_tok_s": 1315.6,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 89.1,
     "ttft_ms_p50": 1659.3,
     "spec": {
      "drafts": 5700,
      "draft_tokens": 28500,
      "accepted_tokens": 10467,
      "mean_acceptance_length": 2.836,
      "draft_acceptance_rate": 0.3673,
      "draft_tokens_per_draft": 5.0,
      "accepted_through_position": [
       0.676,
       0.4588,
       0.32,
       0.2261,
       0.1554
      ]
     }
    }
   ],
   "server_log": [
    "(EngineCore pid=1575) INFO 11-11 05:22:05 [monitor.py:62] [draft_model] torch.compile took 11.46 s in total",
    "(EngineCore pid=1575) INFO 11-11 05:22:05 [monitor.py:91] [draft_model] Initial profiling/warmup run took 0.03 s",
    "(APIServer pid=1552) INFO 11-11 05:22:50 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.45, Accepted throughput: 37.22 tokens/s, Drafted throughput: 76.04 tokens/s, Accepted: 438 tokens, Drafted: 895 tokens, Per-position acceptance rate: 0.777, 0.575, 0.425, 0.374, 0.296, Avg Draft",
    "(APIServer pid=1552) INFO 11-11 05:23:00 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.11, Accepted throughput: 69.59 tokens/s, Drafted throughput: 164.98 tokens/s, Accepted: 696 tokens, Drafted: 1650 tokens, Per-position acceptance rate: 0.718, 0.515, 0.391, 0.279, 0.206, Avg Dra",
    "(APIServer pid=1552) INFO 11-11 05:23:10 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.24, Accepted throughput: 456.34 tokens/s, Drafted throughput: 1020.65 tokens/s, Accepted: 4565 tokens, Drafted: 10210 tokens, Per-position acceptance rate: 0.739, 0.546, 0.411, 0.304, 0.236, Avg",
    "(APIServer pid=1552) INFO 11-11 05:23:20 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.27, Accepted throughput: 2798.46 tokens/s, Drafted throughput: 6176.55 tokens/s, Accepted: 28007 tokens, Drafted: 61815 tokens, Per-position acceptance rate: 0.738, 0.552, 0.419, 0.315, 0.241, A",
    "(APIServer pid=1552) INFO 11-11 05:23:30 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 3.32, Accepted throughput: 2717.52 tokens/s, Drafted throughput: 5851.96 tokens/s, Accepted: 27180 tokens, Drafted: 58530 tokens, Per-position acceptance rate: 0.746, 0.566, 0.429, 0.328, 0.252, A",
    "(APIServer pid=1552) INFO 11-11 05:23:40 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 5.99, Accepted throughput: 291.45 tokens/s, Drafted throughput: 291.95 tokens/s, Accepted: 2915 tokens, Drafted: 2920 tokens, Per-position acceptance rate: 1.000, 1.000, 0.998, 0.997, 0.997, Avg D",
    "(APIServer pid=1552) INFO 11-11 05:23:50 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 5.98, Accepted throughput: 1862.05 tokens/s, Drafted throughput: 1870.05 tokens/s, Accepted: 18620 tokens, Drafted: 18700 tokens, Per-position acceptance rate: 1.000, 0.999, 0.996, 0.993, 0.991, A",
    "(APIServer pid=1552) INFO 11-11 05:24:00 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 5.97, Accepted throughput: 6483.61 tokens/s, Drafted throughput: 6522.11 tokens/s, Accepted: 64840 tokens, Drafted: 65225 tokens, Per-position acceptance rate: 0.999, 0.997, 0.994, 0.991, 0.989, A",
    "(APIServer pid=1552) INFO 11-11 05:24:10 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.98, Accepted throughput: 68.20 tokens/s, Drafted throughput: 172.49 tokens/s, Accepted: 682 tokens, Drafted: 1725 tokens, Per-position acceptance rate: 0.687, 0.499, 0.368, 0.235, 0.188, Avg Dra",
    "(APIServer pid=1552) INFO 11-11 05:24:20 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.80, Accepted throughput: 313.62 tokens/s, Drafted throughput: 869.29 tokens/s, Accepted: 3137 tokens, Drafted: 8695 tokens, Per-position acceptance rate: 0.668, 0.446, 0.316, 0.221, 0.153, Avg D"
   ]
  },
  {
   "config": "synthetic 1.0",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 3,
    "rejection_sample_method": "synthetic",
    "synthetic_acceptance_length": 1.0
   },
   "startup_s": 62.0,
   "kv_pool_tokens": 414032,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 11.68,
     "output_tokens": 1519,
     "output_tok_s": 130.0,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 130.7,
     "ttft_ms_p50": 17.6,
     "spec": {
      "drafts": 1513,
      "draft_tokens": 4539,
      "accepted_tokens": 0,
      "mean_acceptance_length": 1.0,
      "draft_acceptance_rate": 0.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 4.06,
     "output_tokens": 3994,
     "output_tok_s": 984.3,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 249,
     "request_decode_tok_s_p50": 127.8,
     "ttft_ms_p50": 40.0,
     "spec": {
      "drafts": 3978,
      "draft_tokens": 11934,
      "accepted_tokens": 0,
      "mean_acceptance_length": 1.0,
      "draft_acceptance_rate": 0.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 4.64,
     "output_tokens": 16248,
     "output_tok_s": 3502.4,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 114.2,
     "ttft_ms_p50": 81.0,
     "spec": {
      "drafts": 16184,
      "draft_tokens": 48552,
      "accepted_tokens": 0,
      "mean_acceptance_length": 1.0,
      "draft_acceptance_rate": 0.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.0,
       0.0,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 9.17,
     "output_tokens": 64724,
     "output_tok_s": 7059.4,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 58.1,
     "ttft_ms_p50": 181.4,
     "spec": {
      "drafts": 64468,
      "draft_tokens": 193404,
      "accepted_tokens": 0,
      "mean_acceptance_length": 1.0,
      "draft_acceptance_rate": 0.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       0.0,
       0.0,
       0.0
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=1856) INFO 11-11 05:24:48 [model.py:733] Resolved architecture: Eagle3LlamaForCausalLM",
    "(APIServer pid=1856) INFO 11-11 05:24:49 [speculative.py:1669] Overriding draft model max model len from 131072 to 8192",
    "(EngineCore pid=1879) INFO 11-11 05:25:00 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=SpeculativeConfig(method='eagle3', model='RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3', num_spec_tokens=3), tokenizer='unslo",
    "(EngineCore pid=1879) INFO 11-11 05:25:09 [eagle3_utils.py:30] Using Eagle3 auxiliary layers from model: (2, 16, 29)",
    "(EngineCore pid=1879) INFO 11-11 05:25:12 [decorators.py:319] [eagle_head] Directly load AOT compilation from path /root/.cache/vllm/torch_compile_cache/torch_aot_compile/f5991b09c9b01cd08f744a363a59a856f88cee1c337f78fadc7830f87c8a5c66/rank_0_0/model",
    "(EngineCore pid=1879) INFO 11-11 05:25:12 [monitor.py:62] [eagle_head] torch.compile took 0.06 s in total",
    "(EngineCore pid=1879) INFO 11-11 05:25:12 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(APIServer pid=1856) INFO 11-11 05:25:47 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.00, Accepted throughput: 0.00 tokens/s, Drafted throughput: 295.76 tokens/s, Accepted: 0 tokens, Drafted: 3486 tokens, Per-position acceptance rate: 0.000, 0.000, 0.000, Avg Draft acceptance rat",
    "(APIServer pid=1856) INFO 11-11 05:25:57 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.00, Accepted throughput: 0.00 tokens/s, Drafted throughput: 1340.83 tokens/s, Accepted: 0 tokens, Drafted: 13410 tokens, Per-position acceptance rate: 0.000, 0.000, 0.000, Avg Draft acceptance r",
    "(APIServer pid=1856) INFO 11-11 05:26:07 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 1.00, Accepted throughput: 0.00 tokens/s, Drafted throughput: 11115.57 tokens/s, Accepted: 0 tokens, Drafted: 111207 tokens, Per-position acceptance rate: 0.000, 0.000, 0.000, Avg Draft acceptance"
   ]
  },
  {
   "config": "synthetic 2.5",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 3,
    "rejection_sample_method": "synthetic",
    "synthetic_acceptance_length": 2.5
   },
   "startup_s": 62.0,
   "kv_pool_tokens": 414032,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 4.63,
     "output_tokens": 1473,
     "output_tok_s": 317.8,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 249,
     "request_decode_tok_s_p50": 324.3,
     "ttft_ms_p50": 18.9,
     "spec": {
      "drafts": 593,
      "draft_tokens": 1779,
      "accepted_tokens": 876,
      "mean_acceptance_length": 2.477,
      "draft_acceptance_rate": 0.4924,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       0.4772,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.68,
     "output_tokens": 3926,
     "output_tok_s": 2343.8,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 251,
     "request_decode_tok_s_p50": 318.1,
     "ttft_ms_p50": 38.0,
     "spec": {
      "drafts": 1559,
      "draft_tokens": 4677,
      "accepted_tokens": 2362,
      "mean_acceptance_length": 2.515,
      "draft_acceptance_rate": 0.505,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       0.5151,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 2.0,
     "output_tokens": 16260,
     "output_tok_s": 8129.2,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 283.6,
     "ttft_ms_p50": 79.1,
     "spec": {
      "drafts": 6516,
      "draft_tokens": 19548,
      "accepted_tokens": 9738,
      "mean_acceptance_length": 2.494,
      "draft_acceptance_rate": 0.4982,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       0.4945,
       0.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 3.99,
     "output_tokens": 64686,
     "output_tok_s": 16211.1,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 145.4,
     "ttft_ms_p50": 206.5,
     "spec": {
      "drafts": 25886,
      "draft_tokens": 77658,
      "accepted_tokens": 38761,
      "mean_acceptance_length": 2.497,
      "draft_acceptance_rate": 0.4991,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       0.4974,
       0.0
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=2027) INFO 11-11 05:26:33 [model.py:733] Resolved architecture: Eagle3LlamaForCausalLM",
    "(APIServer pid=2027) INFO 11-11 05:26:34 [speculative.py:1669] Overriding draft model max model len from 131072 to 8192",
    "(EngineCore pid=2050) INFO 11-11 05:26:45 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=SpeculativeConfig(method='eagle3', model='RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3', num_spec_tokens=3), tokenizer='unslo",
    "(EngineCore pid=2050) INFO 11-11 05:26:54 [eagle3_utils.py:30] Using Eagle3 auxiliary layers from model: (2, 16, 29)",
    "(EngineCore pid=2050) INFO 11-11 05:26:58 [decorators.py:319] [eagle_head] Directly load AOT compilation from path /root/.cache/vllm/torch_compile_cache/torch_aot_compile/f5991b09c9b01cd08f744a363a59a856f88cee1c337f78fadc7830f87c8a5c66/rank_0_0/model",
    "(EngineCore pid=2050) INFO 11-11 05:26:58 [monitor.py:62] [eagle_head] torch.compile took 0.06 s in total",
    "(EngineCore pid=2050) INFO 11-11 05:26:58 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(APIServer pid=2027) INFO 11-11 05:27:32 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 2.50, Accepted throughput: 287.88 tokens/s, Drafted throughput: 573.94 tokens/s, Accepted: 3324 tokens, Drafted: 6627 tokens, Per-position acceptance rate: 1.000, 0.505, 0.000, Avg Draft acceptanc"
   ]
  },
  {
   "config": "synthetic 4.0",
   "speculative_config": {
    "method": "eagle3",
    "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
    "num_speculative_tokens": 3,
    "rejection_sample_method": "synthetic",
    "synthetic_acceptance_length": 4.0
   },
   "startup_s": 66.0,
   "kv_pool_tokens": 414048,
   "runs": [
    {
     "workload": "chat",
     "concurrency": 1,
     "requests": 6,
     "seconds": 3.01,
     "output_tokens": 1518,
     "output_tok_s": 504.9,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 522.4,
     "ttft_ms_p50": 18.3,
     "spec": {
      "drafts": 380,
      "draft_tokens": 1140,
      "accepted_tokens": 1140,
      "mean_acceptance_length": 4.0,
      "draft_acceptance_rate": 1.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       1.0,
       1.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 8,
     "requests": 16,
     "seconds": 1.08,
     "output_tokens": 3988,
     "output_tok_s": 3696.9,
     "prompt_tokens_p50": 62,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 508.1,
     "ttft_ms_p50": 37.6,
     "spec": {
      "drafts": 998,
      "draft_tokens": 2994,
      "accepted_tokens": 2994,
      "mean_acceptance_length": 4.0,
      "draft_acceptance_rate": 1.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       1.0,
       1.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 32,
     "requests": 64,
     "seconds": 1.32,
     "output_tokens": 16337,
     "output_tok_s": 12357.4,
     "prompt_tokens_p50": 63,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 452.2,
     "ttft_ms_p50": 85.7,
     "spec": {
      "drafts": 4085,
      "draft_tokens": 12255,
      "accepted_tokens": 12255,
      "mean_acceptance_length": 4.0,
      "draft_acceptance_rate": 1.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       1.0,
       1.0
      ]
     }
    },
    {
     "workload": "chat",
     "concurrency": 128,
     "requests": 256,
     "seconds": 2.73,
     "output_tokens": 64790,
     "output_tok_s": 23692.1,
     "prompt_tokens_p50": 64,
     "completion_tokens_p50": 256,
     "request_decode_tok_s_p50": 223.7,
     "ttft_ms_p50": 201.3,
     "spec": {
      "drafts": 16206,
      "draft_tokens": 48618,
      "accepted_tokens": 48618,
      "mean_acceptance_length": 4.0,
      "draft_acceptance_rate": 1.0,
      "draft_tokens_per_draft": 3.0,
      "accepted_through_position": [
       1.0,
       1.0,
       1.0
      ]
     }
    }
   ],
   "server_log": [
    "(APIServer pid=2193) INFO 11-11 05:28:00 [model.py:733] Resolved architecture: Eagle3LlamaForCausalLM",
    "(APIServer pid=2193) INFO 11-11 05:28:01 [speculative.py:1669] Overriding draft model max model len from 131072 to 8192",
    "(EngineCore pid=2216) INFO 11-11 05:28:12 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='unsloth/Meta-Llama-3.1-8B-Instruct', speculative_config=SpeculativeConfig(method='eagle3', model='RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3', num_spec_tokens=3), tokenizer='unslo",
    "(EngineCore pid=2216) INFO 11-11 05:28:22 [eagle3_utils.py:30] Using Eagle3 auxiliary layers from model: (2, 16, 29)",
    "(EngineCore pid=2216) INFO 11-11 05:28:26 [decorators.py:319] [eagle_head] Directly load AOT compilation from path /root/.cache/vllm/torch_compile_cache/torch_aot_compile/f5991b09c9b01cd08f744a363a59a856f88cee1c337f78fadc7830f87c8a5c66/rank_0_0/model",
    "(EngineCore pid=2216) INFO 11-11 05:28:26 [monitor.py:62] [eagle_head] torch.compile took 0.06 s in total",
    "(EngineCore pid=2216) INFO 11-11 05:28:26 [monitor.py:91] [eagle_head] Initial profiling/warmup run took 0.00 s",
    "(APIServer pid=2193) INFO 11-11 05:29:01 [metrics.py:120] SpecDecoding metrics: Mean acceptance length: 4.00, Accepted throughput: 885.32 tokens/s, Drafted throughput: 885.32 tokens/s, Accepted: 10389 tokens, Drafted: 10389 tokens, Per-position acceptance rate: 1.000, 1.000, 1.000, Avg Draft accepta"
   ]
  }
 ]
}
