{"contractVersion":"1.27.0","license":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/","attribution":"AI Compute Tracker (aicomputetracker.com), CC BY 4.0; third-party terms apply to data from Vendor-published inference measurements","sources":[{"id":"vendor-benchmarks","name":"Vendor-published inference measurements","attribution":"Measurements: NVIDIA TensorRT-LLM documentation; vendor-reported synthetic workloads","terms":"https://github.com/NVIDIA/TensorRT-LLM/blob/0c9430e5a530ba958fc9dca561a3ad865ad9f492/LICENSE","termsVersion":"TensorRT-LLM documentation at 0c9430e5a530ba958fc9dca561a3ad865ad9f492","redistribution":"pending-counsel","license":"Third-party terms apply"}]},"schemaVersion":1,"reviewedAt":"2026-11-03","note":"Offline synthetic maximum-load throughput; no interactive latency guarantee or task-quality evaluation.","benchmarks":[{"id":1,"label":"Short output","inputTokens":2048,"outputTokens":128,"outputTokensPerSecond":3275.55,"requestCount":3000,"model":"nvidia/Llama-3.1-8B-Instruct-FP8","precision":"FP8","system":"DGX H100; H100 SXM 80GB","gpus":1,"tensorParallel":1,"concurrency":null,"ttftMs":null,"tpotMs":null,"url":"https://nvidia.github.io/TensorRT-LLM/performance/perf-overview.html","version":"0c9430e5a530ba958fc9dca561a3ad865ad9f492","sourceUpdatedAt":"2025-09-15","reviewedAt":"2026-11-03","measurementDate":null,"evidence":"vendor-measured","engine":"TensorRT-LLM, v0.21 documentation, PyTorch backend","scope":"Offline synthetic maximum-load throughput; no interactive latency guarantee or task-quality evaluation."},{"id":2,"label":"Longer output","inputTokens":2048,"outputTokens":2048,"outputTokensPerSecond":9462.43,"requestCount":1500,"model":"nvidia/Llama-3.1-8B-Instruct-FP8","precision":"FP8","system":"DGX H100; H100 SXM 80GB","gpus":1,"tensorParallel":1,"concurrency":null,"ttftMs":null,"tpotMs":null,"url":"https://nvidia.github.io/TensorRT-LLM/performance/perf-overview.html","version":"0c9430e5a530ba958fc9dca561a3ad865ad9f492","sourceUpdatedAt":"2025-09-15","reviewedAt":"2026-11-03","measurementDate":null,"evidence":"vendor-measured","engine":"TensorRT-LLM, v0.21 documentation, PyTorch backend","scope":"Offline synthetic maximum-load throughput; no interactive latency guarantee or task-quality evaluation."},{"id":3,"label":"Long input","inputTokens":20000,"outputTokens":2000,"outputTokensPerSecond":1340.69,"requestCount":1000,"model":"nvidia/Llama-3.1-8B-Instruct-FP8","precision":"FP8","system":"DGX H100; H100 SXM 80GB","gpus":1,"tensorParallel":1,"concurrency":null,"ttftMs":null,"tpotMs":null,"url":"https://nvidia.github.io/TensorRT-LLM/performance/perf-overview.html","version":"0c9430e5a530ba958fc9dca561a3ad865ad9f492","sourceUpdatedAt":"2025-09-15","reviewedAt":"2026-11-03","measurementDate":null,"evidence":"vendor-measured","engine":"TensorRT-LLM, v0.21 documentation, PyTorch backend","scope":"Offline synthetic maximum-load throughput; no interactive latency guarantee or task-quality evaluation."}]}