261 lines
12 KiB
HTML
261 lines
12 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>FPGABench: Verilog PPA Leaderboard for LLMs</title>
|
|
|
|
<!-- SEO Meta Tags -->
|
|
<meta name="description" content="FPGABench evaluates LLM-generated FPGA Verilog code on 52 hardware design problems, measuring both correctness (Feasible@1) and design quality (MOQ@1) like power, performance, and area.">
|
|
<meta name="keywords" content="FPGABench, Verilog, FPGA, LLM, Leaderboard, AI, Hardware Design, PPA, pass@1, MOQ, EDA, Vivado, Quartus">
|
|
<meta name="author" content="FPGABench Team">
|
|
|
|
<!-- Open Graph / Facebook -->
|
|
<meta property="og:type" content="website">
|
|
<meta property="og:title" content="FPGABench: Verilog PPA Leaderboard for LLMs">
|
|
<meta property="og:description" content="A 52-problem benchmark assessing LLM-generated FPGA Verilog code. Goes beyond functional correctness to Power, Performance, and Area (PPA) quality.">
|
|
|
|
<!-- Twitter -->
|
|
<meta property="twitter:card" content="summary_large_image">
|
|
<meta property="twitter:title" content="FPGABench: Verilog PPA Leaderboard for LLMs">
|
|
<meta property="twitter:description" content="A 52-problem benchmark assessing LLM-generated FPGA Verilog code. Goes beyond functional correctness to Power, Performance, and Area (PPA) quality.">
|
|
|
|
<!-- Google Fonts -->
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Plus+Jakarta+Sans:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
|
|
|
<!-- CSS Stylesheet -->
|
|
|
|
<!-- Chart.js CDN -->
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
<script type="module" crossorigin src="/assets/index-D9BrbtBe.js"></script>
|
|
<link rel="stylesheet" crossorigin href="/assets/index-rRGFXrZW.css">
|
|
</head>
|
|
<body>
|
|
|
|
<!-- Ambient background glow effects -->
|
|
<div class="glow-bg glow-bg-1"></div>
|
|
<div class="glow-bg glow-bg-2"></div>
|
|
|
|
<!-- Top Navigation Header -->
|
|
<header class="app-header">
|
|
<div class="header-container">
|
|
<div class="logo-area">
|
|
<span class="logo-text">FPGA<span class="highlight">Bench</span></span>
|
|
</div>
|
|
<nav class="nav-links">
|
|
<a href="#leaderboard-section">Leaderboard</a>
|
|
<a href="#curve-section">Frontier Curve</a>
|
|
<a href="#about-section">About</a>
|
|
<a href="#submit-section" class="btn-primary-sm">Submit Results</a>
|
|
</nav>
|
|
</div>
|
|
</header>
|
|
|
|
<main class="main-content">
|
|
|
|
<!-- Hero / Intro Section -->
|
|
<section class="hero-section">
|
|
<div class="hero-content">
|
|
<h1>Evaluating Verilog Synthesis & PPA Quality</h1>
|
|
<p class="lead">
|
|
FPGABench is a 52-problem benchmark designed to assess FPGA Verilog code generated by Large Language Models.
|
|
It goes beyond simple functional correctness to evaluate critical hardware design quality metrics.
|
|
</p>
|
|
</div>
|
|
|
|
<!-- Key Statistics Dashboard -->
|
|
<div class="stats-grid">
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Total Problems</span>
|
|
<span class="stat-value">52</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Top Feasible@1</span>
|
|
<span class="stat-value" id="top-feasible-val">82.7%</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Top MOQ@1 Quality</span>
|
|
<span class="stat-value" id="top-moq-val">1.12</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Evaluated Models</span>
|
|
<span class="stat-value" id="total-models-val">13</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Interactive Frontier Curve Chart Section -->
|
|
<section id="curve-section" class="dashboard-panel">
|
|
<div class="panel-header">
|
|
<div class="panel-title-area">
|
|
<h2>Feasibility vs. Design Quality Frontier</h2>
|
|
<p class="panel-desc">Visualizing model capability. The Pareto frontier curve connects models achieving the optimal balance between functional feasibility and compilation/PPA quality.</p>
|
|
</div>
|
|
<div class="chart-legend-custom">
|
|
<span class="legend-item"><span class="legend-dot color-primary"></span>Model Performance</span>
|
|
<span class="legend-item"><span class="legend-line color-secondary"></span>Pareto Frontier</span>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container-wrapper">
|
|
<canvas id="frontierChart"></canvas>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Leaderboard Section -->
|
|
<section id="leaderboard-section" class="dashboard-panel">
|
|
<div class="panel-header table-controls-header">
|
|
<div class="panel-title-area">
|
|
<h2>FPGABench Leaderboard</h2>
|
|
<p class="panel-desc">Sort by columns to find the most cost-effective, high-quality, or functionally correct models.</p>
|
|
</div>
|
|
|
|
<!-- Search and Filter Controls -->
|
|
<div class="filters-container">
|
|
<div class="search-wrapper">
|
|
<input type="text" id="model-search" placeholder="Search models or providers..." aria-label="Search models">
|
|
</div>
|
|
|
|
<div class="filter-dropdown-wrapper">
|
|
<label for="type-filter" class="sr-only">Filter by Access</label>
|
|
<select id="type-filter">
|
|
<option value="all">All License Types</option>
|
|
<option value="Open Source">Open Source</option>
|
|
<option value="Proprietary">Proprietary</option>
|
|
</select>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Leaderboard Table -->
|
|
<div class="table-wrapper">
|
|
<table class="leaderboard-table" id="leaderboard">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col" class="col-rank sortable sorted-asc" data-sort="rank">Rank <span class="sort-indicator">↑</span></th>
|
|
<th scope="col" class="col-name sortable" data-sort="name">Model Name <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-dev sortable" data-sort="developer">Developer <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-type sortable" data-sort="type">License <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-feasible sortable" data-sort="feasible">Feasible@1 <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-moq sortable" data-sort="moq">MOQ@1 <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-action">Details</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="leaderboard-body">
|
|
<!-- Dynamic data injected here via app.js -->
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- About & Metric Definitions Section -->
|
|
<section id="about-section" class="info-section-grid">
|
|
<div class="info-card">
|
|
<h3>What is FPGABench?</h3>
|
|
<p>
|
|
Unlike general code generation benchmarks (e.g., HumanEval) that only verify functional execution, FPGA Verilog development is tightly bound to hardware constraints.
|
|
A design that runs correctly in simulation can still fail synthesis, exceed resource budgets, or run too slowly on a real chip.
|
|
</p>
|
|
<p>
|
|
<strong>FPGABench</strong> introduces 52 diverse, hand-crafted hardware design problems ranging from basic arithmetic blocks (multipliers, ALUs) to complex sequential circuits (FIFO controllers, SPI masters, AES pipelines). It automatically synthesizes generated code to evaluate post-routing hardware metrics.
|
|
</p>
|
|
</div>
|
|
|
|
<div class="info-card">
|
|
<h3>Key Metrics Defined</h3>
|
|
<div class="metric-def">
|
|
<div class="metric-title">
|
|
<span class="metric-badge badge-emerald">Feasible@1</span>
|
|
<strong>Functional & Synthesis Success</strong>
|
|
</div>
|
|
<p>
|
|
The percentage of LLM generations that successfully pass compiler parsing, syntactical checks, compile clean in EDA tools (like Xilinx Vivado or Intel Quartus), and fulfill functional simulation specifications. It is an adaptation of the classic pass@1 metric for hardware.
|
|
</p>
|
|
</div>
|
|
|
|
<div class="metric-def">
|
|
<div class="metric-title">
|
|
<span class="metric-badge badge-purple">MOQ@1 (Mean Quality of Design)</span>
|
|
<strong>PPA Efficiency Index</strong>
|
|
</div>
|
|
<p>
|
|
A composite numerical score normalized around <strong>1.0</strong> (representing reference designs). MOQ@1 evaluates Power, Performance, and Area (PPA) efficiency. It ranks models based on:
|
|
</p>
|
|
<ul>
|
|
<li><strong>Area:</strong> Look-Up Table (LUT) and Flip-Flop usage (lower is better).</li>
|
|
<li><strong>Performance:</strong> Maximum Clock Frequency ($F_{max}$) post-routing (higher is better).</li>
|
|
<li><strong>Power:</strong> Estimated dynamic power consumption (lower is better).</li>
|
|
</ul>
|
|
<p>An MOQ@1 value > 1.0 indicates that the model's design quality surpasses standard reference implementation efficiency.</p>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Submission Guidelines Section -->
|
|
<section id="submit-section" class="submit-section">
|
|
<div class="submit-card">
|
|
<h2>Submit Your Model Results</h2>
|
|
<p>
|
|
We encourage researchers and developers to run FPGABench on new models.
|
|
To add your model to the official leaderboard, please follow the steps below:
|
|
</p>
|
|
|
|
<div class="steps-list">
|
|
<div class="step-item">
|
|
<div class="step-num">1</div>
|
|
<div class="step-detail">
|
|
<strong>Clone the FPGABench Repository:</strong>
|
|
<code>git clone https://github.com/fpgabench/fpgabench.git</code>
|
|
</div>
|
|
</div>
|
|
<div class="step-item">
|
|
<div class="step-num">2</div>
|
|
<div class="step-detail">
|
|
<strong>Run generation & synthesis suite:</strong>
|
|
<code>python run_eval.py --model your-model-name --eda_tool vivado</code>
|
|
</div>
|
|
</div>
|
|
<div class="step-item">
|
|
<div class="step-num">3</div>
|
|
<div class="step-detail">
|
|
<strong>Submit a Pull Request:</strong>
|
|
Upload your generated JSON result artifact under <code>results/your_model.json</code>.
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="submit-actions">
|
|
<a href="https://github.com/fpgabench/fpgabench" target="_blank" rel="noopener noreferrer" class="btn-primary">View GitHub Repository</a>
|
|
<a href="mailto:submit@fpgabench.org" class="btn-secondary">Contact Committee</a>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</main>
|
|
|
|
<footer class="app-footer">
|
|
<div class="footer-container">
|
|
<p>© 2026 FPGABench Project. All rights reserved.</p>
|
|
<div class="footer-meta">
|
|
<span>Evaluated with Xilinx Vivado 2023.2 & Synopsys Design Compiler</span>
|
|
<span class="separator">•</span>
|
|
<a href="#about-section">Documentation</a>
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
<!-- Javascript Logic -->
|
|
</body>
|
|
</html>
|