386 lines
16 KiB
HTML
386 lines
16 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Can LLMs Generate FPGA-Ready Verilog? | FPGABench</title>
|
|
|
|
<!-- SEO Meta Tags -->
|
|
<meta name="description"
|
|
content="FPGABench tests generated hardware designs beyond simulation, measuring whether they can actually synthesise, meet physical FPGA constraints, and optimise for a design objective.">
|
|
<meta name="keywords"
|
|
content="FPGABench, Verilog, FPGA, LLM, Leaderboard, AI, Hardware Design, PPA, Feasible@K, MOQ@K, EDA, Vivado, Quartus">
|
|
<meta name="author" content="FPGABench Team">
|
|
|
|
<!-- Open Graph / Facebook -->
|
|
<meta property="og:type" content="website">
|
|
<meta property="og:title" content="Can LLMs Generate FPGA-Ready Verilog? | FPGABench">
|
|
<meta property="og:description"
|
|
content="FPGABench tests generated hardware designs beyond simulation, measuring whether they can actually synthesise, meet physical FPGA constraints, and optimise for a design objective.">
|
|
|
|
<!-- Twitter -->
|
|
<meta property="twitter:card" content="summary_large_image">
|
|
<meta property="twitter:title" content="Can LLMs Generate FPGA-Ready Verilog? | FPGABench">
|
|
<meta property="twitter:description"
|
|
content="FPGABench tests generated hardware designs beyond simulation, measuring whether they can actually synthesise, meet physical FPGA constraints, and optimise for a design objective.">
|
|
|
|
<!-- Google Fonts -->
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link
|
|
href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=Plus+Jakarta+Sans:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap"
|
|
rel="stylesheet">
|
|
|
|
<!-- CSS Stylesheet -->
|
|
<link rel="stylesheet" href="style.css">
|
|
|
|
<!-- Chart.js CDN -->
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
|
|
<!-- KaTeX CSS & JS for Academic Equations -->
|
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.css">
|
|
<script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.js"></script>
|
|
<script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/contrib/auto-render.min.js"
|
|
onload="renderMathInElement(document.body, {delimiters: [{left: '$$', right: '$$', display: true}, {left: '$', right: '$', display: false}]});"></script>
|
|
</head>
|
|
|
|
<body>
|
|
|
|
<!-- Ambient background glow effects -->
|
|
<div class="glow-bg glow-bg-1"></div>
|
|
<div class="glow-bg glow-bg-2"></div>
|
|
|
|
<!-- Top Navigation Header -->
|
|
<header class="app-header">
|
|
<div class="header-container">
|
|
<div class="logo-area">
|
|
<span class="logo-text">FPGA<span class="highlight">Bench</span></span>
|
|
</div>
|
|
<nav class="nav-links">
|
|
<a href="#leaderboard-section">Leaderboard</a>
|
|
<a href="#curve-section">Frontier Curve</a>
|
|
<a href="#about-section">About</a>
|
|
<a href="/paper.pdf" target="_blank" class="btn-paper">Read Paper</a>
|
|
<a href="#submit-section" class="btn-primary-sm">Submit Results</a>
|
|
</nav>
|
|
</div>
|
|
</header>
|
|
|
|
<main class="main-content">
|
|
|
|
<!-- Hero / Intro Section -->
|
|
<section class="hero-section">
|
|
<div class="hero-content">
|
|
<h1>Can LLMs Generate FPGA-Ready Verilog?</h1>
|
|
<p class="lead">
|
|
FPGABench tests generated hardware designs beyond simulation, measuring whether they can actually synthesise,
|
|
meet physical FPGA constraints, and optimise for a design objective.
|
|
</p>
|
|
<div class="hero-actions" style="margin-top: 1.5rem; display: flex; gap: 1rem;">
|
|
<a href="/paper.pdf" target="_blank" class="btn-primary">Read Paper</a>
|
|
<a href="#leaderboard-section" class="btn-secondary">Explore Leaderboard</a>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Key Statistics Dashboard -->
|
|
<div class="stats-grid">
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Total Problems</span>
|
|
<span class="stat-value">52</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Top Feasible@1</span>
|
|
<span class="stat-value" id="top-feasible-val">82.7%</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Top MOQ@1 Quality</span>
|
|
<span class="stat-value" id="top-moq-val">1.12</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="stat-card">
|
|
<div class="stat-info">
|
|
<span class="stat-label">Evaluated Models</span>
|
|
<span class="stat-value" id="total-models-val">13</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Interactive Frontier Curve Chart Section -->
|
|
<section id="curve-section" class="dashboard-panel">
|
|
<div class="panel-header">
|
|
<div class="panel-title-area">
|
|
<h2>Feasibility vs. Design Quality Frontier</h2>
|
|
<p class="panel-desc">Visualizing model capability. The Pareto frontier curve connects models achieving the
|
|
optimal balance between functional feasibility and compilation/PPA quality.</p>
|
|
</div>
|
|
<div class="chart-legend-custom">
|
|
<span class="legend-item"><span class="legend-dot color-primary"></span>Model Performance</span>
|
|
<span class="legend-item"><span class="legend-line color-secondary"></span>Pareto Frontier</span>
|
|
</div>
|
|
</div>
|
|
<div class="chart-container-wrapper">
|
|
<canvas id="frontierChart"></canvas>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Leaderboard Section -->
|
|
<section id="leaderboard-section" class="dashboard-panel">
|
|
<div class="panel-header table-controls-header">
|
|
<div class="panel-title-area">
|
|
<h2>FPGABench Leaderboard</h2>
|
|
<p class="panel-desc">Sort by columns to find the most cost-effective, high-quality, or functionally correct
|
|
models.</p>
|
|
</div>
|
|
|
|
<!-- Search and Filter Controls -->
|
|
<div class="filters-container">
|
|
<div class="search-wrapper">
|
|
<input type="text" id="model-search" placeholder="Search models or providers..." aria-label="Search models">
|
|
</div>
|
|
|
|
<div class="filter-dropdown-wrapper">
|
|
<label for="type-filter" class="sr-only">Filter by Access</label>
|
|
<select id="type-filter">
|
|
<option value="all">All License Types</option>
|
|
<option value="Open Source">Open Source</option>
|
|
<option value="Proprietary">Proprietary</option>
|
|
</select>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Leaderboard Table -->
|
|
<div class="table-wrapper">
|
|
<table class="leaderboard-table" id="leaderboard">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col" class="col-rank sortable sorted-asc" data-sort="rank">Rank <span
|
|
class="sort-indicator">↑</span></th>
|
|
<th scope="col" class="col-name sortable" data-sort="name">Model Name <span
|
|
class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-dev sortable" data-sort="developer">Developer <span
|
|
class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-type sortable" data-sort="type">License <span class="sort-indicator">↕</span>
|
|
</th>
|
|
<th scope="col" class="col-feasible sortable" data-sort="feasible">Feasible@1 <span
|
|
class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-moq sortable" data-sort="moq">MOQ@1 <span class="sort-indicator">↕</span></th>
|
|
<th scope="col" class="col-action">Details</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="leaderboard-body">
|
|
<!-- Dynamic data injected here via app.js -->
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- About & Metric Definitions Section -->
|
|
<section id="about-section" class="info-section-grid">
|
|
<div class="info-card">
|
|
<h3>What is FPGABench?</h3>
|
|
<p>
|
|
Unlike general code generation benchmarks (e.g., HumanEval) that only verify functional execution, FPGA
|
|
Verilog development is tightly bound to hardware constraints.
|
|
A design that runs correctly in simulation can still fail synthesis, exceed resource budgets, or run too
|
|
slowly on a real chip.
|
|
</p>
|
|
<p>
|
|
<strong>FPGABench</strong> introduces 52 diverse, hand-crafted hardware design problems ranging from basic
|
|
arithmetic blocks (multipliers, ALUs) to complex sequential circuits (FIFO controllers, SPI masters, AES
|
|
pipelines). It automatically synthesizes generated code to evaluate post-routing hardware metrics.
|
|
</p>
|
|
</div>
|
|
|
|
<div class="info-card">
|
|
<h3>Key Metrics Defined</h3>
|
|
<div class="metric-def">
|
|
<div class="metric-title">
|
|
<span class="metric-badge badge-emerald">Feasible@K</span>
|
|
<strong>Functional, Synthesis & Constraint Satisfaction</strong>
|
|
</div>
|
|
<p>
|
|
A candidate design $d$ is feasible for problem $p$ iff it is functionally correct, synthesisable, and meets
|
|
all constraints. $\text{Feasible@}K$ is then the fraction of problems where model $m$ produces a feasible
|
|
design in $K$ attempts:
|
|
</p>
|
|
<div class="equation-block">
|
|
$$\text{feasible}(d, p) = \mathbf{1}[\text{correct}(d, p)] \cdot \mathbf{1}[\text{synth}(d)] \cdot
|
|
\prod_{j=1}^{N_c} \mathbf{1}[d \text{ meets constraint } c_p(j)]$$
|
|
</div>
|
|
<div class="equation-block">
|
|
$$\text{Feasible@}K = \frac{1}{|P|} \sum_{p \in P} \max_{1 \le i \le K} \text{feasible}(d^{(i)}, p)$$
|
|
</div>
|
|
</div>
|
|
|
|
<div class="metric-def">
|
|
<div class="metric-title">
|
|
<span class="metric-badge badge-purple">MOQ@K (Maximum Objective Quality)</span>
|
|
<strong>PPA Optimization Index</strong>
|
|
</div>
|
|
<p>
|
|
$\text{Feasible@}K$ measures whether a model can produce a usable design, but not how well that design
|
|
optimises the target hardware. To quantify synthesis quality among feasible designs, we define the
|
|
<strong>Maximum Objective Quality (MOQ)</strong>.
|
|
</p>
|
|
<p>
|
|
For each problem $p$, let $f_p^*$ denote the objective value achieved by the reference implementation. For a
|
|
feasible generated design $d$, define its normalised quality $q_p(d)$:
|
|
</p>
|
|
<div class="equation-block">
|
|
$$q_p(d) = \begin{cases} \frac{f(d)}{f_p^*} & \text{if } \text{dir}_p = \;\uparrow (F_{\text{max}}) \\
|
|
\frac{f_p^*}{f(d)} & \text{if } \text{dir}_p = \;\downarrow (\text{LUTs}) \end{cases}$$
|
|
</div>
|
|
<p>
|
|
Here, $q_p(d) = 1$ means the generated design matches the reference, $q_p(d) > 1$ means it improves on the
|
|
reference, and $q_p(d) < 1$ indicates underperformance. </p>
|
|
<p>
|
|
Across a benchmark set $P$, $\text{MOQ@}K$ is defined as the geometric mean, over all problems, of the
|
|
best feasible objective ratio found within the first $K$ attempts:
|
|
</p>
|
|
<div class="equation-block">
|
|
$$\text{MOQ@}K = \left( \prod_{p \in P} \max_{\substack{1 \le i \le K \\ \text{feasible}(d^{(i)}, p) =
|
|
1}} q_p(d^{(i)}) \right)^{\frac{1}{|P|}}$$
|
|
</div>
|
|
<p>
|
|
The geometric mean is the natural aggregator for ratio-valued scores: a design $2\times$ better and one
|
|
$0.5\times$ worse correctly average to $1.0$ ($\sqrt{2 \times 0.5} = 1.0$), whereas the arithmetic mean
|
|
would incorrectly report a net gain of $1.25$. $\text{MOQ@}K$ is conditional on feasibility and must be
|
|
interpreted alongside $\text{Feasible@}K$.
|
|
</p>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Scoring Examples Section -->
|
|
<section class="dashboard-panel" style="margin-top: 2rem;">
|
|
<div class="panel-header">
|
|
<div class="panel-title-area">
|
|
<h3>Table 4.3: Scoring examples for the 7-Sample Median Denoising Filter</h3>
|
|
<p class="panel-desc">A detailed illustration of candidate evaluation for a specific hardware design target
|
|
under Feasible@K and MOQ@K metrics.</p>
|
|
</div>
|
|
</div>
|
|
<div class="table-wrapper">
|
|
<table class="leaderboard-table">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col">Candidate ID</th>
|
|
<th scope="col">Functionally Correct</th>
|
|
<th scope="col">Synthesizable</th>
|
|
<th scope="col">Meets Constraints</th>
|
|
<th scope="col">LUT Usage</th>
|
|
<th scope="col">Feasible</th>
|
|
<th scope="col">Quality ratio ($q_p(d)$)</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>Candidate 1 (Optimal)</td>
|
|
<td>✅ Yes</td>
|
|
<td>✅ Yes</td>
|
|
<td>✅ Yes</td>
|
|
<td>80 (Reference: 100)</td>
|
|
<td><span class="badge-type badge-open">Yes</span></td>
|
|
<td><strong>1.25</strong> (Improves PPA)</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Candidate 2 (Suboptimal)</td>
|
|
<td>✅ Yes</td>
|
|
<td>✅ Yes</td>
|
|
<td>✅ Yes</td>
|
|
<td>120 (Reference: 100)</td>
|
|
<td><span class="badge-type badge-open">Yes</span></td>
|
|
<td><strong>0.83</strong> (Underperforms)</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Candidate 3 (Fails Constraint)</td>
|
|
<td>✅ Yes</td>
|
|
<td>✅ Yes</td>
|
|
<td>❌ No (Timing Violation)</td>
|
|
<td>70 (Reference: 100)</td>
|
|
<td><span class="badge-type badge-proprietary">No</span></td>
|
|
<td>— (Infeasible)</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Candidate 4 (Fails Function)</td>
|
|
<td>❌ No</td>
|
|
<td>✅ Yes</td>
|
|
<td>—</td>
|
|
<td>—</td>
|
|
<td><span class="badge-type badge-proprietary">No</span></td>
|
|
<td>— (Infeasible)</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</section>
|
|
|
|
<!-- Submission Guidelines Section -->
|
|
<section id="submit-section" class="submit-section">
|
|
<div class="submit-card">
|
|
<h2>Submit Your Model Results</h2>
|
|
<p>
|
|
We encourage researchers and developers to run FPGABench on new models.
|
|
To add your model to the official leaderboard, please follow the steps below:
|
|
</p>
|
|
|
|
<div class="steps-list">
|
|
<div class="step-item">
|
|
<div class="step-num">1</div>
|
|
<div class="step-detail">
|
|
<strong>Clone the FPGABench Repository:</strong>
|
|
<code>git clone https://github.com/fpgabench/fpgabench.git</code>
|
|
</div>
|
|
</div>
|
|
<div class="step-item">
|
|
<div class="step-num">2</div>
|
|
<div class="step-detail">
|
|
<strong>Run generation & synthesis suite:</strong>
|
|
<code>python run_eval.py --model your-model-name --eda_tool vivado</code>
|
|
</div>
|
|
</div>
|
|
<div class="step-item">
|
|
<div class="step-num">3</div>
|
|
<div class="step-detail">
|
|
<strong>Submit a Pull Request:</strong>
|
|
Upload your generated JSON result artifact under <code>results/your_model.json</code>.
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="submit-actions">
|
|
<a href="https://github.com/fpgabench/fpgabench" target="_blank" rel="noopener noreferrer"
|
|
class="btn-primary">View GitHub Repository</a>
|
|
<a href="mailto:submit@fpgabench.org" class="btn-secondary">Contact Committee</a>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</main>
|
|
|
|
<footer class="app-footer">
|
|
<div class="footer-container">
|
|
<p>© 2026 FPGABench Project. All rights reserved.</p>
|
|
<div class="footer-meta">
|
|
<span>Evaluated with Xilinx Vivado 2023.2 & Synopsys Design Compiler</span>
|
|
<span class="separator">•</span>
|
|
<a href="#about-section">Documentation</a>
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
<!-- Javascript Logic -->
|
|
<script src="app.js" type="module"></script>
|
|
</body>
|
|
|
|
</html> |