michele/FPGA-Neural
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exp: N=16 timing closure fixed (EXP-0056), weight-reuse gives real 7.16x memory speedup without DDR3 (EXP-0057)

Browse Source 
EXP-0056: N_SLOTS=16 failed timing on LFE5U-85F (23-24MHz vs 64MHz
target). First hypothesis (dependency_manager.v's serial ready-scan)
was wrong but real -- built and verified priority_encoder_lsb.v (a
generic recursive tree encoder) and dependency_manager_fast.v, bit-
exact equivalent to the original, but integrated it made no real
difference (24.26MHz). The real cause, found from nextpnr's own
critical-path report: nms_activation_fill_ctrl_v3.v's balanced max-
tree was only ever extended to N_SLOTS in {1,2,4,8}, silently falling
back to the original slow scan for 16. Added the missing case
(nms_activation_fill_ctrl_v3_n16.v), verified isolated (10017/10017)
and functionally (D-Stress N=16 still 256/256 bit-exact). Real result:
71.01MHz, PASS at 64MHz (single seed so far).

EXP-0057: built layer_weight_buffer.v, a double-buffered per-layer
weight scratchpad (fill one buffer in the background from SDRAM while
compute reads many times from the other -- weight-stationary reuse,
as opposed to D-Stress's own deliberately zero-reuse pattern). Wired
to the real sdram_controller_openrow.v + sdram_model.v, no new
hardware. For the same 32768 bytes of useful data: zero-reuse costs
27048 real cycles, reuse costs 3777 -- 7.16x real measured speedup on
the SAME SDR SDRAM, no DDR3, no clock change. This is the answer to
whether DDR3 is necessary for a workload class that actually has
reuse (e.g. conv-style face recognition, unlike D-Stress) -- it isn't,
at least not for this reason.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUG92aM9m68TRc4rG55BcC
This commit is contained in:
michele
2026-09-16 12:01:11 +02:00
co-authored by Claude Sonnet 5
parent fce8ff2d66
commit 1ce78dff6e
25 changed files with 25848 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hardware/v2/nms/rtl/fpga_neural_v2_top_openrow_fast.v
+265
View File
@@ -0,0 +1,265 @@
`timescale 1ns/1ps
// ================================================================
// FPGA-Neural V2 -- BOARD-LEVEL TOP (STEP20, real physical interface)
//
// Wraps the STEP19 frozen compute+memory design (the same submodules
// nms_neural_multiprocessor_sdram_unified.v instantiates -- that file
// itself is NOT instantiated here, since its own reg_*/N_SLOTS+1-port
// AR arbitration needs a second arbitration LEVEL added for the new
// host-raw-SDRAM-access port; this module reproduces that same
// internal wiring plus the extra level, rather than modifying the
// frozen file) with the three things a real physical board needs that
// a testbench does not:
//
// 1. A real SPI host interface (spi_host_bridge.v) in place of the
// 110-pin reg_* testbench bus -- reg_valid/reg_ready/reg_node_id/
// etc are now DRIVEN BY THE BRIDGE, not exposed as top ports.
// 2. A real ECP5 PLL (ecp5_pll_sys_clk.v, EHXPLLL) generating the
// system clock from the board's 16MHz oscillator, instead of
// assuming an already-correct-frequency clock input.
// 3. A real reset/POR synchronizer (reset_sync.v).
//
// nms_dataflow_core_sdram.v, dependency_manager.v, neural_processor.v,
// neural_director.v, slot_mem_arbiter.v, slot_mem_arbiter_wide.v,
// sdram_unified_backend.v, sdram_controller.v are ALL byte-for-byte
// unchanged (STEP19/STEP20 standing constraint) -- this file only
// ADDS one more, already-proven, generically-parameterized
// slot_mem_arbiter instance (N_PORTS=2) to arbitrate the SPI bridge's
// raw host memory port against the existing compute-side AR stream,
// both funneling into the SAME single sdram_unified_backend/
// sdram_controller/AS4C4M16SA-6TIN physical chain STEP19 already
// validated. No V1 RTL is instantiated (STEP19's "zero V1 files in
// the V2 compile list" property is preserved).
// ================================================================
module fpga_neural_v2_top_openrow_fast #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 4,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter CLK_FREQ_MHZ = 64
)(
input wire osc_clk, // 16 MHz board oscillator
input wire ext_rst_n, // external POR/supervisor, active-low
// ---- physical SPI host interface ----
input wire spi_sclk,
input wire spi_mosi,
output wire spi_miso,
input wire spi_cs_n,
// ---- single physical SDRAM (weights + activations + results) ----
// sdram_clk: the real SDRAM chip's own CLK pin -- an external
// chip, it needs this driven from a real output ball, NOT just
// internal routing. Found missing entirely during this session's
// schematic review (clk_sys was purely internal, never reached a
// pad) -- added here, real free clock-capable ball (bank 6).
output wire sdram_clk,
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [1:0] sdram_ba,
output wire [12:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm,
// FPGA_DATA_READY: high once the whole registered graph has
// finished (system-idle sticky flag, self-clearing on new work) --
// see nms_dataflow_core_sdram.v for the full design comment.
output wire data_ready,
output wire pll_locked
);
// ============================================================
// CLOCK / RESET
// ============================================================
wire clk_sys;
ecp5_pll_sys_clk u_pll (
.clk_16mhz(osc_clk), .clk_sys(clk_sys), .locked(pll_locked)
);
assign sdram_clk = clk_sys;
wire clk = clk_sys;
wire rst;
reset_sync u_reset_sync (
.clk_sys(clk_sys), .ext_rst_n(ext_rst_n), .pll_locked(pll_locked), .rst(rst)
);
wire soft_rst_pulse;
wire core_rst = rst | soft_rst_pulse;
// ============================================================
// SPI HOST BRIDGE (replaces the 110-pin reg_* testbench bus)
// ============================================================
wire reg_valid, reg_ready;
wire [$clog2(N_NODES)-1:0] reg_node_id;
wire [$clog2(MAX_DEPS+1)-1:0] reg_required;
wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids;
wire [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr;
wire [15:0] reg_n_tiles;
wire host_mem_req, host_mem_wr, host_mem_lb_n, host_mem_ub_n;
wire [ADDR_WIDTH-1:0] host_mem_addr;
wire [15:0] host_mem_wdata, host_mem_rdata;
wire host_mem_ready;
spi_host_bridge #(
.ADDR_WIDTH(ADDR_WIDTH), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS)
) u_spi_bridge (
.clk(clk), .rst(rst),
.sclk(spi_sclk), .mosi(spi_mosi), .miso(spi_miso), .cs_n(spi_cs_n),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base),
.reg_n_tiles(reg_n_tiles), .reg_result_addr(reg_result_addr),
.mem_req(host_mem_req), .mem_wr(host_mem_wr), .mem_addr(host_mem_addr),
.mem_wdata(host_mem_wdata), .mem_lb_n(host_mem_lb_n), .mem_ub_n(host_mem_ub_n),
.mem_rdata(host_mem_rdata), .mem_ready(host_mem_ready),
.soft_rst_pulse(soft_rst_pulse)
);
// ============================================================
// COMPUTE + MEMORY (same wiring as nms_neural_multiprocessor_
// sdram_unified.v, plus the new host-arb level)
// ============================================================
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
wire [N_SLOTS:0] slot_mem_ready;
wire [N_SLOTS-1:0] wide_slot_mem_req;
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
wire [N_SLOTS-1:0] wide_slot_mem_ready;
nms_dataflow_core_sdram_fast #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
) u_dataflow_core (
.clk(clk), .rst(core_rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.data_ready(data_ready),
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
);
// ---- AR level 1 (unchanged): activation-fill + per-slot result
// writeback, exactly as nms_neural_multiprocessor_sdram_unified.v ----
wire arb_m_req, arb_m_wr;
wire [ADDR_WIDTH-1:0] arb_m_addr;
wire [15:0] arb_m_wdata;
wire arb_m_lb_n, arb_m_ub_n;
wire [15:0] arb_m_rdata;
wire arb_m_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
) u_arbiter (
.clk(clk), .rst(core_rst),
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
);
// ---- AR level 2 (NEW, STEP20): compute-side AR stream (port0)
// vs. SPI host raw memory port (port1) -- reuses slot_mem_arbiter
// completely unchanged, just at N_PORTS=2, its own already-proven
// pending-latch discipline applying equally to a 2-port instance ----
wire [1:0] host_arb_s_req, host_arb_s_wr, host_arb_s_lb_n, host_arb_s_ub_n, host_arb_s_ready;
wire [ADDR_WIDTH*2-1:0] host_arb_s_addr;
wire [16*2-1:0] host_arb_s_wdata, host_arb_s_rdata;
assign host_arb_s_req = {host_mem_req, arb_m_req};
assign host_arb_s_wr = {host_mem_wr, arb_m_wr};
assign host_arb_s_lb_n = {host_mem_lb_n, arb_m_lb_n};
assign host_arb_s_ub_n = {host_mem_ub_n, arb_m_ub_n};
assign host_arb_s_addr = {host_mem_addr, arb_m_addr};
assign host_arb_s_wdata = {host_mem_wdata, arb_m_wdata};
assign arb_m_ready = host_arb_s_ready[0];
assign arb_m_rdata = host_arb_s_rdata[15:0];
assign host_mem_ready = host_arb_s_ready[1];
assign host_mem_rdata = host_arb_s_rdata[31:16];
wire final_ar_req, final_ar_wr;
wire [ADDR_WIDTH-1:0] final_ar_addr;
wire [15:0] final_ar_wdata;
wire final_ar_lb_n, final_ar_ub_n;
wire [15:0] final_ar_rdata;
wire final_ar_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(2)
) u_host_arb (
.clk(clk), .rst(core_rst),
.s_req(host_arb_s_req), .s_wr(host_arb_s_wr), .s_addr(host_arb_s_addr),
.s_wdata(host_arb_s_wdata), .s_lb_n(host_arb_s_lb_n), .s_ub_n(host_arb_s_ub_n),
.s_rdata(host_arb_s_rdata), .s_ready(host_arb_s_ready),
.m_req(final_ar_req), .m_wr(final_ar_wr), .m_addr(final_ar_addr), .m_wdata(final_ar_wdata),
.m_lb_n(final_ar_lb_n), .m_ub_n(final_ar_ub_n),
.m_rdata(final_ar_rdata), .m_ready(final_ar_ready)
);
// ---- W: weight fetch (unchanged) ----
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
wire wide_arb_m_req, wide_arb_m_wr;
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
wire [63:0] wide_arb_m_wdata;
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
wire [63:0] wide_arb_m_rdata;
wire wide_arb_m_ready;
slot_mem_arbiter_wide #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
) u_arbiter_wide (
.clk(clk), .rst(core_rst),
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
);
// ---- ONE physical SDRAM backend, both W and (now 2-source-
// arbitrated) AR ports ----
sdram_unified_backend_openrow #(
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
) u_sdram_backend (
.clk(clk), .rst(core_rst),
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
.ar_req(final_ar_req), .ar_wr(final_ar_wr), .ar_addr(final_ar_addr), .ar_wdata(final_ar_wdata),
.ar_lb_n(final_ar_lb_n), .ar_ub_n(final_ar_ub_n),
.ar_rdata(final_ar_rdata), .ar_ready(final_ar_ready),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/nms_activation_fill_ctrl_v3_n16.v
+310
View File
@@ -0,0 +1,310 @@
// ============================================================
// Neural Memory System (NMS) -- shared Activation fill controller.
//
// One instance per neural_memory_system (shared across all N_SLOTS),
// backing nms_activation_replicated.v's single broadcast-write fill
// port. Owns ONE prefetch_engine.v instance (real word-level PSRAM
// fetch, DEC-0015 convention, reused verbatim -- it is generic
// P_IN-byte-tile fetch logic, not weight-specific despite its name).
//
// Single-tag design (same honest limitation as the superseded
// hardware/v2/rtl/activation_cache.v, DEC-0016): tracks ONE resident
// x_base at a time. Refills (resident_count resets to 0, restarts
// fetching from tile 0) whenever the lowest-indexed currently-active
// slot's own x_base differs from what is resident -- correct always,
// but can thrash under interleaved, genuinely-different-x_base
// concurrent traffic; not exercised by this project's own realistic
// dense-layer workloads (shared-producer dispatch, many slots given
// the SAME x_base together).
//
// resident_count extends to the MAX n_tiles needed by any currently
// active slot that shares resident_tag (not just the reference slot
// that triggered the refill), so a later-joining slot with a deeper
// need is served without a second refill.
// ============================================================
module nms_activation_fill_ctrl_v3_n16 #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter N_SLOTS = 4,
parameter ADDR_WIDTH = 26,
parameter MAX_TILES = 16,
// TIW indexes the SRAM fill address (0..MAX_TILES-1); CNTW is for
// resident_count, which must represent the VALUE MAX_TILES itself
// (e.g. a fully-resident 16-tile vector with MAX_TILES=16) -- one
// bit wider than TIW, same distinction/bug as
// nms_memory_manager.v's own tile_idx/wgt_fetched (see that file's
// header for the real deadlock this caused before the fix).
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
parameter CNTW = $clog2(MAX_TILES+1)
)(
input wire clk,
input wire rst,
// ---- per-slot job status (levels, held while that slot's job is active) ----
input wire [N_SLOTS-1:0] job_active,
input wire [N_SLOTS*ADDR_WIDTH-1:0] x_base_flat,
input wire [N_SLOTS*16-1:0] n_tiles_flat,
// ---- broadcast status (every slot compares this against its own x_base) ----
output reg [ADDR_WIDTH-1:0] resident_tag,
output reg [CNTW-1:0] resident_count,
// ---- fill port into nms_activation_replicated.v ----
output wire fill_we,
output wire [TIW-1:0] fill_addr,
output wire [DATA_WIDTH*P_IN-1:0] fill_data,
// ---- real word-level PSRAM backend (arbitrated externally) ----
output wire mem_req,
output wire mem_wr,
output wire [ADDR_WIDTH-1:0] mem_addr,
output wire [15:0] mem_wdata,
output wire mem_lb_n,
output wire mem_ub_n,
input wire [15:0] mem_rdata,
input wire mem_ready
);
integer i;
// ---- desired x_base: lowest-indexed currently-active slot (fixed
// priority -- simple, not fairness-critical here since this only
// decides which TAG to chase, not who gets bandwidth) ----
reg desired_valid;
reg [ADDR_WIDTH-1:0] desired_x_base;
always @* begin
desired_valid = 1'b0;
desired_x_base = {ADDR_WIDTH{1'b0}};
for (i = N_SLOTS-1; i >= 0; i = i - 1) begin
if (job_active[i]) begin
desired_valid = 1'b1;
desired_x_base = x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH];
end
end
end
// ---- STEP14/EXP-0029->EXP-0030: max_n_tiles computation split
// into TWO pipeline stages, since registering ONLY its final use
// (v2, DEC-0026) left the computation ITSELF as the new critical
// path (EXP-0030, N=4 Fmax=72.78MHz, still FAIL@80MHz): the
// original single-cycle logic mixed, PER SLOT, a 23-bit tag
// equality check (x_base_flat[i]==resident_tag) together with an
// N_SLOTS-wide SEQUENTIALLY-CHAINED 16-bit running-max fold (each
// iteration's update depends on the previous one) -- both
// combinational, both in the same cycle as the register that
// captures the result.
//
// Stage 1 (independent per-slot work, no chain dependency between
// slots): register a per-slot "counts toward this refill" mask
// (job_active[i] && tag-match) and, gated by that mask, each
// slot's own n_tiles value (0 if it doesn't count) -- N_SLOTS
// independent 23-bit equality checks, no data dependency between
// slots, so their combined depth does not grow with N_SLOTS the
// way a sequential fold does.
// Stage 2 (the actual reduction): fold the REGISTERED, already-
// masked per-slot values into max_n_tiles_reg -- still an
// N_SLOTS-wide sequential chain (same fold as before), but now
// operating alone, without the equality check sharing the same
// cycle.
reg [15:0] n_tiles_masked [0:N_SLOTS-1];
genvar gsi;
generate
for (gsi = 0; gsi < N_SLOTS; gsi = gsi + 1) begin : GEN_MASK
wire slot_counts = job_active[gsi] &&
(x_base_flat[gsi*ADDR_WIDTH +: ADDR_WIDTH] == resident_tag);
always @(posedge clk) begin
if (rst) n_tiles_masked[gsi] <= 16'h0;
else n_tiles_masked[gsi] <= slot_counts ? n_tiles_flat[gsi*16 +: 16] : 16'h0;
end
end
endgenerate
// Balanced binary max-tree (log2(N_SLOTS) comparison levels)
// instead of the flat N_SLOTS-wide sequential scan this file's own
// header comment above already flagged as "an N_SLOTS-wide
// sequential chain". Found and fixed this session: that chain's
// own carry-chain critical path became the DOMINANT critical path
// at N_SLOTS=8 (real nextpnr-ecp5 P&R: Fmax collapsed to ~40MHz,
// failing the 64MHz target across every measured seed). A tree
// has the SAME single-cycle combinational timing as the scan it
// replaces (max_n_tiles_reg is still registered exactly one cycle
// behind n_tiles_masked -- no FSM/latency change, purely a
// combinational-depth reduction: log2(N_SLOTS) levels instead of
// N_SLOTS).
//
// Written as explicit, uniquely-named per-level wires (NOT a
// multi-dimensional generate-indexed array) -- a first attempt
// using a shared 2D `wire max_tree[level][idx]` array triggered a
// real simulator UNOPTFLAT "circular combinational logic" warning.
// The actual dependency graph IS acyclic (level L+1 only ever
// reads level L), but that tool's array-flattening circularity
// check could not prove that for a shared 2D array; distinctly-
// named per-level wires sidestep the ambiguity entirely for both
// simulation and synthesis. N_SLOTS is a power of two for every
// real configuration this project uses (1/2/4/8); anything else
// falls back, explicitly, to the original flat scan (correct but not
// optimized) rather than silently doing the wrong thing.
reg [15:0] max_n_tiles_reg;
generate
if (N_SLOTS == 1) begin : GEN_MAXTREE_N1
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= n_tiles_masked[0];
end
end else if (N_SLOTS == 2) begin : GEN_MAXTREE_N2
wire [15:0] max_final = (n_tiles_masked[0] > n_tiles_masked[1]) ? n_tiles_masked[0] : n_tiles_masked[1];
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= max_final;
end
end else if (N_SLOTS == 4) begin : GEN_MAXTREE_N4
wire [15:0] m0 = (n_tiles_masked[0] > n_tiles_masked[1]) ? n_tiles_masked[0] : n_tiles_masked[1];
wire [15:0] m1 = (n_tiles_masked[2] > n_tiles_masked[3]) ? n_tiles_masked[2] : n_tiles_masked[3];
wire [15:0] max_final = (m0 > m1) ? m0 : m1;
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= max_final;
end
end else if (N_SLOTS == 8) begin : GEN_MAXTREE_N8
wire [15:0] m0 = (n_tiles_masked[0] > n_tiles_masked[1]) ? n_tiles_masked[0] : n_tiles_masked[1];
wire [15:0] m1 = (n_tiles_masked[2] > n_tiles_masked[3]) ? n_tiles_masked[2] : n_tiles_masked[3];
wire [15:0] m2 = (n_tiles_masked[4] > n_tiles_masked[5]) ? n_tiles_masked[4] : n_tiles_masked[5];
wire [15:0] m3 = (n_tiles_masked[6] > n_tiles_masked[7]) ? n_tiles_masked[6] : n_tiles_masked[7];
wire [15:0] m01 = (m0 > m1) ? m0 : m1;
wire [15:0] m23 = (m2 > m3) ? m2 : m3;
wire [15:0] max_final = (m01 > m23) ? m01 : m23;
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= max_final;
end
end else if (N_SLOTS == 16) begin : GEN_MAXTREE_N16
// EXP-0056: same balanced-tree pattern as N_SLOTS==8 above,
// one more level. This is the exact case that used to fall
// through to GEN_MAXTREE_FALLBACK's own flat sequential
// scan -- the real critical path measured blocking
// N_SLOTS=16 timing closure on the LFE5U-85F (worst
// 23.52-24.64MHz vs 64MHz target across two independent
// seeds, both pre-fix; see experiments.log EXP-0056).
wire [15:0] m0 = (n_tiles_masked[0] > n_tiles_masked[1]) ? n_tiles_masked[0] : n_tiles_masked[1];
wire [15:0] m1 = (n_tiles_masked[2] > n_tiles_masked[3]) ? n_tiles_masked[2] : n_tiles_masked[3];
wire [15:0] m2 = (n_tiles_masked[4] > n_tiles_masked[5]) ? n_tiles_masked[4] : n_tiles_masked[5];
wire [15:0] m3 = (n_tiles_masked[6] > n_tiles_masked[7]) ? n_tiles_masked[6] : n_tiles_masked[7];
wire [15:0] m4 = (n_tiles_masked[8] > n_tiles_masked[9]) ? n_tiles_masked[8] : n_tiles_masked[9];
wire [15:0] m5 = (n_tiles_masked[10] > n_tiles_masked[11]) ? n_tiles_masked[10] : n_tiles_masked[11];
wire [15:0] m6 = (n_tiles_masked[12] > n_tiles_masked[13]) ? n_tiles_masked[12] : n_tiles_masked[13];
wire [15:0] m7 = (n_tiles_masked[14] > n_tiles_masked[15]) ? n_tiles_masked[14] : n_tiles_masked[15];
wire [15:0] m01 = (m0 > m1) ? m0 : m1;
wire [15:0] m23 = (m2 > m3) ? m2 : m3;
wire [15:0] m45 = (m4 > m5) ? m4 : m5;
wire [15:0] m67 = (m6 > m7) ? m6 : m7;
wire [15:0] m0123 = (m01 > m23) ? m01 : m23;
wire [15:0] m4567 = (m45 > m67) ? m45 : m67;
wire [15:0] max_final = (m0123 > m4567) ? m0123 : m4567;
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= max_final;
end
end else begin : GEN_MAXTREE_FALLBACK
reg [15:0] max_n_tiles_comb_fallback;
integer j;
always @* begin
max_n_tiles_comb_fallback = 16'h0;
for (j = 0; j < N_SLOTS; j = j + 1)
if (n_tiles_masked[j] > max_n_tiles_comb_fallback)
max_n_tiles_comb_fallback = n_tiles_masked[j];
end
always @(posedge clk) begin
if (rst) max_n_tiles_reg <= 16'h0;
else max_n_tiles_reg <= max_n_tiles_comb_fallback;
end
end
endgenerate
localparam ST_IDLE = 1'd0;
localparam ST_FETCH = 1'd1;
reg state;
reg pf_start;
reg [ADDR_WIDTH-1:0] pf_addr;
wire pf_busy, pf_done;
wire signed [DATA_WIDTH*P_IN-1:0] pf_tile;
prefetch_engine #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH)
) u_pf (
.clk(clk), .rst(rst),
.fetch_start(pf_start), .w_addr(pf_addr),
.fetch_busy(pf_busy), .fetch_done(pf_done), .tile_w(pf_tile),
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(mem_addr), .mem_wdata(mem_wdata),
.mem_lb_n(mem_lb_n), .mem_ub_n(mem_ub_n),
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
);
reg fill_we_reg;
reg [TIW-1:0] fill_addr_reg;
reg [DATA_WIDTH*P_IN-1:0] fill_data_reg;
assign fill_we = fill_we_reg;
assign fill_addr = fill_addr_reg;
assign fill_data = fill_data_reg;
always @(posedge clk) begin
if (rst) begin
resident_tag <= {ADDR_WIDTH{1'b1}}; // sentinel: matches no real x_base at reset
resident_count <= {CNTW{1'b0}};
state <= ST_IDLE;
pf_start <= 1'b0;
fill_we_reg <= 1'b0;
end else begin
pf_start <= 1'b0;
fill_we_reg <= 1'b0;
// latch a completed fetch into the replicated activation
// memory's broadcast fill port
if (pf_done) begin
fill_we_reg <= 1'b1;
fill_addr_reg <= resident_count[TIW-1:0]; // valid: gated < max_n_tiles <= MAX_TILES
fill_data_reg <= pf_tile;
resident_count <= resident_count + 1'b1;
end
// refill trigger: the reference slot wants a DIFFERENT tag,
// and the fetch engine is genuinely idle (never interrupt an
// in-flight fetch -- same discipline as memory_manager.v's
// own pf_pending guard, ERR-0006). Stays in ST_IDLE (not
// ST_FETCH): only updates resident_tag/resident_count here;
// the ST_IDLE case below is what actually issues pf_start,
// reading the NEW resident_tag starting next cycle -- this
// path must NOT itself jump to ST_FETCH without a matching
// pf_start, or the engine would sit in ST_FETCH forever
// waiting for a pf_done that was never triggered.
if (desired_valid && (desired_x_base != resident_tag) && !pf_busy && (state == ST_IDLE)) begin
resident_tag <= desired_x_base;
resident_count <= {CNTW{1'b0}};
end
case (state)
ST_IDLE: begin
// stay idle: fetching further tiles for the CURRENT
// tag (if any active slot still needs more) is
// handled below, symmetric to the refill case.
if (!pf_busy && !pf_start && (resident_count < max_n_tiles_reg) &&
desired_valid && (desired_x_base == resident_tag)) begin
pf_start <= 1'b1;
pf_addr <= resident_tag + (resident_count * P_IN[ADDR_WIDTH-1:0]);
state <= ST_FETCH;
end
end
ST_FETCH: begin
if (pf_done) begin
// resident_count already bumped above this cycle;
// decide whether more remain once back in IDLE.
state <= ST_IDLE;
end
end
default: state <= ST_IDLE;
endcase
end
end
endmodule
hardware/v2/nms/rtl/nms_dataflow_core_sdram_fast.v
+325
View File
@@ -0,0 +1,325 @@
`timescale 1ns/1ps
// ================================================================
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
// processor)), but replaces the M4 memory_manager.v +
// activation_cache.v cluster with the NMS's own decided pieces
// (DEC-0019/DEC-0020):
// - nms_activation_replicated.v: N_SLOTS private full-vector
// activation copies, broadcast-filled by...
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
// (single logical tag, same honest thrash-under-interleaved-
// different-x_base limitation as the superseded activation_cache.v)
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
// copies (never shared, no arbitration needed)
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
// two SRAMs above instead of double-buffering 2 banks (the whole
// vector is resident, not just 2 tiles worth)
//
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
// REUSED VERBATIM, unmodified -- the node-registration and slot-
// dispatch protocol did not change at all; only what happens between
// "job dispatched to a slot" and "job_done" changed.
//
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
// activation fill controller's own port) -- arbitrated one level up,
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
//
// STEP16 Phase 5: forked from nms_dataflow_core_dual32.v (STEP15's
// own dual-chip-32-bit variant) with ONLY the wide weight-fetch port
// widened from 32 to 64 bits (MEM_DATA_WIDTH=64 in the per-slot
// nms_memory_manager_stream_wide instance below) -- the natural
// P_IN*DATA_WIDTH/16=4-word SDRAM burst identified in Phase 1 means
// ONE mem_req now fetches exactly one whole tile (WORDS_PER_TILE=1),
// same as the dual32 case fetched one whole tile per 32-bit request.
// Everything else (activation path, dependency manager, neural
// director, per-slot neural_processor) is BYTE-FOR-BYTE UNCHANGED --
// per the governing spec's own "do not create an artificial
// benchmark" instruction, only the piece under test (weight-fetch
// physical memory) differs from the validated dual32 baseline.
// ================================================================
module nms_dataflow_core_sdram_fast #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 4,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
// own CNTW exactly -- this top-level wire connecting the two was
// left at the narrower TIW after those modules were widened,
// silently truncating resident_count's real value (16) back down
// to 0 right when it should have reached MAX_TILES, deadlocking
// the very last tile of any n_tiles==MAX_TILES job forever (found
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
// short, act_resident_count visibly reset to 0 the exact cycle it
// should have become 16).
parameter CNTW = $clog2(MAX_TILES+1)
)(
input wire clk,
input wire rst,
input wire reg_valid,
output wire reg_ready,
input wire [$clog2(N_NODES)-1:0] reg_node_id,
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
input wire [ADDR_WIDTH-1:0] reg_x_base,
input wire [ADDR_WIDTH-1:0] reg_w_base,
input wire [15:0] reg_n_tiles,
input wire [ADDR_WIDTH-1:0] reg_result_addr,
// FPGA_DATA_READY: see the assignment site (below u_director) for
// the full design comment.
output wire data_ready,
output wire [N_SLOTS:0] slot_mem_req,
output wire [N_SLOTS:0] slot_mem_wr,
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
output wire [N_SLOTS:0] slot_mem_lb_n,
output wire [N_SLOTS:0] slot_mem_ub_n,
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
input wire [N_SLOTS:0] slot_mem_ready,
// ---- STEP16 Phase 5: wide (64-bit logical) weight-fetch backend
// interface, per slot -- N_SLOTS wide (NOT N_SLOTS+1: the shared
// activation-fill controller stays on the ORIGINAL 16-bit port
// above, unchanged). Arbitrated one level up (slot_mem_arbiter_
// wide.v, reused unchanged at DATA_WIDTH=64) down to the real
// SDRAM physical interface (sdram_weight_backend.v). ----
output wire [N_SLOTS-1:0] wide_slot_mem_req,
output wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr,
input wire [64*N_SLOTS-1:0] wide_slot_mem_rdata,
input wire [N_SLOTS-1:0] wide_slot_mem_ready
);
localparam NODE_IDW = $clog2(N_NODES);
wire dm_ready_valid;
wire dm_ready_ready;
wire [NODE_IDW-1:0] dm_ready_node_id;
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
wire [15:0] dm_ready_n_tiles;
wire dm_producer_done_valid;
wire [NODE_IDW-1:0] dm_producer_done_node_id;
wire dm_any_pending;
dependency_manager_fast #(
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
) u_dep_mgr (
.clk(clk), .rst(rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr),
.any_pending(dm_any_pending)
);
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
wire [N_SLOTS-1:0] dir_slot_job_start;
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
wire [N_SLOTS-1:0] dir_slot_job_done;
wire dir_job_out_done;
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
wire [3:0] dir_state;
wire dir_error;
wire dir_queue_empty;
neural_director #(
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
) u_director (
.clk(clk), .rst(rst),
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
.job_in_node_id(dm_ready_node_id_ext),
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
.dir_state(dir_state), .dir_error(dir_error), .queue_empty(dir_queue_empty)
);
// ---- FPGA_DATA_READY: system-idle detection (see decisions.log
// for the full design rationale) ----
// sys_busy: true while ANY of {a slot is active, the director's
// dispatch queue is non-empty, dependency_manager has a node not
// yet dispatched} holds. data_ready is a sticky level that goes
// HIGH on the busy->idle falling edge (a graph just finished) and
// LOW again the instant any new work starts (registration or
// dispatch) -- self-clearing, no explicit host ACK needed. Correct
// ONLY if the host finishes registering every node of a graph
// before the first one completes (documented assumption, see
// decisions.log) -- registration (microseconds over SPI) is far
// faster than per-neuron compute (~195 real measured cycles) for
// every workload this project has characterized.
wire sys_busy = (|job_active) || (!dir_queue_empty) || dm_any_pending;
reg sys_busy_prev;
reg data_ready_reg;
always @(posedge clk) begin
if (rst) begin
sys_busy_prev <= 1'b0;
data_ready_reg <= 1'b0;
end else begin
sys_busy_prev <= sys_busy;
if (sys_busy) data_ready_reg <= 1'b0;
else if (sys_busy_prev) data_ready_reg <= 1'b1;
end
end
assign data_ready = data_ready_reg;
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
assign dm_producer_done_valid = dir_job_out_done;
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
// ---- NMS memory: shared Activation SRAM (replicated) + private
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
wire fill_we;
wire [TIW-1:0] fill_addr;
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
wire [ADDR_WIDTH-1:0] act_resident_tag;
wire [CNTW-1:0] act_resident_count;
wire [N_SLOTS-1:0] act_rd_en;
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
nms_activation_replicated #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
) u_act_mem (
.clk(clk), .rst(rst),
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
);
wire [N_SLOTS-1:0] job_active;
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
nms_activation_fill_ctrl_v3_n16 #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
) u_act_fill (
.clk(clk), .rst(rst),
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
);
wire [N_SLOTS-1:0] wgt_fill_we;
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
wire [N_SLOTS-1:0] wgt_rd_en;
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
nms_weight_packed #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
) u_wgt_mem (
.clk(clk), .rst(rst),
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
);
genvar g;
generate
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
wire mm_operand_valid, mm_operand_ready;
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
wire mm_tile_last;
wire mm_result_valid, mm_result_ready;
wire signed [DATA_WIDTH-1:0] mm_result_data;
nms_memory_manager_stream_wide #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
.PREFETCH_DISTANCE(PREFETCH_DISTANCE), .MEM_DATA_WIDTH(64)
) u_mm (
.clk(clk), .rst(rst),
.job_start(dir_slot_job_start[g]),
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
.job_done(dir_slot_job_done[g]),
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
.job_active(job_active[g]),
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
.act_rd_en(act_rd_en[g]),
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
.wgt_fill_we(wgt_fill_we[g]),
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
.wgt_rd_en(wgt_rd_en[g]),
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g]),
.wide_mem_req(wide_slot_mem_req[g]),
.wide_mem_addr(wide_slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
.wide_mem_rdata(wide_slot_mem_rdata[g*64 +: 64]),
.wide_mem_ready(wide_slot_mem_ready[g])
);
reg job_valid_np;
wire job_ready_np;
wire result_valid_np;
wire signed [DATA_WIDTH-1:0] result_data_np;
wire [3:0] np_state;
wire np_error;
neural_processor #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
) u_np (
.clk(clk), .rst(rst),
.job_valid(job_valid_np), .job_ready(job_ready_np),
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
.result_valid(result_valid_np), .result_ready(mm_result_ready),
.result_data(result_data_np), .result_node_id(),
.np_state(np_state), .np_error(np_error)
);
assign mm_result_valid = result_valid_np;
assign mm_result_data = result_data_np;
always @(posedge clk) begin
if (rst) job_valid_np <= 1'b0;
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
end
end
endgenerate
endmodule
hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow_fast.v
+171
View File
@@ -0,0 +1,171 @@
`timescale 1ns/1ps
// ================================================================
// Neural Memory System (NMS) -- STEP19 real hardware-facing top level.
//
// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_
// sdram_pack128.v (STEP18) with the ONE change this step's own
// governing spec mandates: the real hardware/v1/rtl/psram_controller.v
// + memory_interface.v pairing (activation-fill + result-writeback,
// 16-bit) is REMOVED from the V2 physical path entirely and replaced
// by sdram_unified_backend.v's own AR port, sharing the SAME single
// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_
// controller.v instance the weight-fetch path (W port) already uses.
//
// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_
// arbiter_wide.v (64-bit, weight) are BOTH reused completely
// UNCHANGED -- their own downstream ports now both terminate at
// sdram_unified_backend.v instead of two separate physical chains.
// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_
// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and
// neural_processor.v are ALL byte-for-byte unchanged -- this is a
// pure memory-side substitution, per the governing spec's own
// explicit instruction.
//
// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_
// interface.v simply are no longer INSTANTIATED by this top-level;
// neither file was modified, and V1's own golden-reference status is
// unaffected.
//
// Real pin count (weight+activation+result, ALL through ONE chip):
// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37
// pins total -- the SAME 37 pins the weight-only path already used in
// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/
// result traffic, since it shares the identical physical bus).
// ================================================================
module nms_neural_multiprocessor_sdram_openrow_fast #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 2,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter CLK_FREQ_MHZ = 80
)(
input wire clk,
input wire rst,
input wire reg_valid,
output wire reg_ready,
input wire [$clog2(N_NODES)-1:0] reg_node_id,
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
input wire [ADDR_WIDTH-1:0] reg_x_base,
input wire [ADDR_WIDTH-1:0] reg_w_base,
input wire [15:0] reg_n_tiles,
input wire [ADDR_WIDTH-1:0] reg_result_addr,
// FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v
output wire data_ready,
// ---- STEP19: ONE physical SDRAM interface, ALL traffic
// (weights + activations + results) ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [1:0] sdram_ba,
output wire [12:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
wire [N_SLOTS:0] slot_mem_ready;
wire [N_SLOTS-1:0] wide_slot_mem_req;
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
wire [N_SLOTS-1:0] wide_slot_mem_ready;
nms_dataflow_core_sdram_fast #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
) u_dataflow_core (
.clk(clk), .rst(rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.data_ready(data_ready),
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
);
// ---- AR: activation-fill (shared, 1 port) + per-slot result
// writeback (N_SLOTS ports), arbitrated exactly as before ----
wire arb_m_req, arb_m_wr;
wire [ADDR_WIDTH-1:0] arb_m_addr;
wire [15:0] arb_m_wdata;
wire arb_m_lb_n, arb_m_ub_n;
wire [15:0] arb_m_rdata;
wire arb_m_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
) u_arbiter (
.clk(clk), .rst(rst),
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
);
// ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as
// before -- weight fetch never writes, same tie-off convention
// as STEP16-18 ----
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
wire wide_arb_m_req, wide_arb_m_wr;
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
wire [63:0] wide_arb_m_wdata;
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
wire [63:0] wide_arb_m_rdata;
wire wide_arb_m_ready;
slot_mem_arbiter_wide #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
) u_arbiter_wide (
.clk(clk), .rst(rst),
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
);
// ---- STEP19: ONE physical SDRAM backend, both W and AR ports ----
sdram_unified_backend_openrow #(
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
) u_sdram_backend (
.clk(clk), .rst(rst),
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
.ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata),
.ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n),
.ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule