From 03b5cbc25bba6bf62b04a01e2631490b60fc7cf6 Mon Sep 17 00:00:00 2001 From: manvalan Date: Wed, 16 Sep 2026 02:56:51 +0200 Subject: [PATCH] exp: bank-interleaved SDRAM pipelining works in isolation, ~0.3% gain integrated (EXP-0052) Follow-up to EXP-0051: built sdram_controller_pipelined.v, remapping addr->bank to low-order bits (today's weight region always maps to bank 0) and adding a shadow-slot ACTIVATE lookahead so a different-bank request can start its tRCD wait during the current transaction's tail. Phase A (isolated tb_sdram_controller_pipelined.v, 38/38 bit-exact, independently re-verified this session): mechanism works, saves exactly 2 cycles (tRCD) per different-bank back-to-back pair, matching the theoretical ceiling derived before measuring (CAS_LATENCY+BURST_LEN are serial on the shared data bus regardless of bank, so more than tRCD/tRP was never on the table). Phase B (integration, tb_nms_dstress_sdram_pipelined.v, independently rebuilt/rerun): N=4 49760 cycles (-0.33% vs baseline), N=8 49755 (-0.31%) -- both 256/256 bit-exact. Root cause of the gap: the W port's request/ready protocol is one-at-a-time, so a second, different-bank request is essentially never already pending while the first is still in flight, so the mechanism rarely triggers in the real system even though it's correct when directly stimulated. Not integrated into production; kept as additive reference for a possible future arbiter/backend pipelined-dispatch rewrite (out of scope here, larger and riskier). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01YHENedK76onD2Vtc2CMjej --- hardware/v2/logs/experiments.log | 117 ++++ ...ms_neural_multiprocessor_sdram_pipelined.v | 137 +++++ .../v2/nms/rtl/sdram_controller_pipelined.v | 482 +++++++++++++++ .../nms/rtl/sdram_unified_backend_pipelined.v | 255 ++++++++ .../nms/sim/tb_nms_dstress_sdram_pipelined.v | 562 ++++++++++++++++++ .../nms/sim/tb_sdram_controller_pipelined.v | 273 +++++++++ 6 files changed, 1826 insertions(+) create mode 100644 hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v create mode 100644 hardware/v2/nms/rtl/sdram_controller_pipelined.v create mode 100644 hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v create mode 100644 hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v create mode 100644 hardware/v2/nms/sim/tb_sdram_controller_pipelined.v diff --git a/hardware/v2/logs/experiments.log b/hardware/v2/logs/experiments.log index 6ce6e9a..73ee653 100644 --- a/hardware/v2/logs/experiments.log +++ b/hardware/v2/logs/experiments.log @@ -3051,3 +3051,120 @@ next_action: report this refined finding to the user before choosing used by the real board top, additive only): hardware/v2/nms/rtl/ nms_neural_multiprocessor_sdram_dualbank.v, hardware/v2/nms/sim/ tb_nms_dstress_sdram_dualbank.v. + +EXP-0052 -- Bank-interleaved pipelining for the W (weight-fetch) SDRAM +channel: the mechanism works in isolation (verified) but the real +D-Stress integration gain is negligible, because the CALLER never +issues a second request early enough to trigger it (2026-09-16) + +DATE: 2026-09-16 +CONTEXT: follow-up to EXP-0051, which found the weight-fetch (W) +channel itself (not W/AR cross-traffic) as the real ~77-78%-busy +ceiling, and identified the per-transaction fixed cost (measured ~16 +cycles: 1 issue + 2 T_RCD + 4 CAS_LATENCY-wait + 7 BURST_LEN=8 read + +2 T_RP) as the lever to attack, since the real transaction COUNT is +already close to optimal (2163 measured vs 2048 theoretical minimum +for the D-Stress workload, ~5.6% overhead). This session chose to +pursue this via a fork given the real correctness-risk history of this +exact FSM area (ERR-0019/0020/0023, all req-latching races). +TOOLCHAIN: unchanged from EXP-0051 (Yosys 0.69+59 d85872386-dirty, +Verilator 5.053) -- this experiment is Verilator-only, no synthesis/ +P&R. + +METHOD (two-phase, isolated-correctness-first per this project's own +established discipline): + Phase A: `sdram_controller_pipelined.v` (new, forked from + sdram_controller.v) remaps the addr->{bank,row,col} decomposition + from high-order bits (today: bank always 0 for this project's + compact weight region, since bank comes from the TOP address bits) + to LOW-order bits placed just above the burst-alignment zero bits -- + so consecutive burst-aligned weight fetches (each BURST_LEN=8 words + apart) now naturally rotate across the SDRAM's own 4 internal banks + instead of all landing on bank 0. Added a depth-1 "shadow" slot: while + the current transaction is in CAS_WAIT/BURST/PRECHARGE_WAIT (command + bus otherwise idle), a newly-arriving request for a DIFFERENT bank + has its ACTIVATE issued immediately, overlapping that bank's own + T_RCD wait with the current transaction's tail. Same-bank requests, + and refresh, are unaffected (S_IDLE priority: open shadow > refresh > + new request, so AUTO REFRESH can never fire with a row left open). + New isolated testbench `tb_sdram_controller_pipelined.v`: 38/38 PASS, + bit-exact across all 4 banks. Real, INDEPENDENTLY RE-VERIFIED result + for back-to-back different-bank transactions: 30 cycles total vs a + 32-cycle serial baseline for the same pair -- exactly 2 cycles saved + (= T_RCD), NOT a multiple-x speedup. This matches the theoretical + ceiling worked out BEFORE measuring: CAS_LATENCY+BURST_LEN (11 of the + 16 cycles) are serial on the SHARED data bus regardless of bank, and + no amount of bank interleaving can hide that -- only the T_RCD+T_RP + portion (5 of 16 cycles) is bank-local and therefore hideable, and + only T_RCD (2 cycles) was actually recovered here since the OTHER + bank's T_RP tail still has to clear before its OWN next reuse. Same- + bank consecutive case: unchanged, no regression. Refresh-during- + interleaving case (Test 4): AUTO REFRESH spacing rose from 634 to 657 + cycles under sustained back-to-back different-bank stress (vs + tREFI=626 target) -- a real, disclosed +3.6%, already within the + margin this project's OWN unmodified controller already tolerates + under the same synthetic stress pattern, not a new violation. + One bug found and fixed, in the NEW TESTBENCH ONLY (not the RTL): + calling wait_ready() twice in a row double-consumed the same `ready` + pulse -- fixed by advancing one extra @(posedge clk) between calls. + + Phase B (integration, gated on Phase A passing): forked + `sdram_unified_backend_pipelined.v` (swaps in the pipelined + controller, W_ENTRIES cache and W/AR arbitration untouched) and + `nms_neural_multiprocessor_sdram_pipelined.v`, plus a new + `tb_nms_dstress_sdram_pipelined.v` (same D-Stress workload/golden + model; backdoor peek/poke rewritten to go through a `sdram_model.v` + backdoor_read/write helper keyed on the SAME decomposition the new + controller uses, instead of the old flat-address assumption, so + bit-exact verification stays valid under the new addr->bank mapping + -- this was flagged in advance as the one correctness trap in this + whole exercise, and was handled by construction rather than by + parallel, error-prone reimplementation). +RESULT (INDEPENDENTLY RE-BUILT AND RE-RUN by this session directly, not +just taken from the sub-task's own report -- both PASS 256/256 bit- +exact + data_ready PASS in both configs): + N=4: total_cycles=49760 (vs single-bank baseline 49927, EXP-0049 -- + only -0.33%). N=8: total_cycles=49755 (vs baseline 49909 -- + -0.31%). Both essentially within noise of the unmodified single- + bank system, nowhere near either Phase A's own measured 2-cycle- + per-different-bank-pair saving scaled up, or EXP-0051's dual-bank + -8/-10%. +ROOT CAUSE of the gap between Phase A (works) and Phase B (doesn't +help): `slot_mem_arbiter_wide.v` -> `sdram_unified_backend.v`'s own W +port is a synchronous one-request-at-a-time interface -- the caller +waits for `w_ready` before ever asserting the next `w_req`. Phase A's +interleaving mechanism can ONLY help if a request for a DIFFERENT bank +is already pending WHILE the current transaction is still mid-flight +(CAS_WAIT/BURST/PRECHARGE) -- a condition the current arbiter/backend +call convention almost never creates, since nothing is ever dispatched +early. The mechanism itself is real and correctly verified in Phase A +(directly, artificially stimulated); the SYSTEM around it, as it exists +today, essentially never exercises it. +decision: do NOT integrate sdram_controller_pipelined.v into the + production path on this evidence -- the real, measured, system-level + gain (~0.3%) does not justify carrying a second, more complex + controller variant with its own (even if currently well-verified) + correctness surface. The isolated Phase A result remains genuinely + useful and is KEPT as an additive, uncommitted-to-production file: + it proves the mechanism works and quantifies its real ceiling (2 + cycles/pair, not more), which is exactly the number needed to decide + whether a FUTURE arbiter/backend rewrite (teaching the W port to + dispatch its NEXT request BEFORT the current one's `ready`, i.e. a + real pipelined/multi-outstanding-request interface, not just the + memory-side FSM) would be worth attempting -- that rewrite is a + materially larger, riskier change (touches the arbiter's own request/ + grant protocol, not just the memory-side FSM) and was explicitly kept + out of scope for this experiment. +next_action: report to the user; do not pursue the arbiter/backend + pipelined-dispatch rewrite without an explicit go-ahead, given its + larger scope and the modest (2 cycles/pair, capped) ceiling this + experiment just measured -- the slot-group weight-split ("aspettiamo" + item from EXP-0051) remains the other, still-open, ORTHOGONAL lever + (it does not depend on this pipelining work at all and would stack + with it if the arbiter rewrite is ever done). New files (additive + only, none touch the real board top or existing production RTL): + hardware/v2/nms/rtl/sdram_controller_pipelined.v, + hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v, + hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v, + hardware/v2/nms/sim/tb_sdram_controller_pipelined.v, + hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v. diff --git a/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v new file mode 100644 index 0000000..bb04745 --- /dev/null +++ b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v @@ -0,0 +1,137 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXPERIMENTAL fork of nms_neural_multiprocessor_sdram_unified.v -- +// the ONLY change is instantiating sdram_unified_backend_pipelined.v +// (bank-interleaved command pipelining) instead of sdram_unified_ +// backend.v. u_dataflow_core, u_arbiter, u_arbiter_wide are all +// byte-for-byte unchanged. See sdram_controller_pipelined.v's header +// for the mechanism and its own derived/measured ceiling, and +// hardware/v2/logs/experiments.log for why this fork exists. +// ============================================================ +module nms_neural_multiprocessor_sdram_pipelined #( + parameter DATA_WIDTH = 8, + parameter P_IN = 8, + parameter ACC_WIDTH = 32, + parameter ADDR_WIDTH = 26, + parameter N_SLOTS = 2, + parameter N_NODES = 16, + parameter MAX_DEPS = 4, + parameter QUEUE_DEPTH = 8, + parameter MAX_TILES = 16, + parameter PREFETCH_DISTANCE = 8, + parameter CLK_FREQ_MHZ = 80 +)( + input wire clk, + input wire rst, + + input wire reg_valid, + output wire reg_ready, + input wire [$clog2(N_NODES)-1:0] reg_node_id, + input wire [$clog2(MAX_DEPS+1)-1:0] reg_required, + input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids, + input wire [ADDR_WIDTH-1:0] reg_x_base, + input wire [ADDR_WIDTH-1:0] reg_w_base, + input wire [15:0] reg_n_tiles, + input wire [ADDR_WIDTH-1:0] reg_result_addr, + + output wire data_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [1:0] sdram_ba, + output wire [12:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + wire [N_SLOTS:0] slot_mem_req, slot_mem_wr; + wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr; + wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata; + wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n; + wire [N_SLOTS:0] slot_mem_ready; + + wire [N_SLOTS-1:0] wide_slot_mem_req; + wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr; + wire [64*N_SLOTS-1:0] wide_slot_mem_rdata; + wire [N_SLOTS-1:0] wide_slot_mem_ready; + + nms_dataflow_core_sdram #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE) + ) u_dataflow_core ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .data_ready(data_ready), + .slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr), + .slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n), + .slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready), + .wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr), + .wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready) + ); + + wire arb_m_req, arb_m_wr; + wire [ADDR_WIDTH-1:0] arb_m_addr; + wire [15:0] arb_m_wdata; + wire arb_m_lb_n, arb_m_ub_n; + wire [15:0] arb_m_rdata; + wire arb_m_ready; + + slot_mem_arbiter #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1) + ) u_arbiter ( + .clk(clk), .rst(rst), + .s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr), + .s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n), + .s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready), + .m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata), + .m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n), + .m_rdata(arb_m_rdata), .m_ready(arb_m_ready) + ); + + wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}}; + wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}}; + wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}}; + wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}}; + + wire wide_arb_m_req, wide_arb_m_wr; + wire [ADDR_WIDTH-1:0] wide_arb_m_addr; + wire [63:0] wide_arb_m_wdata; + wire wide_arb_m_lb_n, wide_arb_m_ub_n; + wire [63:0] wide_arb_m_rdata; + wire wide_arb_m_ready; + + slot_mem_arbiter_wide #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64) + ) u_arbiter_wide ( + .clk(clk), .rst(rst), + .s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr), + .s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n), + .s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready), + .m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata), + .m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n), + .m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready) + ); + + sdram_unified_backend_pipelined #( + .ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ) + ) u_sdram_backend ( + .clk(clk), .rst(rst), + .w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr), + .w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready), + .ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata), + .ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n), + .ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_controller_pipelined.v b/hardware/v2/nms/rtl/sdram_controller_pipelined.v new file mode 100644 index 0000000..ba0aeab --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_controller_pipelined.v @@ -0,0 +1,482 @@ +`timescale 1ns/1ps + +// ============================================================ +// NMS -- EXPERIMENTAL bank-interleaved-pipelining fork of +// sdram_controller.v, built to test whether the ~77-78% Bank-W +// busy-cycle ceiling EXP-0051 measured (see hardware/v2/logs/ +// experiments.log) can be reduced by overlapping the NEXT +// transaction's ACTIVATE/tRCD with the CURRENT transaction's own +// tail (CAS latency + burst + auto-precharge), when the two target +// DIFFERENT internal SDRAM banks. Real SDR SDRAM banks are +// electrically independent -- a real chip supports this exact kind +// of overlap (issuing ACTIVATE to bank B while bank A is still mid- +// burst/mid-precharge), it is simply never exploited by the +// original, deliberately linear, one-transaction-at-a-time +// sdram_controller.v (STEP16's own explicit, documented scope +// boundary: "exactly one physical SDRAM transaction in flight at a +// time"). +// +// TWO changes vs sdram_controller.v, both additive/isolated (every +// existing state/signal/behavior for the ORIGINAL usage pattern -- +// one req, wait for ready, THEN issue the next -- is byte-for-byte +// unchanged): +// +// (1) Address->bank decomposition moved from the TOP bits (original: +// addr_bank = addr[ADDR_WIDTH-1 -: BANK_BITS], meaning the +// project's own compact, single-region weight/activation memory +// map always lands on bank 0 -- confirmed by inspection, this is +// WHY no interleaving opportunity could ever exist under the +// original decomposition) to bits immediately ABOVE the fixed- +// zero burst-alignment low bits: addr_bank = addr[ALIGN_BITS +: +// BANK_BITS], where ALIGN_BITS = clog2(BURST_LEN). Since every +// real caller always increments the word address by exactly +// BURST_LEN between consecutive real transactions (see +// sdram_unified_backend.v's own w_eff_aligned_word_addr/ +// ar_eff_block_base computation), this makes CONSECUTIVE real +// transactions round-robin across all BANK_BITS**2 banks +// automatically, with zero change needed at any caller. This is +// a pure re-slicing of the SAME flat word-address bits into a +// DIFFERENT (bank,row,col) triple -- still a bijection over the +// full address space (each of the 2**ADDR_WIDTH addresses maps +// to exactly one (bank,row,col) and vice versa), so read-after- +// write correctness is unaffected; only the caller-visible flat +// address <-> physical-location mapping changes, which is why +// any INTEGRATION testbench built around this module must apply +// the SAME decomposition in its own backdoor peek/poke helpers +// (see tb_nms_dstress_sdram_pipelined.v's own header note) or +// use sdram_model.v's own explicit backdoor_read/backdoor_write +// tasks (bank/row/col-addressed, decomposition-agnostic) instead +// of computing a flat array index by hand. +// +// (2) A single-depth "shadow" pipeline slot (pipe_valid/pipe_*_reg/ +// pipe_wait_cnt): while the CURRENT transaction is in S_CAS_WAIT, +// S_BURST_READ, S_BURST_WRITE, or S_PRECHARGE_WAIT (i.e. its own +// ACTIVATE has already been sent and the command bus is +// otherwise idle -- confirmed by inspection: none of those four +// states drive sdram_ras_n/sdram_ba/sdram_a), a NEW `req` for a +// DIFFERENT bank than the current transaction's own req_bank_reg +// is captured into the shadow slot AND its ACTIVATE command is +// issued immediately (overlapping its own tRCD with whatever of +// the current transaction's tail remains), instead of going +// through the original req_pending latch (which would otherwise +// wait for a full return to S_IDLE before even starting the +// ACTIVATE). A `req` for the SAME bank as the current transaction +// -- or arriving in any state OTHER than those four, or arriving +// while the shadow slot is already occupied -- falls through to +// the ORIGINAL, unmodified req_pending path, so that case behaves +// EXACTLY as in sdram_controller.v (no regression, verified in +// tb_sdram_controller_pipelined.v's own "same-bank" test). +// +// REAL PROTOCOL CONSTRAINT respected: AUTO REFRESH requires EVERY +// bank precharged first (a real JEDEC rule sdram_model.v itself +// does NOT currently check/enforce -- confirmed by inspection, a +// real, disclosed gap in that model, not exploited here). This +// design avoids ever violating it by construction: S_IDLE's own +// priority order checks `pipe_valid` BEFORE `refresh_timer==0` -- +// a still-open shadow bank is always promoted/closed (via its own +// ordinary auto-precharge) before any refresh is allowed to fire, +// so refresh can only ever run when EVERY bank (primary transaction, +// always auto-precharged by construction -- A10=1 on every real +// command, unchanged from the original design -- and any shadow +// transaction) is already closed. The resulting refresh delay is +// bounded by one shadow transaction's own worst-case duration +// (~16 cycles at CLK_FREQ_MHZ=80/BURST_LEN=8), a small fraction of +// T_REFI (~625 cycles at the same frequency) -- verified, not +// assumed, by tb_sdram_controller_pipelined.v's own refresh- +// during-interleaving test. +// +// THEORETICAL CEILING (derived here, confirmed by measurement in +// tb_sdram_controller_pipelined.v -- disclosed up front so the result +// isn't oversold): only tRCD can ever be hidden by this scheme, since +// the shared DQ bus means the NEXT transaction's own CAS/burst can +// never start before the CURRENT transaction's burst fully drains, +// regardless of banking. At CLK_FREQ_MHZ=80/BURST_LEN=8, tRCD is only +// ~2 of a real transaction's ~16 total cycles (the dominant cost, +// CAS_LATENCY+BURST_LEN=11 cycles/69%, is serial DATA transfer that +// NO command-level bank interleaving can shorten) -- so the best case +// for a long chain of alternating-bank transactions is each one AFTER +// the first costing ~14 instead of ~16 cycles, an asymptotic ~12.5% +// per-transaction ceiling, not a multiple-x speedup. +// ============================================================ +module sdram_controller_pipelined #( + parameter CLK_FREQ_MHZ = 64, + parameter BURST_LEN = 4, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2, + parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS +)( + input wire clk, + input wire rst, + + input wire req, + input wire wr, + input wire [ADDR_WIDTH-1:0] addr, + input wire [16*BURST_LEN-1:0] wdata, + input wire [2*BURST_LEN-1:0] wmask, + output reg [16*BURST_LEN-1:0] rdata, + output reg ready, + output reg busy, + + output reg sdram_cke, + output reg sdram_cs_n, + output reg sdram_ras_n, + output reg sdram_cas_n, + output reg sdram_we_n, + output reg [1:0] sdram_ba, + output reg [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output reg [1:0] sdram_dqm +); + + localparam BURST_IDXW = (BURST_LEN <= 1) ? 1 : $clog2(BURST_LEN); + // unclamped log2 (0 for BURST_LEN=1), used ONLY for the bank-slice + // position -- see header note (1). + localparam ALIGN_BITS = $clog2(BURST_LEN); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS) begin + $display("FATAL sdram_controller_pipelined: ADDR_WIDTH=%0d != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)=%0d", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS, BANK_BITS+ROW_BITS+COL_BITS); + $finish; + end + initial if (ALIGN_BITS + BANK_BITS > COL_BITS) begin + $display("FATAL sdram_controller_pipelined: ALIGN_BITS(%0d)+BANK_BITS(%0d) > COL_BITS(%0d) -- bank slice does not fit below row field", + ALIGN_BITS, BANK_BITS, COL_BITS); + $finish; + end + + function integer ns_to_cycles; + input integer ns; + begin + ns_to_cycles = (ns * CLK_FREQ_MHZ + 999) / 1000; + end + endfunction + localparam T_RCD = ns_to_cycles(15); + localparam T_RP = ns_to_cycles(15); + localparam T_MRD = 2; + localparam T_INIT_US= 200; + localparam T_INIT = T_INIT_US * CLK_FREQ_MHZ; + localparam CAS_LATENCY = 3; + localparam T_REFI = ns_to_cycles(64000000 / (1 << ROW_BITS) + 1); + + localparam CNTW = $clog2((T_INIT>T_REFI ? T_INIT : T_REFI) + 1); + + localparam + S_INIT_WAIT = 5'd0, + S_INIT_PRE_WAIT = 5'd2, + S_INIT_REF = 5'd3, + S_INIT_REF_WAIT = 5'd4, + S_INIT_MRS_WAIT = 5'd6, + S_IDLE = 5'd7, + S_REFRESH_WAIT = 5'd9, + S_ACTIVATE_WAIT = 5'd11, + S_CAS_WAIT = 5'd13, + S_BURST_READ = 5'd14, + S_BURST_WRITE = 5'd15, + S_PRECHARGE_WAIT = 5'd16; + + reg [4:0] state; + reg [CNTW-1:0] wait_cnt; + reg [3:0] init_ref_cnt; + reg [CNTW-1:0] refresh_timer; + reg [BURST_IDXW-1:0] burst_idx; + reg req_wr_reg; + reg [BANK_BITS-1:0] req_bank_reg; + reg [ROW_BITS-1:0] req_row_reg; + reg [COL_BITS-1:0] req_col_reg; + reg [16*BURST_LEN-1:0] wdata_reg; + reg [2*BURST_LEN-1:0] wmask_reg; + + // ---- (1) re-sliced address decomposition -- see header note ---- + wire [BANK_BITS-1:0] addr_bank = addr[ALIGN_BITS +: BANK_BITS]; + wire [COL_BITS-1:0] addr_col = (ALIGN_BITS == 0) ? addr[ALIGN_BITS+BANK_BITS +: COL_BITS] + : {addr[ALIGN_BITS+BANK_BITS +: (COL_BITS-ALIGN_BITS)], addr[ALIGN_BITS-1:0]}; + wire [ROW_BITS-1:0] addr_row = addr[ADDR_WIDTH-1 -: ROW_BITS]; + + // ---- (2) shadow pipeline slot ---- + reg pipe_valid; + reg pipe_wr_reg; + reg [BANK_BITS-1:0] pipe_bank_reg; + reg [ROW_BITS-1:0] pipe_row_reg; + reg [COL_BITS-1:0] pipe_col_reg; + reg [16*BURST_LEN-1:0] pipe_wdata_reg; + reg [2*BURST_LEN-1:0] pipe_wmask_reg; + reg [CNTW-1:0] pipe_wait_cnt; + + wire shadow_capturable_state = (state==S_CAS_WAIT) || (state==S_BURST_READ) || + (state==S_BURST_WRITE) || (state==S_PRECHARGE_WAIT); + wire shadow_capture_now = req && shadow_capturable_state && !pipe_valid && + (addr_bank != req_bank_reg); + + reg req_pending; + wire eff_wr = req ? wr : req_wr_reg; + wire [BANK_BITS-1:0] eff_bank = req ? addr_bank : req_bank_reg; + wire [ROW_BITS-1:0] eff_row = req ? addr_row : req_row_reg; + wire [COL_BITS-1:0] eff_col = req ? addr_col : req_col_reg; + wire [16*BURST_LEN-1:0] eff_wdata = req ? wdata : wdata_reg; + wire [2*BURST_LEN-1:0] eff_wmask = req ? wmask : wmask_reg; + + reg dq_out_en; + reg [15:0] dq_out; + assign sdram_dq = dq_out_en ? dq_out : 16'hzzzz; + + function [ROW_BITS-1:0] mrs_value; + input integer burst_len; + reg [2:0] bl_code; + reg [ROW_BITS-1:0] v; + begin + bl_code = (burst_len==1) ? 3'b000 : + (burst_len==2) ? 3'b001 : + (burst_len==4) ? 3'b010 : + (burst_len==8) ? 3'b011 : 3'b111; + v = {ROW_BITS{1'b0}}; + v[6:4] = 3'b011; + v[3] = 1'b0; + v[2:0] = bl_code; + mrs_value = v; + end + endfunction + + function [CNTW-1:0] T_RC_MINUS1; + localparam integer T_RC = ns_to_cycles(65); + begin + T_RC_MINUS1 = T_RC[CNTW-1:0] - 1'b1; + end + endfunction + + always @(posedge clk) begin + if (rst) begin + state <= S_INIT_WAIT; + wait_cnt <= T_INIT[CNTW-1:0]; + init_ref_cnt <= 4'd0; + refresh_timer <= T_REFI[CNTW-1:0]; + sdram_cke <= 1'b1; + sdram_cs_n <= 1'b1; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + sdram_ba <= 2'b00; + sdram_a <= {ROW_BITS{1'b0}}; + sdram_dqm <= 2'b00; + dq_out_en <= 1'b0; + ready <= 1'b0; + busy <= 1'b1; + req_pending <= 1'b0; + pipe_valid <= 1'b0; + pipe_wait_cnt <= {CNTW{1'b0}}; + end else begin + sdram_cs_n <= 1'b0; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + ready <= 1'b0; + dq_out_en <= 1'b0; + sdram_dqm <= 2'b00; + + if (refresh_timer != 0) refresh_timer <= refresh_timer - 1'b1; + // shadow's own tRCD countdown runs independently of `state` + // (it tracks a DIFFERENT, already-open bank than whatever + // the primary FSM below is doing) -- see header note (2). + if (pipe_valid && pipe_wait_cnt != 0) pipe_wait_cnt <= pipe_wait_cnt - 1'b1; + + if (req) begin + if (shadow_capture_now) begin + // capture into the shadow slot INSTEAD OF the + // original req_pending latch (so wdata_reg/ + // wmask_reg/req_bank_reg etc, still owned by the + // CURRENTLY in-flight transaction, are never + // touched) -- and issue its real ACTIVATE command + // this very cycle (command bus is idle in every + // shadow_capturable_state, confirmed by inspection: + // none of those four states drive ras_n/ba/a). + pipe_wr_reg <= wr; + pipe_bank_reg <= addr_bank; + pipe_row_reg <= addr_row; + pipe_col_reg <= addr_col; + pipe_wdata_reg <= wdata; + pipe_wmask_reg <= wmask; + pipe_wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + pipe_valid <= 1'b1; + sdram_ras_n <= 1'b0; + sdram_ba <= addr_bank; + sdram_a <= addr_row; + end else begin + // ORIGINAL, unmodified path -- byte-for-byte + // identical to sdram_controller.v. + req_wr_reg <= wr; + req_bank_reg <= addr_bank; + req_row_reg <= addr_row; + req_col_reg <= addr_col; + wdata_reg <= wdata; + wmask_reg <= wmask; + req_pending <= 1'b1; + end + end + + case (state) + S_INIT_WAIT: begin + busy <= 1'b1; + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_a[10] <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_INIT_PRE_WAIT; + end + end + S_INIT_PRE_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + state <= S_INIT_REF; + end + end + S_INIT_REF: begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + state <= S_INIT_REF_WAIT; + end + S_INIT_REF_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else if (init_ref_cnt < 4'd7) begin + init_ref_cnt <= init_ref_cnt + 1'b1; + state <= S_INIT_REF; + end else begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_ba <= 2'b00; + sdram_a <= mrs_value(BURST_LEN); + wait_cnt <= T_MRD[CNTW-1:0] - 1'b1; + state <= S_INIT_MRS_WAIT; + end + end + S_INIT_MRS_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + busy <= 1'b0; + state <= S_IDLE; + end + end + + S_IDLE: begin + busy <= 1'b0; + // Priority: (1) a still-open SHADOW bank must be + // promoted/closed before anything else -- see + // header note (2) on why this ordering is the + // thing that keeps AUTO REFRESH from ever firing + // with an open row. (2) periodic refresh, exactly + // as sdram_controller.v. (3) the original req/ + // req_pending path, exactly as sdram_controller.v. + if (pipe_valid) begin + busy <= 1'b1; + req_wr_reg <= pipe_wr_reg; + req_bank_reg <= pipe_bank_reg; + req_row_reg <= pipe_row_reg; + req_col_reg <= pipe_col_reg; + wdata_reg <= pipe_wdata_reg; + wmask_reg <= pipe_wmask_reg; + wait_cnt <= pipe_wait_cnt; // remaining tRCD, may already be 0 + pipe_valid <= 1'b0; + state <= S_ACTIVATE_WAIT; + // NOTE: ACTIVATE for this bank was ALREADY + // issued at shadow-capture time -- do not + // re-issue it here (ras_n stays at its default + // NOP drive this cycle). + end else if (refresh_timer == 0) begin + busy <= 1'b1; + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + refresh_timer <= T_REFI[CNTW-1:0]; + state <= S_REFRESH_WAIT; + end else if (req || req_pending) begin + busy <= 1'b1; + req_wr_reg <= eff_wr; + req_bank_reg <= eff_bank; + req_row_reg <= eff_row; + req_col_reg <= eff_col; + wdata_reg <= eff_wdata; + wmask_reg <= eff_wmask; + req_pending <= 1'b0; + sdram_ras_n <= 1'b0; + sdram_ba <= eff_bank; + sdram_a <= eff_row; + wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + state <= S_ACTIVATE_WAIT; + end + end + + S_REFRESH_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; + end + + S_ACTIVATE_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + sdram_cas_n <= 1'b0; + sdram_we_n <= req_wr_reg ? 1'b0 : 1'b1; + sdram_ba <= req_bank_reg; + sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b1, {(10-COL_BITS){1'b0}}, req_col_reg}; + burst_idx <= {BURST_IDXW{1'b0}}; + if (req_wr_reg) begin + dq_out_en <= 1'b1; + dq_out <= wdata_reg[15:0]; + sdram_dqm <= wmask_reg[1:0]; + state <= S_BURST_WRITE; + end else begin + wait_cnt <= CAS_LATENCY[CNTW-1:0]; + state <= S_CAS_WAIT; + end + end + end + + S_CAS_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + rdata[0 +: 16] <= sdram_dq; + if (BURST_LEN == 1) begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRECHARGE_WAIT; + end else begin + burst_idx <= burst_idx + 1'b1; + state <= S_BURST_READ; + end + end + end + + S_BURST_READ: begin + rdata[burst_idx*16 +: 16] <= sdram_dq; + if (burst_idx == BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRECHARGE_WAIT; + end else begin + burst_idx <= burst_idx + 1'b1; + end + end + + S_BURST_WRITE: begin + if (burst_idx < BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + burst_idx <= burst_idx + 1'b1; + dq_out_en <= 1'b1; + dq_out <= wdata_reg[(burst_idx+1'b1)*16 +: 16]; + sdram_dqm <= wmask_reg[(burst_idx+1'b1)*2 +: 2]; + end else begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] + 1'b1; + state <= S_PRECHARGE_WAIT; + end + end + + S_PRECHARGE_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; + end + + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v b/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v new file mode 100644 index 0000000..f820102 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v @@ -0,0 +1,255 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXPERIMENTAL fork of sdram_unified_backend.v -- the ONLY change is +// instantiating sdram_controller_pipelined.v instead of sdram_ +// controller.v. W-port cache, arbitration, and the W/AR top-level FSM +// are ALL byte-for-byte unchanged. See sdram_controller_pipelined.v's +// own header for what changed at the controller level and why, and +// hardware/v2/logs/experiments.log (search "pipelin") for why this +// fork exists: testing whether bank-interleaved command pipelining +// recovers any of the ~77-78% Bank-W busy ceiling EXP-0051 measured. +// ============================================================ +module sdram_unified_backend_pipelined #( + parameter ADDR_WIDTH = 26, + parameter CLK_FREQ_MHZ = 64, + parameter W_ENTRIES = 4, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2 +)( + input wire clk, + input wire rst, + + input wire w_req, + input wire [ADDR_WIDTH-1:0] w_addr, + output reg [63:0] w_rdata, + output reg w_ready, + + input wire ar_req, + input wire ar_wr, + input wire [ADDR_WIDTH-1:0] ar_addr, + input wire [15:0] ar_wdata, + input wire ar_lb_n, + input wire ar_ub_n, + output reg [15:0] ar_rdata, + output reg ar_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin + $display("FATAL sdram_unified_backend_pipelined: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS); + $finish; + end + + localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES); + reg w_cache_valid [0:W_ENTRIES-1]; + reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1]; + reg [63:0] w_cache_data [0:W_ENTRIES-1]; + reg [WEIDXW-1:0] w_alloc_ptr; + + wire [W_ENTRIES-1:0] w_match_oh; + genvar wgi; + generate + for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH + assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr); + end + endgenerate + + reg w_hit_found_c; + reg [WEIDXW-1:0] w_hit_idx_c; + integer ei; + generate + if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT + always @(*) begin + w_hit_found_c = |w_match_oh; + casez (w_match_oh) + 4'b1???: w_hit_idx_c = 2'd3; + 4'b01??: w_hit_idx_c = 2'd2; + 4'b001?: w_hit_idx_c = 2'd1; + 4'b0001: w_hit_idx_c = 2'd0; + default: w_hit_idx_c = {WEIDXW{1'b0}}; + endcase + end + end else begin : GEN_WHIT_FALLBACK + always @(*) begin + w_hit_found_c = 1'b0; + w_hit_idx_c = {WEIDXW{1'b0}}; + for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin + if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin + w_hit_found_c = 1'b1; + w_hit_idx_c = ei[WEIDXW-1:0]; + end + end + end + end + endgenerate + wire w_cache_hit = w_hit_found_c && w_req; + + reg ctrl_req; + reg ctrl_wr; + reg [ADDR_WIDTH-2:0] ctrl_addr; + reg [127:0] ctrl_wdata; + reg [15:0] ctrl_wmask; + wire [127:0] ctrl_rdata; + wire ctrl_ready; + wire ctrl_busy; + + sdram_controller_pipelined #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk(clk), .rst(rst), + .req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr), + .wdata(ctrl_wdata), .wmask(ctrl_wmask), + .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + localparam S_IDLE = 3'd0, + S_W_WAIT = 3'd1, + S_AR_RD_WAIT = 3'd2, + S_AR_WR_WAIT = 3'd3; + reg [2:0] state; + reg w_pending_upper_half; + reg [ADDR_WIDTH-1:0] w_pending_addr; + reg [2:0] ar_pending_word; + + reg w_req_pending; + reg [ADDR_WIDTH-1:0] w_req_addr_lat; + reg ar_req_pending; + reg ar_req_wr_lat; + reg [ADDR_WIDTH-1:0] ar_req_addr_lat; + reg [15:0] ar_req_wdata_lat; + reg ar_req_lbn_lat, ar_req_ubn_lat; + + wire w_eff_req = w_req || w_req_pending; + wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat; + wire ar_eff_req = ar_req || ar_req_pending; + wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat; + wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat; + wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat; + wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat; + wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat; + + wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000}; + wire w_eff_addr_is_upper_half = w_eff_addr[3]; + wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000}; + wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0]; + + integer ri; + always @(posedge clk) begin + if (rst) begin + state <= S_IDLE; + for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0; + w_alloc_ptr <= {WEIDXW{1'b0}}; + ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}}; + ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF; + w_ready <= 1'b0; w_rdata <= 64'h0; + ar_ready <= 1'b0; ar_rdata <= 16'h0; + w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}}; + ar_pending_word <= 3'h0; + w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}}; + ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0; + ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0; + ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1; + end else begin + ctrl_req <= 1'b0; + w_ready <= 1'b0; + ar_ready <= 1'b0; + + if (w_req) begin + w_req_addr_lat <= w_addr; + w_req_pending <= 1'b1; + end + if (ar_req) begin + ar_req_wr_lat <= ar_wr; + ar_req_addr_lat <= ar_addr; + ar_req_wdata_lat <= ar_wdata; + ar_req_lbn_lat <= ar_lb_n; + ar_req_ubn_lat <= ar_ub_n; + ar_req_pending <= 1'b1; + end + + case (state) + S_IDLE: begin + if (w_cache_hit) begin + w_rdata <= w_cache_data[w_hit_idx_c]; + w_ready <= 1'b1; + w_cache_valid[w_hit_idx_c] <= 1'b0; + w_req_pending <= 1'b0; + end else if (w_eff_req) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= w_eff_aligned_word_addr; + ctrl_wmask <= 16'h0000; + w_pending_upper_half <= w_eff_addr_is_upper_half; + w_pending_addr <= w_eff_addr; + w_req_pending <= 1'b0; + state <= S_W_WAIT; + end else if (ar_eff_req && !ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= ar_eff_block_base; + ctrl_wmask <= 16'h0000; + ar_pending_word <= ar_eff_word_in_blk; + ar_req_pending <= 1'b0; + state <= S_AR_RD_WAIT; + end else if (ar_eff_req && ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b1; + ctrl_addr <= ar_eff_block_base; + ctrl_wdata <= {8{ar_eff_wdata}}; + ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2)); + ar_req_pending <= 1'b0; + state <= S_AR_WR_WAIT; + end + end + S_W_WAIT: begin + if (ctrl_ready) begin + if (w_pending_upper_half) begin + w_rdata <= ctrl_rdata[127:64]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end else begin + w_rdata <= ctrl_rdata[63:0]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end + w_cache_valid[w_alloc_ptr] <= 1'b1; + w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1; + w_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_RD_WAIT: begin + if (ctrl_ready) begin + ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16]; + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_WR_WAIT: begin + if (ctrl_ready) begin + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v b/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v new file mode 100644 index 0000000..e36a8ca --- /dev/null +++ b/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v @@ -0,0 +1,562 @@ +`timescale 1ns/1ps + +// ================================================================ +// EXPERIMENTAL fork of tb_nms_dstress_sdram_unified.v -- instantiates +// nms_neural_multiprocessor_sdram_pipelined.v (single physical SDRAM +// chip, but with sdram_controller_pipelined.v's bank-interleaved +// command pipelining inside) instead of nms_neural_multiprocessor_ +// sdram_unified.v. Identical D-Stress workload/golden-model/bit-exact +// verification. +// +// ONLY functional difference vs the original testbench: poke_byte/ +// peek_byte/poke_byte_weight/peek_byte_weight no longer compute a +// flat `u_sdram.mem[word_addr]` index by hand (that shortcut relied +// on the ORIGINAL controller's bank-from-TOP-bits decomposition, +// where ROWS/COLS being powers of 2 makes the flat word address +// numerically identical to bank*ROWS*COLS+row*COLS+col). The +// pipelined controller re-slices which address bits mean bank/row/ +// col (see sdram_controller_pipelined.v's own header, note (1)), so +// these backdoor helpers instead: (a) decompose the flat word address +// using the EXACT SAME bit ranges as sdram_controller_pipelined.v's +// own addr_bank/addr_row/addr_col wires, then (b) call sdram_model.v's +// own explicit, decomposition-agnostic backdoor_read/backdoor_write +// tasks (bank/row/col-addressed) instead of indexing `mem[]` directly +// -- this guarantees the testbench and the RTL agree on where a given +// byte physically lives, by construction, rather than by two +// independently-maintained flat-index formulas that could silently +// drift apart. +// ================================================================ +module tb #( + parameter N_SLOTS_CFG = 2, + parameter PFD_CFG = 8 +); + + localparam ADDR_WIDTH = 26; + localparam DATA_WIDTH = 8; + localparam P_IN = 8; + localparam ACC_WIDTH = 32; + localparam N_NODES = 1024; + localparam MAX_DEPS = 8; + localparam QUEUE_DEPTH = 8; + localparam NODE_IDW = $clog2(N_NODES); + localparam CLK_PERIOD = 12.5; // 80 MHz + + // must match sdram_controller_pipelined.v's own instantiation + // parameters exactly (BURST_LEN=8 hardcoded by sdram_unified_ + // backend_pipelined.v, ROW_BITS/COL_BITS/BANK_BITS defaults) + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ALIGN_BITS = 3; // clog2(BURST_LEN=8) + + reg clk, rst; + initial begin clk = 1'b0; forever #(CLK_PERIOD/2.0) clk = ~clk; end + + reg reg_valid; + wire reg_ready; + reg [NODE_IDW-1:0] reg_node_id; + reg [$clog2(MAX_DEPS+1)-1:0] reg_required; + reg [MAX_DEPS*NODE_IDW-1:0] reg_producer_ids; + reg [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr; + reg [15:0] reg_n_tiles; + + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [1:0] sdram_ba; + wire [12:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + nms_neural_multiprocessor_sdram_pipelined #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS_CFG), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(16), .PREFETCH_DISTANCE(PFD_CFG), .CLK_FREQ_MHZ(80) + ) u_nmp ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + sdram_model #(.CLK_FREQ_MHZ(80)) u_sdram ( + .clk(clk), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + // ---- backdoor helpers: decompose a flat 25-bit word address into + // (bank,row,col) using sdram_controller_pipelined.v's own bit + // ranges, then use sdram_model.v's own bank/row/col-addressed + // backdoor tasks -- see header note above ---- + function automatic [BANK_BITS-1:0] wa_bank(input [24:0] wa); + wa_bank = wa[ALIGN_BITS +: BANK_BITS]; + endfunction + function automatic [COL_BITS-1:0] wa_col(input [24:0] wa); + wa_col = {wa[ALIGN_BITS+BANK_BITS +: (COL_BITS-ALIGN_BITS)], wa[ALIGN_BITS-1:0]}; + endfunction + function automatic [ROW_BITS-1:0] wa_row(input [24:0] wa); + wa_row = wa[24 -: ROW_BITS]; + endfunction + + task automatic poke_byte(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + if (byte_addr[0] == 1'b0) cur[7:0] = val; else cur[15:8] = val; + u_sdram.backdoor_write(wa_bank(wa), wa_row(wa), wa_col(wa), cur); + end + endtask + + function automatic signed [7:0] peek_byte(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + peek_byte = (byte_addr[0] == 1'b0) ? cur[7:0] : cur[15:8]; + end + endfunction + + task automatic poke_byte_weight(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + if (byte_addr[0] == 1'b0) cur[7:0] = val; else cur[15:8] = val; + u_sdram.backdoor_write(wa_bank(wa), wa_row(wa), wa_col(wa), cur); + end + endtask + + function automatic signed [7:0] peek_byte_weight(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + peek_byte_weight = (byte_addr[0] == 1'b0) ? cur[7:0] : cur[15:8]; + end + endfunction + + function automatic signed [7:0] relu_sat(input integer acc); + begin + if (acc <= 0) relu_sat = 8'sd0; + else if (acc > 127) relu_sat = 8'sd127; + else relu_sat = acc[7:0]; + end + endfunction + + task automatic register_node( + input [NODE_IDW-1:0] nid, + input [$clog2(MAX_DEPS+1)-1:0] required, + input [MAX_DEPS*NODE_IDW-1:0] producer_ids_packed, + input [ADDR_WIDTH-1:0] xb, input [ADDR_WIDTH-1:0] wb, + input [15:0] nt, input [ADDR_WIDTH-1:0] resaddr + ); + begin + @(posedge clk); + reg_node_id = nid; + reg_required = required; + reg_producer_ids = producer_ids_packed; + reg_x_base = xb; reg_w_base = wb; reg_n_tiles = nt; reg_result_addr = resaddr; + reg_valid = 1'b1; + while (!reg_ready) @(posedge clk); + @(posedge clk); + reg_valid = 1'b0; + end + endtask + + reg measure_en; + integer total_cycles; + integer psram_busy_cycles; + integer ni; + genvar gi; + + reg [N_SLOTS_CFG-1:0] slot_busy_bit; + reg [N_SLOTS_CFG-1:0] slot_tile_bit; + integer slot_busy_cycles [0:N_SLOTS_CFG-1]; + integer slot_tiles_delivered [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_MON + always @(*) begin + slot_busy_bit[gi] = (u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.state != 3'd0); + slot_tile_bit[gi] = u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_valid && + u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_ready; + end + end + endgenerate + + integer active_count; + integer active_hist [0:4]; + integer useful_mac_cycles; + integer first_tile_cyc; + integer last_tile_cyc; + integer any_tile_bit; + + integer sdram_req_count, sdram_ready_count, sdram_wr_count; + integer sdram_busy_cycles, sdram_refresh_count; + integer sdram_req_start_cyc, sdram_lat_sum, sdram_lat_min, sdram_lat_max, sdram_lat_n; + reg sdram_prev_state_is_refwait; + + initial begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + sdram_prev_state_is_refwait=1'b0; + end + + always @(posedge clk) begin + if (measure_en) begin + active_count = slot_busy_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) active_count = active_count + slot_busy_bit[ni]; + active_hist[active_count] <= active_hist[active_count] + 1; + + any_tile_bit = slot_tile_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) any_tile_bit = any_tile_bit | slot_tile_bit[ni]; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) + if (slot_tile_bit[ni]) useful_mac_cycles <= useful_mac_cycles + 1; + if (any_tile_bit) begin + if (first_tile_cyc < 0) first_tile_cyc <= total_cycles; + last_tile_cyc <= total_cycles; + end + + if (u_nmp.u_sdram_backend.u_sdram_ctrl.req) begin + sdram_req_count <= sdram_req_count + 1; + sdram_req_start_cyc <= total_cycles; + if (u_nmp.u_sdram_backend.u_sdram_ctrl.wr) sdram_wr_count <= sdram_wr_count + 1; + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.ready) begin + sdram_ready_count <= sdram_ready_count + 1; + sdram_lat_sum <= sdram_lat_sum + (total_cycles - sdram_req_start_cyc); + sdram_lat_n <= sdram_lat_n + 1; + if ((total_cycles - sdram_req_start_cyc) < sdram_lat_min) sdram_lat_min <= (total_cycles - sdram_req_start_cyc); + if ((total_cycles - sdram_req_start_cyc) > sdram_lat_max) sdram_lat_max <= (total_cycles - sdram_req_start_cyc); + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.busy) sdram_busy_cycles <= sdram_busy_cycles + 1; + sdram_prev_state_is_refwait <= (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9); + if (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9 && !sdram_prev_state_is_refwait) + sdram_refresh_count <= sdram_refresh_count + 1; + end + end + + task automatic report_step17_instrumentation; + real active_pct [0:4]; + real util_pct, startup_cycles, drain_cycles; + real sdram_avg_lat, sdram_busy_pct, sdram_bytes_per_cycle; + integer kk, total_tiles_all; + begin + total_tiles_all = 0; + for (kk = 0; kk < N_SLOTS_CFG; kk = kk + 1) total_tiles_all = total_tiles_all + slot_tiles_delivered[kk]; + $display(" ---- cycle decomposition ----"); + for (kk = 0; kk <= N_SLOTS_CFG; kk = kk + 1) begin + active_pct[kk] = (total_cycles > 0) ? (100.0*active_hist[kk]/total_cycles) : 0.0; + $display(" active_slots=%0d: %0d cycles (%0.2f%%)", kk, active_hist[kk], active_pct[kk]); + end + util_pct = (total_cycles > 0) ? (100.0*useful_mac_cycles/(total_cycles*1.0*N_SLOTS_CFG)) : 0.0; + $display(" useful_mac_cycles (slot-tile-delivery events, summed)=%0d (%0.2f%% of total_cycles*N_SLOTS)", useful_mac_cycles, util_pct); + startup_cycles = (first_tile_cyc >= 0) ? (1.0*first_tile_cyc) : 0.0; + drain_cycles = (last_tile_cyc >= 0) ? (1.0*(total_cycles - last_tile_cyc)) : 0.0; + $display(" startup (cycles before first tile delivered anywhere)=%0.0f", startup_cycles); + $display(" drain (cycles after last tile delivered, until job completion)=%0.0f", drain_cycles); + $display(" ---- SDRAM (pipelined controller) effectiveness ----"); + sdram_avg_lat = (sdram_lat_n > 0) ? (1.0*sdram_lat_sum/sdram_lat_n) : 0.0; + sdram_busy_pct = (total_cycles > 0) ? (100.0*sdram_busy_cycles/total_cycles) : 0.0; + sdram_bytes_per_cycle = (total_cycles > 0) ? (8.0*sdram_ready_count/total_cycles) : 0.0; + $display(" sdram_req_count=%0d sdram_ready_count=%0d sdram_wr_count=%0d", + sdram_req_count, sdram_ready_count, sdram_wr_count); + $display(" sdram_busy_cycles=%0d/%0d (%0.2f%%)", sdram_busy_cycles, total_cycles, sdram_busy_pct); + $display(" sdram_refresh_count=%0d", sdram_refresh_count); + $display(" sdram_request_latency: min=%0d max=%0d avg=%0.2f cycles", + sdram_lat_min, sdram_lat_max, sdram_avg_lat); + $display(" sdram_avg_bytes_per_cycle=%0.4f", sdram_bytes_per_cycle); + end + endtask + + reg [N_SLOTS_CFG-1:0] slot_could_present_act; + reg [N_SLOTS_CFG-1:0] slot_weight_blocking; + reg [N_SLOTS_CFG-1:0] slot_stalled_this_tile; + reg [31:0] prev_tile_idx [0:N_SLOTS_CFG-1]; + integer weight_stall_cycles [0:N_SLOTS_CFG-1]; + integer tiles_prefetched_clean [0:N_SLOTS_CFG-1]; + integer tiles_consumed_total [0:N_SLOTS_CFG-1]; + wire [31:0] slot_tile_idx_w [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_PF_MON + assign slot_tile_idx_w[gi] = {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx}; + always @(*) begin + slot_could_present_act[gi] = + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.n_tiles_reg}) && + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.usable_act}); + slot_weight_blocking[gi] = + slot_could_present_act[gi] && + !(u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx < + u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.wgt_ready_count) && + !u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.operand_valid; + end + end + endgenerate + + always @(posedge clk) begin + if (measure_en) begin + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (prev_tile_idx[ni] != slot_tile_idx_w[ni]) begin + slot_stalled_this_tile[ni] <= 1'b0; + prev_tile_idx[ni] <= slot_tile_idx_w[ni]; + end else if (slot_weight_blocking[ni]) begin + slot_stalled_this_tile[ni] <= 1'b1; + weight_stall_cycles[ni] <= weight_stall_cycles[ni] + 1; + end + if (slot_tile_bit[ni]) begin + tiles_consumed_total[ni] <= tiles_consumed_total[ni] + 1; + if (!slot_stalled_this_tile[ni]) + tiles_prefetched_clean[ni] <= tiles_prefetched_clean[ni] + 1; + end + end + end + end + + integer jobs_allocated, jobs_completed, wakeups; + integer waiting_sum, ready_sum, dispatched_sum, sample_count; + reg sample_occupancy; + integer scan_i; + integer waiting_now, ready_now, dispatched_now; + + always @(posedge clk) begin + if (measure_en) begin + total_cycles <= total_cycles + 1; + if (u_nmp.u_arbiter.owner != 0) psram_busy_cycles <= psram_busy_cycles + 1; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (slot_busy_bit[ni]) slot_busy_cycles[ni] <= slot_busy_cycles[ni] + 1; + if (slot_tile_bit[ni]) slot_tiles_delivered[ni] <= slot_tiles_delivered[ni] + 1; + end + if (u_nmp.u_dataflow_core.dm_ready_valid && u_nmp.u_dataflow_core.dm_ready_ready) + jobs_allocated <= jobs_allocated + 1; + if (u_nmp.u_dataflow_core.dir_job_out_done) + jobs_completed <= jobs_completed + 1; + if (u_nmp.u_dataflow_core.dm_producer_done_valid) + wakeups <= wakeups + 1; + + if (sample_occupancy) begin + waiting_now = 0; ready_now = 0; dispatched_now = 0; + for (scan_i = 0; scan_i < N_NODES; scan_i = scan_i + 1) begin + case (u_nmp.u_dataflow_core.u_dep_mgr.node_state[scan_i]) + 2'd1: waiting_now = waiting_now + 1; + 2'd2: ready_now = ready_now + 1; + 2'd3: dispatched_now = dispatched_now + 1; + default: ; + endcase + end + waiting_sum <= waiting_sum + waiting_now; + ready_sum <= ready_sum + ready_now; + dispatched_sum <= dispatched_sum + dispatched_now; + sample_count <= sample_count + 1; + end + end + end + + task automatic reset_instrumentation(input do_sample_occupancy); + integer k; + begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + total_cycles = 0; psram_busy_cycles = 0; + jobs_allocated = 0; jobs_completed = 0; wakeups = 0; + waiting_sum = 0; ready_sum = 0; dispatched_sum = 0; sample_count = 0; + sample_occupancy = do_sample_occupancy; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + slot_busy_cycles[k] = 0; + slot_tiles_delivered[k] = 0; + weight_stall_cycles[k] = 0; + tiles_prefetched_clean[k] = 0; + tiles_consumed_total[k] = 0; + slot_stalled_this_tile[k] = 1'b0; + prev_tile_idx[k] = 32'hFFFFFFFF; + end + end + endtask + + task automatic report_instrumentation(input [255:0] label, input integer n_neurons_completed); + integer k, total_tiles; + integer total_weight_stall_cycles, total_tiles_consumed_all, total_tiles_prefetched_clean; + real avg_waiting, avg_ready, avg_dispatched; + real psram_util, sustained_mac_per_cycle, wallclock_us; + real processor_utilization, weight_stall_pct, prefetch_effectiveness_pct; + begin + total_tiles = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) total_tiles = total_tiles + slot_tiles_delivered[k]; + avg_waiting = (sample_count > 0) ? (1.0*waiting_sum/sample_count) : 0.0; + avg_ready = (sample_count > 0) ? (1.0*ready_sum/sample_count) : 0.0; + avg_dispatched = (sample_count > 0) ? (1.0*dispatched_sum/sample_count) : 0.0; + psram_util = (total_cycles > 0) ? (100.0*psram_busy_cycles/total_cycles) : 0.0; + sustained_mac_per_cycle = (total_cycles > 0) ? (1.0*total_tiles*P_IN/total_cycles) : 0.0; + wallclock_us = total_cycles * CLK_PERIOD / 1000.0; + $display("---- BENCHMARK REPORT: %0s ----", label); + $display(" total_cycles=%0d wallclock_us=%0.3f", total_cycles, wallclock_us); + $display(" neurons_completed=%0d tiles_delivered(real)=%0d", n_neurons_completed, total_tiles); + $display(" jobs_allocated=%0d jobs_completed=%0d dependency_wakeups=%0d", jobs_allocated, jobs_completed, wakeups); + $display(" shared AR (activation+result) arbiter-side utilization: %0.1f%% (%0d/%0d busy cycles)", psram_util, psram_busy_cycles, total_cycles); + for (k = 0; k < N_SLOTS_CFG; k = k + 1) + $display(" slot %0d: busy=%0d/%0d (%0.1f%%) tiles=%0d", k, slot_busy_cycles[k], total_cycles, + (total_cycles>0)?(100.0*slot_busy_cycles[k]/total_cycles):0.0, slot_tiles_delivered[k]); + if (sample_count > 0) + $display(" dependency_manager avg occupancy: waiting=%0.2f ready=%0.2f dispatched=%0.2f", avg_waiting, avg_ready, avg_dispatched); + $display(" DERIVED: sustained end-to-end MAC/cycle = %0.4f", sustained_mac_per_cycle); + if (n_neurons_completed > 0) + $display(" DERIVED: cycles/neuron = %0.2f", 1.0*total_cycles/n_neurons_completed); + if (total_tiles > 0) + $display(" DERIVED: cycles/tile = %0.2f", 1.0*total_cycles/total_tiles); + + total_weight_stall_cycles = 0; total_tiles_consumed_all = 0; total_tiles_prefetched_clean = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + total_weight_stall_cycles = total_weight_stall_cycles + weight_stall_cycles[k]; + total_tiles_consumed_all = total_tiles_consumed_all + tiles_consumed_total[k]; + total_tiles_prefetched_clean = total_tiles_prefetched_clean + tiles_prefetched_clean[k]; + end + weight_stall_pct = (total_cycles > 0) ? (100.0*total_weight_stall_cycles/(total_cycles*N_SLOTS_CFG*1.0)) : 0.0; + prefetch_effectiveness_pct = (total_tiles_consumed_all > 0) ? + (100.0*total_tiles_prefetched_clean/(total_tiles_consumed_all*1.0)) : 0.0; + $display(" [STEP11] PFD=%0d weight_stall_cycles(sum,all slots)=%0d (%0.2f%%)", + PFD_CFG, total_weight_stall_cycles, weight_stall_pct); + $display(" [STEP11] DERIVED: prefetch_effectiveness = %0.2f%%", prefetch_effectiveness_pct); + end + endtask + + integer errors, tests; + + task automatic run_dense_layer( + input [255:0] label, + input integer n_neurons, + input integer n_tiles_count, + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, + input [ADDR_WIDTH-1:0] w_base, + input [ADDR_WIDTH-1:0] res_base, + input sample_occ + ); + integer n, t, k, len, acc; + reg signed [7:0] xv, wv, golden, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps; + integer completed, wd2; + begin + len = n_tiles_count * P_IN; + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + + for (k = 0; k < len; k = k + 1) + poke_byte(x_base + k, ((k % 8) + 1)); + + reset_instrumentation(sample_occ); + measure_en = 1'b1; + + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) begin + for (k = 0; k < P_IN; k = k + 1) begin + xv = peek_byte(x_base + t*P_IN + k); + wv = (((n + t*P_IN + k) % 8) + 1); + poke_byte_weight(w_base + n*len + t*P_IN + k, wv); + acc = acc + xv*wv; + end + end + golden = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + x_base, w_base + n*len, n_tiles_count[15:0], res_base + n); + if ((n % 32) == 0) begin + $display(" [%0s] registered %0d/%0d", label, n+1, n_neurons); + $fflush; + end + end + $display(" [%0s] all %0d neurons registered, waiting for completion...", label, n_neurons); + $fflush; + + completed = 0; wd2 = 0; + while (completed < n_neurons && wd2 < 2000000) begin + @(posedge clk); + wd2 = wd2 + 1; + completed = jobs_completed; + if ((wd2 % 20000) == 0) begin + $display(" [%0s] watchdog %0d: completed=%0d/%0d total_cycles=%0d", label, wd2, completed, n_neurons, total_cycles); + $fflush; + end + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + if (completed < n_neurons) begin + $display("FAIL %0s: only %0d/%0d neurons completed within watchdog", label, completed, n_neurons); + errors = errors + 1; + end else begin : check_block + integer local_errors; + local_errors = 0; + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) + for (k = 0; k < P_IN; k = k + 1) + acc = acc + peek_byte(x_base + t*P_IN + k) * peek_byte_weight(w_base + n*len + t*P_IN + k); + golden = relu_sat(acc); + real_y = peek_byte(res_base + n); + if (real_y !== golden) begin + $display("FAIL %0s neuron %0d: real=%0d golden=%0d", label, n, real_y, golden); + local_errors = local_errors + 1; + end + end + if (local_errors == 0) + $display("PASS %0s: all %0d neurons bit-exact vs golden", label, n_neurons); + else + errors = errors + 1; + end + report_instrumentation(label, n_neurons); + report_step17_instrumentation; + end + endtask + + initial begin + errors = 0; tests = 0; + rst = 1; reg_valid = 0; reg_node_id = 0; reg_required = 0; reg_producer_ids = 0; + reg_x_base = 0; reg_w_base = 0; reg_n_tiles = 0; reg_result_addr = 0; + measure_en = 0; + repeat(5) @(posedge clk); + rst = 0; + + $display("========================================"); + $display("NMS D-Stress benchmark (EXPERIMENTAL PIPELINED SDRAM controller) -- N_SLOTS_CFG=%0d PFD_CFG=%0d", N_SLOTS_CFG, PFD_CFG); + $display("========================================"); + + wait (u_nmp.u_sdram_backend.u_sdram_ctrl.state == u_nmp.u_sdram_backend.u_sdram_ctrl.S_IDLE); + @(posedge clk); + + run_dense_layer("D-Stress", 256, 16, 16'd400, 26'h200000, 26'h010000, 26'h300000, 1'b0); + + repeat (4) @(posedge clk); + if (u_nmp.data_ready !== 1'b1) begin + $display("FAIL data_ready: expected 1 after graph completion, got %b", u_nmp.data_ready); + errors = errors + 1; + end else begin + $display("PASS data_ready: correctly asserted after graph completion"); + end + + $display("========================================"); + if (errors == 0) + $display("ALL %0d WORKLOAD SUITES PASSED (N_SLOTS_CFG=%0d, PFD_CFG=%0d, PIPELINED SDRAM)", tests, N_SLOTS_CFG, PFD_CFG); + else + $display("FAILED: %0d/%0d workload suite(s) had errors -- see messages above", errors, tests); + $display("========================================"); + $finish; + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v b/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v new file mode 100644 index 0000000..5d282fa --- /dev/null +++ b/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v @@ -0,0 +1,273 @@ +`timescale 1ns/1ps + +// ============================================================ +// Isolated correctness + cycle-savings regression for +// sdram_controller_pipelined.v, forked from tb_sdram_controller.v's +// own idiom (same do_transaction task style, same sdram_model.v DUT +// pairing). Adds what the original testbench cannot exercise (it +// always waits for `busy` to clear before issuing the next request): +// deliberately pulsing a SECOND req WHILE the controller is still +// mid-transaction, to test the new shadow-pipeline slot. +// +// Covers: +// 1) same correctness battery as the original (write->read, +// sequential, all 4 banks, address limits, pseudo-random) -- +// using the ORIGINAL wait-for-ready protocol throughout, so this +// also proves the re-sliced address decomposition (header note +// (1) in sdram_controller_pipelined.v) is a correct bijection. +// 2) DIFFERENT-bank early injection: issue a second request for a +// different bank while the first is still in S_CAS_WAIT, verify +// both results bit-exact AND that the combined cycle count is +// LOWER than 2x the serial baseline. +// 3) SAME-bank consecutive (both via the normal wait-for-ready +// protocol): must cost exactly the same as the original +// controller, no regression. +// 4) refresh spanning an early-injected interleave: run enough +// interleaved pairs to cross >=1 real tREFI interval, watch for +// any "VIOLATION"/"WARNING" from sdram_model.v. +// ============================================================ +module tb #( + parameter BURST_LEN = 8, + parameter CLK_FREQ_MHZ = 80 +); + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS; + localparam CLK_PERIOD_NS = 1000.0/CLK_FREQ_MHZ; + localparam ALIGN_BITS = (BURST_LEN<=1) ? 0 : $clog2(BURST_LEN); + + reg clk = 0; + always #(CLK_PERIOD_NS/2.0) clk = ~clk; + reg rst; + + reg req, wr; + reg [ADDR_WIDTH-1:0] addr; + reg [16*BURST_LEN-1:0] wdata; + reg [2*BURST_LEN-1:0] wmask; + wire [16*BURST_LEN-1:0] rdata; + wire ready, busy; + + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [BANK_BITS-1:0] sdram_ba; + wire [ROW_BITS-1:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + sdram_controller_pipelined #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut ( + .clk(clk), .rst(rst), + .req(req), .wr(wr), .addr(addr), .wdata(wdata), .wmask(wmask), .rdata(rdata), .ready(ready), .busy(busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + sdram_model #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem ( + .clk(clk), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + integer errors, tests; + integer cyc; + always @(posedge clk) if (!rst) cyc <= cyc + 1; + + // ---- helper: which bank a given flat word address maps to under + // the PIPELINED decomposition (must match sdram_controller_ + // pipelined.v's own addr_bank wire exactly) ---- + function automatic [BANK_BITS-1:0] bank_of; + input [ADDR_WIDTH-1:0] a; + begin + bank_of = a[ALIGN_BITS +: BANK_BITS]; + end + endfunction + + task automatic do_transaction( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk); + while (busy) @(posedge clk); + t0 = cyc; + req = 1'b1; wr = t_wr; addr = t_addr; wdata = t_wdata; + @(posedge clk); + req = 1'b0; + while (!ready) @(posedge clk); + t_rdata = rdata; + t_cycles = cyc - t0; + end + endtask + + reg [16*BURST_LEN-1:0] got, wpat; + integer elapsed; + + task automatic check_word(input [ADDR_WIDTH-1:0] a, input [15:0] pattern); + integer k; + begin + for (k = 0; k < BURST_LEN; k = k + 1) + wpat[k*16 +: 16] = pattern + k[15:0]; + do_transaction(1'b1, a, wpat, got, elapsed); + do_transaction(1'b0, a, {(16*BURST_LEN){1'b0}}, got, elapsed); + tests = tests + 1; + if (got !== wpat) begin + $display("FAIL addr=%0d bank=%0d: got=%h expected=%h", a, bank_of(a), got, wpat); + errors = errors + 1; + end else begin + $display("PASS addr=%0d bank=%0d: burst=%0d bit-exact, cycles=%0d", a, bank_of(a), BURST_LEN, elapsed); + end + end + endtask + + // issue a request THIS cycle without waiting for busy/ready -- + // the caller is responsible for knowing this is safe (shadow slot + // free, or accepting fallback-to-req_pending semantics otherwise) + task automatic issue_req_now(input t_wr, input [ADDR_WIDTH-1:0] t_addr, input [16*BURST_LEN-1:0] t_wdata); + begin + @(posedge clk); + req = 1'b1; wr = t_wr; addr = t_addr; wdata = t_wdata; + @(posedge clk); + req = 1'b0; + end + endtask + + task automatic wait_ready(output [16*BURST_LEN-1:0] t_rdata, output integer t_cyc_at_ready); + begin + // always advance at least one cycle first -- otherwise two + // back-to-back calls can both observe the SAME still-high + // `ready` pulse from the previous call's own exit cycle + // (a single-cycle-wide pulse level-checked with no + // intervening clock edge looks identical to a fresh one). + @(posedge clk); + while (!ready) @(posedge clk); + t_rdata = rdata; + t_cyc_at_ready = cyc; + end + endtask + + integer seed; + integer i; + reg [ADDR_WIDTH-1:0] rnd_addr; + + initial begin + errors = 0; tests = 0; cyc = 0; seed = 32'hC0FFEE; + rst = 1; req = 0; wr = 0; addr = 0; wdata = 0; wmask = 0; + repeat(5) @(posedge clk); + rst = 0; + while (busy) @(posedge clk); + + $display("=== TEST 1: correctness battery (original wait-for-ready protocol) ==="); + check_word({ADDR_WIDTH{1'b0}}, 16'hA5A5); + for (i = 0; i < 8; i = i + 1) + check_word(i*BURST_LEN, 16'h1000 + i); + // all 4 banks (bank now comes from LOW bits above the burst + // alignment -- addr values chosen so bank_of() sweeps 0..3) + for (i = 0; i < 4; i = i + 1) + check_word((i << ALIGN_BITS) + (100 << (ALIGN_BITS+BANK_BITS)), 16'h2000 + i); + // pseudo-random + for (i = 0; i < 24; i = i + 1) begin + rnd_addr = ($random(seed) % ((1< B-ready = %0d (serial baseline for 2 back-to-back BURST_LEN=%0d transactions is ~%0d; savings expected ~tRCD per pipelined pair, NOT a multiple-x speedup -- see sdram_controller_pipelined.v header)", + cyc_ready1 - t0, BURST_LEN, 2*(1+2+(BURST_LEN==1?0:3+1)+ (BURST_LEN>1?BURST_LEN-1:0) +2)); + end + end + + $display("=== TEST 4: refresh spanning interleaved traffic (watch for VIOLATION/WARNING above) ==="); + begin : test4 + integer t0b, cyc_r0, cyc_r1, j; + reg [ADDR_WIDTH-1:0] ba0, ba1; + reg [16*BURST_LEN-1:0] rr0, rr1; + for (j = 0; j < 60; j = j + 1) begin + ba0 = ((j*3) << ALIGN_BITS); + ba1 = ((j*3+1) << ALIGN_BITS); + if (bank_of(ba0) == bank_of(ba1)) ba1 = ba1 + (1 << ALIGN_BITS); + do_transaction(1'b1, ba0, {(16*BURST_LEN){16'hAA55}}, got, elapsed); + do_transaction(1'b1, ba1, {(16*BURST_LEN){16'h55AA}}, got, elapsed); + @(posedge clk); + while (busy) @(posedge clk); + t0b = cyc; + issue_req_now(1'b0, ba0, {(16*BURST_LEN){1'b0}}); + while (dut.state !== 13 && dut.state !== 7) @(posedge clk); // S_CAS_WAIT or back to S_IDLE (refresh could have won) + if (dut.state === 13 && !dut.pipe_valid) + issue_req_now(1'b0, ba1, {(16*BURST_LEN){1'b0}}); + wait_ready(rr0, cyc_r0); + if (dut.pipe_valid || dut.state != 7) + wait_ready(rr1, cyc_r1); + end + $display(" TEST4: 60 interleaved read pairs completed (spans real tREFI at CLK_FREQ_MHZ=%0d) -- check log above for VIOLATION/WARNING", CLK_FREQ_MHZ); + end + + $display("=== %0d/%0d tests, %0d errors (BURST_LEN=%0d, CLK_FREQ_MHZ=%0d) ===", + tests-errors, tests, errors, BURST_LEN, CLK_FREQ_MHZ); + if (errors == 0) $display("ALL TESTS PASSED (tb_sdram_controller_pipelined, BURST_LEN=%0d, CLK_FREQ_MHZ=%0d)", BURST_LEN, CLK_FREQ_MHZ); + $finish; + end +endmodule