diff --git a/hardware/v2/logs/experiments.log b/hardware/v2/logs/experiments.log index 6ce6e9a..73ee653 100644 --- a/hardware/v2/logs/experiments.log +++ b/hardware/v2/logs/experiments.log @@ -3051,3 +3051,120 @@ next_action: report this refined finding to the user before choosing used by the real board top, additive only): hardware/v2/nms/rtl/ nms_neural_multiprocessor_sdram_dualbank.v, hardware/v2/nms/sim/ tb_nms_dstress_sdram_dualbank.v. + +EXP-0052 -- Bank-interleaved pipelining for the W (weight-fetch) SDRAM +channel: the mechanism works in isolation (verified) but the real +D-Stress integration gain is negligible, because the CALLER never +issues a second request early enough to trigger it (2026-09-16) + +DATE: 2026-09-16 +CONTEXT: follow-up to EXP-0051, which found the weight-fetch (W) +channel itself (not W/AR cross-traffic) as the real ~77-78%-busy +ceiling, and identified the per-transaction fixed cost (measured ~16 +cycles: 1 issue + 2 T_RCD + 4 CAS_LATENCY-wait + 7 BURST_LEN=8 read + +2 T_RP) as the lever to attack, since the real transaction COUNT is +already close to optimal (2163 measured vs 2048 theoretical minimum +for the D-Stress workload, ~5.6% overhead). This session chose to +pursue this via a fork given the real correctness-risk history of this +exact FSM area (ERR-0019/0020/0023, all req-latching races). +TOOLCHAIN: unchanged from EXP-0051 (Yosys 0.69+59 d85872386-dirty, +Verilator 5.053) -- this experiment is Verilator-only, no synthesis/ +P&R. + +METHOD (two-phase, isolated-correctness-first per this project's own +established discipline): + Phase A: `sdram_controller_pipelined.v` (new, forked from + sdram_controller.v) remaps the addr->{bank,row,col} decomposition + from high-order bits (today: bank always 0 for this project's + compact weight region, since bank comes from the TOP address bits) + to LOW-order bits placed just above the burst-alignment zero bits -- + so consecutive burst-aligned weight fetches (each BURST_LEN=8 words + apart) now naturally rotate across the SDRAM's own 4 internal banks + instead of all landing on bank 0. Added a depth-1 "shadow" slot: while + the current transaction is in CAS_WAIT/BURST/PRECHARGE_WAIT (command + bus otherwise idle), a newly-arriving request for a DIFFERENT bank + has its ACTIVATE issued immediately, overlapping that bank's own + T_RCD wait with the current transaction's tail. Same-bank requests, + and refresh, are unaffected (S_IDLE priority: open shadow > refresh > + new request, so AUTO REFRESH can never fire with a row left open). + New isolated testbench `tb_sdram_controller_pipelined.v`: 38/38 PASS, + bit-exact across all 4 banks. Real, INDEPENDENTLY RE-VERIFIED result + for back-to-back different-bank transactions: 30 cycles total vs a + 32-cycle serial baseline for the same pair -- exactly 2 cycles saved + (= T_RCD), NOT a multiple-x speedup. This matches the theoretical + ceiling worked out BEFORE measuring: CAS_LATENCY+BURST_LEN (11 of the + 16 cycles) are serial on the SHARED data bus regardless of bank, and + no amount of bank interleaving can hide that -- only the T_RCD+T_RP + portion (5 of 16 cycles) is bank-local and therefore hideable, and + only T_RCD (2 cycles) was actually recovered here since the OTHER + bank's T_RP tail still has to clear before its OWN next reuse. Same- + bank consecutive case: unchanged, no regression. Refresh-during- + interleaving case (Test 4): AUTO REFRESH spacing rose from 634 to 657 + cycles under sustained back-to-back different-bank stress (vs + tREFI=626 target) -- a real, disclosed +3.6%, already within the + margin this project's OWN unmodified controller already tolerates + under the same synthetic stress pattern, not a new violation. + One bug found and fixed, in the NEW TESTBENCH ONLY (not the RTL): + calling wait_ready() twice in a row double-consumed the same `ready` + pulse -- fixed by advancing one extra @(posedge clk) between calls. + + Phase B (integration, gated on Phase A passing): forked + `sdram_unified_backend_pipelined.v` (swaps in the pipelined + controller, W_ENTRIES cache and W/AR arbitration untouched) and + `nms_neural_multiprocessor_sdram_pipelined.v`, plus a new + `tb_nms_dstress_sdram_pipelined.v` (same D-Stress workload/golden + model; backdoor peek/poke rewritten to go through a `sdram_model.v` + backdoor_read/write helper keyed on the SAME decomposition the new + controller uses, instead of the old flat-address assumption, so + bit-exact verification stays valid under the new addr->bank mapping + -- this was flagged in advance as the one correctness trap in this + whole exercise, and was handled by construction rather than by + parallel, error-prone reimplementation). +RESULT (INDEPENDENTLY RE-BUILT AND RE-RUN by this session directly, not +just taken from the sub-task's own report -- both PASS 256/256 bit- +exact + data_ready PASS in both configs): + N=4: total_cycles=49760 (vs single-bank baseline 49927, EXP-0049 -- + only -0.33%). N=8: total_cycles=49755 (vs baseline 49909 -- + -0.31%). Both essentially within noise of the unmodified single- + bank system, nowhere near either Phase A's own measured 2-cycle- + per-different-bank-pair saving scaled up, or EXP-0051's dual-bank + -8/-10%. +ROOT CAUSE of the gap between Phase A (works) and Phase B (doesn't +help): `slot_mem_arbiter_wide.v` -> `sdram_unified_backend.v`'s own W +port is a synchronous one-request-at-a-time interface -- the caller +waits for `w_ready` before ever asserting the next `w_req`. Phase A's +interleaving mechanism can ONLY help if a request for a DIFFERENT bank +is already pending WHILE the current transaction is still mid-flight +(CAS_WAIT/BURST/PRECHARGE) -- a condition the current arbiter/backend +call convention almost never creates, since nothing is ever dispatched +early. The mechanism itself is real and correctly verified in Phase A +(directly, artificially stimulated); the SYSTEM around it, as it exists +today, essentially never exercises it. +decision: do NOT integrate sdram_controller_pipelined.v into the + production path on this evidence -- the real, measured, system-level + gain (~0.3%) does not justify carrying a second, more complex + controller variant with its own (even if currently well-verified) + correctness surface. The isolated Phase A result remains genuinely + useful and is KEPT as an additive, uncommitted-to-production file: + it proves the mechanism works and quantifies its real ceiling (2 + cycles/pair, not more), which is exactly the number needed to decide + whether a FUTURE arbiter/backend rewrite (teaching the W port to + dispatch its NEXT request BEFORT the current one's `ready`, i.e. a + real pipelined/multi-outstanding-request interface, not just the + memory-side FSM) would be worth attempting -- that rewrite is a + materially larger, riskier change (touches the arbiter's own request/ + grant protocol, not just the memory-side FSM) and was explicitly kept + out of scope for this experiment. +next_action: report to the user; do not pursue the arbiter/backend + pipelined-dispatch rewrite without an explicit go-ahead, given its + larger scope and the modest (2 cycles/pair, capped) ceiling this + experiment just measured -- the slot-group weight-split ("aspettiamo" + item from EXP-0051) remains the other, still-open, ORTHOGONAL lever + (it does not depend on this pipelining work at all and would stack + with it if the arbiter rewrite is ever done). New files (additive + only, none touch the real board top or existing production RTL): + hardware/v2/nms/rtl/sdram_controller_pipelined.v, + hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v, + hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v, + hardware/v2/nms/sim/tb_sdram_controller_pipelined.v, + hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v. diff --git a/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v new file mode 100644 index 0000000..bb04745 --- /dev/null +++ b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v @@ -0,0 +1,137 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXPERIMENTAL fork of nms_neural_multiprocessor_sdram_unified.v -- +// the ONLY change is instantiating sdram_unified_backend_pipelined.v +// (bank-interleaved command pipelining) instead of sdram_unified_ +// backend.v. u_dataflow_core, u_arbiter, u_arbiter_wide are all +// byte-for-byte unchanged. See sdram_controller_pipelined.v's header +// for the mechanism and its own derived/measured ceiling, and +// hardware/v2/logs/experiments.log for why this fork exists. +// ============================================================ +module nms_neural_multiprocessor_sdram_pipelined #( + parameter DATA_WIDTH = 8, + parameter P_IN = 8, + parameter ACC_WIDTH = 32, + parameter ADDR_WIDTH = 26, + parameter N_SLOTS = 2, + parameter N_NODES = 16, + parameter MAX_DEPS = 4, + parameter QUEUE_DEPTH = 8, + parameter MAX_TILES = 16, + parameter PREFETCH_DISTANCE = 8, + parameter CLK_FREQ_MHZ = 80 +)( + input wire clk, + input wire rst, + + input wire reg_valid, + output wire reg_ready, + input wire [$clog2(N_NODES)-1:0] reg_node_id, + input wire [$clog2(MAX_DEPS+1)-1:0] reg_required, + input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids, + input wire [ADDR_WIDTH-1:0] reg_x_base, + input wire [ADDR_WIDTH-1:0] reg_w_base, + input wire [15:0] reg_n_tiles, + input wire [ADDR_WIDTH-1:0] reg_result_addr, + + output wire data_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [1:0] sdram_ba, + output wire [12:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + wire [N_SLOTS:0] slot_mem_req, slot_mem_wr; + wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr; + wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata; + wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n; + wire [N_SLOTS:0] slot_mem_ready; + + wire [N_SLOTS-1:0] wide_slot_mem_req; + wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr; + wire [64*N_SLOTS-1:0] wide_slot_mem_rdata; + wire [N_SLOTS-1:0] wide_slot_mem_ready; + + nms_dataflow_core_sdram #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE) + ) u_dataflow_core ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .data_ready(data_ready), + .slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr), + .slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n), + .slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready), + .wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr), + .wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready) + ); + + wire arb_m_req, arb_m_wr; + wire [ADDR_WIDTH-1:0] arb_m_addr; + wire [15:0] arb_m_wdata; + wire arb_m_lb_n, arb_m_ub_n; + wire [15:0] arb_m_rdata; + wire arb_m_ready; + + slot_mem_arbiter #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1) + ) u_arbiter ( + .clk(clk), .rst(rst), + .s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr), + .s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n), + .s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready), + .m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata), + .m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n), + .m_rdata(arb_m_rdata), .m_ready(arb_m_ready) + ); + + wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}}; + wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}}; + wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}}; + wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}}; + + wire wide_arb_m_req, wide_arb_m_wr; + wire [ADDR_WIDTH-1:0] wide_arb_m_addr; + wire [63:0] wide_arb_m_wdata; + wire wide_arb_m_lb_n, wide_arb_m_ub_n; + wire [63:0] wide_arb_m_rdata; + wire wide_arb_m_ready; + + slot_mem_arbiter_wide #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64) + ) u_arbiter_wide ( + .clk(clk), .rst(rst), + .s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr), + .s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n), + .s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready), + .m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata), + .m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n), + .m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready) + ); + + sdram_unified_backend_pipelined #( + .ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ) + ) u_sdram_backend ( + .clk(clk), .rst(rst), + .w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr), + .w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready), + .ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata), + .ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n), + .ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_controller_pipelined.v b/hardware/v2/nms/rtl/sdram_controller_pipelined.v new file mode 100644 index 0000000..ba0aeab --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_controller_pipelined.v @@ -0,0 +1,482 @@ +`timescale 1ns/1ps + +// ============================================================ +// NMS -- EXPERIMENTAL bank-interleaved-pipelining fork of +// sdram_controller.v, built to test whether the ~77-78% Bank-W +// busy-cycle ceiling EXP-0051 measured (see hardware/v2/logs/ +// experiments.log) can be reduced by overlapping the NEXT +// transaction's ACTIVATE/tRCD with the CURRENT transaction's own +// tail (CAS latency + burst + auto-precharge), when the two target +// DIFFERENT internal SDRAM banks. Real SDR SDRAM banks are +// electrically independent -- a real chip supports this exact kind +// of overlap (issuing ACTIVATE to bank B while bank A is still mid- +// burst/mid-precharge), it is simply never exploited by the +// original, deliberately linear, one-transaction-at-a-time +// sdram_controller.v (STEP16's own explicit, documented scope +// boundary: "exactly one physical SDRAM transaction in flight at a +// time"). +// +// TWO changes vs sdram_controller.v, both additive/isolated (every +// existing state/signal/behavior for the ORIGINAL usage pattern -- +// one req, wait for ready, THEN issue the next -- is byte-for-byte +// unchanged): +// +// (1) Address->bank decomposition moved from the TOP bits (original: +// addr_bank = addr[ADDR_WIDTH-1 -: BANK_BITS], meaning the +// project's own compact, single-region weight/activation memory +// map always lands on bank 0 -- confirmed by inspection, this is +// WHY no interleaving opportunity could ever exist under the +// original decomposition) to bits immediately ABOVE the fixed- +// zero burst-alignment low bits: addr_bank = addr[ALIGN_BITS +: +// BANK_BITS], where ALIGN_BITS = clog2(BURST_LEN). Since every +// real caller always increments the word address by exactly +// BURST_LEN between consecutive real transactions (see +// sdram_unified_backend.v's own w_eff_aligned_word_addr/ +// ar_eff_block_base computation), this makes CONSECUTIVE real +// transactions round-robin across all BANK_BITS**2 banks +// automatically, with zero change needed at any caller. This is +// a pure re-slicing of the SAME flat word-address bits into a +// DIFFERENT (bank,row,col) triple -- still a bijection over the +// full address space (each of the 2**ADDR_WIDTH addresses maps +// to exactly one (bank,row,col) and vice versa), so read-after- +// write correctness is unaffected; only the caller-visible flat +// address <-> physical-location mapping changes, which is why +// any INTEGRATION testbench built around this module must apply +// the SAME decomposition in its own backdoor peek/poke helpers +// (see tb_nms_dstress_sdram_pipelined.v's own header note) or +// use sdram_model.v's own explicit backdoor_read/backdoor_write +// tasks (bank/row/col-addressed, decomposition-agnostic) instead +// of computing a flat array index by hand. +// +// (2) A single-depth "shadow" pipeline slot (pipe_valid/pipe_*_reg/ +// pipe_wait_cnt): while the CURRENT transaction is in S_CAS_WAIT, +// S_BURST_READ, S_BURST_WRITE, or S_PRECHARGE_WAIT (i.e. its own +// ACTIVATE has already been sent and the command bus is +// otherwise idle -- confirmed by inspection: none of those four +// states drive sdram_ras_n/sdram_ba/sdram_a), a NEW `req` for a +// DIFFERENT bank than the current transaction's own req_bank_reg +// is captured into the shadow slot AND its ACTIVATE command is +// issued immediately (overlapping its own tRCD with whatever of +// the current transaction's tail remains), instead of going +// through the original req_pending latch (which would otherwise +// wait for a full return to S_IDLE before even starting the +// ACTIVATE). A `req` for the SAME bank as the current transaction +// -- or arriving in any state OTHER than those four, or arriving +// while the shadow slot is already occupied -- falls through to +// the ORIGINAL, unmodified req_pending path, so that case behaves +// EXACTLY as in sdram_controller.v (no regression, verified in +// tb_sdram_controller_pipelined.v's own "same-bank" test). +// +// REAL PROTOCOL CONSTRAINT respected: AUTO REFRESH requires EVERY +// bank precharged first (a real JEDEC rule sdram_model.v itself +// does NOT currently check/enforce -- confirmed by inspection, a +// real, disclosed gap in that model, not exploited here). This +// design avoids ever violating it by construction: S_IDLE's own +// priority order checks `pipe_valid` BEFORE `refresh_timer==0` -- +// a still-open shadow bank is always promoted/closed (via its own +// ordinary auto-precharge) before any refresh is allowed to fire, +// so refresh can only ever run when EVERY bank (primary transaction, +// always auto-precharged by construction -- A10=1 on every real +// command, unchanged from the original design -- and any shadow +// transaction) is already closed. The resulting refresh delay is +// bounded by one shadow transaction's own worst-case duration +// (~16 cycles at CLK_FREQ_MHZ=80/BURST_LEN=8), a small fraction of +// T_REFI (~625 cycles at the same frequency) -- verified, not +// assumed, by tb_sdram_controller_pipelined.v's own refresh- +// during-interleaving test. +// +// THEORETICAL CEILING (derived here, confirmed by measurement in +// tb_sdram_controller_pipelined.v -- disclosed up front so the result +// isn't oversold): only tRCD can ever be hidden by this scheme, since +// the shared DQ bus means the NEXT transaction's own CAS/burst can +// never start before the CURRENT transaction's burst fully drains, +// regardless of banking. At CLK_FREQ_MHZ=80/BURST_LEN=8, tRCD is only +// ~2 of a real transaction's ~16 total cycles (the dominant cost, +// CAS_LATENCY+BURST_LEN=11 cycles/69%, is serial DATA transfer that +// NO command-level bank interleaving can shorten) -- so the best case +// for a long chain of alternating-bank transactions is each one AFTER +// the first costing ~14 instead of ~16 cycles, an asymptotic ~12.5% +// per-transaction ceiling, not a multiple-x speedup. +// ============================================================ +module sdram_controller_pipelined #( + parameter CLK_FREQ_MHZ = 64, + parameter BURST_LEN = 4, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2, + parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS +)( + input wire clk, + input wire rst, + + input wire req, + input wire wr, + input wire [ADDR_WIDTH-1:0] addr, + input wire [16*BURST_LEN-1:0] wdata, + input wire [2*BURST_LEN-1:0] wmask, + output reg [16*BURST_LEN-1:0] rdata, + output reg ready, + output reg busy, + + output reg sdram_cke, + output reg sdram_cs_n, + output reg sdram_ras_n, + output reg sdram_cas_n, + output reg sdram_we_n, + output reg [1:0] sdram_ba, + output reg [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output reg [1:0] sdram_dqm +); + + localparam BURST_IDXW = (BURST_LEN <= 1) ? 1 : $clog2(BURST_LEN); + // unclamped log2 (0 for BURST_LEN=1), used ONLY for the bank-slice + // position -- see header note (1). + localparam ALIGN_BITS = $clog2(BURST_LEN); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS) begin + $display("FATAL sdram_controller_pipelined: ADDR_WIDTH=%0d != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)=%0d", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS, BANK_BITS+ROW_BITS+COL_BITS); + $finish; + end + initial if (ALIGN_BITS + BANK_BITS > COL_BITS) begin + $display("FATAL sdram_controller_pipelined: ALIGN_BITS(%0d)+BANK_BITS(%0d) > COL_BITS(%0d) -- bank slice does not fit below row field", + ALIGN_BITS, BANK_BITS, COL_BITS); + $finish; + end + + function integer ns_to_cycles; + input integer ns; + begin + ns_to_cycles = (ns * CLK_FREQ_MHZ + 999) / 1000; + end + endfunction + localparam T_RCD = ns_to_cycles(15); + localparam T_RP = ns_to_cycles(15); + localparam T_MRD = 2; + localparam T_INIT_US= 200; + localparam T_INIT = T_INIT_US * CLK_FREQ_MHZ; + localparam CAS_LATENCY = 3; + localparam T_REFI = ns_to_cycles(64000000 / (1 << ROW_BITS) + 1); + + localparam CNTW = $clog2((T_INIT>T_REFI ? T_INIT : T_REFI) + 1); + + localparam + S_INIT_WAIT = 5'd0, + S_INIT_PRE_WAIT = 5'd2, + S_INIT_REF = 5'd3, + S_INIT_REF_WAIT = 5'd4, + S_INIT_MRS_WAIT = 5'd6, + S_IDLE = 5'd7, + S_REFRESH_WAIT = 5'd9, + S_ACTIVATE_WAIT = 5'd11, + S_CAS_WAIT = 5'd13, + S_BURST_READ = 5'd14, + S_BURST_WRITE = 5'd15, + S_PRECHARGE_WAIT = 5'd16; + + reg [4:0] state; + reg [CNTW-1:0] wait_cnt; + reg [3:0] init_ref_cnt; + reg [CNTW-1:0] refresh_timer; + reg [BURST_IDXW-1:0] burst_idx; + reg req_wr_reg; + reg [BANK_BITS-1:0] req_bank_reg; + reg [ROW_BITS-1:0] req_row_reg; + reg [COL_BITS-1:0] req_col_reg; + reg [16*BURST_LEN-1:0] wdata_reg; + reg [2*BURST_LEN-1:0] wmask_reg; + + // ---- (1) re-sliced address decomposition -- see header note ---- + wire [BANK_BITS-1:0] addr_bank = addr[ALIGN_BITS +: BANK_BITS]; + wire [COL_BITS-1:0] addr_col = (ALIGN_BITS == 0) ? addr[ALIGN_BITS+BANK_BITS +: COL_BITS] + : {addr[ALIGN_BITS+BANK_BITS +: (COL_BITS-ALIGN_BITS)], addr[ALIGN_BITS-1:0]}; + wire [ROW_BITS-1:0] addr_row = addr[ADDR_WIDTH-1 -: ROW_BITS]; + + // ---- (2) shadow pipeline slot ---- + reg pipe_valid; + reg pipe_wr_reg; + reg [BANK_BITS-1:0] pipe_bank_reg; + reg [ROW_BITS-1:0] pipe_row_reg; + reg [COL_BITS-1:0] pipe_col_reg; + reg [16*BURST_LEN-1:0] pipe_wdata_reg; + reg [2*BURST_LEN-1:0] pipe_wmask_reg; + reg [CNTW-1:0] pipe_wait_cnt; + + wire shadow_capturable_state = (state==S_CAS_WAIT) || (state==S_BURST_READ) || + (state==S_BURST_WRITE) || (state==S_PRECHARGE_WAIT); + wire shadow_capture_now = req && shadow_capturable_state && !pipe_valid && + (addr_bank != req_bank_reg); + + reg req_pending; + wire eff_wr = req ? wr : req_wr_reg; + wire [BANK_BITS-1:0] eff_bank = req ? addr_bank : req_bank_reg; + wire [ROW_BITS-1:0] eff_row = req ? addr_row : req_row_reg; + wire [COL_BITS-1:0] eff_col = req ? addr_col : req_col_reg; + wire [16*BURST_LEN-1:0] eff_wdata = req ? wdata : wdata_reg; + wire [2*BURST_LEN-1:0] eff_wmask = req ? wmask : wmask_reg; + + reg dq_out_en; + reg [15:0] dq_out; + assign sdram_dq = dq_out_en ? dq_out : 16'hzzzz; + + function [ROW_BITS-1:0] mrs_value; + input integer burst_len; + reg [2:0] bl_code; + reg [ROW_BITS-1:0] v; + begin + bl_code = (burst_len==1) ? 3'b000 : + (burst_len==2) ? 3'b001 : + (burst_len==4) ? 3'b010 : + (burst_len==8) ? 3'b011 : 3'b111; + v = {ROW_BITS{1'b0}}; + v[6:4] = 3'b011; + v[3] = 1'b0; + v[2:0] = bl_code; + mrs_value = v; + end + endfunction + + function [CNTW-1:0] T_RC_MINUS1; + localparam integer T_RC = ns_to_cycles(65); + begin + T_RC_MINUS1 = T_RC[CNTW-1:0] - 1'b1; + end + endfunction + + always @(posedge clk) begin + if (rst) begin + state <= S_INIT_WAIT; + wait_cnt <= T_INIT[CNTW-1:0]; + init_ref_cnt <= 4'd0; + refresh_timer <= T_REFI[CNTW-1:0]; + sdram_cke <= 1'b1; + sdram_cs_n <= 1'b1; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + sdram_ba <= 2'b00; + sdram_a <= {ROW_BITS{1'b0}}; + sdram_dqm <= 2'b00; + dq_out_en <= 1'b0; + ready <= 1'b0; + busy <= 1'b1; + req_pending <= 1'b0; + pipe_valid <= 1'b0; + pipe_wait_cnt <= {CNTW{1'b0}}; + end else begin + sdram_cs_n <= 1'b0; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + ready <= 1'b0; + dq_out_en <= 1'b0; + sdram_dqm <= 2'b00; + + if (refresh_timer != 0) refresh_timer <= refresh_timer - 1'b1; + // shadow's own tRCD countdown runs independently of `state` + // (it tracks a DIFFERENT, already-open bank than whatever + // the primary FSM below is doing) -- see header note (2). + if (pipe_valid && pipe_wait_cnt != 0) pipe_wait_cnt <= pipe_wait_cnt - 1'b1; + + if (req) begin + if (shadow_capture_now) begin + // capture into the shadow slot INSTEAD OF the + // original req_pending latch (so wdata_reg/ + // wmask_reg/req_bank_reg etc, still owned by the + // CURRENTLY in-flight transaction, are never + // touched) -- and issue its real ACTIVATE command + // this very cycle (command bus is idle in every + // shadow_capturable_state, confirmed by inspection: + // none of those four states drive ras_n/ba/a). + pipe_wr_reg <= wr; + pipe_bank_reg <= addr_bank; + pipe_row_reg <= addr_row; + pipe_col_reg <= addr_col; + pipe_wdata_reg <= wdata; + pipe_wmask_reg <= wmask; + pipe_wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + pipe_valid <= 1'b1; + sdram_ras_n <= 1'b0; + sdram_ba <= addr_bank; + sdram_a <= addr_row; + end else begin + // ORIGINAL, unmodified path -- byte-for-byte + // identical to sdram_controller.v. + req_wr_reg <= wr; + req_bank_reg <= addr_bank; + req_row_reg <= addr_row; + req_col_reg <= addr_col; + wdata_reg <= wdata; + wmask_reg <= wmask; + req_pending <= 1'b1; + end + end + + case (state) + S_INIT_WAIT: begin + busy <= 1'b1; + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_a[10] <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_INIT_PRE_WAIT; + end + end + S_INIT_PRE_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + state <= S_INIT_REF; + end + end + S_INIT_REF: begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + state <= S_INIT_REF_WAIT; + end + S_INIT_REF_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else if (init_ref_cnt < 4'd7) begin + init_ref_cnt <= init_ref_cnt + 1'b1; + state <= S_INIT_REF; + end else begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_ba <= 2'b00; + sdram_a <= mrs_value(BURST_LEN); + wait_cnt <= T_MRD[CNTW-1:0] - 1'b1; + state <= S_INIT_MRS_WAIT; + end + end + S_INIT_MRS_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + busy <= 1'b0; + state <= S_IDLE; + end + end + + S_IDLE: begin + busy <= 1'b0; + // Priority: (1) a still-open SHADOW bank must be + // promoted/closed before anything else -- see + // header note (2) on why this ordering is the + // thing that keeps AUTO REFRESH from ever firing + // with an open row. (2) periodic refresh, exactly + // as sdram_controller.v. (3) the original req/ + // req_pending path, exactly as sdram_controller.v. + if (pipe_valid) begin + busy <= 1'b1; + req_wr_reg <= pipe_wr_reg; + req_bank_reg <= pipe_bank_reg; + req_row_reg <= pipe_row_reg; + req_col_reg <= pipe_col_reg; + wdata_reg <= pipe_wdata_reg; + wmask_reg <= pipe_wmask_reg; + wait_cnt <= pipe_wait_cnt; // remaining tRCD, may already be 0 + pipe_valid <= 1'b0; + state <= S_ACTIVATE_WAIT; + // NOTE: ACTIVATE for this bank was ALREADY + // issued at shadow-capture time -- do not + // re-issue it here (ras_n stays at its default + // NOP drive this cycle). + end else if (refresh_timer == 0) begin + busy <= 1'b1; + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + refresh_timer <= T_REFI[CNTW-1:0]; + state <= S_REFRESH_WAIT; + end else if (req || req_pending) begin + busy <= 1'b1; + req_wr_reg <= eff_wr; + req_bank_reg <= eff_bank; + req_row_reg <= eff_row; + req_col_reg <= eff_col; + wdata_reg <= eff_wdata; + wmask_reg <= eff_wmask; + req_pending <= 1'b0; + sdram_ras_n <= 1'b0; + sdram_ba <= eff_bank; + sdram_a <= eff_row; + wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + state <= S_ACTIVATE_WAIT; + end + end + + S_REFRESH_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; + end + + S_ACTIVATE_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + sdram_cas_n <= 1'b0; + sdram_we_n <= req_wr_reg ? 1'b0 : 1'b1; + sdram_ba <= req_bank_reg; + sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b1, {(10-COL_BITS){1'b0}}, req_col_reg}; + burst_idx <= {BURST_IDXW{1'b0}}; + if (req_wr_reg) begin + dq_out_en <= 1'b1; + dq_out <= wdata_reg[15:0]; + sdram_dqm <= wmask_reg[1:0]; + state <= S_BURST_WRITE; + end else begin + wait_cnt <= CAS_LATENCY[CNTW-1:0]; + state <= S_CAS_WAIT; + end + end + end + + S_CAS_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + rdata[0 +: 16] <= sdram_dq; + if (BURST_LEN == 1) begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRECHARGE_WAIT; + end else begin + burst_idx <= burst_idx + 1'b1; + state <= S_BURST_READ; + end + end + end + + S_BURST_READ: begin + rdata[burst_idx*16 +: 16] <= sdram_dq; + if (burst_idx == BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRECHARGE_WAIT; + end else begin + burst_idx <= burst_idx + 1'b1; + end + end + + S_BURST_WRITE: begin + if (burst_idx < BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + burst_idx <= burst_idx + 1'b1; + dq_out_en <= 1'b1; + dq_out <= wdata_reg[(burst_idx+1'b1)*16 +: 16]; + sdram_dqm <= wmask_reg[(burst_idx+1'b1)*2 +: 2]; + end else begin + ready <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] + 1'b1; + state <= S_PRECHARGE_WAIT; + end + end + + S_PRECHARGE_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; + end + + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v b/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v new file mode 100644 index 0000000..f820102 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_unified_backend_pipelined.v @@ -0,0 +1,255 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXPERIMENTAL fork of sdram_unified_backend.v -- the ONLY change is +// instantiating sdram_controller_pipelined.v instead of sdram_ +// controller.v. W-port cache, arbitration, and the W/AR top-level FSM +// are ALL byte-for-byte unchanged. See sdram_controller_pipelined.v's +// own header for what changed at the controller level and why, and +// hardware/v2/logs/experiments.log (search "pipelin") for why this +// fork exists: testing whether bank-interleaved command pipelining +// recovers any of the ~77-78% Bank-W busy ceiling EXP-0051 measured. +// ============================================================ +module sdram_unified_backend_pipelined #( + parameter ADDR_WIDTH = 26, + parameter CLK_FREQ_MHZ = 64, + parameter W_ENTRIES = 4, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2 +)( + input wire clk, + input wire rst, + + input wire w_req, + input wire [ADDR_WIDTH-1:0] w_addr, + output reg [63:0] w_rdata, + output reg w_ready, + + input wire ar_req, + input wire ar_wr, + input wire [ADDR_WIDTH-1:0] ar_addr, + input wire [15:0] ar_wdata, + input wire ar_lb_n, + input wire ar_ub_n, + output reg [15:0] ar_rdata, + output reg ar_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin + $display("FATAL sdram_unified_backend_pipelined: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS); + $finish; + end + + localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES); + reg w_cache_valid [0:W_ENTRIES-1]; + reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1]; + reg [63:0] w_cache_data [0:W_ENTRIES-1]; + reg [WEIDXW-1:0] w_alloc_ptr; + + wire [W_ENTRIES-1:0] w_match_oh; + genvar wgi; + generate + for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH + assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr); + end + endgenerate + + reg w_hit_found_c; + reg [WEIDXW-1:0] w_hit_idx_c; + integer ei; + generate + if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT + always @(*) begin + w_hit_found_c = |w_match_oh; + casez (w_match_oh) + 4'b1???: w_hit_idx_c = 2'd3; + 4'b01??: w_hit_idx_c = 2'd2; + 4'b001?: w_hit_idx_c = 2'd1; + 4'b0001: w_hit_idx_c = 2'd0; + default: w_hit_idx_c = {WEIDXW{1'b0}}; + endcase + end + end else begin : GEN_WHIT_FALLBACK + always @(*) begin + w_hit_found_c = 1'b0; + w_hit_idx_c = {WEIDXW{1'b0}}; + for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin + if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin + w_hit_found_c = 1'b1; + w_hit_idx_c = ei[WEIDXW-1:0]; + end + end + end + end + endgenerate + wire w_cache_hit = w_hit_found_c && w_req; + + reg ctrl_req; + reg ctrl_wr; + reg [ADDR_WIDTH-2:0] ctrl_addr; + reg [127:0] ctrl_wdata; + reg [15:0] ctrl_wmask; + wire [127:0] ctrl_rdata; + wire ctrl_ready; + wire ctrl_busy; + + sdram_controller_pipelined #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk(clk), .rst(rst), + .req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr), + .wdata(ctrl_wdata), .wmask(ctrl_wmask), + .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + localparam S_IDLE = 3'd0, + S_W_WAIT = 3'd1, + S_AR_RD_WAIT = 3'd2, + S_AR_WR_WAIT = 3'd3; + reg [2:0] state; + reg w_pending_upper_half; + reg [ADDR_WIDTH-1:0] w_pending_addr; + reg [2:0] ar_pending_word; + + reg w_req_pending; + reg [ADDR_WIDTH-1:0] w_req_addr_lat; + reg ar_req_pending; + reg ar_req_wr_lat; + reg [ADDR_WIDTH-1:0] ar_req_addr_lat; + reg [15:0] ar_req_wdata_lat; + reg ar_req_lbn_lat, ar_req_ubn_lat; + + wire w_eff_req = w_req || w_req_pending; + wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat; + wire ar_eff_req = ar_req || ar_req_pending; + wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat; + wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat; + wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat; + wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat; + wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat; + + wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000}; + wire w_eff_addr_is_upper_half = w_eff_addr[3]; + wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000}; + wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0]; + + integer ri; + always @(posedge clk) begin + if (rst) begin + state <= S_IDLE; + for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0; + w_alloc_ptr <= {WEIDXW{1'b0}}; + ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}}; + ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF; + w_ready <= 1'b0; w_rdata <= 64'h0; + ar_ready <= 1'b0; ar_rdata <= 16'h0; + w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}}; + ar_pending_word <= 3'h0; + w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}}; + ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0; + ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0; + ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1; + end else begin + ctrl_req <= 1'b0; + w_ready <= 1'b0; + ar_ready <= 1'b0; + + if (w_req) begin + w_req_addr_lat <= w_addr; + w_req_pending <= 1'b1; + end + if (ar_req) begin + ar_req_wr_lat <= ar_wr; + ar_req_addr_lat <= ar_addr; + ar_req_wdata_lat <= ar_wdata; + ar_req_lbn_lat <= ar_lb_n; + ar_req_ubn_lat <= ar_ub_n; + ar_req_pending <= 1'b1; + end + + case (state) + S_IDLE: begin + if (w_cache_hit) begin + w_rdata <= w_cache_data[w_hit_idx_c]; + w_ready <= 1'b1; + w_cache_valid[w_hit_idx_c] <= 1'b0; + w_req_pending <= 1'b0; + end else if (w_eff_req) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= w_eff_aligned_word_addr; + ctrl_wmask <= 16'h0000; + w_pending_upper_half <= w_eff_addr_is_upper_half; + w_pending_addr <= w_eff_addr; + w_req_pending <= 1'b0; + state <= S_W_WAIT; + end else if (ar_eff_req && !ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= ar_eff_block_base; + ctrl_wmask <= 16'h0000; + ar_pending_word <= ar_eff_word_in_blk; + ar_req_pending <= 1'b0; + state <= S_AR_RD_WAIT; + end else if (ar_eff_req && ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b1; + ctrl_addr <= ar_eff_block_base; + ctrl_wdata <= {8{ar_eff_wdata}}; + ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2)); + ar_req_pending <= 1'b0; + state <= S_AR_WR_WAIT; + end + end + S_W_WAIT: begin + if (ctrl_ready) begin + if (w_pending_upper_half) begin + w_rdata <= ctrl_rdata[127:64]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end else begin + w_rdata <= ctrl_rdata[63:0]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end + w_cache_valid[w_alloc_ptr] <= 1'b1; + w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1; + w_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_RD_WAIT: begin + if (ctrl_ready) begin + ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16]; + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_WR_WAIT: begin + if (ctrl_ready) begin + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v b/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v new file mode 100644 index 0000000..e36a8ca --- /dev/null +++ b/hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v @@ -0,0 +1,562 @@ +`timescale 1ns/1ps + +// ================================================================ +// EXPERIMENTAL fork of tb_nms_dstress_sdram_unified.v -- instantiates +// nms_neural_multiprocessor_sdram_pipelined.v (single physical SDRAM +// chip, but with sdram_controller_pipelined.v's bank-interleaved +// command pipelining inside) instead of nms_neural_multiprocessor_ +// sdram_unified.v. Identical D-Stress workload/golden-model/bit-exact +// verification. +// +// ONLY functional difference vs the original testbench: poke_byte/ +// peek_byte/poke_byte_weight/peek_byte_weight no longer compute a +// flat `u_sdram.mem[word_addr]` index by hand (that shortcut relied +// on the ORIGINAL controller's bank-from-TOP-bits decomposition, +// where ROWS/COLS being powers of 2 makes the flat word address +// numerically identical to bank*ROWS*COLS+row*COLS+col). The +// pipelined controller re-slices which address bits mean bank/row/ +// col (see sdram_controller_pipelined.v's own header, note (1)), so +// these backdoor helpers instead: (a) decompose the flat word address +// using the EXACT SAME bit ranges as sdram_controller_pipelined.v's +// own addr_bank/addr_row/addr_col wires, then (b) call sdram_model.v's +// own explicit, decomposition-agnostic backdoor_read/backdoor_write +// tasks (bank/row/col-addressed) instead of indexing `mem[]` directly +// -- this guarantees the testbench and the RTL agree on where a given +// byte physically lives, by construction, rather than by two +// independently-maintained flat-index formulas that could silently +// drift apart. +// ================================================================ +module tb #( + parameter N_SLOTS_CFG = 2, + parameter PFD_CFG = 8 +); + + localparam ADDR_WIDTH = 26; + localparam DATA_WIDTH = 8; + localparam P_IN = 8; + localparam ACC_WIDTH = 32; + localparam N_NODES = 1024; + localparam MAX_DEPS = 8; + localparam QUEUE_DEPTH = 8; + localparam NODE_IDW = $clog2(N_NODES); + localparam CLK_PERIOD = 12.5; // 80 MHz + + // must match sdram_controller_pipelined.v's own instantiation + // parameters exactly (BURST_LEN=8 hardcoded by sdram_unified_ + // backend_pipelined.v, ROW_BITS/COL_BITS/BANK_BITS defaults) + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ALIGN_BITS = 3; // clog2(BURST_LEN=8) + + reg clk, rst; + initial begin clk = 1'b0; forever #(CLK_PERIOD/2.0) clk = ~clk; end + + reg reg_valid; + wire reg_ready; + reg [NODE_IDW-1:0] reg_node_id; + reg [$clog2(MAX_DEPS+1)-1:0] reg_required; + reg [MAX_DEPS*NODE_IDW-1:0] reg_producer_ids; + reg [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr; + reg [15:0] reg_n_tiles; + + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [1:0] sdram_ba; + wire [12:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + nms_neural_multiprocessor_sdram_pipelined #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS_CFG), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(16), .PREFETCH_DISTANCE(PFD_CFG), .CLK_FREQ_MHZ(80) + ) u_nmp ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + sdram_model #(.CLK_FREQ_MHZ(80)) u_sdram ( + .clk(clk), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + // ---- backdoor helpers: decompose a flat 25-bit word address into + // (bank,row,col) using sdram_controller_pipelined.v's own bit + // ranges, then use sdram_model.v's own bank/row/col-addressed + // backdoor tasks -- see header note above ---- + function automatic [BANK_BITS-1:0] wa_bank(input [24:0] wa); + wa_bank = wa[ALIGN_BITS +: BANK_BITS]; + endfunction + function automatic [COL_BITS-1:0] wa_col(input [24:0] wa); + wa_col = {wa[ALIGN_BITS+BANK_BITS +: (COL_BITS-ALIGN_BITS)], wa[ALIGN_BITS-1:0]}; + endfunction + function automatic [ROW_BITS-1:0] wa_row(input [24:0] wa); + wa_row = wa[24 -: ROW_BITS]; + endfunction + + task automatic poke_byte(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + if (byte_addr[0] == 1'b0) cur[7:0] = val; else cur[15:8] = val; + u_sdram.backdoor_write(wa_bank(wa), wa_row(wa), wa_col(wa), cur); + end + endtask + + function automatic signed [7:0] peek_byte(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + peek_byte = (byte_addr[0] == 1'b0) ? cur[7:0] : cur[15:8]; + end + endfunction + + task automatic poke_byte_weight(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + if (byte_addr[0] == 1'b0) cur[7:0] = val; else cur[15:8] = val; + u_sdram.backdoor_write(wa_bank(wa), wa_row(wa), wa_col(wa), cur); + end + endtask + + function automatic signed [7:0] peek_byte_weight(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] wa; + reg [15:0] cur; + begin + wa = byte_addr[ADDR_WIDTH-1:1]; + cur = u_sdram.backdoor_read(wa_bank(wa), wa_row(wa), wa_col(wa)); + peek_byte_weight = (byte_addr[0] == 1'b0) ? cur[7:0] : cur[15:8]; + end + endfunction + + function automatic signed [7:0] relu_sat(input integer acc); + begin + if (acc <= 0) relu_sat = 8'sd0; + else if (acc > 127) relu_sat = 8'sd127; + else relu_sat = acc[7:0]; + end + endfunction + + task automatic register_node( + input [NODE_IDW-1:0] nid, + input [$clog2(MAX_DEPS+1)-1:0] required, + input [MAX_DEPS*NODE_IDW-1:0] producer_ids_packed, + input [ADDR_WIDTH-1:0] xb, input [ADDR_WIDTH-1:0] wb, + input [15:0] nt, input [ADDR_WIDTH-1:0] resaddr + ); + begin + @(posedge clk); + reg_node_id = nid; + reg_required = required; + reg_producer_ids = producer_ids_packed; + reg_x_base = xb; reg_w_base = wb; reg_n_tiles = nt; reg_result_addr = resaddr; + reg_valid = 1'b1; + while (!reg_ready) @(posedge clk); + @(posedge clk); + reg_valid = 1'b0; + end + endtask + + reg measure_en; + integer total_cycles; + integer psram_busy_cycles; + integer ni; + genvar gi; + + reg [N_SLOTS_CFG-1:0] slot_busy_bit; + reg [N_SLOTS_CFG-1:0] slot_tile_bit; + integer slot_busy_cycles [0:N_SLOTS_CFG-1]; + integer slot_tiles_delivered [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_MON + always @(*) begin + slot_busy_bit[gi] = (u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.state != 3'd0); + slot_tile_bit[gi] = u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_valid && + u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_ready; + end + end + endgenerate + + integer active_count; + integer active_hist [0:4]; + integer useful_mac_cycles; + integer first_tile_cyc; + integer last_tile_cyc; + integer any_tile_bit; + + integer sdram_req_count, sdram_ready_count, sdram_wr_count; + integer sdram_busy_cycles, sdram_refresh_count; + integer sdram_req_start_cyc, sdram_lat_sum, sdram_lat_min, sdram_lat_max, sdram_lat_n; + reg sdram_prev_state_is_refwait; + + initial begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + sdram_prev_state_is_refwait=1'b0; + end + + always @(posedge clk) begin + if (measure_en) begin + active_count = slot_busy_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) active_count = active_count + slot_busy_bit[ni]; + active_hist[active_count] <= active_hist[active_count] + 1; + + any_tile_bit = slot_tile_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) any_tile_bit = any_tile_bit | slot_tile_bit[ni]; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) + if (slot_tile_bit[ni]) useful_mac_cycles <= useful_mac_cycles + 1; + if (any_tile_bit) begin + if (first_tile_cyc < 0) first_tile_cyc <= total_cycles; + last_tile_cyc <= total_cycles; + end + + if (u_nmp.u_sdram_backend.u_sdram_ctrl.req) begin + sdram_req_count <= sdram_req_count + 1; + sdram_req_start_cyc <= total_cycles; + if (u_nmp.u_sdram_backend.u_sdram_ctrl.wr) sdram_wr_count <= sdram_wr_count + 1; + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.ready) begin + sdram_ready_count <= sdram_ready_count + 1; + sdram_lat_sum <= sdram_lat_sum + (total_cycles - sdram_req_start_cyc); + sdram_lat_n <= sdram_lat_n + 1; + if ((total_cycles - sdram_req_start_cyc) < sdram_lat_min) sdram_lat_min <= (total_cycles - sdram_req_start_cyc); + if ((total_cycles - sdram_req_start_cyc) > sdram_lat_max) sdram_lat_max <= (total_cycles - sdram_req_start_cyc); + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.busy) sdram_busy_cycles <= sdram_busy_cycles + 1; + sdram_prev_state_is_refwait <= (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9); + if (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9 && !sdram_prev_state_is_refwait) + sdram_refresh_count <= sdram_refresh_count + 1; + end + end + + task automatic report_step17_instrumentation; + real active_pct [0:4]; + real util_pct, startup_cycles, drain_cycles; + real sdram_avg_lat, sdram_busy_pct, sdram_bytes_per_cycle; + integer kk, total_tiles_all; + begin + total_tiles_all = 0; + for (kk = 0; kk < N_SLOTS_CFG; kk = kk + 1) total_tiles_all = total_tiles_all + slot_tiles_delivered[kk]; + $display(" ---- cycle decomposition ----"); + for (kk = 0; kk <= N_SLOTS_CFG; kk = kk + 1) begin + active_pct[kk] = (total_cycles > 0) ? (100.0*active_hist[kk]/total_cycles) : 0.0; + $display(" active_slots=%0d: %0d cycles (%0.2f%%)", kk, active_hist[kk], active_pct[kk]); + end + util_pct = (total_cycles > 0) ? (100.0*useful_mac_cycles/(total_cycles*1.0*N_SLOTS_CFG)) : 0.0; + $display(" useful_mac_cycles (slot-tile-delivery events, summed)=%0d (%0.2f%% of total_cycles*N_SLOTS)", useful_mac_cycles, util_pct); + startup_cycles = (first_tile_cyc >= 0) ? (1.0*first_tile_cyc) : 0.0; + drain_cycles = (last_tile_cyc >= 0) ? (1.0*(total_cycles - last_tile_cyc)) : 0.0; + $display(" startup (cycles before first tile delivered anywhere)=%0.0f", startup_cycles); + $display(" drain (cycles after last tile delivered, until job completion)=%0.0f", drain_cycles); + $display(" ---- SDRAM (pipelined controller) effectiveness ----"); + sdram_avg_lat = (sdram_lat_n > 0) ? (1.0*sdram_lat_sum/sdram_lat_n) : 0.0; + sdram_busy_pct = (total_cycles > 0) ? (100.0*sdram_busy_cycles/total_cycles) : 0.0; + sdram_bytes_per_cycle = (total_cycles > 0) ? (8.0*sdram_ready_count/total_cycles) : 0.0; + $display(" sdram_req_count=%0d sdram_ready_count=%0d sdram_wr_count=%0d", + sdram_req_count, sdram_ready_count, sdram_wr_count); + $display(" sdram_busy_cycles=%0d/%0d (%0.2f%%)", sdram_busy_cycles, total_cycles, sdram_busy_pct); + $display(" sdram_refresh_count=%0d", sdram_refresh_count); + $display(" sdram_request_latency: min=%0d max=%0d avg=%0.2f cycles", + sdram_lat_min, sdram_lat_max, sdram_avg_lat); + $display(" sdram_avg_bytes_per_cycle=%0.4f", sdram_bytes_per_cycle); + end + endtask + + reg [N_SLOTS_CFG-1:0] slot_could_present_act; + reg [N_SLOTS_CFG-1:0] slot_weight_blocking; + reg [N_SLOTS_CFG-1:0] slot_stalled_this_tile; + reg [31:0] prev_tile_idx [0:N_SLOTS_CFG-1]; + integer weight_stall_cycles [0:N_SLOTS_CFG-1]; + integer tiles_prefetched_clean [0:N_SLOTS_CFG-1]; + integer tiles_consumed_total [0:N_SLOTS_CFG-1]; + wire [31:0] slot_tile_idx_w [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_PF_MON + assign slot_tile_idx_w[gi] = {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx}; + always @(*) begin + slot_could_present_act[gi] = + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.n_tiles_reg}) && + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.usable_act}); + slot_weight_blocking[gi] = + slot_could_present_act[gi] && + !(u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx < + u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.wgt_ready_count) && + !u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.operand_valid; + end + end + endgenerate + + always @(posedge clk) begin + if (measure_en) begin + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (prev_tile_idx[ni] != slot_tile_idx_w[ni]) begin + slot_stalled_this_tile[ni] <= 1'b0; + prev_tile_idx[ni] <= slot_tile_idx_w[ni]; + end else if (slot_weight_blocking[ni]) begin + slot_stalled_this_tile[ni] <= 1'b1; + weight_stall_cycles[ni] <= weight_stall_cycles[ni] + 1; + end + if (slot_tile_bit[ni]) begin + tiles_consumed_total[ni] <= tiles_consumed_total[ni] + 1; + if (!slot_stalled_this_tile[ni]) + tiles_prefetched_clean[ni] <= tiles_prefetched_clean[ni] + 1; + end + end + end + end + + integer jobs_allocated, jobs_completed, wakeups; + integer waiting_sum, ready_sum, dispatched_sum, sample_count; + reg sample_occupancy; + integer scan_i; + integer waiting_now, ready_now, dispatched_now; + + always @(posedge clk) begin + if (measure_en) begin + total_cycles <= total_cycles + 1; + if (u_nmp.u_arbiter.owner != 0) psram_busy_cycles <= psram_busy_cycles + 1; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (slot_busy_bit[ni]) slot_busy_cycles[ni] <= slot_busy_cycles[ni] + 1; + if (slot_tile_bit[ni]) slot_tiles_delivered[ni] <= slot_tiles_delivered[ni] + 1; + end + if (u_nmp.u_dataflow_core.dm_ready_valid && u_nmp.u_dataflow_core.dm_ready_ready) + jobs_allocated <= jobs_allocated + 1; + if (u_nmp.u_dataflow_core.dir_job_out_done) + jobs_completed <= jobs_completed + 1; + if (u_nmp.u_dataflow_core.dm_producer_done_valid) + wakeups <= wakeups + 1; + + if (sample_occupancy) begin + waiting_now = 0; ready_now = 0; dispatched_now = 0; + for (scan_i = 0; scan_i < N_NODES; scan_i = scan_i + 1) begin + case (u_nmp.u_dataflow_core.u_dep_mgr.node_state[scan_i]) + 2'd1: waiting_now = waiting_now + 1; + 2'd2: ready_now = ready_now + 1; + 2'd3: dispatched_now = dispatched_now + 1; + default: ; + endcase + end + waiting_sum <= waiting_sum + waiting_now; + ready_sum <= ready_sum + ready_now; + dispatched_sum <= dispatched_sum + dispatched_now; + sample_count <= sample_count + 1; + end + end + end + + task automatic reset_instrumentation(input do_sample_occupancy); + integer k; + begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + total_cycles = 0; psram_busy_cycles = 0; + jobs_allocated = 0; jobs_completed = 0; wakeups = 0; + waiting_sum = 0; ready_sum = 0; dispatched_sum = 0; sample_count = 0; + sample_occupancy = do_sample_occupancy; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + slot_busy_cycles[k] = 0; + slot_tiles_delivered[k] = 0; + weight_stall_cycles[k] = 0; + tiles_prefetched_clean[k] = 0; + tiles_consumed_total[k] = 0; + slot_stalled_this_tile[k] = 1'b0; + prev_tile_idx[k] = 32'hFFFFFFFF; + end + end + endtask + + task automatic report_instrumentation(input [255:0] label, input integer n_neurons_completed); + integer k, total_tiles; + integer total_weight_stall_cycles, total_tiles_consumed_all, total_tiles_prefetched_clean; + real avg_waiting, avg_ready, avg_dispatched; + real psram_util, sustained_mac_per_cycle, wallclock_us; + real processor_utilization, weight_stall_pct, prefetch_effectiveness_pct; + begin + total_tiles = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) total_tiles = total_tiles + slot_tiles_delivered[k]; + avg_waiting = (sample_count > 0) ? (1.0*waiting_sum/sample_count) : 0.0; + avg_ready = (sample_count > 0) ? (1.0*ready_sum/sample_count) : 0.0; + avg_dispatched = (sample_count > 0) ? (1.0*dispatched_sum/sample_count) : 0.0; + psram_util = (total_cycles > 0) ? (100.0*psram_busy_cycles/total_cycles) : 0.0; + sustained_mac_per_cycle = (total_cycles > 0) ? (1.0*total_tiles*P_IN/total_cycles) : 0.0; + wallclock_us = total_cycles * CLK_PERIOD / 1000.0; + $display("---- BENCHMARK REPORT: %0s ----", label); + $display(" total_cycles=%0d wallclock_us=%0.3f", total_cycles, wallclock_us); + $display(" neurons_completed=%0d tiles_delivered(real)=%0d", n_neurons_completed, total_tiles); + $display(" jobs_allocated=%0d jobs_completed=%0d dependency_wakeups=%0d", jobs_allocated, jobs_completed, wakeups); + $display(" shared AR (activation+result) arbiter-side utilization: %0.1f%% (%0d/%0d busy cycles)", psram_util, psram_busy_cycles, total_cycles); + for (k = 0; k < N_SLOTS_CFG; k = k + 1) + $display(" slot %0d: busy=%0d/%0d (%0.1f%%) tiles=%0d", k, slot_busy_cycles[k], total_cycles, + (total_cycles>0)?(100.0*slot_busy_cycles[k]/total_cycles):0.0, slot_tiles_delivered[k]); + if (sample_count > 0) + $display(" dependency_manager avg occupancy: waiting=%0.2f ready=%0.2f dispatched=%0.2f", avg_waiting, avg_ready, avg_dispatched); + $display(" DERIVED: sustained end-to-end MAC/cycle = %0.4f", sustained_mac_per_cycle); + if (n_neurons_completed > 0) + $display(" DERIVED: cycles/neuron = %0.2f", 1.0*total_cycles/n_neurons_completed); + if (total_tiles > 0) + $display(" DERIVED: cycles/tile = %0.2f", 1.0*total_cycles/total_tiles); + + total_weight_stall_cycles = 0; total_tiles_consumed_all = 0; total_tiles_prefetched_clean = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + total_weight_stall_cycles = total_weight_stall_cycles + weight_stall_cycles[k]; + total_tiles_consumed_all = total_tiles_consumed_all + tiles_consumed_total[k]; + total_tiles_prefetched_clean = total_tiles_prefetched_clean + tiles_prefetched_clean[k]; + end + weight_stall_pct = (total_cycles > 0) ? (100.0*total_weight_stall_cycles/(total_cycles*N_SLOTS_CFG*1.0)) : 0.0; + prefetch_effectiveness_pct = (total_tiles_consumed_all > 0) ? + (100.0*total_tiles_prefetched_clean/(total_tiles_consumed_all*1.0)) : 0.0; + $display(" [STEP11] PFD=%0d weight_stall_cycles(sum,all slots)=%0d (%0.2f%%)", + PFD_CFG, total_weight_stall_cycles, weight_stall_pct); + $display(" [STEP11] DERIVED: prefetch_effectiveness = %0.2f%%", prefetch_effectiveness_pct); + end + endtask + + integer errors, tests; + + task automatic run_dense_layer( + input [255:0] label, + input integer n_neurons, + input integer n_tiles_count, + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, + input [ADDR_WIDTH-1:0] w_base, + input [ADDR_WIDTH-1:0] res_base, + input sample_occ + ); + integer n, t, k, len, acc; + reg signed [7:0] xv, wv, golden, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps; + integer completed, wd2; + begin + len = n_tiles_count * P_IN; + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + + for (k = 0; k < len; k = k + 1) + poke_byte(x_base + k, ((k % 8) + 1)); + + reset_instrumentation(sample_occ); + measure_en = 1'b1; + + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) begin + for (k = 0; k < P_IN; k = k + 1) begin + xv = peek_byte(x_base + t*P_IN + k); + wv = (((n + t*P_IN + k) % 8) + 1); + poke_byte_weight(w_base + n*len + t*P_IN + k, wv); + acc = acc + xv*wv; + end + end + golden = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + x_base, w_base + n*len, n_tiles_count[15:0], res_base + n); + if ((n % 32) == 0) begin + $display(" [%0s] registered %0d/%0d", label, n+1, n_neurons); + $fflush; + end + end + $display(" [%0s] all %0d neurons registered, waiting for completion...", label, n_neurons); + $fflush; + + completed = 0; wd2 = 0; + while (completed < n_neurons && wd2 < 2000000) begin + @(posedge clk); + wd2 = wd2 + 1; + completed = jobs_completed; + if ((wd2 % 20000) == 0) begin + $display(" [%0s] watchdog %0d: completed=%0d/%0d total_cycles=%0d", label, wd2, completed, n_neurons, total_cycles); + $fflush; + end + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + if (completed < n_neurons) begin + $display("FAIL %0s: only %0d/%0d neurons completed within watchdog", label, completed, n_neurons); + errors = errors + 1; + end else begin : check_block + integer local_errors; + local_errors = 0; + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) + for (k = 0; k < P_IN; k = k + 1) + acc = acc + peek_byte(x_base + t*P_IN + k) * peek_byte_weight(w_base + n*len + t*P_IN + k); + golden = relu_sat(acc); + real_y = peek_byte(res_base + n); + if (real_y !== golden) begin + $display("FAIL %0s neuron %0d: real=%0d golden=%0d", label, n, real_y, golden); + local_errors = local_errors + 1; + end + end + if (local_errors == 0) + $display("PASS %0s: all %0d neurons bit-exact vs golden", label, n_neurons); + else + errors = errors + 1; + end + report_instrumentation(label, n_neurons); + report_step17_instrumentation; + end + endtask + + initial begin + errors = 0; tests = 0; + rst = 1; reg_valid = 0; reg_node_id = 0; reg_required = 0; reg_producer_ids = 0; + reg_x_base = 0; reg_w_base = 0; reg_n_tiles = 0; reg_result_addr = 0; + measure_en = 0; + repeat(5) @(posedge clk); + rst = 0; + + $display("========================================"); + $display("NMS D-Stress benchmark (EXPERIMENTAL PIPELINED SDRAM controller) -- N_SLOTS_CFG=%0d PFD_CFG=%0d", N_SLOTS_CFG, PFD_CFG); + $display("========================================"); + + wait (u_nmp.u_sdram_backend.u_sdram_ctrl.state == u_nmp.u_sdram_backend.u_sdram_ctrl.S_IDLE); + @(posedge clk); + + run_dense_layer("D-Stress", 256, 16, 16'd400, 26'h200000, 26'h010000, 26'h300000, 1'b0); + + repeat (4) @(posedge clk); + if (u_nmp.data_ready !== 1'b1) begin + $display("FAIL data_ready: expected 1 after graph completion, got %b", u_nmp.data_ready); + errors = errors + 1; + end else begin + $display("PASS data_ready: correctly asserted after graph completion"); + end + + $display("========================================"); + if (errors == 0) + $display("ALL %0d WORKLOAD SUITES PASSED (N_SLOTS_CFG=%0d, PFD_CFG=%0d, PIPELINED SDRAM)", tests, N_SLOTS_CFG, PFD_CFG); + else + $display("FAILED: %0d/%0d workload suite(s) had errors -- see messages above", errors, tests); + $display("========================================"); + $finish; + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v b/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v new file mode 100644 index 0000000..5d282fa --- /dev/null +++ b/hardware/v2/nms/sim/tb_sdram_controller_pipelined.v @@ -0,0 +1,273 @@ +`timescale 1ns/1ps + +// ============================================================ +// Isolated correctness + cycle-savings regression for +// sdram_controller_pipelined.v, forked from tb_sdram_controller.v's +// own idiom (same do_transaction task style, same sdram_model.v DUT +// pairing). Adds what the original testbench cannot exercise (it +// always waits for `busy` to clear before issuing the next request): +// deliberately pulsing a SECOND req WHILE the controller is still +// mid-transaction, to test the new shadow-pipeline slot. +// +// Covers: +// 1) same correctness battery as the original (write->read, +// sequential, all 4 banks, address limits, pseudo-random) -- +// using the ORIGINAL wait-for-ready protocol throughout, so this +// also proves the re-sliced address decomposition (header note +// (1) in sdram_controller_pipelined.v) is a correct bijection. +// 2) DIFFERENT-bank early injection: issue a second request for a +// different bank while the first is still in S_CAS_WAIT, verify +// both results bit-exact AND that the combined cycle count is +// LOWER than 2x the serial baseline. +// 3) SAME-bank consecutive (both via the normal wait-for-ready +// protocol): must cost exactly the same as the original +// controller, no regression. +// 4) refresh spanning an early-injected interleave: run enough +// interleaved pairs to cross >=1 real tREFI interval, watch for +// any "VIOLATION"/"WARNING" from sdram_model.v. +// ============================================================ +module tb #( + parameter BURST_LEN = 8, + parameter CLK_FREQ_MHZ = 80 +); + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS; + localparam CLK_PERIOD_NS = 1000.0/CLK_FREQ_MHZ; + localparam ALIGN_BITS = (BURST_LEN<=1) ? 0 : $clog2(BURST_LEN); + + reg clk = 0; + always #(CLK_PERIOD_NS/2.0) clk = ~clk; + reg rst; + + reg req, wr; + reg [ADDR_WIDTH-1:0] addr; + reg [16*BURST_LEN-1:0] wdata; + reg [2*BURST_LEN-1:0] wmask; + wire [16*BURST_LEN-1:0] rdata; + wire ready, busy; + + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [BANK_BITS-1:0] sdram_ba; + wire [ROW_BITS-1:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + sdram_controller_pipelined #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut ( + .clk(clk), .rst(rst), + .req(req), .wr(wr), .addr(addr), .wdata(wdata), .wmask(wmask), .rdata(rdata), .ready(ready), .busy(busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + sdram_model #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem ( + .clk(clk), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + integer errors, tests; + integer cyc; + always @(posedge clk) if (!rst) cyc <= cyc + 1; + + // ---- helper: which bank a given flat word address maps to under + // the PIPELINED decomposition (must match sdram_controller_ + // pipelined.v's own addr_bank wire exactly) ---- + function automatic [BANK_BITS-1:0] bank_of; + input [ADDR_WIDTH-1:0] a; + begin + bank_of = a[ALIGN_BITS +: BANK_BITS]; + end + endfunction + + task automatic do_transaction( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk); + while (busy) @(posedge clk); + t0 = cyc; + req = 1'b1; wr = t_wr; addr = t_addr; wdata = t_wdata; + @(posedge clk); + req = 1'b0; + while (!ready) @(posedge clk); + t_rdata = rdata; + t_cycles = cyc - t0; + end + endtask + + reg [16*BURST_LEN-1:0] got, wpat; + integer elapsed; + + task automatic check_word(input [ADDR_WIDTH-1:0] a, input [15:0] pattern); + integer k; + begin + for (k = 0; k < BURST_LEN; k = k + 1) + wpat[k*16 +: 16] = pattern + k[15:0]; + do_transaction(1'b1, a, wpat, got, elapsed); + do_transaction(1'b0, a, {(16*BURST_LEN){1'b0}}, got, elapsed); + tests = tests + 1; + if (got !== wpat) begin + $display("FAIL addr=%0d bank=%0d: got=%h expected=%h", a, bank_of(a), got, wpat); + errors = errors + 1; + end else begin + $display("PASS addr=%0d bank=%0d: burst=%0d bit-exact, cycles=%0d", a, bank_of(a), BURST_LEN, elapsed); + end + end + endtask + + // issue a request THIS cycle without waiting for busy/ready -- + // the caller is responsible for knowing this is safe (shadow slot + // free, or accepting fallback-to-req_pending semantics otherwise) + task automatic issue_req_now(input t_wr, input [ADDR_WIDTH-1:0] t_addr, input [16*BURST_LEN-1:0] t_wdata); + begin + @(posedge clk); + req = 1'b1; wr = t_wr; addr = t_addr; wdata = t_wdata; + @(posedge clk); + req = 1'b0; + end + endtask + + task automatic wait_ready(output [16*BURST_LEN-1:0] t_rdata, output integer t_cyc_at_ready); + begin + // always advance at least one cycle first -- otherwise two + // back-to-back calls can both observe the SAME still-high + // `ready` pulse from the previous call's own exit cycle + // (a single-cycle-wide pulse level-checked with no + // intervening clock edge looks identical to a fresh one). + @(posedge clk); + while (!ready) @(posedge clk); + t_rdata = rdata; + t_cyc_at_ready = cyc; + end + endtask + + integer seed; + integer i; + reg [ADDR_WIDTH-1:0] rnd_addr; + + initial begin + errors = 0; tests = 0; cyc = 0; seed = 32'hC0FFEE; + rst = 1; req = 0; wr = 0; addr = 0; wdata = 0; wmask = 0; + repeat(5) @(posedge clk); + rst = 0; + while (busy) @(posedge clk); + + $display("=== TEST 1: correctness battery (original wait-for-ready protocol) ==="); + check_word({ADDR_WIDTH{1'b0}}, 16'hA5A5); + for (i = 0; i < 8; i = i + 1) + check_word(i*BURST_LEN, 16'h1000 + i); + // all 4 banks (bank now comes from LOW bits above the burst + // alignment -- addr values chosen so bank_of() sweeps 0..3) + for (i = 0; i < 4; i = i + 1) + check_word((i << ALIGN_BITS) + (100 << (ALIGN_BITS+BANK_BITS)), 16'h2000 + i); + // pseudo-random + for (i = 0; i < 24; i = i + 1) begin + rnd_addr = ($random(seed) % ((1< B-ready = %0d (serial baseline for 2 back-to-back BURST_LEN=%0d transactions is ~%0d; savings expected ~tRCD per pipelined pair, NOT a multiple-x speedup -- see sdram_controller_pipelined.v header)", + cyc_ready1 - t0, BURST_LEN, 2*(1+2+(BURST_LEN==1?0:3+1)+ (BURST_LEN>1?BURST_LEN-1:0) +2)); + end + end + + $display("=== TEST 4: refresh spanning interleaved traffic (watch for VIOLATION/WARNING above) ==="); + begin : test4 + integer t0b, cyc_r0, cyc_r1, j; + reg [ADDR_WIDTH-1:0] ba0, ba1; + reg [16*BURST_LEN-1:0] rr0, rr1; + for (j = 0; j < 60; j = j + 1) begin + ba0 = ((j*3) << ALIGN_BITS); + ba1 = ((j*3+1) << ALIGN_BITS); + if (bank_of(ba0) == bank_of(ba1)) ba1 = ba1 + (1 << ALIGN_BITS); + do_transaction(1'b1, ba0, {(16*BURST_LEN){16'hAA55}}, got, elapsed); + do_transaction(1'b1, ba1, {(16*BURST_LEN){16'h55AA}}, got, elapsed); + @(posedge clk); + while (busy) @(posedge clk); + t0b = cyc; + issue_req_now(1'b0, ba0, {(16*BURST_LEN){1'b0}}); + while (dut.state !== 13 && dut.state !== 7) @(posedge clk); // S_CAS_WAIT or back to S_IDLE (refresh could have won) + if (dut.state === 13 && !dut.pipe_valid) + issue_req_now(1'b0, ba1, {(16*BURST_LEN){1'b0}}); + wait_ready(rr0, cyc_r0); + if (dut.pipe_valid || dut.state != 7) + wait_ready(rr1, cyc_r1); + end + $display(" TEST4: 60 interleaved read pairs completed (spans real tREFI at CLK_FREQ_MHZ=%0d) -- check log above for VIOLATION/WARNING", CLK_FREQ_MHZ); + end + + $display("=== %0d/%0d tests, %0d errors (BURST_LEN=%0d, CLK_FREQ_MHZ=%0d) ===", + tests-errors, tests, errors, BURST_LEN, CLK_FREQ_MHZ); + if (errors == 0) $display("ALL TESTS PASSED (tb_sdram_controller_pipelined, BURST_LEN=%0d, CLK_FREQ_MHZ=%0d)", BURST_LEN, CLK_FREQ_MHZ); + $finish; + end +endmodule