diff --git a/hardware/v2/logs/experiments.log b/hardware/v2/logs/experiments.log index 73ee653..11cd548 100644 --- a/hardware/v2/logs/experiments.log +++ b/hardware/v2/logs/experiments.log @@ -3168,3 +3168,180 @@ next_action: report to the user; do not pursue the arbiter/backend hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_pipelined.v, hardware/v2/nms/sim/tb_sdram_controller_pipelined.v, hardware/v2/nms/sim/tb_nms_dstress_sdram_pipelined.v. + +EXP-0053 -- SDRAM clock-domain-crossing bridge: decouple the physical +SDRAM clock from the 64MHz compute domain (2026-09-16) + +DATE: 2026-09-16 +CONTEXT: user asked for the inverse roofline calculation (given N-core +compute demand, what memory bandwidth would be needed) after EXP-0052 +closed with only 0.3% real gain; derived requirement ~1GB/s/slot at +64MHz, current single 16-bit SDR SDRAM chip (Alliance AS4C32M16SB-7, +143MHz max) delivers ~90-130MB/s at the board's real 64MHz clk_sys. +Found via real ecppll -i 16 -o 64 --clkout1 sweep (OSS CAD +Suite, now installed at ~/tools_cache/oss-cad-suite, added to PATH via +~/.bashrc this session) that the board's existing PLL VCO is fixed at +576MHz by the ALREADY-VERIFIED 64MHz CLKOP config -- the only clean +integer VCO/N divisors near the chip's ceiling are 576/4=144MHz (0.8% +OVER the real 143MHz max, rejected) and 576/5=115.2MHz (~19% real +margin). 115.2MHz chosen as the fast SDRAM clock, derivable from the +SAME PLL with a second CLKOS output, zero new board components. + +METHOD: sdram_cdc_bridge.v -- toggle+last-seen two-flop-synchronizer +handshake (slow-domain caller interface, fast-domain sdram_ +controller.v instance), safe because this project's own req/busy/ +ready protocol never has more than one transaction outstanding (see +module header for the full quasi-static-bus argument). Isolated tb +(tb_sdram_cdc_bridge.v): 64MHz vs 115.2MHz, deliberately non-integer +ratio, no lucky alignment. + +RESULT (Phase A, isolated): 137/137 tests, 0 errors, including +back-to-back stress. REAL measured total-cycle speedup over 40 +transactions: 1.095x -- NOT the naive 1.8x clock-ratio estimate. Root +cause: the CDC handshake's own synchronizer round-trip (~4-6 +slow-cycle-equivalent per transaction) is a FIXED tax that eats most +of the benefit when the underlying transaction is short (~13 cycles at +BURST_LEN=8). Ad-hoc check at BURST_LEN=32 showed speedup rising to +1.45x (fixed tax amortized over more useful cycles) but also exposed a +real, disclosed, pre-existing controller limitation (see EXP-0054). +decision: correctness verified; real-system integration deferred to +EXP-0055 (composed with EXP-0054). New files (additive only): +hardware/v2/nms/rtl/sdram_cdc_bridge.v, +hardware/v2/nms/sim/tb_sdram_cdc_bridge.v. + +EXP-0054 -- open-row (page-hit/keep-row-open) SDRAM controller policy +(2026-09-16) + +DATE: 2026-09-16 +CONTEXT: investigating why BURST_LEN=32 broke (EXP-0053's ad-hoc +check) led to the real root cause: sdram_controller.v's own mrs_value +function only encodes JEDEC burst-length 1/2/4/8 -- any other value +silently falls through to burst-length code 3'b111 ("full page"), +a real, disclosed, unimplemented-elsewhere scope limit, not a bug to +fix. This redirected the effort toward the controller's OWN header, +which already named the real next lever: "ALWAYS uses auto-precharge +... NOT the fastest possible design (no page-hit/keep-row-open +optimization)". weight_prefetch_engine_wide.v (confirmed via grep, +NOT dead/exploratory code as its own stale header claims -- real +production traffic, instantiated by nms_dataflow_core_sdram.v, +PREFETCH_DISTANCE=8) issues strictly sequential per-job tile addresses +that mostly stay within one SDRAM row (1024 cols/row = 256 tile-blocks +at BURST_LEN=8) -- closing/reopening that row on every single tile +(today's fixed policy) pays tRP+tRCD twice per transaction for no +reason when the next transaction hits the same row anyway. + +METHOD: sdram_controller_openrow.v, forked from sdram_controller.v. +Never auto-precharges; tracks the single currently-open bank+row +(same one-transaction-in-flight scope as the original); on the next +request: ROW HIT (same bank+row) skips ACTIVATE entirely (saves +tRCD); ROW MISS with a row open issues an explicit PRECHARGE first, +same total cost as today's auto-precharge, just paid on-demand. Two +real correctness hazards this policy introduces vs the original +(both fixed, not assumed safe): + (1) JEDEC AUTO REFRESH requires all banks precharged first -- the + original design's own comment ("no row is ever left open...") + no longer holds; fixed via a new S_PRE_THEN_REF_WAIT state. + (2) tWR (write recovery, 2 CLK, real datasheet value) was folded + into the original's always-paid post-write precharge wait -- + now paid alone via a new S_WRITE_RECOVERY_WAIT state. +DISCLOSED, NOT independently verified: read-to-read/read-to-write +same-row turnaround has no extra wait beyond the existing 1-cycle +S_IDLE minimum (standard JEDEC page-mode reasoning) -- sdram_model.v +does NOT itself assert tCCD/tRTW/tWTR (confirmed by inspection), so +this relies on DATA-correctness checks (tb_sdram_controller_openrow.v +TEST 4) rather than an independent timing oracle. + +RESULT (Phase A, isolated, vs sdram_controller.v baseline, same +sdram_model.v-checked correctness harness): 154/154 tests, 0 errors, +0 protocol VIOLATIONs -- including refresh-while-row-open (the one +real new hazard) across 80 write/read pairs spanning real tREFI. +REAL measured speedup, 32 sequential same-row tile reads (the actual +weight_prefetch_engine_wide.v access pattern): 1.141x. +decision: correctness verified; real-system integration in EXP-0055. +New files (additive only): +hardware/v2/nms/rtl/sdram_controller_openrow.v, +hardware/v2/nms/sim/tb_sdram_controller_openrow.v. + +EXP-0055 -- Phase B integration: EXP-0053 (CDC) + EXP-0054 (open-row), +isolated combination AND real D-Stress system, N=4/N=8 -- combined +result is WORSE than baseline; open-row ALONE is a real, disclosed win +(2026-09-16) + +DATE: 2026-09-16 +CONTEXT: per user direction ("procediamo"/"implementiamo queste"), +integrate both mechanisms and measure the real combined effect on the +actual D-Stress benchmark, following this project's own established +Phase A (isolated) -> Phase B (integration) discipline. + +METHOD: sdram_cdc_bridge_openrow.v (EXP-0053's CDC composed with +EXP-0054's page-hit controller as the fast-domain DUT -- CDC handshake +itself unchanged, treats the controller as a black box). Isolated tb +(tb_sdram_cdc_bridge_openrow.v): 149/149 tests, 0 errors, 0 +VIOLATIONs. REAL measured combined speedup, same 32-tile same-row +sequential pattern: 1.158x -- LOWER than the naive product of the two +isolated numbers (1.095 x 1.141 = 1.25), a real, disclosed, non-linear +interaction (the CDC's fixed tax becomes a proportionally BIGGER +fraction of an already-shorter open-row transaction), not assumed. + +Phase B (full system, forked exactly as EXP-0052's own minimal-diff +pattern: sdram_unified_backend_combined.v + nms_neural_multiprocessor_ +sdram_combined.v + tb_nms_dstress_sdram_combined.v, real D-Stress +workload, 256/256 bit-exact + data_ready PASS in every configuration +below): + + baseline (today, real): N=4: 49927 cyc N=8: 49909 cyc + CDC alone (no open-row): N=4: 54096 cyc (+8.35%, WORSE) + open-row alone (no CDC): N=4: 47445 cyc N=8: 47468 cyc + (-4.97% / -4.89%, REAL GAIN) + combined (CDC + open-row): N=4: 51931 cyc N=8: 51943 cyc + (+4.02% / +4.10%, still WORSE) + +ROOT CAUSE of the combined regression: the CDC bridge's synchronizer +round-trip is a FIXED tax paid on EVERY transaction, hit or miss, +regardless of benefit -- unlike EXP-0052's pipelining mechanism (which +simply reverts to baseline-equivalent cost when its condition doesn't +trigger), this tax is not "free when unused". The real D-Stress +traffic is NOT purely sequential same-row (sdram_unified_backend.v's +own 2-way W/AR priority arbitration interleaves weight-fetch and +activation/result traffic, which live in different address regions -- +see its own header, "W granted priority when both pending, AR never +starved" -- meaning the physical channel alternates row context far +more often than the open-row mechanism's own isolated same-row-sweep +test exercised). Open-row's real per-transaction saving (real, ~5%, +confirmed at both N=4 and N=8) is not enough to offset the CDC's own +per-transaction cost once row hits become less frequent under real +interleaved traffic. + +DECISION: do NOT adopt the CDC clock-domain-crossing approach (EXP- +0053) -- measured net negative in the real system despite passing +isolated correctness and even showing a real isolated speedup on its +own synthetic same-row test. ADOPT-CANDIDATE: sdram_controller_ +openrow.v (EXP-0054) alone, without any clock change -- real, +consistent ~5% D-Stress cycle-count improvement at both N=4 and N=8, +zero new clock domains, zero CDC correctness surface, single-variable +change. Not yet promoted to production (that would mean swapping +sdram_controller.v itself in the real board top, fpga_neural_v2_top.v +-- an explicit go-ahead item, not assumed here). This ~5% is +consistent with, and stacks multiplicatively with, EXP-0051's +dual-bank ~9% (different mechanism, same physical-floor-efficiency +class) if both are ever combined -- not measured together in this +session, an open item for a future experiment, not claimed here. + +next_action: report combined finding to the user (CDC bridge measured +net-negative despite being individually correct and individually +faster in isolation -- do not pursue further without new evidence); +open-row is the one real, disclosed win from this whole EXP-0053/54/55 +line and is the candidate worth promoting toward production if the +user wants that next. New files (additive only, none touch the real +board top or existing production RTL): +hardware/v2/nms/rtl/sdram_cdc_bridge_openrow.v, +hardware/v2/nms/rtl/sdram_unified_backend_combined.v, +hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_combined.v, +hardware/v2/nms/rtl/sdram_unified_backend_openrow.v, +hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow.v, +hardware/v2/nms/rtl/sdram_unified_backend_cdc.v, +hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_cdc.v, +hardware/v2/nms/sim/tb_sdram_cdc_bridge_openrow.v, +hardware/v2/nms/sim/tb_nms_dstress_sdram_combined.v, +hardware/v2/nms/sim/tb_nms_dstress_sdram_openrow.v, +hardware/v2/nms/sim/tb_nms_dstress_sdram_cdc.v. diff --git a/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_cdc.v b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_cdc.v new file mode 100644 index 0000000..9802452 --- /dev/null +++ b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_cdc.v @@ -0,0 +1,173 @@ +`timescale 1ns/1ps + +// ================================================================ +// Neural Memory System (NMS) -- STEP19 real hardware-facing top level. +// +// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_ +// sdram_pack128.v (STEP18) with the ONE change this step's own +// governing spec mandates: the real hardware/v1/rtl/psram_controller.v +// + memory_interface.v pairing (activation-fill + result-writeback, +// 16-bit) is REMOVED from the V2 physical path entirely and replaced +// by sdram_unified_backend.v's own AR port, sharing the SAME single +// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_ +// controller.v instance the weight-fetch path (W port) already uses. +// +// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_ +// arbiter_wide.v (64-bit, weight) are BOTH reused completely +// UNCHANGED -- their own downstream ports now both terminate at +// sdram_unified_backend.v instead of two separate physical chains. +// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_ +// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and +// neural_processor.v are ALL byte-for-byte unchanged -- this is a +// pure memory-side substitution, per the governing spec's own +// explicit instruction. +// +// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_ +// interface.v simply are no longer INSTANTIATED by this top-level; +// neither file was modified, and V1's own golden-reference status is +// unaffected. +// +// Real pin count (weight+activation+result, ALL through ONE chip): +// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37 +// pins total -- the SAME 37 pins the weight-only path already used in +// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/ +// result traffic, since it shares the identical physical bus). +// ================================================================ + +module nms_neural_multiprocessor_sdram_cdc #( + parameter DATA_WIDTH = 8, + parameter P_IN = 8, + parameter ACC_WIDTH = 32, + parameter ADDR_WIDTH = 26, + parameter N_SLOTS = 2, + parameter N_NODES = 16, + parameter MAX_DEPS = 4, + parameter QUEUE_DEPTH = 8, + parameter MAX_TILES = 16, + parameter PREFETCH_DISTANCE = 8, + parameter CLK_FREQ_MHZ = 80 +)( + input wire clk, + input wire rst, + input wire clk_fast, + input wire rst_fast, + + input wire reg_valid, + output wire reg_ready, + input wire [$clog2(N_NODES)-1:0] reg_node_id, + input wire [$clog2(MAX_DEPS+1)-1:0] reg_required, + input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids, + input wire [ADDR_WIDTH-1:0] reg_x_base, + input wire [ADDR_WIDTH-1:0] reg_w_base, + input wire [15:0] reg_n_tiles, + input wire [ADDR_WIDTH-1:0] reg_result_addr, + + // FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v + output wire data_ready, + + // ---- STEP19: ONE physical SDRAM interface, ALL traffic + // (weights + activations + results) ---- + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [1:0] sdram_ba, + output wire [12:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + wire [N_SLOTS:0] slot_mem_req, slot_mem_wr; + wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr; + wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata; + wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n; + wire [N_SLOTS:0] slot_mem_ready; + + wire [N_SLOTS-1:0] wide_slot_mem_req; + wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr; + wire [64*N_SLOTS-1:0] wide_slot_mem_rdata; + wire [N_SLOTS-1:0] wide_slot_mem_ready; + + nms_dataflow_core_sdram #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE) + ) u_dataflow_core ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .data_ready(data_ready), + .slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr), + .slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n), + .slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready), + .wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr), + .wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready) + ); + + // ---- AR: activation-fill (shared, 1 port) + per-slot result + // writeback (N_SLOTS ports), arbitrated exactly as before ---- + wire arb_m_req, arb_m_wr; + wire [ADDR_WIDTH-1:0] arb_m_addr; + wire [15:0] arb_m_wdata; + wire arb_m_lb_n, arb_m_ub_n; + wire [15:0] arb_m_rdata; + wire arb_m_ready; + + slot_mem_arbiter #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1) + ) u_arbiter ( + .clk(clk), .rst(rst), + .s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr), + .s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n), + .s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready), + .m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata), + .m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n), + .m_rdata(arb_m_rdata), .m_ready(arb_m_ready) + ); + + // ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as + // before -- weight fetch never writes, same tie-off convention + // as STEP16-18 ---- + wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}}; + wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}}; + wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}}; + wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}}; + + wire wide_arb_m_req, wide_arb_m_wr; + wire [ADDR_WIDTH-1:0] wide_arb_m_addr; + wire [63:0] wide_arb_m_wdata; + wire wide_arb_m_lb_n, wide_arb_m_ub_n; + wire [63:0] wide_arb_m_rdata; + wire wide_arb_m_ready; + + slot_mem_arbiter_wide #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64) + ) u_arbiter_wide ( + .clk(clk), .rst(rst), + .s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr), + .s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n), + .s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready), + .m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata), + .m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n), + .m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready) + ); + + // ---- STEP19: ONE physical SDRAM backend, both W and AR ports ---- + sdram_unified_backend_cdc #( + .ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ) + ) u_sdram_backend ( + .clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast), + .w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr), + .w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready), + .ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata), + .ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n), + .ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_combined.v b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_combined.v new file mode 100644 index 0000000..ba34baf --- /dev/null +++ b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_combined.v @@ -0,0 +1,186 @@ +`timescale 1ns/1ps + +// ================================================================ +// EXP-0055 -- combined fork of nms_neural_multiprocessor_sdram_ +// unified.v (STEP19), same minimal-diff discipline as the pipelined +// fork (EXP-0052): the ONLY changes are two new clk_fast/rst_fast +// ports (forwarded straight through) and instantiating sdram_unified_ +// backend_combined.v (EXP-0055's CDC+page-hit backend) instead of +// sdram_unified_backend.v. u_dataflow_core, both arbiters, are all +// byte-for-byte unchanged. +// +// Everything below is STEP19's own original header, preserved as-is: +// +// Neural Memory System (NMS) -- STEP19 real hardware-facing top level. +// +// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_ +// sdram_pack128.v (STEP18) with the ONE change this step's own +// governing spec mandates: the real hardware/v1/rtl/psram_controller.v +// + memory_interface.v pairing (activation-fill + result-writeback, +// 16-bit) is REMOVED from the V2 physical path entirely and replaced +// by sdram_unified_backend.v's own AR port, sharing the SAME single +// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_ +// controller.v instance the weight-fetch path (W port) already uses. +// +// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_ +// arbiter_wide.v (64-bit, weight) are BOTH reused completely +// UNCHANGED -- their own downstream ports now both terminate at +// sdram_unified_backend.v instead of two separate physical chains. +// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_ +// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and +// neural_processor.v are ALL byte-for-byte unchanged -- this is a +// pure memory-side substitution, per the governing spec's own +// explicit instruction. +// +// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_ +// interface.v simply are no longer INSTANTIATED by this top-level; +// neither file was modified, and V1's own golden-reference status is +// unaffected. +// +// Real pin count (weight+activation+result, ALL through ONE chip): +// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37 +// pins total -- the SAME 37 pins the weight-only path already used in +// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/ +// result traffic, since it shares the identical physical bus). +// ================================================================ + +module nms_neural_multiprocessor_sdram_combined #( + parameter DATA_WIDTH = 8, + parameter P_IN = 8, + parameter ACC_WIDTH = 32, + parameter ADDR_WIDTH = 26, + parameter N_SLOTS = 2, + parameter N_NODES = 16, + parameter MAX_DEPS = 4, + parameter QUEUE_DEPTH = 8, + parameter MAX_TILES = 16, + parameter PREFETCH_DISTANCE = 8, + parameter CLK_FREQ_MHZ = 80 +)( + input wire clk, + input wire rst, + // ---- EXP-0055: fast SDRAM clock domain, forwarded straight + // through to sdram_unified_backend_combined.v -- see that + // module's own header ---- + input wire clk_fast, + input wire rst_fast, + + input wire reg_valid, + output wire reg_ready, + input wire [$clog2(N_NODES)-1:0] reg_node_id, + input wire [$clog2(MAX_DEPS+1)-1:0] reg_required, + input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids, + input wire [ADDR_WIDTH-1:0] reg_x_base, + input wire [ADDR_WIDTH-1:0] reg_w_base, + input wire [15:0] reg_n_tiles, + input wire [ADDR_WIDTH-1:0] reg_result_addr, + + // FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v + output wire data_ready, + + // ---- STEP19: ONE physical SDRAM interface, ALL traffic + // (weights + activations + results) ---- + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [1:0] sdram_ba, + output wire [12:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + wire [N_SLOTS:0] slot_mem_req, slot_mem_wr; + wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr; + wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata; + wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n; + wire [N_SLOTS:0] slot_mem_ready; + + wire [N_SLOTS-1:0] wide_slot_mem_req; + wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr; + wire [64*N_SLOTS-1:0] wide_slot_mem_rdata; + wire [N_SLOTS-1:0] wide_slot_mem_ready; + + nms_dataflow_core_sdram #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE) + ) u_dataflow_core ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .data_ready(data_ready), + .slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr), + .slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n), + .slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready), + .wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr), + .wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready) + ); + + // ---- AR: activation-fill (shared, 1 port) + per-slot result + // writeback (N_SLOTS ports), arbitrated exactly as before ---- + wire arb_m_req, arb_m_wr; + wire [ADDR_WIDTH-1:0] arb_m_addr; + wire [15:0] arb_m_wdata; + wire arb_m_lb_n, arb_m_ub_n; + wire [15:0] arb_m_rdata; + wire arb_m_ready; + + slot_mem_arbiter #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1) + ) u_arbiter ( + .clk(clk), .rst(rst), + .s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr), + .s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n), + .s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready), + .m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata), + .m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n), + .m_rdata(arb_m_rdata), .m_ready(arb_m_ready) + ); + + // ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as + // before -- weight fetch never writes, same tie-off convention + // as STEP16-18 ---- + wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}}; + wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}}; + wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}}; + wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}}; + + wire wide_arb_m_req, wide_arb_m_wr; + wire [ADDR_WIDTH-1:0] wide_arb_m_addr; + wire [63:0] wide_arb_m_wdata; + wire wide_arb_m_lb_n, wide_arb_m_ub_n; + wire [63:0] wide_arb_m_rdata; + wire wide_arb_m_ready; + + slot_mem_arbiter_wide #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64) + ) u_arbiter_wide ( + .clk(clk), .rst(rst), + .s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr), + .s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n), + .s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready), + .m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata), + .m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n), + .m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready) + ); + + // ---- STEP19: ONE physical SDRAM backend, both W and AR ports ---- + sdram_unified_backend_combined #( + .ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ) + ) u_sdram_backend ( + .clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast), + .w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr), + .w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready), + .ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata), + .ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n), + .ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow.v b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow.v new file mode 100644 index 0000000..15b947e --- /dev/null +++ b/hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow.v @@ -0,0 +1,171 @@ +`timescale 1ns/1ps + +// ================================================================ +// Neural Memory System (NMS) -- STEP19 real hardware-facing top level. +// +// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_ +// sdram_pack128.v (STEP18) with the ONE change this step's own +// governing spec mandates: the real hardware/v1/rtl/psram_controller.v +// + memory_interface.v pairing (activation-fill + result-writeback, +// 16-bit) is REMOVED from the V2 physical path entirely and replaced +// by sdram_unified_backend.v's own AR port, sharing the SAME single +// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_ +// controller.v instance the weight-fetch path (W port) already uses. +// +// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_ +// arbiter_wide.v (64-bit, weight) are BOTH reused completely +// UNCHANGED -- their own downstream ports now both terminate at +// sdram_unified_backend.v instead of two separate physical chains. +// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_ +// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and +// neural_processor.v are ALL byte-for-byte unchanged -- this is a +// pure memory-side substitution, per the governing spec's own +// explicit instruction. +// +// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_ +// interface.v simply are no longer INSTANTIATED by this top-level; +// neither file was modified, and V1's own golden-reference status is +// unaffected. +// +// Real pin count (weight+activation+result, ALL through ONE chip): +// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37 +// pins total -- the SAME 37 pins the weight-only path already used in +// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/ +// result traffic, since it shares the identical physical bus). +// ================================================================ + +module nms_neural_multiprocessor_sdram_openrow #( + parameter DATA_WIDTH = 8, + parameter P_IN = 8, + parameter ACC_WIDTH = 32, + parameter ADDR_WIDTH = 26, + parameter N_SLOTS = 2, + parameter N_NODES = 16, + parameter MAX_DEPS = 4, + parameter QUEUE_DEPTH = 8, + parameter MAX_TILES = 16, + parameter PREFETCH_DISTANCE = 8, + parameter CLK_FREQ_MHZ = 80 +)( + input wire clk, + input wire rst, + + input wire reg_valid, + output wire reg_ready, + input wire [$clog2(N_NODES)-1:0] reg_node_id, + input wire [$clog2(MAX_DEPS+1)-1:0] reg_required, + input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids, + input wire [ADDR_WIDTH-1:0] reg_x_base, + input wire [ADDR_WIDTH-1:0] reg_w_base, + input wire [15:0] reg_n_tiles, + input wire [ADDR_WIDTH-1:0] reg_result_addr, + + // FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v + output wire data_ready, + + // ---- STEP19: ONE physical SDRAM interface, ALL traffic + // (weights + activations + results) ---- + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [1:0] sdram_ba, + output wire [12:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + wire [N_SLOTS:0] slot_mem_req, slot_mem_wr; + wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr; + wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata; + wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n; + wire [N_SLOTS:0] slot_mem_ready; + + wire [N_SLOTS-1:0] wide_slot_mem_req; + wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr; + wire [64*N_SLOTS-1:0] wide_slot_mem_rdata; + wire [N_SLOTS-1:0] wide_slot_mem_ready; + + nms_dataflow_core_sdram #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE) + ) u_dataflow_core ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .data_ready(data_ready), + .slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr), + .slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n), + .slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready), + .wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr), + .wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready) + ); + + // ---- AR: activation-fill (shared, 1 port) + per-slot result + // writeback (N_SLOTS ports), arbitrated exactly as before ---- + wire arb_m_req, arb_m_wr; + wire [ADDR_WIDTH-1:0] arb_m_addr; + wire [15:0] arb_m_wdata; + wire arb_m_lb_n, arb_m_ub_n; + wire [15:0] arb_m_rdata; + wire arb_m_ready; + + slot_mem_arbiter #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1) + ) u_arbiter ( + .clk(clk), .rst(rst), + .s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr), + .s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n), + .s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready), + .m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata), + .m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n), + .m_rdata(arb_m_rdata), .m_ready(arb_m_ready) + ); + + // ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as + // before -- weight fetch never writes, same tie-off convention + // as STEP16-18 ---- + wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}}; + wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}}; + wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}}; + wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}}; + + wire wide_arb_m_req, wide_arb_m_wr; + wire [ADDR_WIDTH-1:0] wide_arb_m_addr; + wire [63:0] wide_arb_m_wdata; + wire wide_arb_m_lb_n, wide_arb_m_ub_n; + wire [63:0] wide_arb_m_rdata; + wire wide_arb_m_ready; + + slot_mem_arbiter_wide #( + .ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64) + ) u_arbiter_wide ( + .clk(clk), .rst(rst), + .s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr), + .s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n), + .s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready), + .m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata), + .m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n), + .m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready) + ); + + // ---- STEP19: ONE physical SDRAM backend, both W and AR ports ---- + sdram_unified_backend_openrow #( + .ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ) + ) u_sdram_backend ( + .clk(clk), .rst(rst), + .w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr), + .w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready), + .ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata), + .ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n), + .ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_cdc_bridge.v b/hardware/v2/nms/rtl/sdram_cdc_bridge.v new file mode 100644 index 0000000..99d22e0 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_cdc_bridge.v @@ -0,0 +1,253 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXP-0053 -- SDRAM clock-domain-crossing bridge. +// +// PURPOSE: let sdram_controller.v run on its OWN, faster clock +// (target: 115.2MHz, derived from the SAME PLL VCO as the existing +// 64MHz clk_sys -- see ecp5_pll_sys_clk_dualclk.v) while every +// existing caller (sdram_unified_backend.v's ctrl_req/ctrl_addr/... +// signals) stays on the unchanged 64MHz compute domain. Presents the +// EXACT same req/wr/addr/wdata/wmask -> rdata/ready/busy contract as +// sdram_controller.v itself, so it is a drop-in replacement for the +// direct sdram_controller instantiation at that one call site +// (verified by the isolated tb_sdram_cdc_bridge.v before any +// integration). +// +// WHY 115.2MHz and not the chip's rated 143MHz max (AS4C32M16SA-7, +// tCK=7ns min): the board's single PLL VCO is fixed at 576MHz by the +// existing, already-verified 64MHz CLKOP config (CLKFB_DIV=4, +// CLKOP_DIV=9 -- unchanged, not touched by this experiment). The only +// integer VCO/N divisors near the chip's ceiling are 576/4=144MHz +// (0.8% OVER the 143MHz max -- rejected, not "correctness first") and +// 576/5=115.2MHz (real ~19% margin under the rated max). 115.2MHz is +// therefore the fastest SAFE clock obtainable from this board's +// existing PLL without touching the verified 64MHz compute domain. +// Real measured gain vs the current 64MHz single-domain design is +// therefore 115.2/64 = 1.8x raw controller-clock speedup, NOT the 2.2x +// a naive 143MHz assumption would suggest -- this correction is +// intentional, verified against real ecppll output, not estimated. +// +// PROTOCOL: single-outstanding-request only (matches every existing +// caller's own req/busy/ready idiom exactly -- this bridge does NOT +// add multi-request pipelining; that is EXP-0052's explicitly +// deferred, larger, riskier follow-up, out of scope here). Because at +// most one transaction is ever in flight, a classic two-domain +// "toggle + last-seen" handshake is provably safe: +// - the requesting (slow) domain latches addr/wr/wdata/wmask and +// flips req_toggle_slow on the SAME clock edge, then holds ALL of +// those signals perfectly stable (no new request is ever issued +// while busy=1) until the response toggle comes back; +// - the fast domain double-flop-synchronizes req_toggle_slow (2 FF, +// standard metastability margin) and compares it against its own +// "last serviced" copy -- a mismatch means a new request is +// pending. Because addr/wr/wdata/wmask changed on the SAME edge +// that flipped the toggle, and never change again before the +// response, they are safe to sample directly (no per-bit +// synchronizer needed) once the synchronized toggle has visibly +// changed -- this is the standard "quasi-static bus + toggle" +// CDC idiom, not a shortcut. +// - the same reasoning applies in reverse for ack_toggle_fast/rdata +// going back to the slow domain. +// Reset: rst_slow and rst_fast are separate inputs, each assumed +// ALREADY synchronized to its own clock domain by the caller (this +// module does not itself synchronize an async reset -- matches this +// project's existing convention of a single, pre-synchronized `rst` +// per clock domain, see ecp5_pll_sys_clk.v's own reset handling). +// ============================================================ +module sdram_cdc_bridge #( + parameter CLK_FREQ_MHZ_FAST = 115, // deliberately rounded DOWN from + // the real 115.2MHz (never over- + // count available ns/cycle -- + // same "ceiling division" spirit + // as sdram_controller.v's own + // ns_to_cycles), so every derived + // timing constant (T_RCD/T_RP/...) + // gets AT LEAST as many cycles as + // the real, slightly-faster clock + // requires. + parameter BURST_LEN = 8, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2, + parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS +)( + input wire clk_slow, + input wire rst_slow, // pre-synchronized to clk_slow + input wire clk_fast, + input wire rst_fast, // pre-synchronized to clk_fast + + // ---- slow-domain caller interface (identical shape to + // sdram_controller.v's own ports) ---- + input wire req, + input wire wr, + input wire [ADDR_WIDTH-1:0] addr, + input wire [16*BURST_LEN-1:0] wdata, + input wire [2*BURST_LEN-1:0] wmask, + output reg [16*BURST_LEN-1:0] rdata, + output reg ready, + output wire busy, + + // ---- real SDRAM pins, driven directly by the fast-domain + // sdram_controller instance ---- + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + // ============================================================ + // Slow domain: capture request, drive toggle, wait for ack + // ============================================================ + reg busy_slow; + reg req_toggle_slow; + reg wr_lat; + reg [ADDR_WIDTH-1:0] addr_lat; + reg [16*BURST_LEN-1:0] wdata_lat; + reg [2*BURST_LEN-1:0] wmask_lat; + + assign busy = busy_slow; + + // synchronize ack_toggle_fast (fast domain) into the slow domain + wire ack_toggle_fast; + reg ack_toggle_sync1, ack_toggle_sync2; + always @(posedge clk_slow) begin + if (rst_slow) begin + ack_toggle_sync1 <= 1'b0; + ack_toggle_sync2 <= 1'b0; + end else begin + ack_toggle_sync1 <= ack_toggle_fast; + ack_toggle_sync2 <= ack_toggle_sync1; + end + end + + reg last_ack_toggle_seen_slow; + wire [16*BURST_LEN-1:0] rdata_fast_captured; + + always @(posedge clk_slow) begin + if (rst_slow) begin + busy_slow <= 1'b0; + req_toggle_slow <= 1'b0; + last_ack_toggle_seen_slow <= 1'b0; + ready <= 1'b0; + rdata <= {(16*BURST_LEN){1'b0}}; + wr_lat <= 1'b0; + addr_lat <= {ADDR_WIDTH{1'b0}}; + wdata_lat <= {(16*BURST_LEN){1'b0}}; + wmask_lat <= {(2*BURST_LEN){1'b0}}; + end else begin + ready <= 1'b0; + + if (req && !busy_slow) begin + wr_lat <= wr; + addr_lat <= addr; + wdata_lat <= wdata; + wmask_lat <= wmask; + req_toggle_slow <= ~req_toggle_slow; + busy_slow <= 1'b1; + end + + if (busy_slow && (ack_toggle_sync2 != last_ack_toggle_seen_slow)) begin + last_ack_toggle_seen_slow <= ack_toggle_sync2; + rdata <= rdata_fast_captured; + ready <= 1'b1; + busy_slow <= 1'b0; + end + end + end + + // ============================================================ + // Fast domain: synchronize request toggle, drive the real + // sdram_controller, capture response, drive ack toggle back + // ============================================================ + reg ctrl_req_f; + reg ctrl_wr_f; + reg [ADDR_WIDTH-1:0] ctrl_addr_f; + reg [16*BURST_LEN-1:0] ctrl_wdata_f; + reg [2*BURST_LEN-1:0] ctrl_wmask_f; + wire [16*BURST_LEN-1:0] ctrl_rdata_f; + wire ctrl_ready_f, ctrl_busy_f; + + reg req_toggle_sync1, req_toggle_sync2; + always @(posedge clk_fast) begin + if (rst_fast) begin + req_toggle_sync1 <= 1'b0; + req_toggle_sync2 <= 1'b0; + end else begin + req_toggle_sync1 <= req_toggle_slow; + req_toggle_sync2 <= req_toggle_sync1; + end + end + + localparam F_IDLE = 1'b0, F_WAIT = 1'b1; + reg f_state; + reg last_req_toggle_seen_fast; + reg ack_toggle_fast_r; + reg [16*BURST_LEN-1:0] rdata_fast_captured_r; + + assign ack_toggle_fast = ack_toggle_fast_r; + assign rdata_fast_captured = rdata_fast_captured_r; + + always @(posedge clk_fast) begin + if (rst_fast) begin + f_state <= F_IDLE; + last_req_toggle_seen_fast <= 1'b0; + ack_toggle_fast_r <= 1'b0; + rdata_fast_captured_r <= {(16*BURST_LEN){1'b0}}; + ctrl_req_f <= 1'b0; + ctrl_wr_f <= 1'b0; + ctrl_addr_f <= {ADDR_WIDTH{1'b0}}; + ctrl_wdata_f<= {(16*BURST_LEN){1'b0}}; + ctrl_wmask_f<= {(2*BURST_LEN){1'b0}}; + end else begin + ctrl_req_f <= 1'b0; + case (f_state) + F_IDLE: begin + if (req_toggle_sync2 != last_req_toggle_seen_fast) begin + // addr_lat/wr_lat/wdata_lat/wmask_lat (slow- + // domain regs) are quasi-static: they changed + // on the exact same slow-domain edge that + // flipped req_toggle_slow, and will not change + // again until busy_slow deasserts (long after + // this transaction completes) -- safe to + // sample directly, see module header. + ctrl_req_f <= 1'b1; + ctrl_wr_f <= wr_lat; + ctrl_addr_f <= addr_lat; + ctrl_wdata_f<= wdata_lat; + ctrl_wmask_f<= wmask_lat; + last_req_toggle_seen_fast <= req_toggle_sync2; + f_state <= F_WAIT; + end + end + F_WAIT: begin + if (ctrl_ready_f) begin + rdata_fast_captured_r <= ctrl_rdata_f; + ack_toggle_fast_r <= ~ack_toggle_fast_r; + f_state <= F_IDLE; + end + end + default: f_state <= F_IDLE; + endcase + end + end + + sdram_controller #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ_FAST), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk(clk_fast), .rst(rst_fast), + .req(ctrl_req_f), .wr(ctrl_wr_f), .addr(ctrl_addr_f), + .wdata(ctrl_wdata_f), .wmask(ctrl_wmask_f), + .rdata(ctrl_rdata_f), .ready(ctrl_ready_f), .busy(ctrl_busy_f), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_cdc_bridge_openrow.v b/hardware/v2/nms/rtl/sdram_cdc_bridge_openrow.v new file mode 100644 index 0000000..d4bf466 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_cdc_bridge_openrow.v @@ -0,0 +1,262 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXP-0055 -- combined fork of sdram_cdc_bridge.v (EXP-0053): the ONLY +// change is instantiating sdram_controller_openrow.v (EXP-0054, +// page-hit/keep-row-open policy) instead of plain sdram_controller.v +// as the fast-domain controller. The CDC handshake itself (toggle + +// last-seen, quasi-static bus sampling) is byte-for-byte unchanged -- +// it treats whatever fast-domain controller it wraps as a black box +// behind the same req/wr/addr/wdata/wmask -> rdata/ready/busy +// contract, so this combination was expected to compose cleanly, and +// is verified as such by tb_sdram_cdc_bridge_openrow.v before any +// further integration. +// +// PURPOSE: let sdram_controller.v run on its OWN, faster clock +// (target: 115.2MHz, derived from the SAME PLL VCO as the existing +// 64MHz clk_sys -- see ecp5_pll_sys_clk_dualclk.v) while every +// existing caller (sdram_unified_backend.v's ctrl_req/ctrl_addr/... +// signals) stays on the unchanged 64MHz compute domain. Presents the +// EXACT same req/wr/addr/wdata/wmask -> rdata/ready/busy contract as +// sdram_controller.v itself, so it is a drop-in replacement for the +// direct sdram_controller instantiation at that one call site +// (verified by the isolated tb_sdram_cdc_bridge.v before any +// integration). +// +// WHY 115.2MHz and not the chip's rated 143MHz max (AS4C32M16SA-7, +// tCK=7ns min): the board's single PLL VCO is fixed at 576MHz by the +// existing, already-verified 64MHz CLKOP config (CLKFB_DIV=4, +// CLKOP_DIV=9 -- unchanged, not touched by this experiment). The only +// integer VCO/N divisors near the chip's ceiling are 576/4=144MHz +// (0.8% OVER the 143MHz max -- rejected, not "correctness first") and +// 576/5=115.2MHz (real ~19% margin under the rated max). 115.2MHz is +// therefore the fastest SAFE clock obtainable from this board's +// existing PLL without touching the verified 64MHz compute domain. +// Real measured gain vs the current 64MHz single-domain design is +// therefore 115.2/64 = 1.8x raw controller-clock speedup, NOT the 2.2x +// a naive 143MHz assumption would suggest -- this correction is +// intentional, verified against real ecppll output, not estimated. +// +// PROTOCOL: single-outstanding-request only (matches every existing +// caller's own req/busy/ready idiom exactly -- this bridge does NOT +// add multi-request pipelining; that is EXP-0052's explicitly +// deferred, larger, riskier follow-up, out of scope here). Because at +// most one transaction is ever in flight, a classic two-domain +// "toggle + last-seen" handshake is provably safe: +// - the requesting (slow) domain latches addr/wr/wdata/wmask and +// flips req_toggle_slow on the SAME clock edge, then holds ALL of +// those signals perfectly stable (no new request is ever issued +// while busy=1) until the response toggle comes back; +// - the fast domain double-flop-synchronizes req_toggle_slow (2 FF, +// standard metastability margin) and compares it against its own +// "last serviced" copy -- a mismatch means a new request is +// pending. Because addr/wr/wdata/wmask changed on the SAME edge +// that flipped the toggle, and never change again before the +// response, they are safe to sample directly (no per-bit +// synchronizer needed) once the synchronized toggle has visibly +// changed -- this is the standard "quasi-static bus + toggle" +// CDC idiom, not a shortcut. +// - the same reasoning applies in reverse for ack_toggle_fast/rdata +// going back to the slow domain. +// Reset: rst_slow and rst_fast are separate inputs, each assumed +// ALREADY synchronized to its own clock domain by the caller (this +// module does not itself synchronize an async reset -- matches this +// project's existing convention of a single, pre-synchronized `rst` +// per clock domain, see ecp5_pll_sys_clk.v's own reset handling). +// ============================================================ +module sdram_cdc_bridge_openrow #( + parameter CLK_FREQ_MHZ_FAST = 115, // deliberately rounded DOWN from + // the real 115.2MHz (never over- + // count available ns/cycle -- + // same "ceiling division" spirit + // as sdram_controller.v's own + // ns_to_cycles), so every derived + // timing constant (T_RCD/T_RP/...) + // gets AT LEAST as many cycles as + // the real, slightly-faster clock + // requires. + parameter BURST_LEN = 8, + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2, + parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS +)( + input wire clk_slow, + input wire rst_slow, // pre-synchronized to clk_slow + input wire clk_fast, + input wire rst_fast, // pre-synchronized to clk_fast + + // ---- slow-domain caller interface (identical shape to + // sdram_controller.v's own ports) ---- + input wire req, + input wire wr, + input wire [ADDR_WIDTH-1:0] addr, + input wire [16*BURST_LEN-1:0] wdata, + input wire [2*BURST_LEN-1:0] wmask, + output reg [16*BURST_LEN-1:0] rdata, + output reg ready, + output wire busy, + + // ---- real SDRAM pins, driven directly by the fast-domain + // sdram_controller instance ---- + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + // ============================================================ + // Slow domain: capture request, drive toggle, wait for ack + // ============================================================ + reg busy_slow; + reg req_toggle_slow; + reg wr_lat; + reg [ADDR_WIDTH-1:0] addr_lat; + reg [16*BURST_LEN-1:0] wdata_lat; + reg [2*BURST_LEN-1:0] wmask_lat; + + assign busy = busy_slow; + + // synchronize ack_toggle_fast (fast domain) into the slow domain + wire ack_toggle_fast; + reg ack_toggle_sync1, ack_toggle_sync2; + always @(posedge clk_slow) begin + if (rst_slow) begin + ack_toggle_sync1 <= 1'b0; + ack_toggle_sync2 <= 1'b0; + end else begin + ack_toggle_sync1 <= ack_toggle_fast; + ack_toggle_sync2 <= ack_toggle_sync1; + end + end + + reg last_ack_toggle_seen_slow; + wire [16*BURST_LEN-1:0] rdata_fast_captured; + + always @(posedge clk_slow) begin + if (rst_slow) begin + busy_slow <= 1'b0; + req_toggle_slow <= 1'b0; + last_ack_toggle_seen_slow <= 1'b0; + ready <= 1'b0; + rdata <= {(16*BURST_LEN){1'b0}}; + wr_lat <= 1'b0; + addr_lat <= {ADDR_WIDTH{1'b0}}; + wdata_lat <= {(16*BURST_LEN){1'b0}}; + wmask_lat <= {(2*BURST_LEN){1'b0}}; + end else begin + ready <= 1'b0; + + if (req && !busy_slow) begin + wr_lat <= wr; + addr_lat <= addr; + wdata_lat <= wdata; + wmask_lat <= wmask; + req_toggle_slow <= ~req_toggle_slow; + busy_slow <= 1'b1; + end + + if (busy_slow && (ack_toggle_sync2 != last_ack_toggle_seen_slow)) begin + last_ack_toggle_seen_slow <= ack_toggle_sync2; + rdata <= rdata_fast_captured; + ready <= 1'b1; + busy_slow <= 1'b0; + end + end + end + + // ============================================================ + // Fast domain: synchronize request toggle, drive the real + // sdram_controller, capture response, drive ack toggle back + // ============================================================ + reg ctrl_req_f; + reg ctrl_wr_f; + reg [ADDR_WIDTH-1:0] ctrl_addr_f; + reg [16*BURST_LEN-1:0] ctrl_wdata_f; + reg [2*BURST_LEN-1:0] ctrl_wmask_f; + wire [16*BURST_LEN-1:0] ctrl_rdata_f; + wire ctrl_ready_f, ctrl_busy_f; + + reg req_toggle_sync1, req_toggle_sync2; + always @(posedge clk_fast) begin + if (rst_fast) begin + req_toggle_sync1 <= 1'b0; + req_toggle_sync2 <= 1'b0; + end else begin + req_toggle_sync1 <= req_toggle_slow; + req_toggle_sync2 <= req_toggle_sync1; + end + end + + localparam F_IDLE = 1'b0, F_WAIT = 1'b1; + reg f_state; + reg last_req_toggle_seen_fast; + reg ack_toggle_fast_r; + reg [16*BURST_LEN-1:0] rdata_fast_captured_r; + + assign ack_toggle_fast = ack_toggle_fast_r; + assign rdata_fast_captured = rdata_fast_captured_r; + + always @(posedge clk_fast) begin + if (rst_fast) begin + f_state <= F_IDLE; + last_req_toggle_seen_fast <= 1'b0; + ack_toggle_fast_r <= 1'b0; + rdata_fast_captured_r <= {(16*BURST_LEN){1'b0}}; + ctrl_req_f <= 1'b0; + ctrl_wr_f <= 1'b0; + ctrl_addr_f <= {ADDR_WIDTH{1'b0}}; + ctrl_wdata_f<= {(16*BURST_LEN){1'b0}}; + ctrl_wmask_f<= {(2*BURST_LEN){1'b0}}; + end else begin + ctrl_req_f <= 1'b0; + case (f_state) + F_IDLE: begin + if (req_toggle_sync2 != last_req_toggle_seen_fast) begin + // addr_lat/wr_lat/wdata_lat/wmask_lat (slow- + // domain regs) are quasi-static: they changed + // on the exact same slow-domain edge that + // flipped req_toggle_slow, and will not change + // again until busy_slow deasserts (long after + // this transaction completes) -- safe to + // sample directly, see module header. + ctrl_req_f <= 1'b1; + ctrl_wr_f <= wr_lat; + ctrl_addr_f <= addr_lat; + ctrl_wdata_f<= wdata_lat; + ctrl_wmask_f<= wmask_lat; + last_req_toggle_seen_fast <= req_toggle_sync2; + f_state <= F_WAIT; + end + end + F_WAIT: begin + if (ctrl_ready_f) begin + rdata_fast_captured_r <= ctrl_rdata_f; + ack_toggle_fast_r <= ~ack_toggle_fast_r; + f_state <= F_IDLE; + end + end + default: f_state <= F_IDLE; + endcase + end + end + + sdram_controller_openrow #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ_FAST), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk(clk_fast), .rst(rst_fast), + .req(ctrl_req_f), .wr(ctrl_wr_f), .addr(ctrl_addr_f), + .wdata(ctrl_wdata_f), .wmask(ctrl_wmask_f), + .rdata(ctrl_rdata_f), .ready(ctrl_ready_f), .busy(ctrl_busy_f), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_controller_openrow.v b/hardware/v2/nms/rtl/sdram_controller_openrow.v new file mode 100644 index 0000000..d1cd585 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_controller_openrow.v @@ -0,0 +1,460 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXP-0054 -- open-row (page-hit) SDR SDRAM controller, forked from +// sdram_controller.v (STEP16). Implements the "page-hit/keep-row-open +// optimization" that sdram_controller.v's own header explicitly +// deferred: +// "ALWAYS uses auto-precharge... NOT the fastest possible design +// (no page-hit/keep-row-open optimization, unlike psram_ +// controller.v's own real page-mode), but it is trivially correct" +// +// MOTIVATION: weight_prefetch_engine_wide.v (real production traffic, +// instantiated by nms_dataflow_core_sdram.v, PREFETCH_DISTANCE=8) +// already issues a stream of STRICTLY SEQUENTIAL tile addresses per +// job. With ROW_BITS=13/COL_BITS=10 (AS4C32M16SA: 1024 columns/row, +// 4 words/tile at 16-bit words -- see sdram_controller.v's own TILE +// comment), a single row holds 256 consecutive tiles before crossing +// a row boundary -- most real jobs' weight streams never leave the +// row they started in. Closing and reopening that row on EVERY single +// tile (today's fixed auto-precharge policy) pays tRP+tRCD twice per +// transaction for no reason when the next transaction is going to hit +// the SAME row anyway. +// +// POLICY: never auto-precharge (A10=0 on every READ/WRITE). Track the +// single currently-open bank+row (this controller has always modeled +// "one transaction in flight" -- this experiment keeps that same +// single-open-row scope, not per-bank tracking across multiple +// simultaneously-open banks, matching the project's own established +// risk posture). On the NEXT request (evaluated in S_IDLE, exactly +// where every prior request was already evaluated): +// - SAME bank+row as currently open ("row hit"): skip ACTIVATE +// entirely -- issue READ/WRITE directly, saving tRCD. +// - DIFFERENT bank+row while a row IS open ("row miss"): issue an +// explicit PRECHARGE first (this controller no longer gets that +// for free via auto-precharge), wait tRP, THEN activate the new +// row exactly as before -- same total cost as today's design, +// just paid on-demand instead of unconditionally after every +// transaction. +// - no row open (e.g. right after reset/refresh): activate directly, +// unchanged from today. +// +// REFRESH INTERACTION (the one real correctness hazard this policy +// introduces, absent from the original always-precharged design): +// JEDEC AUTO REFRESH requires ALL banks precharged first. The +// original S_IDLE refresh branch's own comment ("no row is ever left +// open between transactions... so we can refresh immediately") is no +// longer true under this policy -- fixed here by precharging first +// (S_PRE_THEN_REF_WAIT) whenever row_open is set at the moment +// refresh comes due, before issuing AUTO REFRESH exactly as before. +// +// WRITE RECOVERY (tWR): the original design folded tWR into its +// always-paid post-burst precharge wait ("T_RP + 1'b1 // tWR folded +// in conservatively"). This design no longer precharges after every +// write, so tWR is now paid explicitly and alone (T_WR=2 CLK, real +// AS4C32M16SA datasheet value, same explicit-CLK-units treatment as +// T_MRD) via a new S_WRITE_RECOVERY_WAIT state, before the row-open +// path returns to S_IDLE and can accept a same-row follow-on command. +// +// DISCLOSED, NOT INDEPENDENTLY VERIFIED: read-burst-end -> next +// command (read-to-read or read-to-write, same open row) has NO extra +// wait beyond the existing 1-cycle-minimum S_IDLE turnaround, on the +// reasoning that JEDEC SDR SDRAM page-mode reads support back-to-back +// column access with no additional bubble. sdram_model.v (this +// project's own real-command-sequence checker) does NOT itself assert +// tCCD/tRTW/tWTR -- it only checks ACTIVATE-while-active, tRP, tRAS +// (min), refresh spacing, and access-with-no-active-row (see its own +// VIOLATION messages). tb_sdram_controller_openrow.v exercises +// read-after-read and write-after-read same-row sequences explicitly +// and checks DATA correctness, but a genuine read-to-write DQ bus +// turnaround hazard would not be caught by sdram_model.v itself if +// present -- flagged here exactly as this project's own convention +// requires, not silently assumed safe. +// +// Every timing constant, the mrs_value encoding, the req_pending +// unconditional-latch fix, and the address decomposition are carried +// over UNCHANGED from sdram_controller.v -- only the state machine's +// precharge policy and the two new wait states are new. +// ============================================================ +module sdram_controller_openrow #( + parameter CLK_FREQ_MHZ = 64, + parameter BURST_LEN = 4, // 1, 4, or 8 -- same real scope as sdram_controller.v (see its own mrs_value) + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2, + parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS +)( + input wire clk, + input wire rst, + + input wire req, + input wire wr, + input wire [ADDR_WIDTH-1:0] addr, + input wire [16*BURST_LEN-1:0] wdata, + input wire [2*BURST_LEN-1:0] wmask, + output reg [16*BURST_LEN-1:0] rdata, + output reg ready, + output reg busy, + + output reg sdram_cke, + output reg sdram_cs_n, + output reg sdram_ras_n, + output reg sdram_cas_n, + output reg sdram_we_n, + output reg [1:0] sdram_ba, + output reg [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output reg [1:0] sdram_dqm +); + + localparam BURST_IDXW = (BURST_LEN <= 1) ? 1 : $clog2(BURST_LEN); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS) begin + $display("FATAL sdram_controller_openrow: ADDR_WIDTH=%0d != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)=%0d", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS, BANK_BITS+ROW_BITS+COL_BITS); + $finish; + end + + function integer ns_to_cycles; + input integer ns; + begin + ns_to_cycles = (ns * CLK_FREQ_MHZ + 999) / 1000; + end + endfunction + localparam T_RCD = ns_to_cycles(15); + localparam T_RP = ns_to_cycles(15); + localparam T_MRD = 2; + localparam T_WR = 2; // real AS4C32M16SA datasheet value, explicit CLK units (same treatment as T_MRD) + localparam T_INIT_US= 200; + localparam T_INIT = T_INIT_US * CLK_FREQ_MHZ; + localparam CAS_LATENCY = 3; + localparam T_REFI = ns_to_cycles(64000000 / (1 << ROW_BITS) + 1); + + localparam CNTW = $clog2((T_INIT>T_REFI ? T_INIT : T_REFI) + 1); + + function [CNTW-1:0] T_RC_MINUS1; + localparam integer T_RC = ns_to_cycles(65); + begin + T_RC_MINUS1 = T_RC[CNTW-1:0] - 1'b1; + end + endfunction + + localparam + S_INIT_WAIT = 5'd0, + S_INIT_PRE_WAIT = 5'd2, + S_INIT_REF = 5'd3, + S_INIT_REF_WAIT = 5'd4, + S_INIT_MRS_WAIT = 5'd6, + S_IDLE = 5'd7, + S_REFRESH_WAIT = 5'd9, + S_ACTIVATE_WAIT = 5'd11, + S_CAS_WAIT = 5'd13, + S_BURST_READ = 5'd14, + S_BURST_WRITE = 5'd15, + S_PRE_THEN_ACT_WAIT = 5'd17, + S_PRE_THEN_REF_WAIT = 5'd18, + S_WRITE_RECOVERY_WAIT= 5'd19; + + reg [4:0] state; + reg [CNTW-1:0] wait_cnt; + reg [3:0] init_ref_cnt; + reg [CNTW-1:0] refresh_timer; + reg [BURST_IDXW-1:0] burst_idx; + reg req_wr_reg; + reg [BANK_BITS-1:0] req_bank_reg; + reg [ROW_BITS-1:0] req_row_reg; + reg [COL_BITS-1:0] req_col_reg; + reg [16*BURST_LEN-1:0] wdata_reg; + reg [2*BURST_LEN-1:0] wmask_reg; + + // ---- open-row tracking (new vs sdram_controller.v) ---- + reg row_open; + reg [BANK_BITS-1:0] open_bank; + reg [ROW_BITS-1:0] open_row; + + wire [BANK_BITS-1:0] addr_bank = addr[ADDR_WIDTH-1 -: BANK_BITS]; + wire [ROW_BITS-1:0] addr_row = addr[ADDR_WIDTH-BANK_BITS-1 -: ROW_BITS]; + wire [COL_BITS-1:0] addr_col = addr[COL_BITS-1:0]; + + reg req_pending; + wire eff_wr = req ? wr : req_wr_reg; + wire [BANK_BITS-1:0] eff_bank = req ? addr_bank : req_bank_reg; + wire [ROW_BITS-1:0] eff_row = req ? addr_row : req_row_reg; + wire [COL_BITS-1:0] eff_col = req ? addr_col : req_col_reg; + wire [16*BURST_LEN-1:0] eff_wdata = req ? wdata : wdata_reg; + wire [2*BURST_LEN-1:0] eff_wmask = req ? wmask : wmask_reg; + + reg dq_out_en; + reg [15:0] dq_out; + assign sdram_dq = dq_out_en ? dq_out : 16'hzzzz; + + function [ROW_BITS-1:0] mrs_value; + input integer burst_len; + reg [2:0] bl_code; + reg [ROW_BITS-1:0] v; + begin + bl_code = (burst_len==1) ? 3'b000 : + (burst_len==2) ? 3'b001 : + (burst_len==4) ? 3'b010 : + (burst_len==8) ? 3'b011 : 3'b111; + v = {ROW_BITS{1'b0}}; + v[6:4] = 3'b011; + v[3] = 1'b0; + v[2:0] = bl_code; + mrs_value = v; + end + endfunction + + always @(posedge clk) begin + if (rst) begin + state <= S_INIT_WAIT; + wait_cnt <= T_INIT[CNTW-1:0]; + init_ref_cnt <= 4'd0; + refresh_timer <= T_REFI[CNTW-1:0]; + sdram_cke <= 1'b1; + sdram_cs_n <= 1'b1; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + sdram_ba <= 2'b00; + sdram_a <= {ROW_BITS{1'b0}}; + sdram_dqm <= 2'b00; + dq_out_en <= 1'b0; + ready <= 1'b0; + busy <= 1'b1; + req_pending <= 1'b0; + row_open <= 1'b0; + open_bank <= {BANK_BITS{1'b0}}; + open_row <= {ROW_BITS{1'b0}}; + end else begin + sdram_cs_n <= 1'b0; + sdram_ras_n <= 1'b1; + sdram_cas_n <= 1'b1; + sdram_we_n <= 1'b1; + ready <= 1'b0; + dq_out_en <= 1'b0; + sdram_dqm <= 2'b00; + + if (refresh_timer != 0) refresh_timer <= refresh_timer - 1'b1; + + if (req) begin + req_wr_reg <= wr; + req_bank_reg <= addr_bank; + req_row_reg <= addr_row; + req_col_reg <= addr_col; + wdata_reg <= wdata; + wmask_reg <= wmask; + req_pending <= 1'b1; + end + + case (state) + S_INIT_WAIT: begin + busy <= 1'b1; + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_a[10] <= 1'b1; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_INIT_PRE_WAIT; + end + end + S_INIT_PRE_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_INIT_REF; + end + S_INIT_REF: begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + state <= S_INIT_REF_WAIT; + end + S_INIT_REF_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else if (init_ref_cnt < 4'd7) begin + init_ref_cnt <= init_ref_cnt + 1'b1; + state <= S_INIT_REF; + end else begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_ba <= 2'b00; + sdram_a <= mrs_value(BURST_LEN); + wait_cnt <= T_MRD[CNTW-1:0] - 1'b1; + state <= S_INIT_MRS_WAIT; + end + end + S_INIT_MRS_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + busy <= 1'b0; + state <= S_IDLE; + end + end + + S_IDLE: begin + busy <= 1'b0; + if (refresh_timer == 0) begin + busy <= 1'b1; + if (row_open) begin + // JEDEC: all banks must be precharged before + // AUTO REFRESH -- no longer free/automatic + // under the open-row policy (see header). + sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_ba <= open_bank; + sdram_a[10] <= 1'b1; + row_open <= 1'b0; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRE_THEN_REF_WAIT; + end else begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + refresh_timer <= T_REFI[CNTW-1:0]; + state <= S_REFRESH_WAIT; + end + end else if (req || req_pending) begin + busy <= 1'b1; + req_wr_reg <= eff_wr; + req_bank_reg <= eff_bank; + req_row_reg <= eff_row; + req_col_reg <= eff_col; + wdata_reg <= eff_wdata; + wmask_reg <= eff_wmask; + req_pending <= 1'b0; + + if (row_open && eff_bank == open_bank && eff_row == open_row) begin + // ROW HIT: skip ACTIVATE entirely, saves tRCD. + burst_idx <= {BURST_IDXW{1'b0}}; + sdram_cas_n <= 1'b0; + sdram_we_n <= eff_wr ? 1'b0 : 1'b1; + sdram_ba <= eff_bank; + sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b0, {(10-COL_BITS){1'b0}}, eff_col}; // A10=0: no auto-precharge + if (eff_wr) begin + dq_out_en <= 1'b1; + dq_out <= eff_wdata[15:0]; + sdram_dqm <= eff_wmask[1:0]; + state <= S_BURST_WRITE; + end else begin + wait_cnt <= CAS_LATENCY[CNTW-1:0]; + state <= S_CAS_WAIT; + end + end else if (row_open) begin + // ROW MISS, a different row is open: precharge + // it first (paid on-demand, same total cost as + // today's unconditional auto-precharge, just + // deferred until actually needed). + sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0; + sdram_ba <= open_bank; + sdram_a[10] <= 1'b1; + row_open <= 1'b0; + wait_cnt <= T_RP[CNTW-1:0] - 1'b1; + state <= S_PRE_THEN_ACT_WAIT; + end else begin + // no row open at all: activate directly. + sdram_ras_n <= 1'b0; + sdram_ba <= eff_bank; + sdram_a <= eff_row; + wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + state <= S_ACTIVATE_WAIT; + end + end + end + + S_PRE_THEN_REF_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; + wait_cnt <= T_RC_MINUS1(); + refresh_timer <= T_REFI[CNTW-1:0]; + state <= S_REFRESH_WAIT; + end + end + + S_PRE_THEN_ACT_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else begin + sdram_ras_n <= 1'b0; + sdram_ba <= req_bank_reg; + sdram_a <= req_row_reg; + wait_cnt <= T_RCD[CNTW-1:0] - 1'b1; + state <= S_ACTIVATE_WAIT; + end + end + + S_REFRESH_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; + end + + S_ACTIVATE_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + sdram_cas_n <= 1'b0; + sdram_we_n <= req_wr_reg ? 1'b0 : 1'b1; + sdram_ba <= req_bank_reg; + sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b0, {(10-COL_BITS){1'b0}}, req_col_reg}; // A10=0 + burst_idx <= {BURST_IDXW{1'b0}}; + row_open <= 1'b1; + open_bank <= req_bank_reg; + open_row <= req_row_reg; + if (req_wr_reg) begin + dq_out_en <= 1'b1; + dq_out <= wdata_reg[15:0]; + sdram_dqm <= wmask_reg[1:0]; + state <= S_BURST_WRITE; + end else begin + wait_cnt <= CAS_LATENCY[CNTW-1:0]; + state <= S_CAS_WAIT; + end + end + end + + S_CAS_WAIT: begin + if (wait_cnt != 0) begin + wait_cnt <= wait_cnt - 1'b1; + end else begin + rdata[0 +: 16] <= sdram_dq; + if (BURST_LEN == 1) begin + ready <= 1'b1; + state <= S_IDLE; // row stays open, no precharge + end else begin + burst_idx <= burst_idx + 1'b1; + state <= S_BURST_READ; + end + end + end + + S_BURST_READ: begin + rdata[burst_idx*16 +: 16] <= sdram_dq; + if (burst_idx == BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + ready <= 1'b1; + state <= S_IDLE; // row stays open, no precharge + end else begin + burst_idx <= burst_idx + 1'b1; + end + end + + S_BURST_WRITE: begin + if (burst_idx < BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin + burst_idx <= burst_idx + 1'b1; + dq_out_en <= 1'b1; + dq_out <= wdata_reg[(burst_idx+1'b1)*16 +: 16]; + sdram_dqm <= wmask_reg[(burst_idx+1'b1)*2 +: 2]; + end else begin + ready <= 1'b1; + // tWR now paid alone (no longer folded with tRP, + // since we no longer precharge unconditionally -- + // see header). + wait_cnt <= T_WR[CNTW-1:0] - 1'b1; + state <= S_WRITE_RECOVERY_WAIT; + end + end + + S_WRITE_RECOVERY_WAIT: begin + if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1; + else state <= S_IDLE; // row stays open, no precharge + end + + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_unified_backend_cdc.v b/hardware/v2/nms/rtl/sdram_unified_backend_cdc.v new file mode 100644 index 0000000..147b28c --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_unified_backend_cdc.v @@ -0,0 +1,385 @@ +`timescale 1ns/1ps + +// ============================================================ +// NMS STEP19 -- UNIFIED single-SDRAM memory backend. +// +// Replaces BOTH physical memory paths that existed through STEP18 +// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/ +// memory_interface.v + psram_controller.v for activation-fill/result- +// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE +// sdram_controller.v instance (BURST_LEN=8), serving THREE logical +// traffic classes through TWO external ports that exactly match what +// the existing, UNCHANGED consumers already drive: +// +// W port (64-bit): weight_prefetch_engine_wide.v's own real +// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external +// contract to STEP18's sdram_weight_backend_pack128.v (byte +// address in, 64-bit mem_rdata out), and internally reuses that +// module's own validated N_ENTRIES=4 "other half" cache +// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here). +// +// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's +// own activation reads AND every per-slot nms_memory_manager_ +// stream_wide.v's own result writes, via slot_mem_arbiter.v -- +// IDENTICAL external contract to the real V1 psram_controller.v +// port it replaces (word address in, 16-bit mem_wdata/mem_rdata, +// mem_lb_n/mem_ub_n byte-lane write masking). Neither +// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_ +// wide.v needed ANY change -- they already produce a WORD +// address and already drive lb_n/ub_n exactly as the real V1 +// PSRAM controller expected. +// +// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3. +// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed +// AT ALL for this step -- this is a pure memory-side substitution, +// per the governing spec's own explicit instruction. +// +// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE +// write mask (STEP19's own real, tested extension to sdram_ +// controller.v's `wmask` port) -- this lets a single-BYTE result +// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by +// masking out every byte except the one/two the caller actually wants +// written, with NO read-modify-write needed at all (the real SDRAM +// chip itself leaves masked bytes untouched, by JEDEC definition). +// Activation reads need no such trick -- a full 128-bit block is +// fetched and the caller's own requested 16-bit word is extracted +// combinationally from it. +// +// Arbitration: simple, correctness-first 2-way priority (weight +// traffic strongly dominates real measured traffic -- STEP17 showed +// the activation/result path at <=7.2% of all external-memory +// activity -- so W is granted priority when both are pending, AR is +// never starved since W's own real traffic pattern always eventually +// idles between tiles/jobs). Exactly one physical SDRAM transaction +// in flight at a time (matches sdram_controller.v's own inherent +// single-transaction design, STEP18 Part E's own documented, accepted +// scope boundary -- not revisited here). +// ============================================================ +module sdram_unified_backend_cdc #( + parameter ADDR_WIDTH = 26, // byte address width (W port convention) + parameter CLK_FREQ_MHZ = 64, + parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS + // physical SDRAM geometry, forwarded directly to sdram_controller.v + // (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9, + // 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 == + // BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit), + // asserted at elaboration below. + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2 +)( + input wire clk, + input wire rst, + input wire clk_fast, + input wire rst_fast, + + // ---- W: weight fetch (64-bit, byte address, read-only) ---- + input wire w_req, + input wire [ADDR_WIDTH-1:0] w_addr, + output reg [63:0] w_rdata, + output reg w_ready, + + // ---- AR: activation-fill (read) + result-writeback (write), + // 16-bit, WORD address (matches the real V1 psram_controller.v + // convention this port replaces exactly) ---- + input wire ar_req, + input wire ar_wr, + input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful + // (matches slot_mem_arbiter.v's own + // m_addr width convention exactly -- + // that arbiter's real callers only ever + // drive a 22-bit-significant word + // address into an ADDR_WIDTH-wide bus) + input wire [15:0] ar_wdata, + input wire ar_lb_n, + input wire ar_ub_n, + output reg [15:0] ar_rdata, + output reg ar_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin + $display("FATAL sdram_unified_backend: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS); + $finish; + end + + // ============================================================ + // W-port cache (identical logic to sdram_weight_backend_pack128.v + // -- an N_ENTRIES-deep, fully-associative "other half" cache, + // round-robin allocated; safe under any sizing, see that module's + // own header/ERR-0022 for the full rationale, not repeated here) + // ============================================================ + localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES); + reg w_cache_valid [0:W_ENTRIES-1]; + reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1]; + reg [63:0] w_cache_data [0:W_ENTRIES-1]; + reg [WEIDXW-1:0] w_alloc_ptr; + + // ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R: + // worst seed1 total delay 17.909ns, 84% routing, dominant hop + // 2.5-2.8ns): the original RTL used a sequential for-loop that + // overwrites w_hit_idx_c on every match ("last valid+matching entry + // wins"), which Yosys/nextpnr synthesized as a serially-dependent + // cascade of PFUMX/OFX fast-mux primitives -- each entry's result + // depends on the previous one, forcing nextpnr to place the whole + // chain along one physical path with no freedom to shorten it. This + // is the same architectural fix class as ERR-0028 (activation_fill_ + // ctrl's max-tree): replace the serial dependency chain with a flat + // one-hot compare (fully parallel, W_ENTRIES=4 comparators, no + // inter-entry dependency) followed by a single-level priority-encode + // casez, preserving the EXACT original "highest index wins" semantics + // bit-for-bit (verified: original loop always ends on the highest ei + // that matched, since ei counts up without break). + wire [W_ENTRIES-1:0] w_match_oh; + genvar wgi; + generate + for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH + assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr); + end + endgenerate + + reg w_hit_found_c; + reg [WEIDXW-1:0] w_hit_idx_c; + integer ei; + generate + if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT + // real, measured configuration (see ERR-0029) -- flat, + // single-level priority encode over the parallel one-hot + // compare above, no serial inter-entry dependency. + always @(*) begin + w_hit_found_c = |w_match_oh; + casez (w_match_oh) + 4'b1???: w_hit_idx_c = 2'd3; + 4'b01??: w_hit_idx_c = 2'd2; + 4'b001?: w_hit_idx_c = 2'd1; + 4'b0001: w_hit_idx_c = 2'd0; + default: w_hit_idx_c = {WEIDXW{1'b0}}; + endcase + end + end else begin : GEN_WHIT_FALLBACK + // any other W_ENTRIES value: fall back to the original, + // functionally-equivalent (but serially-dependent) scan -- + // not the measured/optimized configuration this project + // actually builds, kept only for parametric safety. + always @(*) begin + w_hit_found_c = 1'b0; + w_hit_idx_c = {WEIDXW{1'b0}}; + for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin + if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin + w_hit_found_c = 1'b1; + w_hit_idx_c = ei[WEIDXW-1:0]; + end + end + end + end + endgenerate + wire w_cache_hit = w_hit_found_c && w_req; + + // ============================================================ + // Shared physical controller, BURST_LEN=8 (128-bit/16-byte real + // SDRAM transactions), reused UNCHANGED from STEP16-18. + // ============================================================ + reg ctrl_req; + reg ctrl_wr; + reg [ADDR_WIDTH-2:0] ctrl_addr; + reg [127:0] ctrl_wdata; + reg [15:0] ctrl_wmask; + wire [127:0] ctrl_rdata; + wire ctrl_ready; + wire ctrl_busy; + + sdram_cdc_bridge #( + .CLK_FREQ_MHZ_FAST(115), .BURST_LEN(8), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk_slow(clk), .rst_slow(rst), + .clk_fast(clk_fast), .rst_fast(rst_fast), + .req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr), + .wdata(ctrl_wdata), .wmask(ctrl_wmask), + .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + localparam S_IDLE = 3'd0, + S_W_WAIT = 3'd1, + S_AR_RD_WAIT = 3'd2, + S_AR_WR_WAIT = 3'd3; + reg [2:0] state; + reg w_pending_upper_half; + reg [ADDR_WIDTH-1:0] w_pending_addr; + reg [2:0] ar_pending_word; + + // ---- req_pending latches (same fix class as sdram_controller.v's + // own ERR-0019/ERR-0020): this backend's own top-level S_IDLE + // arbitration can only START a new transaction when it is + // genuinely idle. A single-cycle w_req/ar_req pulse (this + // project's own established mem_req convention) arriving on a + // cycle this backend happens to be busy servicing the OTHER port + // would otherwise be silently dropped -- the caller has no idea, + // waits forever for a `ready` that never comes. Found the hard way + // (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked + // at 0/256 neurons, jobs_allocated stuck at 12, because the very + // first activation-fill read raced against weight-prefetch traffic + // and was lost exactly this way. Fix: latch EVERY req's own fields + // unconditionally, every cycle, regardless of current state (not + // just from S_IDLE), mirroring sdram_controller.v's own corrected + // fix exactly (ERR-0020: the FIRST attempt only latched from + // S_IDLE, which was still not enough -- latch unconditionally). + reg w_req_pending; + reg [ADDR_WIDTH-1:0] w_req_addr_lat; + reg ar_req_pending; + reg ar_req_wr_lat; + reg [ADDR_WIDTH-1:0] ar_req_addr_lat; + reg [15:0] ar_req_wdata_lat; + reg ar_req_lbn_lat, ar_req_ubn_lat; + + wire w_eff_req = w_req || w_req_pending; + wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat; + wire ar_eff_req = ar_req || ar_req_pending; + wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat; + wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat; + wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat; + wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat; + wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat; + + wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000}; + wire w_eff_addr_is_upper_half = w_eff_addr[3]; + wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000}; + wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0]; + + integer ri; + always @(posedge clk) begin + if (rst) begin + state <= S_IDLE; + for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0; + w_alloc_ptr <= {WEIDXW{1'b0}}; + ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}}; + ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF; + w_ready <= 1'b0; w_rdata <= 64'h0; + ar_ready <= 1'b0; ar_rdata <= 16'h0; + w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}}; + ar_pending_word <= 3'h0; + w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}}; + ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0; + ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0; + ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1; + end else begin + ctrl_req <= 1'b0; + w_ready <= 1'b0; + ar_ready <= 1'b0; + + // latch fresh requests unconditionally, every cycle, + // regardless of state (see req_pending's own comment above) + if (w_req) begin + w_req_addr_lat <= w_addr; + w_req_pending <= 1'b1; + end + if (ar_req) begin + ar_req_wr_lat <= ar_wr; + ar_req_addr_lat <= ar_addr; + ar_req_wdata_lat <= ar_wdata; + ar_req_lbn_lat <= ar_lb_n; + ar_req_ubn_lat <= ar_ub_n; + ar_req_pending <= 1'b1; + end + + case (state) + S_IDLE: begin + // W has priority when both are pending (real + // measured traffic: weight >>> activation+result, + // STEP17 EXP-0045 -- AR is never starved since W's + // own real access pattern idles between tiles). + if (w_cache_hit) begin + // fully serviced THIS cycle -- must also cancel + // the unconditional latch above, which just set + // w_req_pending<=1 for this SAME w_req pulse + // (real bug found via full regression, EXP-0048 + // /ERR-0023: without this the latch survives + // uncontested, and next cycle w_eff_req reads + // true from STALE w_req_pending/w_req_addr_lat, + // issuing a bogus extra fetch that shifts every + // subsequent response by one). + w_rdata <= w_cache_data[w_hit_idx_c]; + w_ready <= 1'b1; + w_cache_valid[w_hit_idx_c] <= 1'b0; + w_req_pending <= 1'b0; + end else if (w_eff_req) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= w_eff_aligned_word_addr; + ctrl_wmask <= 16'h0000; + w_pending_upper_half <= w_eff_addr_is_upper_half; + w_pending_addr <= w_eff_addr; + w_req_pending <= 1'b0; + state <= S_W_WAIT; + end else if (ar_eff_req && !ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= ar_eff_block_base; + ctrl_wmask <= 16'h0000; + ar_pending_word <= ar_eff_word_in_blk; + ar_req_pending <= 1'b0; + state <= S_AR_RD_WAIT; + end else if (ar_eff_req && ar_eff_wr) begin + // mask every word except the target one; within + // the target word, pass ar_lb_n/ar_ub_n through + // directly (same active-low "write this byte" + // polarity as real SDRAM DQM: lb_n=0 -> DQM=0 + // -> byte written; lb_n=1 -> DQM=1 -> masked). + ctrl_req <= 1'b1; + ctrl_wr <= 1'b1; + ctrl_addr <= ar_eff_block_base; + ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter + ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2)); + ar_req_pending <= 1'b0; + state <= S_AR_WR_WAIT; + end + end + S_W_WAIT: begin + if (ctrl_ready) begin + if (w_pending_upper_half) begin + w_rdata <= ctrl_rdata[127:64]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end else begin + w_rdata <= ctrl_rdata[63:0]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end + w_cache_valid[w_alloc_ptr] <= 1'b1; + w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1; + w_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_RD_WAIT: begin + if (ctrl_ready) begin + ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16]; + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_WR_WAIT: begin + if (ctrl_ready) begin + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_unified_backend_combined.v b/hardware/v2/nms/rtl/sdram_unified_backend_combined.v new file mode 100644 index 0000000..11bfac4 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_unified_backend_combined.v @@ -0,0 +1,404 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXP-0055 -- combined fork of sdram_unified_backend.v (STEP19) -- the +// ONLY changes are: (1) two new clk_fast/rst_fast ports, (2) the +// physical sdram_controller.v instance replaced by sdram_cdc_bridge_ +// openrow.v (EXP-0053 CDC + EXP-0054 page-hit, composed together). +// W-port cache, arbitration, and the W/AR top-level FSM are ALL +// byte-for-byte unchanged, per the same fork discipline sdram_ +// unified_backend_pipelined.v (EXP-0052) already established. See +// hardware/v2/logs/experiments.log (search "EXP-0055") for why this +// fork exists and its own isolated (tb_sdram_cdc_bridge_openrow.v) +// measured number before this integration step. +// +// Everything below this point is STEP19's own original header, +// preserved for the W/AR port contract description (still accurate -- +// only the physical controller behind ctrl_req/.../ctrl_ready changed): +// +// NMS STEP19 -- UNIFIED single-SDRAM memory backend. +// +// Replaces BOTH physical memory paths that existed through STEP18 +// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/ +// memory_interface.v + psram_controller.v for activation-fill/result- +// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE +// sdram_controller.v instance (BURST_LEN=8), serving THREE logical +// traffic classes through TWO external ports that exactly match what +// the existing, UNCHANGED consumers already drive: +// +// W port (64-bit): weight_prefetch_engine_wide.v's own real +// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external +// contract to STEP18's sdram_weight_backend_pack128.v (byte +// address in, 64-bit mem_rdata out), and internally reuses that +// module's own validated N_ENTRIES=4 "other half" cache +// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here). +// +// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's +// own activation reads AND every per-slot nms_memory_manager_ +// stream_wide.v's own result writes, via slot_mem_arbiter.v -- +// IDENTICAL external contract to the real V1 psram_controller.v +// port it replaces (word address in, 16-bit mem_wdata/mem_rdata, +// mem_lb_n/mem_ub_n byte-lane write masking). Neither +// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_ +// wide.v needed ANY change -- they already produce a WORD +// address and already drive lb_n/ub_n exactly as the real V1 +// PSRAM controller expected. +// +// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3. +// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed +// AT ALL for this step -- this is a pure memory-side substitution, +// per the governing spec's own explicit instruction. +// +// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE +// write mask (STEP19's own real, tested extension to sdram_ +// controller.v's `wmask` port) -- this lets a single-BYTE result +// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by +// masking out every byte except the one/two the caller actually wants +// written, with NO read-modify-write needed at all (the real SDRAM +// chip itself leaves masked bytes untouched, by JEDEC definition). +// Activation reads need no such trick -- a full 128-bit block is +// fetched and the caller's own requested 16-bit word is extracted +// combinationally from it. +// +// Arbitration: simple, correctness-first 2-way priority (weight +// traffic strongly dominates real measured traffic -- STEP17 showed +// the activation/result path at <=7.2% of all external-memory +// activity -- so W is granted priority when both are pending, AR is +// never starved since W's own real traffic pattern always eventually +// idles between tiles/jobs). Exactly one physical SDRAM transaction +// in flight at a time (matches sdram_controller.v's own inherent +// single-transaction design, STEP18 Part E's own documented, accepted +// scope boundary -- not revisited here). +// ============================================================ +module sdram_unified_backend_combined #( + parameter ADDR_WIDTH = 26, // byte address width (W port convention) + parameter CLK_FREQ_MHZ = 64, + parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS + // physical SDRAM geometry, forwarded directly to sdram_controller.v + // (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9, + // 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 == + // BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit), + // asserted at elaboration below. + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2 +)( + input wire clk, + input wire rst, + // ---- EXP-0055: fast SDRAM clock domain (115.2MHz-class), see + // sdram_cdc_bridge_openrow.v -- everything else in this module + // (W/AR ports, W-cache, top-level FSM) stays on clk/rst exactly + // as before ---- + input wire clk_fast, + input wire rst_fast, + + // ---- W: weight fetch (64-bit, byte address, read-only) ---- + input wire w_req, + input wire [ADDR_WIDTH-1:0] w_addr, + output reg [63:0] w_rdata, + output reg w_ready, + + // ---- AR: activation-fill (read) + result-writeback (write), + // 16-bit, WORD address (matches the real V1 psram_controller.v + // convention this port replaces exactly) ---- + input wire ar_req, + input wire ar_wr, + input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful + // (matches slot_mem_arbiter.v's own + // m_addr width convention exactly -- + // that arbiter's real callers only ever + // drive a 22-bit-significant word + // address into an ADDR_WIDTH-wide bus) + input wire [15:0] ar_wdata, + input wire ar_lb_n, + input wire ar_ub_n, + output reg [15:0] ar_rdata, + output reg ar_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin + $display("FATAL sdram_unified_backend_combined: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS); + $finish; + end + + // ============================================================ + // W-port cache (identical logic to sdram_weight_backend_pack128.v + // -- an N_ENTRIES-deep, fully-associative "other half" cache, + // round-robin allocated; safe under any sizing, see that module's + // own header/ERR-0022 for the full rationale, not repeated here) + // ============================================================ + localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES); + reg w_cache_valid [0:W_ENTRIES-1]; + reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1]; + reg [63:0] w_cache_data [0:W_ENTRIES-1]; + reg [WEIDXW-1:0] w_alloc_ptr; + + // ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R: + // worst seed1 total delay 17.909ns, 84% routing, dominant hop + // 2.5-2.8ns): the original RTL used a sequential for-loop that + // overwrites w_hit_idx_c on every match ("last valid+matching entry + // wins"), which Yosys/nextpnr synthesized as a serially-dependent + // cascade of PFUMX/OFX fast-mux primitives -- each entry's result + // depends on the previous one, forcing nextpnr to place the whole + // chain along one physical path with no freedom to shorten it. This + // is the same architectural fix class as ERR-0028 (activation_fill_ + // ctrl's max-tree): replace the serial dependency chain with a flat + // one-hot compare (fully parallel, W_ENTRIES=4 comparators, no + // inter-entry dependency) followed by a single-level priority-encode + // casez, preserving the EXACT original "highest index wins" semantics + // bit-for-bit (verified: original loop always ends on the highest ei + // that matched, since ei counts up without break). + wire [W_ENTRIES-1:0] w_match_oh; + genvar wgi; + generate + for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH + assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr); + end + endgenerate + + reg w_hit_found_c; + reg [WEIDXW-1:0] w_hit_idx_c; + integer ei; + generate + if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT + // real, measured configuration (see ERR-0029) -- flat, + // single-level priority encode over the parallel one-hot + // compare above, no serial inter-entry dependency. + always @(*) begin + w_hit_found_c = |w_match_oh; + casez (w_match_oh) + 4'b1???: w_hit_idx_c = 2'd3; + 4'b01??: w_hit_idx_c = 2'd2; + 4'b001?: w_hit_idx_c = 2'd1; + 4'b0001: w_hit_idx_c = 2'd0; + default: w_hit_idx_c = {WEIDXW{1'b0}}; + endcase + end + end else begin : GEN_WHIT_FALLBACK + // any other W_ENTRIES value: fall back to the original, + // functionally-equivalent (but serially-dependent) scan -- + // not the measured/optimized configuration this project + // actually builds, kept only for parametric safety. + always @(*) begin + w_hit_found_c = 1'b0; + w_hit_idx_c = {WEIDXW{1'b0}}; + for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin + if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin + w_hit_found_c = 1'b1; + w_hit_idx_c = ei[WEIDXW-1:0]; + end + end + end + end + endgenerate + wire w_cache_hit = w_hit_found_c && w_req; + + // ============================================================ + // Shared physical controller, BURST_LEN=8 (128-bit/16-byte real + // SDRAM transactions), reused UNCHANGED from STEP16-18. + // ============================================================ + reg ctrl_req; + reg ctrl_wr; + reg [ADDR_WIDTH-2:0] ctrl_addr; + reg [127:0] ctrl_wdata; + reg [15:0] ctrl_wmask; + wire [127:0] ctrl_rdata; + wire ctrl_ready; + wire ctrl_busy; + + sdram_cdc_bridge_openrow #( + .CLK_FREQ_MHZ_FAST(115), .BURST_LEN(8), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk_slow(clk), .rst_slow(rst), + .clk_fast(clk_fast), .rst_fast(rst_fast), + .req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr), + .wdata(ctrl_wdata), .wmask(ctrl_wmask), + .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + localparam S_IDLE = 3'd0, + S_W_WAIT = 3'd1, + S_AR_RD_WAIT = 3'd2, + S_AR_WR_WAIT = 3'd3; + reg [2:0] state; + reg w_pending_upper_half; + reg [ADDR_WIDTH-1:0] w_pending_addr; + reg [2:0] ar_pending_word; + + // ---- req_pending latches (same fix class as sdram_controller.v's + // own ERR-0019/ERR-0020): this backend's own top-level S_IDLE + // arbitration can only START a new transaction when it is + // genuinely idle. A single-cycle w_req/ar_req pulse (this + // project's own established mem_req convention) arriving on a + // cycle this backend happens to be busy servicing the OTHER port + // would otherwise be silently dropped -- the caller has no idea, + // waits forever for a `ready` that never comes. Found the hard way + // (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked + // at 0/256 neurons, jobs_allocated stuck at 12, because the very + // first activation-fill read raced against weight-prefetch traffic + // and was lost exactly this way. Fix: latch EVERY req's own fields + // unconditionally, every cycle, regardless of current state (not + // just from S_IDLE), mirroring sdram_controller.v's own corrected + // fix exactly (ERR-0020: the FIRST attempt only latched from + // S_IDLE, which was still not enough -- latch unconditionally). + reg w_req_pending; + reg [ADDR_WIDTH-1:0] w_req_addr_lat; + reg ar_req_pending; + reg ar_req_wr_lat; + reg [ADDR_WIDTH-1:0] ar_req_addr_lat; + reg [15:0] ar_req_wdata_lat; + reg ar_req_lbn_lat, ar_req_ubn_lat; + + wire w_eff_req = w_req || w_req_pending; + wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat; + wire ar_eff_req = ar_req || ar_req_pending; + wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat; + wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat; + wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat; + wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat; + wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat; + + wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000}; + wire w_eff_addr_is_upper_half = w_eff_addr[3]; + wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000}; + wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0]; + + integer ri; + always @(posedge clk) begin + if (rst) begin + state <= S_IDLE; + for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0; + w_alloc_ptr <= {WEIDXW{1'b0}}; + ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}}; + ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF; + w_ready <= 1'b0; w_rdata <= 64'h0; + ar_ready <= 1'b0; ar_rdata <= 16'h0; + w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}}; + ar_pending_word <= 3'h0; + w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}}; + ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0; + ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0; + ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1; + end else begin + ctrl_req <= 1'b0; + w_ready <= 1'b0; + ar_ready <= 1'b0; + + // latch fresh requests unconditionally, every cycle, + // regardless of state (see req_pending's own comment above) + if (w_req) begin + w_req_addr_lat <= w_addr; + w_req_pending <= 1'b1; + end + if (ar_req) begin + ar_req_wr_lat <= ar_wr; + ar_req_addr_lat <= ar_addr; + ar_req_wdata_lat <= ar_wdata; + ar_req_lbn_lat <= ar_lb_n; + ar_req_ubn_lat <= ar_ub_n; + ar_req_pending <= 1'b1; + end + + case (state) + S_IDLE: begin + // W has priority when both are pending (real + // measured traffic: weight >>> activation+result, + // STEP17 EXP-0045 -- AR is never starved since W's + // own real access pattern idles between tiles). + if (w_cache_hit) begin + // fully serviced THIS cycle -- must also cancel + // the unconditional latch above, which just set + // w_req_pending<=1 for this SAME w_req pulse + // (real bug found via full regression, EXP-0048 + // /ERR-0023: without this the latch survives + // uncontested, and next cycle w_eff_req reads + // true from STALE w_req_pending/w_req_addr_lat, + // issuing a bogus extra fetch that shifts every + // subsequent response by one). + w_rdata <= w_cache_data[w_hit_idx_c]; + w_ready <= 1'b1; + w_cache_valid[w_hit_idx_c] <= 1'b0; + w_req_pending <= 1'b0; + end else if (w_eff_req) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= w_eff_aligned_word_addr; + ctrl_wmask <= 16'h0000; + w_pending_upper_half <= w_eff_addr_is_upper_half; + w_pending_addr <= w_eff_addr; + w_req_pending <= 1'b0; + state <= S_W_WAIT; + end else if (ar_eff_req && !ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= ar_eff_block_base; + ctrl_wmask <= 16'h0000; + ar_pending_word <= ar_eff_word_in_blk; + ar_req_pending <= 1'b0; + state <= S_AR_RD_WAIT; + end else if (ar_eff_req && ar_eff_wr) begin + // mask every word except the target one; within + // the target word, pass ar_lb_n/ar_ub_n through + // directly (same active-low "write this byte" + // polarity as real SDRAM DQM: lb_n=0 -> DQM=0 + // -> byte written; lb_n=1 -> DQM=1 -> masked). + ctrl_req <= 1'b1; + ctrl_wr <= 1'b1; + ctrl_addr <= ar_eff_block_base; + ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter + ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2)); + ar_req_pending <= 1'b0; + state <= S_AR_WR_WAIT; + end + end + S_W_WAIT: begin + if (ctrl_ready) begin + if (w_pending_upper_half) begin + w_rdata <= ctrl_rdata[127:64]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end else begin + w_rdata <= ctrl_rdata[63:0]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end + w_cache_valid[w_alloc_ptr] <= 1'b1; + w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1; + w_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_RD_WAIT: begin + if (ctrl_ready) begin + ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16]; + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_WR_WAIT: begin + if (ctrl_ready) begin + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/rtl/sdram_unified_backend_openrow.v b/hardware/v2/nms/rtl/sdram_unified_backend_openrow.v new file mode 100644 index 0000000..0cd3c11 --- /dev/null +++ b/hardware/v2/nms/rtl/sdram_unified_backend_openrow.v @@ -0,0 +1,382 @@ +`timescale 1ns/1ps + +// ============================================================ +// NMS STEP19 -- UNIFIED single-SDRAM memory backend. +// +// Replaces BOTH physical memory paths that existed through STEP18 +// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/ +// memory_interface.v + psram_controller.v for activation-fill/result- +// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE +// sdram_controller.v instance (BURST_LEN=8), serving THREE logical +// traffic classes through TWO external ports that exactly match what +// the existing, UNCHANGED consumers already drive: +// +// W port (64-bit): weight_prefetch_engine_wide.v's own real +// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external +// contract to STEP18's sdram_weight_backend_pack128.v (byte +// address in, 64-bit mem_rdata out), and internally reuses that +// module's own validated N_ENTRIES=4 "other half" cache +// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here). +// +// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's +// own activation reads AND every per-slot nms_memory_manager_ +// stream_wide.v's own result writes, via slot_mem_arbiter.v -- +// IDENTICAL external contract to the real V1 psram_controller.v +// port it replaces (word address in, 16-bit mem_wdata/mem_rdata, +// mem_lb_n/mem_ub_n byte-lane write masking). Neither +// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_ +// wide.v needed ANY change -- they already produce a WORD +// address and already drive lb_n/ub_n exactly as the real V1 +// PSRAM controller expected. +// +// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3. +// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed +// AT ALL for this step -- this is a pure memory-side substitution, +// per the governing spec's own explicit instruction. +// +// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE +// write mask (STEP19's own real, tested extension to sdram_ +// controller.v's `wmask` port) -- this lets a single-BYTE result +// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by +// masking out every byte except the one/two the caller actually wants +// written, with NO read-modify-write needed at all (the real SDRAM +// chip itself leaves masked bytes untouched, by JEDEC definition). +// Activation reads need no such trick -- a full 128-bit block is +// fetched and the caller's own requested 16-bit word is extracted +// combinationally from it. +// +// Arbitration: simple, correctness-first 2-way priority (weight +// traffic strongly dominates real measured traffic -- STEP17 showed +// the activation/result path at <=7.2% of all external-memory +// activity -- so W is granted priority when both are pending, AR is +// never starved since W's own real traffic pattern always eventually +// idles between tiles/jobs). Exactly one physical SDRAM transaction +// in flight at a time (matches sdram_controller.v's own inherent +// single-transaction design, STEP18 Part E's own documented, accepted +// scope boundary -- not revisited here). +// ============================================================ +module sdram_unified_backend_openrow #( + parameter ADDR_WIDTH = 26, // byte address width (W port convention) + parameter CLK_FREQ_MHZ = 64, + parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS + // physical SDRAM geometry, forwarded directly to sdram_controller.v + // (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9, + // 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 == + // BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit), + // asserted at elaboration below. + parameter ROW_BITS = 13, + parameter COL_BITS = 10, + parameter BANK_BITS = 2 +)( + input wire clk, + input wire rst, + + // ---- W: weight fetch (64-bit, byte address, read-only) ---- + input wire w_req, + input wire [ADDR_WIDTH-1:0] w_addr, + output reg [63:0] w_rdata, + output reg w_ready, + + // ---- AR: activation-fill (read) + result-writeback (write), + // 16-bit, WORD address (matches the real V1 psram_controller.v + // convention this port replaces exactly) ---- + input wire ar_req, + input wire ar_wr, + input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful + // (matches slot_mem_arbiter.v's own + // m_addr width convention exactly -- + // that arbiter's real callers only ever + // drive a 22-bit-significant word + // address into an ADDR_WIDTH-wide bus) + input wire [15:0] ar_wdata, + input wire ar_lb_n, + input wire ar_ub_n, + output reg [15:0] ar_rdata, + output reg ar_ready, + + output wire sdram_cke, + output wire sdram_cs_n, + output wire sdram_ras_n, + output wire sdram_cas_n, + output wire sdram_we_n, + output wire [BANK_BITS-1:0] sdram_ba, + output wire [ROW_BITS-1:0] sdram_a, + inout wire [15:0] sdram_dq, + output wire [1:0] sdram_dqm +); + + initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin + $display("FATAL sdram_unified_backend: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1", + ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS); + $finish; + end + + // ============================================================ + // W-port cache (identical logic to sdram_weight_backend_pack128.v + // -- an N_ENTRIES-deep, fully-associative "other half" cache, + // round-robin allocated; safe under any sizing, see that module's + // own header/ERR-0022 for the full rationale, not repeated here) + // ============================================================ + localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES); + reg w_cache_valid [0:W_ENTRIES-1]; + reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1]; + reg [63:0] w_cache_data [0:W_ENTRIES-1]; + reg [WEIDXW-1:0] w_alloc_ptr; + + // ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R: + // worst seed1 total delay 17.909ns, 84% routing, dominant hop + // 2.5-2.8ns): the original RTL used a sequential for-loop that + // overwrites w_hit_idx_c on every match ("last valid+matching entry + // wins"), which Yosys/nextpnr synthesized as a serially-dependent + // cascade of PFUMX/OFX fast-mux primitives -- each entry's result + // depends on the previous one, forcing nextpnr to place the whole + // chain along one physical path with no freedom to shorten it. This + // is the same architectural fix class as ERR-0028 (activation_fill_ + // ctrl's max-tree): replace the serial dependency chain with a flat + // one-hot compare (fully parallel, W_ENTRIES=4 comparators, no + // inter-entry dependency) followed by a single-level priority-encode + // casez, preserving the EXACT original "highest index wins" semantics + // bit-for-bit (verified: original loop always ends on the highest ei + // that matched, since ei counts up without break). + wire [W_ENTRIES-1:0] w_match_oh; + genvar wgi; + generate + for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH + assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr); + end + endgenerate + + reg w_hit_found_c; + reg [WEIDXW-1:0] w_hit_idx_c; + integer ei; + generate + if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT + // real, measured configuration (see ERR-0029) -- flat, + // single-level priority encode over the parallel one-hot + // compare above, no serial inter-entry dependency. + always @(*) begin + w_hit_found_c = |w_match_oh; + casez (w_match_oh) + 4'b1???: w_hit_idx_c = 2'd3; + 4'b01??: w_hit_idx_c = 2'd2; + 4'b001?: w_hit_idx_c = 2'd1; + 4'b0001: w_hit_idx_c = 2'd0; + default: w_hit_idx_c = {WEIDXW{1'b0}}; + endcase + end + end else begin : GEN_WHIT_FALLBACK + // any other W_ENTRIES value: fall back to the original, + // functionally-equivalent (but serially-dependent) scan -- + // not the measured/optimized configuration this project + // actually builds, kept only for parametric safety. + always @(*) begin + w_hit_found_c = 1'b0; + w_hit_idx_c = {WEIDXW{1'b0}}; + for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin + if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin + w_hit_found_c = 1'b1; + w_hit_idx_c = ei[WEIDXW-1:0]; + end + end + end + end + endgenerate + wire w_cache_hit = w_hit_found_c && w_req; + + // ============================================================ + // Shared physical controller, BURST_LEN=8 (128-bit/16-byte real + // SDRAM transactions), reused UNCHANGED from STEP16-18. + // ============================================================ + reg ctrl_req; + reg ctrl_wr; + reg [ADDR_WIDTH-2:0] ctrl_addr; + reg [127:0] ctrl_wdata; + reg [15:0] ctrl_wmask; + wire [127:0] ctrl_rdata; + wire ctrl_ready; + wire ctrl_busy; + + sdram_controller_openrow #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) u_sdram_ctrl ( + .clk(clk), .rst(rst), + .req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr), + .wdata(ctrl_wdata), .wmask(ctrl_wmask), + .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + localparam S_IDLE = 3'd0, + S_W_WAIT = 3'd1, + S_AR_RD_WAIT = 3'd2, + S_AR_WR_WAIT = 3'd3; + reg [2:0] state; + reg w_pending_upper_half; + reg [ADDR_WIDTH-1:0] w_pending_addr; + reg [2:0] ar_pending_word; + + // ---- req_pending latches (same fix class as sdram_controller.v's + // own ERR-0019/ERR-0020): this backend's own top-level S_IDLE + // arbitration can only START a new transaction when it is + // genuinely idle. A single-cycle w_req/ar_req pulse (this + // project's own established mem_req convention) arriving on a + // cycle this backend happens to be busy servicing the OTHER port + // would otherwise be silently dropped -- the caller has no idea, + // waits forever for a `ready` that never comes. Found the hard way + // (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked + // at 0/256 neurons, jobs_allocated stuck at 12, because the very + // first activation-fill read raced against weight-prefetch traffic + // and was lost exactly this way. Fix: latch EVERY req's own fields + // unconditionally, every cycle, regardless of current state (not + // just from S_IDLE), mirroring sdram_controller.v's own corrected + // fix exactly (ERR-0020: the FIRST attempt only latched from + // S_IDLE, which was still not enough -- latch unconditionally). + reg w_req_pending; + reg [ADDR_WIDTH-1:0] w_req_addr_lat; + reg ar_req_pending; + reg ar_req_wr_lat; + reg [ADDR_WIDTH-1:0] ar_req_addr_lat; + reg [15:0] ar_req_wdata_lat; + reg ar_req_lbn_lat, ar_req_ubn_lat; + + wire w_eff_req = w_req || w_req_pending; + wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat; + wire ar_eff_req = ar_req || ar_req_pending; + wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat; + wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat; + wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat; + wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat; + wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat; + + wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000}; + wire w_eff_addr_is_upper_half = w_eff_addr[3]; + wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000}; + wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0]; + + integer ri; + always @(posedge clk) begin + if (rst) begin + state <= S_IDLE; + for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0; + w_alloc_ptr <= {WEIDXW{1'b0}}; + ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}}; + ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF; + w_ready <= 1'b0; w_rdata <= 64'h0; + ar_ready <= 1'b0; ar_rdata <= 16'h0; + w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}}; + ar_pending_word <= 3'h0; + w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}}; + ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0; + ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0; + ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1; + end else begin + ctrl_req <= 1'b0; + w_ready <= 1'b0; + ar_ready <= 1'b0; + + // latch fresh requests unconditionally, every cycle, + // regardless of state (see req_pending's own comment above) + if (w_req) begin + w_req_addr_lat <= w_addr; + w_req_pending <= 1'b1; + end + if (ar_req) begin + ar_req_wr_lat <= ar_wr; + ar_req_addr_lat <= ar_addr; + ar_req_wdata_lat <= ar_wdata; + ar_req_lbn_lat <= ar_lb_n; + ar_req_ubn_lat <= ar_ub_n; + ar_req_pending <= 1'b1; + end + + case (state) + S_IDLE: begin + // W has priority when both are pending (real + // measured traffic: weight >>> activation+result, + // STEP17 EXP-0045 -- AR is never starved since W's + // own real access pattern idles between tiles). + if (w_cache_hit) begin + // fully serviced THIS cycle -- must also cancel + // the unconditional latch above, which just set + // w_req_pending<=1 for this SAME w_req pulse + // (real bug found via full regression, EXP-0048 + // /ERR-0023: without this the latch survives + // uncontested, and next cycle w_eff_req reads + // true from STALE w_req_pending/w_req_addr_lat, + // issuing a bogus extra fetch that shifts every + // subsequent response by one). + w_rdata <= w_cache_data[w_hit_idx_c]; + w_ready <= 1'b1; + w_cache_valid[w_hit_idx_c] <= 1'b0; + w_req_pending <= 1'b0; + end else if (w_eff_req) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= w_eff_aligned_word_addr; + ctrl_wmask <= 16'h0000; + w_pending_upper_half <= w_eff_addr_is_upper_half; + w_pending_addr <= w_eff_addr; + w_req_pending <= 1'b0; + state <= S_W_WAIT; + end else if (ar_eff_req && !ar_eff_wr) begin + ctrl_req <= 1'b1; + ctrl_wr <= 1'b0; + ctrl_addr <= ar_eff_block_base; + ctrl_wmask <= 16'h0000; + ar_pending_word <= ar_eff_word_in_blk; + ar_req_pending <= 1'b0; + state <= S_AR_RD_WAIT; + end else if (ar_eff_req && ar_eff_wr) begin + // mask every word except the target one; within + // the target word, pass ar_lb_n/ar_ub_n through + // directly (same active-low "write this byte" + // polarity as real SDRAM DQM: lb_n=0 -> DQM=0 + // -> byte written; lb_n=1 -> DQM=1 -> masked). + ctrl_req <= 1'b1; + ctrl_wr <= 1'b1; + ctrl_addr <= ar_eff_block_base; + ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter + ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2)); + ar_req_pending <= 1'b0; + state <= S_AR_WR_WAIT; + end + end + S_W_WAIT: begin + if (ctrl_ready) begin + if (w_pending_upper_half) begin + w_rdata <= ctrl_rdata[127:64]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end else begin + w_rdata <= ctrl_rdata[63:0]; + w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64]; + w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8}; + end + w_cache_valid[w_alloc_ptr] <= 1'b1; + w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1; + w_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_RD_WAIT: begin + if (ctrl_ready) begin + ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16]; + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + S_AR_WR_WAIT: begin + if (ctrl_ready) begin + ar_ready <= 1'b1; + state <= S_IDLE; + end + end + default: state <= S_IDLE; + endcase + end + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_nms_dstress_sdram_cdc.v b/hardware/v2/nms/sim/tb_nms_dstress_sdram_cdc.v new file mode 100644 index 0000000..9f9f70e --- /dev/null +++ b/hardware/v2/nms/sim/tb_nms_dstress_sdram_cdc.v @@ -0,0 +1,918 @@ +`timescale 1ns/1ps + +// ================================================================ +// EXP-0055 -- Phase B integration: real D-Stress run through +// nms_neural_multiprocessor_sdram_cdc.v (CDC-to-115.2MHz + +// page-hit SDRAM backend, EXP-0053+EXP-0054 composed together). +// Forked from tb_nms_dstress_sdram_unified.v -- the ONLY changes are: +// (1) a second free-running clk_fast (115.2MHz, real value; matches +// what EXP-0053 separately confirmed is derivable from the real +// board's own PLL VCO relative to its 64MHz clk_sys -- this specific +// testbench family's own convention runs the SLOW/system domain at +// 80MHz, not 64MHz, to stay directly comparable to EXP-0051/0052's +// own historical baseline numbers measured at that same 80MHz; 115.2 +// MHz is reused here as an absolute value, not re-derived from an +// 80MHz-rooted PLL) with its own rst_fast, (2) the DUT swapped for +// nms_neural_multiprocessor_sdram_cdc.v with clk_fast/rst_fast +// wired through, (3) sdram_model.v moved onto clk_fast (the real +// physical SDRAM pins now toggle in the fast domain, inside the +// bridge). No other change -- workloads, golden model, cycle +// accounting, and backdoor peek/poke helpers are IDENTICAL: sdram_ +// controller_openrow.v's address decomposition (bank/row/col bit +// ranges) is byte-for-byte the same as the original sdram_ +// controller.v's (unlike EXP-0052's re-sliced pipelined variant), so +// no backdoor-helper rework was needed here. +// +// Everything below this point is the ORIGINAL testbench's own header, +// preserved as-is: +// +// FPGA-Neural V2 -- Final Benchmark Campaign (post-M10, real +// end-to-end characterization, docs/v2-description.md §22/§30/§32) +// +// One testbench, compiled once per N_SLOTS configuration (N_SLOTS_CFG +// parameter, overridden at Verilator invocation via -GN_SLOTS_CFG=N), +// running SIX representative workloads back-to-back through the REAL +// neural_multiprocessor.v (M8: dataflow_core + slot_mem_arbiter + the +// real, unmodified V1 PSRAM chain), with: +// - a software "golden" model replicating neural_processor.v's exact +// integer math (sum(x*w) over all tiles, ReLU + INT8 saturate -- +// dataflow_core.v hardcodes bias=0/ACT_RELU for every job, so the +// golden model only needs to replicate that one path) +// - bit-exact verification of EVERY neuron's real result against +// that golden model (peek_byte from the real psram_model backing +// array -- an oracle independent of the RTL under test) +// - real cycle-accounting instrumentation (testbench-only, no RTL +// touched): per-slot busy/idle cycles, shared PSRAM port busy/idle +// cycles, REAL tiles delivered per slot (operand_valid&& +// operand_ready pulses -- one pulse = one whole P_IN-wide tile +// consumed by neural_processor, NOT one byte), director/dependency +// bookkeeping (jobs allocated/completed, ready-queue occupancy, +// WAITING/READY/DISPATCHED node counts, producer-done wakeups) +// +// Workloads (node_id ranges are disjoint across all six so the WHOLE +// campaign runs in ONE continuous simulation -- only ONE real PSRAM +// power-up wait, no reset between phases, closer to real sustained +// operation than resetting between every workload): +// A) Small -- 16 independent neurons, 8 inputs each +// B) Medium -- 64 independent neurons, 32 inputs each +// C) Large -- 128 independent neurons, 128 inputs each +// D) Stress -- 256 independent neurons, 128 inputs each +// E) Multilayer -- 8 layer-1 neurons (RANDOM data, logged seed) feed +// a shared 8-byte hidden vector; 2 layer-2 neurons +// consume that vector (real cross-node data +// forwarding through real PSRAM, real dependency +// wake-up, "shared producer/multiple consumers") +// F) DAG -- 6-node diamond+fan-in graph (A,B independent; C +// dep on A; D dep on B; E dep on BOTH C and D +// [2-hop transitive wake-up]; F dep on A,B,C [mixed +// direct+1-hop, 3 producers]) +// +// All workloads A-D use a REALISTIC dense-layer shape: one shared +// input activation vector, N independent weight vectors (one per +// neuron) -- exactly how a real fully-connected layer's neurons share +// their layer's input. This is not an isolated synthetic microbench. +// +// Verified with Verilator (decisions.log DEC-0004). +// ================================================================ + +// ================================================================ +// STEP11 variant: identical D-Stress workload/golden-model/correctness +// criteria as tb_nms_dstress.v (STEP9's own official benchmark), but +// instantiating nms_neural_multiprocessor_pf (REAL weight prefetch +// engine, weight_prefetch_engine.v) instead of the baseline +// nms_neural_multiprocessor.v, with an added PFD_CFG (PREFETCH_DISTANCE) +// parameter, plus NEW instrumentation (testbench-only, no RTL touched) +// for the two STEP11-mandated metrics that cannot be derived from the +// STEP9 instrumentation alone: +// weight_stall_cycles = cycles a slot is otherwise ready to +// present a tile (activation resident, +// in bounds) but blocked purely because +// tile_idx >= wgt_ready_count +// prefetch_effectiveness = tiles consumed with ZERO such +// weight-blocking cycles beforehand +// (i.e. the weight was ALREADY resident +// the moment the tile became eligible) +// / total tiles consumed +// per STEP11's own explicit metric definitions. +// ================================================================ +module tb #( + parameter N_SLOTS_CFG = 2, + parameter PFD_CFG = 8 +); + + localparam ADDR_WIDTH = 26; // AS4C32M16SA: 25-bit word address + 1 byte-select bit + localparam DATA_WIDTH = 8; + localparam P_IN = 8; + localparam ACC_WIDTH = 32; + // N_NODES must exceed the HIGHEST node_id used by ANY workload + // (node_base + count - 1) -- workload D's own range alone + // (node_base=400, 256 neurons) reaches id 655. An earlier draft + // used N_NODES=512: D's ids silently wrapped (9-bit truncation) + // past id 511, colliding with workload A's already-DISPATCHED + // node 0 (dependency_manager never reclaims dispatched node slots, + // DEC-0008) and deadlocking register_node's reg_ready wait + // forever. A real consequence of DEC-0008's design choice, not an + // RTL bug -- fixed here by sizing N_NODES generously above the + // real id range used below (see decisions.log DEC-0008 and the + // final benchmark report's Limitations section). + localparam N_NODES = 1024; + localparam MAX_DEPS = 8; + localparam QUEUE_DEPTH = 8; + localparam NODE_IDW = $clog2(N_NODES); + localparam CLK_PERIOD = 12.5; // 80 MHz, matches psram_controller's CLK_FREQ_MHZ + + reg clk, rst; + initial begin clk = 1'b0; forever #(CLK_PERIOD/2.0) clk = ~clk; end + + // EXP-0055: second, independent fast clock for the SDRAM domain + // (real 115.2MHz value, non-integer ratio vs the 80MHz slow + // domain -- deliberately not a lucky-alignment case, see + // sdram_cdc_bridge.v's own header on why this is the harder test). + localparam real CLK_FREQ_FAST_REAL = 115.2; + localparam real FAST_PERIOD_NS = 1000.0/CLK_FREQ_FAST_REAL; + reg clk_fast, rst_fast; + initial begin clk_fast = 1'b0; forever #(FAST_PERIOD_NS/2.0) clk_fast = ~clk_fast; end + + reg reg_valid; + wire reg_ready; + reg [NODE_IDW-1:0] reg_node_id; + reg [$clog2(MAX_DEPS+1)-1:0] reg_required; + reg [MAX_DEPS*NODE_IDW-1:0] reg_producer_ids; + reg [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr; + reg [15:0] reg_n_tiles; + + // STEP19: ONE physical SDRAM interface. weights, activations, and + // results ALL share this single bus/chip now -- no PSRAM anywhere. + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [1:0] sdram_ba; + wire [12:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + nms_neural_multiprocessor_sdram_cdc #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS_CFG), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(16), .PREFETCH_DISTANCE(PFD_CFG), .CLK_FREQ_MHZ(80) + ) u_nmp ( + .clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + // EXP-0055: the physical SDRAM pins now live in the FAST domain + // (inside the bridge) -- the behavioral chip model must be clocked + // accordingly, not by the slow/system clk anymore. + sdram_model #(.CLK_FREQ_MHZ(115)) u_sdram ( + .clk(clk_fast), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + // ============================================================ + // STEP19: byte-level backdoor access (test setup/verification + // only) -- weights, activations, AND results now ALL live on the + // single real SDRAM physical interface (u_sdram); there is no + // PSRAM anywhere in this system anymore. poke_byte/peek_byte (used + // by activation+result call sites) and poke_byte_weight/peek_byte + // _weight (used by weight call sites) are now identical in + // implementation -- kept as two names rather than merged, to avoid + // touching every one of their many existing call sites for a + // cosmetic rename; both correctly target the same u_sdram.mem + // backing array via the same byte_addr>>1 / byte_addr[0] pattern. + // ============================================================ + task automatic poke_byte(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // sdram_model.v's own `mem` array is flat-indexed by the 25-bit + // word address directly (bank*ROWS*COLS + row*COLS + col, which, + // given ROWS=8192/COLS=1024 are both powers of 2, is numerically + // IDENTICAL to treating the address as one flat 25-bit integer -- + // confirmed against sdram_model.v's own BANKS/ROWS/COLS localparams + // before writing this, not assumed) -- so this is the exact same + // byte_addr>>1 / byte_addr[0] pattern as the original single-chip + // poke_byte/peek_byte above, just against u_sdram.mem instead of + // u_psram.mem. + task automatic poke_byte_weight(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte_weight(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte_weight = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // Golden model: exactly replicates neural_processor.v's real path + // through dataflow_core (bias=0, ACT_RELU always -- see + // dataflow_core.v's own hardcoded job_bias/job_activation). + function automatic signed [7:0] relu_sat(input integer acc); + begin + if (acc <= 0) relu_sat = 8'sd0; + else if (acc > 127) relu_sat = 8'sd127; + else relu_sat = acc[7:0]; + end + endfunction + + // ============================================================ + // Node registration (generalized to MAX_DEPS=8 producers, passed + // as a packed array; n_producers of them are meaningful, the rest + // ignored since reg_required gates how many entries the RTL + // actually reads). + // ============================================================ + task automatic register_node( + input [NODE_IDW-1:0] nid, + input [$clog2(MAX_DEPS+1)-1:0] required, + input [MAX_DEPS*NODE_IDW-1:0] producer_ids_packed, + input [ADDR_WIDTH-1:0] xb, input [ADDR_WIDTH-1:0] wb, + input [15:0] nt, input [ADDR_WIDTH-1:0] resaddr + ); + begin + @(posedge clk); + reg_node_id = nid; + reg_required = required; + reg_producer_ids = producer_ids_packed; + reg_x_base = xb; reg_w_base = wb; reg_n_tiles = nt; reg_result_addr = resaddr; + reg_valid = 1'b1; + while (!reg_ready) @(posedge clk); + @(posedge clk); + reg_valid = 1'b0; + end + endtask + + // ============================================================ + // M10+ real cycle-accounting instrumentation (testbench-only, no + // RTL touched -- same idiom as EXP-0013). + // ============================================================ + reg measure_en; + integer total_cycles; + integer psram_busy_cycles; + integer ni; // moved up from its original later declaration point + // (STEP20 tooling-compatibility fix, zero behavior + // change -- see nms_memory_manager_stream_wide.v's own + // header note on icarus 13.0's stricter declared- + // before-use rule for procedural blocks) + genvar gi; + + reg [N_SLOTS_CFG-1:0] slot_busy_bit; // memory_manager.state != MM_IDLE, this cycle + reg [N_SLOTS_CFG-1:0] slot_tile_bit; // operand_valid && operand_ready, this cycle + integer slot_busy_cycles [0:N_SLOTS_CFG-1]; + integer slot_tiles_delivered [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_MON + always @(*) begin + slot_busy_bit[gi] = (u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.state != 3'd0); + slot_tile_bit[gi] = u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_valid && + u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_ready; + end + end + endgenerate + + // ============================================================ + // STEP17 Part B/C: cycle-decomposition + SDRAM effectiveness + // instrumentation (testbench-only, no RTL touched). + // ============================================================ + integer active_count; // popcount(slot_busy_bit) this cycle + integer active_hist [0:4]; // cycles with exactly k active slots, k=0..4 + integer useful_mac_cycles; // sum over cycles of (#slots with slot_tile_bit this cycle) + integer first_tile_cyc; // total_cycles value at the first tile ever delivered (startup boundary) + integer last_tile_cyc; // total_cycles value at the most recent tile delivered (drain boundary) + integer any_tile_bit; + + // SDRAM controller-port instrumentation (real signals on the + // actual sdram_controller.v instance servicing all weight fetch) + integer sdram_req_count, sdram_ready_count, sdram_wr_count; + integer sdram_busy_cycles, sdram_refresh_count; + integer sdram_req_start_cyc, sdram_lat_sum, sdram_lat_min, sdram_lat_max, sdram_lat_n; + reg sdram_prev_state_is_refwait; + + initial begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + sdram_prev_state_is_refwait=1'b0; + end + + always @(posedge clk) begin + if (measure_en) begin + active_count = slot_busy_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) active_count = active_count + slot_busy_bit[ni]; + active_hist[active_count] <= active_hist[active_count] + 1; + + any_tile_bit = slot_tile_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) any_tile_bit = any_tile_bit | slot_tile_bit[ni]; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) + if (slot_tile_bit[ni]) useful_mac_cycles <= useful_mac_cycles + 1; + if (any_tile_bit) begin + if (first_tile_cyc < 0) first_tile_cyc <= total_cycles; + last_tile_cyc <= total_cycles; + end + + // ---- real SDRAM controller port (single physical chip, + // all weight-fetch traffic funnels through this one + // instance) ---- + if (u_nmp.u_sdram_backend.u_sdram_ctrl.req) begin + sdram_req_count <= sdram_req_count + 1; + sdram_req_start_cyc <= total_cycles; + if (u_nmp.u_sdram_backend.u_sdram_ctrl.wr) sdram_wr_count <= sdram_wr_count + 1; + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.ready) begin + sdram_ready_count <= sdram_ready_count + 1; + sdram_lat_sum <= sdram_lat_sum + (total_cycles - sdram_req_start_cyc); + sdram_lat_n <= sdram_lat_n + 1; + if ((total_cycles - sdram_req_start_cyc) < sdram_lat_min) sdram_lat_min <= (total_cycles - sdram_req_start_cyc); + if ((total_cycles - sdram_req_start_cyc) > sdram_lat_max) sdram_lat_max <= (total_cycles - sdram_req_start_cyc); + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.busy) sdram_busy_cycles <= sdram_busy_cycles + 1; + sdram_prev_state_is_refwait <= (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == 5'd9); + if (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == 5'd9 && !sdram_prev_state_is_refwait) + sdram_refresh_count <= sdram_refresh_count + 1; + end + end + + task automatic report_step17_instrumentation; + real active_pct [0:4]; + real util_pct, startup_cycles, drain_cycles; + real sdram_avg_lat, sdram_busy_pct, sdram_bytes_per_cycle; + integer kk, total_tiles_all; + begin + total_tiles_all = 0; + for (kk = 0; kk < N_SLOTS_CFG; kk = kk + 1) total_tiles_all = total_tiles_all + slot_tiles_delivered[kk]; + $display(" ---- STEP17 Part B: cycle decomposition ----"); + for (kk = 0; kk <= N_SLOTS_CFG; kk = kk + 1) begin + active_pct[kk] = (total_cycles > 0) ? (100.0*active_hist[kk]/total_cycles) : 0.0; + $display(" active_slots=%0d: %0d cycles (%0.2f%%)", kk, active_hist[kk], active_pct[kk]); + end + util_pct = (total_cycles > 0) ? (100.0*useful_mac_cycles/(total_cycles*1.0*N_SLOTS_CFG)) : 0.0; + $display(" useful_mac_cycles (slot-tile-delivery events, summed)=%0d (%0.2f%% of total_cycles*N_SLOTS)", useful_mac_cycles, util_pct); + startup_cycles = (first_tile_cyc >= 0) ? (1.0*first_tile_cyc) : 0.0; + drain_cycles = (last_tile_cyc >= 0) ? (1.0*(total_cycles - last_tile_cyc)) : 0.0; + $display(" startup (cycles before first tile delivered anywhere)=%0.0f", startup_cycles); + $display(" drain (cycles after last tile delivered, until job completion)=%0.0f", drain_cycles); + $display(" ---- STEP17 Part C: SDRAM effectiveness ----"); + sdram_avg_lat = (sdram_lat_n > 0) ? (1.0*sdram_lat_sum/sdram_lat_n) : 0.0; + sdram_busy_pct = (total_cycles > 0) ? (100.0*sdram_busy_cycles/total_cycles) : 0.0; + sdram_bytes_per_cycle = (total_cycles > 0) ? (8.0*sdram_ready_count/total_cycles) : 0.0; + $display(" sdram_req_count=%0d sdram_ready_count=%0d sdram_wr_count=%0d (real reads vs writes)", + sdram_req_count, sdram_ready_count, sdram_wr_count); + $display(" sdram_busy_cycles=%0d/%0d (%0.2f%%)", sdram_busy_cycles, total_cycles, sdram_busy_pct); + $display(" sdram_refresh_count=%0d (real AUTO REFRESH commands issued)", sdram_refresh_count); + $display(" sdram_request_latency: min=%0d max=%0d avg=%0.2f cycles (req-to-ready, single controller port)", + sdram_lat_min, sdram_lat_max, sdram_avg_lat); + $display(" sdram_avg_bytes_per_cycle (8 bytes/transaction * ready_count / total_cycles)=%0.4f", sdram_bytes_per_cycle); + end + endtask + + // ---- STEP11: weight-stall / prefetch-effectiveness instrumentation ---- + // slot_could_present_act: this slot's tile_idx is in-bounds and the + // activation operand for it is already resident -- i.e. everything + // EXCEPT the weight is ready. slot_weight_blocking: on top of that, + // the weight specifically is NOT yet ready (tile_idx>=wgt_ready_count) + // and the FSM is genuinely stalled on it (not mid-read-pipeline, not + // already holding a valid operand). + reg [N_SLOTS_CFG-1:0] slot_could_present_act; + reg [N_SLOTS_CFG-1:0] slot_weight_blocking; + reg [N_SLOTS_CFG-1:0] slot_stalled_this_tile; // sticky per current tile_idx + reg [31:0] prev_tile_idx [0:N_SLOTS_CFG-1]; + integer weight_stall_cycles [0:N_SLOTS_CFG-1]; + integer tiles_prefetched_clean [0:N_SLOTS_CFG-1]; // consumed w/ zero weight-blocking cycles + integer tiles_consumed_total [0:N_SLOTS_CFG-1]; + // plain (non-hierarchical) mirrors of each slot's tile_idx, populated + // combinationally inside the genvar-indexed generate block below -- + // a generate-block instance array (GEN_SLOT[.]) can only be indexed + // by a constant genvar, not a runtime `for` variable, so the + // sequential accumulation loop reads these plain arrays instead of + // reaching back into the hierarchy with a runtime index. + wire [31:0] slot_tile_idx_w [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_PF_MON + assign slot_tile_idx_w[gi] = {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx}; + always @(*) begin + slot_could_present_act[gi] = + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.n_tiles_reg}) && + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.usable_act}); + // nms_memory_manager_stream.v has no read_issued/ + // read_ready states (replaced by the rd_ptr/rd_pending + // read-ahead pipeline) -- the equivalent "blocked + // purely on weight readiness, nothing buffered yet" + // condition is simply: consumption pointer in bounds, + // activation ready, weight NOT ready, and no operand + // currently held in the skid buffer awaiting NP. + slot_weight_blocking[gi] = + slot_could_present_act[gi] && + !(u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx < + u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.wgt_ready_count) && + !u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.operand_valid; + end + end + endgenerate + + always @(posedge clk) begin + if (measure_en) begin + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (prev_tile_idx[ni] != slot_tile_idx_w[ni]) begin + // moved on to a new tile: clear the sticky flag for it + slot_stalled_this_tile[ni] <= 1'b0; + prev_tile_idx[ni] <= slot_tile_idx_w[ni]; + end else if (slot_weight_blocking[ni]) begin + slot_stalled_this_tile[ni] <= 1'b1; + weight_stall_cycles[ni] <= weight_stall_cycles[ni] + 1; + end + if (slot_tile_bit[ni]) begin + tiles_consumed_total[ni] <= tiles_consumed_total[ni] + 1; + if (!slot_stalled_this_tile[ni]) + tiles_prefetched_clean[ni] <= tiles_prefetched_clean[ni] + 1; + end + end + end + end + + // Director/dependency bookkeeping + integer jobs_allocated, jobs_completed, wakeups; + integer waiting_sum, ready_sum, dispatched_sum, sample_count; + + // Occupancy sampling is EXPENSIVE (a full N_NODES=512 scan) and is + // only needed for the small/structural workloads (A/B/E/F), not + // for the large neuron counts (C/D) where it would dominate + // simulation wall-time for no real benefit (per-slot/PSRAM/tile + // counters below are cheap and always collected). Gated by + // sample_occupancy, set per-workload. + reg sample_occupancy; + integer scan_i; + integer waiting_now, ready_now, dispatched_now; + + always @(posedge clk) begin + if (measure_en) begin + total_cycles <= total_cycles + 1; + if (u_nmp.u_arbiter.owner != 0) psram_busy_cycles <= psram_busy_cycles + 1; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (slot_busy_bit[ni]) slot_busy_cycles[ni] <= slot_busy_cycles[ni] + 1; + if (slot_tile_bit[ni]) slot_tiles_delivered[ni] <= slot_tiles_delivered[ni] + 1; + end + if (u_nmp.u_dataflow_core.dm_ready_valid && u_nmp.u_dataflow_core.dm_ready_ready) + jobs_allocated <= jobs_allocated + 1; + if (u_nmp.u_dataflow_core.dir_job_out_done) + jobs_completed <= jobs_completed + 1; + if (u_nmp.u_dataflow_core.dm_producer_done_valid) + wakeups <= wakeups + 1; + + if (sample_occupancy) begin + waiting_now = 0; ready_now = 0; dispatched_now = 0; + for (scan_i = 0; scan_i < N_NODES; scan_i = scan_i + 1) begin + case (u_nmp.u_dataflow_core.u_dep_mgr.node_state[scan_i]) + 2'd1: waiting_now = waiting_now + 1; + 2'd2: ready_now = ready_now + 1; + 2'd3: dispatched_now = dispatched_now + 1; + default: ; + endcase + end + waiting_sum <= waiting_sum + waiting_now; + ready_sum <= ready_sum + ready_now; + dispatched_sum <= dispatched_sum + dispatched_now; + sample_count <= sample_count + 1; + end + end + end + + task automatic reset_instrumentation(input do_sample_occupancy); + integer k; + begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + total_cycles = 0; psram_busy_cycles = 0; + jobs_allocated = 0; jobs_completed = 0; wakeups = 0; + waiting_sum = 0; ready_sum = 0; dispatched_sum = 0; sample_count = 0; + sample_occupancy = do_sample_occupancy; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + slot_busy_cycles[k] = 0; + slot_tiles_delivered[k] = 0; + weight_stall_cycles[k] = 0; + tiles_prefetched_clean[k] = 0; + tiles_consumed_total[k] = 0; + slot_stalled_this_tile[k] = 1'b0; + prev_tile_idx[k] = 32'hFFFFFFFF; + end + end + endtask + + task automatic report_instrumentation(input [255:0] label, input integer n_neurons_completed); + integer k, total_tiles; + integer total_weight_stall_cycles, total_tiles_consumed_all, total_tiles_prefetched_clean; + real avg_waiting, avg_ready, avg_dispatched; + real psram_util, sustained_mac_per_cycle, wallclock_us; + real processor_utilization, weight_stall_pct, prefetch_effectiveness_pct; + begin + total_tiles = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) total_tiles = total_tiles + slot_tiles_delivered[k]; + avg_waiting = (sample_count > 0) ? (1.0*waiting_sum/sample_count) : 0.0; + avg_ready = (sample_count > 0) ? (1.0*ready_sum/sample_count) : 0.0; + avg_dispatched = (sample_count > 0) ? (1.0*dispatched_sum/sample_count) : 0.0; + psram_util = (total_cycles > 0) ? (100.0*psram_busy_cycles/total_cycles) : 0.0; + sustained_mac_per_cycle = (total_cycles > 0) ? (1.0*total_tiles*P_IN/total_cycles) : 0.0; + wallclock_us = total_cycles * CLK_PERIOD / 1000.0; + $display("---- BENCHMARK REPORT: %0s ----", label); + $display(" total_cycles=%0d wallclock_us=%0.3f", total_cycles, wallclock_us); + $display(" neurons_completed=%0d tiles_delivered(real)=%0d", n_neurons_completed, total_tiles); + $display(" jobs_allocated=%0d jobs_completed=%0d dependency_wakeups=%0d", jobs_allocated, jobs_completed, wakeups); + $display(" shared AR (activation+result) arbiter-side utilization: %0.1f%% (%0d/%0d busy cycles)", psram_util, psram_busy_cycles, total_cycles); + for (k = 0; k < N_SLOTS_CFG; k = k + 1) + $display(" slot %0d: busy=%0d/%0d (%0.1f%%) tiles=%0d", k, slot_busy_cycles[k], total_cycles, + (total_cycles>0)?(100.0*slot_busy_cycles[k]/total_cycles):0.0, slot_tiles_delivered[k]); + if (sample_count > 0) + $display(" dependency_manager avg occupancy (sampled every measured cycle): waiting=%0.2f ready=%0.2f dispatched=%0.2f", avg_waiting, avg_ready, avg_dispatched); + else + $display(" dependency_manager occupancy: NOT SAMPLED for this workload (N_NODES scan skipped for large neuron counts to keep simulation time reasonable)"); + $display(" DERIVED: sustained end-to-end MAC/cycle = %0.4f (real tiles*%0d / real total_cycles)", sustained_mac_per_cycle, P_IN); + if (n_neurons_completed > 0) + $display(" DERIVED: cycles/neuron = %0.2f", 1.0*total_cycles/n_neurons_completed); + if (total_tiles > 0) + $display(" DERIVED: cycles/tile = %0.2f", 1.0*total_cycles/total_tiles); + + // ---- STEP11 metrics ---- + total_weight_stall_cycles = 0; total_tiles_consumed_all = 0; total_tiles_prefetched_clean = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + total_weight_stall_cycles = total_weight_stall_cycles + weight_stall_cycles[k]; + total_tiles_consumed_all = total_tiles_consumed_all + tiles_consumed_total[k]; + total_tiles_prefetched_clean = total_tiles_prefetched_clean + tiles_prefetched_clean[k]; + end + processor_utilization = (total_cycles > 0) ? (100.0*total_tiles/(total_cycles*1.0)) : 0.0; + weight_stall_pct = (total_cycles > 0) ? (100.0*total_weight_stall_cycles/(total_cycles*N_SLOTS_CFG*1.0)) : 0.0; + prefetch_effectiveness_pct = (total_tiles_consumed_all > 0) ? + (100.0*total_tiles_prefetched_clean/(total_tiles_consumed_all*1.0)) : 0.0; + $display(" [STEP11] PFD=%0d weight_stall_cycles(sum,all slots)=%0d (%0.2f%% of total_cycles*N_SLOTS)", + PFD_CFG, total_weight_stall_cycles, weight_stall_pct); + $display(" [STEP11] tiles_consumed=%0d tiles_prefetched_clean(zero weight-block before consumption)=%0d", + total_tiles_consumed_all, total_tiles_prefetched_clean); + $display(" [STEP11] DERIVED: prefetch_effectiveness = %0.2f%%", prefetch_effectiveness_pct); + $display(" [STEP11] DERIVED: processor_utilization (tiles*P_IN-equivalent proxy, see sustained MAC/cycle) reference sustained_mac_per_cycle=%0.4f", sustained_mac_per_cycle); + end + endtask + + // ============================================================ + // Workload generators + // ============================================================ + integer errors, tests; + integer wd; + + // A/B/C/D: shared-input dense layer. Generates the shared X + // vector, then N independent (neuron, weight-vector) jobs, each + // verified bit-exact against the golden model. + task automatic run_dense_layer( + input [255:0] label, + input integer n_neurons, + input integer n_tiles_count, + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, + input [ADDR_WIDTH-1:0] w_base, + input [ADDR_WIDTH-1:0] res_base, + input sample_occ + ); + integer n, t, k, len, acc; + reg signed [7:0] xv, wv, golden, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps; + integer completed, wd2; + begin + len = n_tiles_count * P_IN; + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + + // shared input vector + for (k = 0; k < len; k = k + 1) + poke_byte(x_base + k, ((k % 8) + 1)); + + reset_instrumentation(sample_occ); + measure_en = 1'b1; + + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) begin + for (k = 0; k < P_IN; k = k + 1) begin + xv = peek_byte(x_base + t*P_IN + k); + wv = (((n + t*P_IN + k) % 8) + 1); + poke_byte_weight(w_base + n*len + t*P_IN + k, wv); + acc = acc + xv*wv; + end + end + golden = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); // poison, must NOT still be 0 after completion (unless golden IS 0 -- checked separately) + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + x_base, w_base + n*len, n_tiles_count[15:0], res_base + n); + if ((n % 32) == 0) begin + $display(" [%0s] registered %0d/%0d", label, n+1, n_neurons); + $fflush; + end + end + $display(" [%0s] all %0d neurons registered, waiting for completion...", label, n_neurons); + $fflush; + + // wait for all n_neurons completions + completed = 0; wd2 = 0; + while (completed < n_neurons && wd2 < 2000000) begin + @(posedge clk); + wd2 = wd2 + 1; + completed = jobs_completed; + if ((wd2 % 20000) == 0) begin + $display(" [%0s] watchdog %0d: completed=%0d/%0d total_cycles=%0d", label, wd2, completed, n_neurons, total_cycles); +`ifdef STEP16_DEBUG_TRACE + $display(" slot0: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_ready); + $display(" slot1: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_ready); + $display(" sdram: req=%0d busy=%0d ready=%0d req_pending=%0d state=%0d | arb: owner=%0d pending=%0b wide_req=%0b wide_ready=%0b", + u_nmp.u_sdram_backend.u_sdram_ctrl.req, + u_nmp.u_sdram_backend.u_sdram_ctrl.busy, + u_nmp.u_sdram_backend.u_sdram_ctrl.ready, + u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.req_pending, + u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state, + u_nmp.u_arbiter_wide.owner, + u_nmp.u_arbiter_wide.pending, + u_nmp.wide_slot_mem_req, + u_nmp.wide_slot_mem_ready); +`endif + $fflush; + end + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + if (completed < n_neurons) begin + $display("FAIL %0s: only %0d/%0d neurons completed within watchdog", label, completed, n_neurons); + errors = errors + 1; + end else begin : check_block + integer local_errors; + local_errors = 0; + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) + for (k = 0; k < P_IN; k = k + 1) + acc = acc + peek_byte(x_base + t*P_IN + k) * peek_byte_weight(w_base + n*len + t*P_IN + k); + golden = relu_sat(acc); + real_y = peek_byte(res_base + n); + if (real_y !== golden) begin + $display("FAIL %0s neuron %0d: real=%0d golden=%0d", label, n, real_y, golden); + local_errors = local_errors + 1; + end + end + if (local_errors == 0) + $display("PASS %0s: all %0d neurons bit-exact vs golden", label, n_neurons); + else + errors = errors + 1; + end + report_instrumentation(label, n_neurons); + report_step17_instrumentation; + end + endtask + + // E: Multilayer (8 layer-1 random neurons -> shared hidden vector + // -> 2 layer-2 neurons consuming it, real dependency wake-up + + // real cross-node data forwarding through real PSRAM). + localparam L1_N = 8; + localparam L2_N = 2; + integer rand_seed; + + task automatic run_multilayer( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] l1x_base, input [ADDR_WIDTH-1:0] l1w_base, + input [ADDR_WIDTH-1:0] hidden_base, + input [ADDR_WIDTH-1:0] l2w_base, input [ADDR_WIDTH-1:0] l2res_base + ); + integer n, k, acc, completed, wd2; + reg signed [7:0] xv, wv, golden_l1 [0:L1_N-1], golden_l2, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps, l2_deps; + integer local_errors; + begin + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + l2_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + for (n = 0; n < L1_N; n = n + 1) + l2_deps[n*NODE_IDW +: NODE_IDW] = node_base + n[NODE_IDW-1:0]; + + rand_seed = 32'hC0FFEE01; + $display("RANDOM SEED (workload E, layer-1 data) = 32'h%08h", rand_seed); + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + for (n = 0; n < L1_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + xv = $random(rand_seed) % 9; // deterministic PRNG stream, range roughly [-8,8] + wv = $random(rand_seed) % 9; + poke_byte(l1x_base + n*P_IN + k, xv); + poke_byte(l1w_base + n*P_IN + k, wv); + acc = acc + xv*wv; + end + golden_l1[n] = relu_sat(acc); + poke_byte(hidden_base + n, 8'sd0); // poison hidden slot + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + l1x_base + n*P_IN, l1w_base + n*P_IN, 16'd1, hidden_base + n); + end + + for (n = 0; n < L2_N; n = n + 1) begin + for (k = 0; k < P_IN; k = k + 1) + poke_byte(l2w_base + n*P_IN + k, ((n + k) % 6) + 1); + register_node(node_base + L1_N[NODE_IDW-1:0] + n[NODE_IDW-1:0], L1_N[$clog2(MAX_DEPS+1)-1:0], l2_deps, + hidden_base, l2w_base + n*P_IN, 16'd1, l2res_base + n); + end + + completed = 0; wd2 = 0; + while (completed < (L1_N+L2_N) && wd2 < 2000000) begin + @(posedge clk); wd2 = wd2 + 1; completed = jobs_completed; + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < (L1_N+L2_N)) begin + $display("FAIL Multilayer: only %0d/%0d nodes completed", completed, L1_N+L2_N); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < L1_N; n = n + 1) begin + real_y = peek_byte(hidden_base + n); + if (real_y !== golden_l1[n]) begin + $display("FAIL Multilayer L1 neuron %0d: real=%0d golden=%0d", n, real_y, golden_l1[n]); + local_errors = local_errors + 1; + end + end + for (n = 0; n < L2_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) + acc = acc + golden_l1[k] * peek_byte(l2w_base + n*P_IN + k); + golden_l2 = relu_sat(acc); + real_y = peek_byte(l2res_base + n); + if (real_y !== golden_l2) begin + $display("FAIL Multilayer L2 neuron %0d: real=%0d golden=%0d (using REAL L1 hidden values)", n, real_y, golden_l2); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS Multilayer: 8 L1 (random) -> 2 L2 neurons, all bit-exact, real cross-node forwarding via real PSRAM"); + else errors = errors + 1; + report_instrumentation("E-Multilayer", L1_N+L2_N); + end + endtask + + // F: DAG diamond+fan-in (A,B indep; C dep-A; D dep-B; E dep-C&D + // [2-hop]; F dep-A,B,C [mixed, 3 producers]) + task automatic run_dag( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, input [ADDR_WIDTH-1:0] w_base, input [ADDR_WIDTH-1:0] res_base + ); + integer n, k, acc, completed, wd2, local_errors; + reg signed [7:0] golden [0:5]; + reg signed [7:0] real_y; + reg [MAX_DEPS*NODE_IDW-1:0] deps; + reg [NODE_IDW-1:0] idA, idB, idC, idD, idE, idF; + begin + idA = node_base+0; idB = node_base+1; idC = node_base+2; + idD = node_base+3; idE = node_base+4; idF = node_base+5; + + // Each of the 6 nodes: its own small independent 8-input + // job (deterministic, distinct per node) -- dependencies + // here are purely about SCHEDULING/wake-up order, not + // data forwarding (workload E already covers that). + for (n = 0; n < 6; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + poke_byte(x_base + n*P_IN + k, ((n+k)%4)+1); + poke_byte(w_base + n*P_IN + k, ((n+k)%5)+1); + acc = acc + peek_byte(x_base+n*P_IN+k)*peek_byte(w_base+n*P_IN+k); + end + golden[n] = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); + end + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + register_node(idA, 0, deps, x_base+0*P_IN, w_base+0*P_IN, 16'd1, res_base+0); + register_node(idB, 0, deps, x_base+1*P_IN, w_base+1*P_IN, 16'd1, res_base+1); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; + register_node(idC, 1, deps, x_base+2*P_IN, w_base+2*P_IN, 16'd1, res_base+2); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idB; + register_node(idD, 1, deps, x_base+3*P_IN, w_base+3*P_IN, 16'd1, res_base+3); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idC; deps[1*NODE_IDW+:NODE_IDW] = idD; + register_node(idE, 2, deps, x_base+4*P_IN, w_base+4*P_IN, 16'd1, res_base+4); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; deps[1*NODE_IDW+:NODE_IDW] = idB; deps[2*NODE_IDW+:NODE_IDW] = idC; + register_node(idF, 3, deps, x_base+5*P_IN, w_base+5*P_IN, 16'd1, res_base+5); + + completed = 0; wd2 = 0; + while (completed < 6 && wd2 < 2000000) begin @(posedge clk); wd2=wd2+1; completed = jobs_completed; end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < 6) begin + $display("FAIL DAG: only %0d/6 nodes completed", completed); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < 6; n = n + 1) begin + real_y = peek_byte(res_base+n); + if (real_y !== golden[n]) begin + $display("FAIL DAG node %0d: real=%0d golden=%0d", n, real_y, golden[n]); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS DAG: 6-node diamond+fan-in (2-hop transitive wake-up, 3-producer mixed-depth dependency), all bit-exact"); + else errors = errors + 1; + report_instrumentation("F-DAG", 6); + end + endtask + + initial begin + errors = 0; tests = 0; + rst = 1; rst_fast = 1; reg_valid = 0; reg_node_id = 0; reg_required = 0; reg_producer_ids = 0; + reg_x_base = 0; reg_w_base = 0; reg_n_tiles = 0; reg_result_addr = 0; + measure_en = 0; + repeat(5) @(posedge clk); + repeat(5) @(posedge clk_fast); + rst = 0; rst_fast = 0; + + $display("========================================"); + $display("NMS D-Stress benchmark (STEP19, SINGLE SDRAM (AS4C4M16SA-6TIN) for weights+activations+results, no PSRAM anywhere) -- N_SLOTS_CFG=%0d PFD_CFG=%0d", N_SLOTS_CFG, PFD_CFG); + $display("========================================"); + + wait (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.S_IDLE); + @(posedge clk); + + // Official V2 memory map (datasheet ch.5): weights @ 0x010000, + // activations @ 0x200000, results @ 0x300000 -- non-overlapping + // 1MB-aligned regions in the single SDRAM. + run_dense_layer("D-Stress", 256, 16, 16'd400, 26'h200000, 26'h010000, 26'h300000, 1'b0); + + // FPGA_DATA_READY check: the whole graph (256 nodes) just + // finished and no new work has been registered -- data_ready + // must be asserted (system-idle sticky flag, see + // nms_dataflow_core_sdram.v). A few idle cycles for the + // busy->idle edge to settle before sampling. + repeat (4) @(posedge clk); + if (u_nmp.data_ready !== 1'b1) begin + $display("FAIL data_ready: expected 1 after graph completion, got %b", u_nmp.data_ready); + errors = errors + 1; + end else begin + $display("PASS data_ready: correctly asserted after graph completion"); + end + + $display("========================================"); + if (errors == 0) + $display("ALL %0d WORKLOAD SUITES PASSED (N_SLOTS_CFG=%0d, PFD_CFG=%0d, SINGLE SDRAM for weights+activations+results, no PSRAM)", tests, N_SLOTS_CFG, PFD_CFG); + else + $display("FAILED: %0d/%0d workload suite(s) had errors -- see messages above", errors, tests); + $display("========================================"); + $finish; + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_nms_dstress_sdram_combined.v b/hardware/v2/nms/sim/tb_nms_dstress_sdram_combined.v new file mode 100644 index 0000000..90ea958 --- /dev/null +++ b/hardware/v2/nms/sim/tb_nms_dstress_sdram_combined.v @@ -0,0 +1,918 @@ +`timescale 1ns/1ps + +// ================================================================ +// EXP-0055 -- Phase B integration: real D-Stress run through +// nms_neural_multiprocessor_sdram_combined.v (CDC-to-115.2MHz + +// page-hit SDRAM backend, EXP-0053+EXP-0054 composed together). +// Forked from tb_nms_dstress_sdram_unified.v -- the ONLY changes are: +// (1) a second free-running clk_fast (115.2MHz, real value; matches +// what EXP-0053 separately confirmed is derivable from the real +// board's own PLL VCO relative to its 64MHz clk_sys -- this specific +// testbench family's own convention runs the SLOW/system domain at +// 80MHz, not 64MHz, to stay directly comparable to EXP-0051/0052's +// own historical baseline numbers measured at that same 80MHz; 115.2 +// MHz is reused here as an absolute value, not re-derived from an +// 80MHz-rooted PLL) with its own rst_fast, (2) the DUT swapped for +// nms_neural_multiprocessor_sdram_combined.v with clk_fast/rst_fast +// wired through, (3) sdram_model.v moved onto clk_fast (the real +// physical SDRAM pins now toggle in the fast domain, inside the +// bridge). No other change -- workloads, golden model, cycle +// accounting, and backdoor peek/poke helpers are IDENTICAL: sdram_ +// controller_openrow.v's address decomposition (bank/row/col bit +// ranges) is byte-for-byte the same as the original sdram_ +// controller.v's (unlike EXP-0052's re-sliced pipelined variant), so +// no backdoor-helper rework was needed here. +// +// Everything below this point is the ORIGINAL testbench's own header, +// preserved as-is: +// +// FPGA-Neural V2 -- Final Benchmark Campaign (post-M10, real +// end-to-end characterization, docs/v2-description.md §22/§30/§32) +// +// One testbench, compiled once per N_SLOTS configuration (N_SLOTS_CFG +// parameter, overridden at Verilator invocation via -GN_SLOTS_CFG=N), +// running SIX representative workloads back-to-back through the REAL +// neural_multiprocessor.v (M8: dataflow_core + slot_mem_arbiter + the +// real, unmodified V1 PSRAM chain), with: +// - a software "golden" model replicating neural_processor.v's exact +// integer math (sum(x*w) over all tiles, ReLU + INT8 saturate -- +// dataflow_core.v hardcodes bias=0/ACT_RELU for every job, so the +// golden model only needs to replicate that one path) +// - bit-exact verification of EVERY neuron's real result against +// that golden model (peek_byte from the real psram_model backing +// array -- an oracle independent of the RTL under test) +// - real cycle-accounting instrumentation (testbench-only, no RTL +// touched): per-slot busy/idle cycles, shared PSRAM port busy/idle +// cycles, REAL tiles delivered per slot (operand_valid&& +// operand_ready pulses -- one pulse = one whole P_IN-wide tile +// consumed by neural_processor, NOT one byte), director/dependency +// bookkeeping (jobs allocated/completed, ready-queue occupancy, +// WAITING/READY/DISPATCHED node counts, producer-done wakeups) +// +// Workloads (node_id ranges are disjoint across all six so the WHOLE +// campaign runs in ONE continuous simulation -- only ONE real PSRAM +// power-up wait, no reset between phases, closer to real sustained +// operation than resetting between every workload): +// A) Small -- 16 independent neurons, 8 inputs each +// B) Medium -- 64 independent neurons, 32 inputs each +// C) Large -- 128 independent neurons, 128 inputs each +// D) Stress -- 256 independent neurons, 128 inputs each +// E) Multilayer -- 8 layer-1 neurons (RANDOM data, logged seed) feed +// a shared 8-byte hidden vector; 2 layer-2 neurons +// consume that vector (real cross-node data +// forwarding through real PSRAM, real dependency +// wake-up, "shared producer/multiple consumers") +// F) DAG -- 6-node diamond+fan-in graph (A,B independent; C +// dep on A; D dep on B; E dep on BOTH C and D +// [2-hop transitive wake-up]; F dep on A,B,C [mixed +// direct+1-hop, 3 producers]) +// +// All workloads A-D use a REALISTIC dense-layer shape: one shared +// input activation vector, N independent weight vectors (one per +// neuron) -- exactly how a real fully-connected layer's neurons share +// their layer's input. This is not an isolated synthetic microbench. +// +// Verified with Verilator (decisions.log DEC-0004). +// ================================================================ + +// ================================================================ +// STEP11 variant: identical D-Stress workload/golden-model/correctness +// criteria as tb_nms_dstress.v (STEP9's own official benchmark), but +// instantiating nms_neural_multiprocessor_pf (REAL weight prefetch +// engine, weight_prefetch_engine.v) instead of the baseline +// nms_neural_multiprocessor.v, with an added PFD_CFG (PREFETCH_DISTANCE) +// parameter, plus NEW instrumentation (testbench-only, no RTL touched) +// for the two STEP11-mandated metrics that cannot be derived from the +// STEP9 instrumentation alone: +// weight_stall_cycles = cycles a slot is otherwise ready to +// present a tile (activation resident, +// in bounds) but blocked purely because +// tile_idx >= wgt_ready_count +// prefetch_effectiveness = tiles consumed with ZERO such +// weight-blocking cycles beforehand +// (i.e. the weight was ALREADY resident +// the moment the tile became eligible) +// / total tiles consumed +// per STEP11's own explicit metric definitions. +// ================================================================ +module tb #( + parameter N_SLOTS_CFG = 2, + parameter PFD_CFG = 8 +); + + localparam ADDR_WIDTH = 26; // AS4C32M16SA: 25-bit word address + 1 byte-select bit + localparam DATA_WIDTH = 8; + localparam P_IN = 8; + localparam ACC_WIDTH = 32; + // N_NODES must exceed the HIGHEST node_id used by ANY workload + // (node_base + count - 1) -- workload D's own range alone + // (node_base=400, 256 neurons) reaches id 655. An earlier draft + // used N_NODES=512: D's ids silently wrapped (9-bit truncation) + // past id 511, colliding with workload A's already-DISPATCHED + // node 0 (dependency_manager never reclaims dispatched node slots, + // DEC-0008) and deadlocking register_node's reg_ready wait + // forever. A real consequence of DEC-0008's design choice, not an + // RTL bug -- fixed here by sizing N_NODES generously above the + // real id range used below (see decisions.log DEC-0008 and the + // final benchmark report's Limitations section). + localparam N_NODES = 1024; + localparam MAX_DEPS = 8; + localparam QUEUE_DEPTH = 8; + localparam NODE_IDW = $clog2(N_NODES); + localparam CLK_PERIOD = 12.5; // 80 MHz, matches psram_controller's CLK_FREQ_MHZ + + reg clk, rst; + initial begin clk = 1'b0; forever #(CLK_PERIOD/2.0) clk = ~clk; end + + // EXP-0055: second, independent fast clock for the SDRAM domain + // (real 115.2MHz value, non-integer ratio vs the 80MHz slow + // domain -- deliberately not a lucky-alignment case, see + // sdram_cdc_bridge.v's own header on why this is the harder test). + localparam real CLK_FREQ_FAST_REAL = 115.2; + localparam real FAST_PERIOD_NS = 1000.0/CLK_FREQ_FAST_REAL; + reg clk_fast, rst_fast; + initial begin clk_fast = 1'b0; forever #(FAST_PERIOD_NS/2.0) clk_fast = ~clk_fast; end + + reg reg_valid; + wire reg_ready; + reg [NODE_IDW-1:0] reg_node_id; + reg [$clog2(MAX_DEPS+1)-1:0] reg_required; + reg [MAX_DEPS*NODE_IDW-1:0] reg_producer_ids; + reg [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr; + reg [15:0] reg_n_tiles; + + // STEP19: ONE physical SDRAM interface. weights, activations, and + // results ALL share this single bus/chip now -- no PSRAM anywhere. + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [1:0] sdram_ba; + wire [12:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + nms_neural_multiprocessor_sdram_combined #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS_CFG), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(16), .PREFETCH_DISTANCE(PFD_CFG), .CLK_FREQ_MHZ(80) + ) u_nmp ( + .clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + // EXP-0055: the physical SDRAM pins now live in the FAST domain + // (inside the bridge) -- the behavioral chip model must be clocked + // accordingly, not by the slow/system clk anymore. + sdram_model #(.CLK_FREQ_MHZ(115)) u_sdram ( + .clk(clk_fast), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + // ============================================================ + // STEP19: byte-level backdoor access (test setup/verification + // only) -- weights, activations, AND results now ALL live on the + // single real SDRAM physical interface (u_sdram); there is no + // PSRAM anywhere in this system anymore. poke_byte/peek_byte (used + // by activation+result call sites) and poke_byte_weight/peek_byte + // _weight (used by weight call sites) are now identical in + // implementation -- kept as two names rather than merged, to avoid + // touching every one of their many existing call sites for a + // cosmetic rename; both correctly target the same u_sdram.mem + // backing array via the same byte_addr>>1 / byte_addr[0] pattern. + // ============================================================ + task automatic poke_byte(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // sdram_model.v's own `mem` array is flat-indexed by the 25-bit + // word address directly (bank*ROWS*COLS + row*COLS + col, which, + // given ROWS=8192/COLS=1024 are both powers of 2, is numerically + // IDENTICAL to treating the address as one flat 25-bit integer -- + // confirmed against sdram_model.v's own BANKS/ROWS/COLS localparams + // before writing this, not assumed) -- so this is the exact same + // byte_addr>>1 / byte_addr[0] pattern as the original single-chip + // poke_byte/peek_byte above, just against u_sdram.mem instead of + // u_psram.mem. + task automatic poke_byte_weight(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte_weight(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte_weight = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // Golden model: exactly replicates neural_processor.v's real path + // through dataflow_core (bias=0, ACT_RELU always -- see + // dataflow_core.v's own hardcoded job_bias/job_activation). + function automatic signed [7:0] relu_sat(input integer acc); + begin + if (acc <= 0) relu_sat = 8'sd0; + else if (acc > 127) relu_sat = 8'sd127; + else relu_sat = acc[7:0]; + end + endfunction + + // ============================================================ + // Node registration (generalized to MAX_DEPS=8 producers, passed + // as a packed array; n_producers of them are meaningful, the rest + // ignored since reg_required gates how many entries the RTL + // actually reads). + // ============================================================ + task automatic register_node( + input [NODE_IDW-1:0] nid, + input [$clog2(MAX_DEPS+1)-1:0] required, + input [MAX_DEPS*NODE_IDW-1:0] producer_ids_packed, + input [ADDR_WIDTH-1:0] xb, input [ADDR_WIDTH-1:0] wb, + input [15:0] nt, input [ADDR_WIDTH-1:0] resaddr + ); + begin + @(posedge clk); + reg_node_id = nid; + reg_required = required; + reg_producer_ids = producer_ids_packed; + reg_x_base = xb; reg_w_base = wb; reg_n_tiles = nt; reg_result_addr = resaddr; + reg_valid = 1'b1; + while (!reg_ready) @(posedge clk); + @(posedge clk); + reg_valid = 1'b0; + end + endtask + + // ============================================================ + // M10+ real cycle-accounting instrumentation (testbench-only, no + // RTL touched -- same idiom as EXP-0013). + // ============================================================ + reg measure_en; + integer total_cycles; + integer psram_busy_cycles; + integer ni; // moved up from its original later declaration point + // (STEP20 tooling-compatibility fix, zero behavior + // change -- see nms_memory_manager_stream_wide.v's own + // header note on icarus 13.0's stricter declared- + // before-use rule for procedural blocks) + genvar gi; + + reg [N_SLOTS_CFG-1:0] slot_busy_bit; // memory_manager.state != MM_IDLE, this cycle + reg [N_SLOTS_CFG-1:0] slot_tile_bit; // operand_valid && operand_ready, this cycle + integer slot_busy_cycles [0:N_SLOTS_CFG-1]; + integer slot_tiles_delivered [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_MON + always @(*) begin + slot_busy_bit[gi] = (u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.state != 3'd0); + slot_tile_bit[gi] = u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_valid && + u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_ready; + end + end + endgenerate + + // ============================================================ + // STEP17 Part B/C: cycle-decomposition + SDRAM effectiveness + // instrumentation (testbench-only, no RTL touched). + // ============================================================ + integer active_count; // popcount(slot_busy_bit) this cycle + integer active_hist [0:4]; // cycles with exactly k active slots, k=0..4 + integer useful_mac_cycles; // sum over cycles of (#slots with slot_tile_bit this cycle) + integer first_tile_cyc; // total_cycles value at the first tile ever delivered (startup boundary) + integer last_tile_cyc; // total_cycles value at the most recent tile delivered (drain boundary) + integer any_tile_bit; + + // SDRAM controller-port instrumentation (real signals on the + // actual sdram_controller.v instance servicing all weight fetch) + integer sdram_req_count, sdram_ready_count, sdram_wr_count; + integer sdram_busy_cycles, sdram_refresh_count; + integer sdram_req_start_cyc, sdram_lat_sum, sdram_lat_min, sdram_lat_max, sdram_lat_n; + reg sdram_prev_state_is_refwait; + + initial begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + sdram_prev_state_is_refwait=1'b0; + end + + always @(posedge clk) begin + if (measure_en) begin + active_count = slot_busy_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) active_count = active_count + slot_busy_bit[ni]; + active_hist[active_count] <= active_hist[active_count] + 1; + + any_tile_bit = slot_tile_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) any_tile_bit = any_tile_bit | slot_tile_bit[ni]; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) + if (slot_tile_bit[ni]) useful_mac_cycles <= useful_mac_cycles + 1; + if (any_tile_bit) begin + if (first_tile_cyc < 0) first_tile_cyc <= total_cycles; + last_tile_cyc <= total_cycles; + end + + // ---- real SDRAM controller port (single physical chip, + // all weight-fetch traffic funnels through this one + // instance) ---- + if (u_nmp.u_sdram_backend.u_sdram_ctrl.req) begin + sdram_req_count <= sdram_req_count + 1; + sdram_req_start_cyc <= total_cycles; + if (u_nmp.u_sdram_backend.u_sdram_ctrl.wr) sdram_wr_count <= sdram_wr_count + 1; + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.ready) begin + sdram_ready_count <= sdram_ready_count + 1; + sdram_lat_sum <= sdram_lat_sum + (total_cycles - sdram_req_start_cyc); + sdram_lat_n <= sdram_lat_n + 1; + if ((total_cycles - sdram_req_start_cyc) < sdram_lat_min) sdram_lat_min <= (total_cycles - sdram_req_start_cyc); + if ((total_cycles - sdram_req_start_cyc) > sdram_lat_max) sdram_lat_max <= (total_cycles - sdram_req_start_cyc); + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.busy) sdram_busy_cycles <= sdram_busy_cycles + 1; + sdram_prev_state_is_refwait <= (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == 5'd9); + if (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == 5'd9 && !sdram_prev_state_is_refwait) + sdram_refresh_count <= sdram_refresh_count + 1; + end + end + + task automatic report_step17_instrumentation; + real active_pct [0:4]; + real util_pct, startup_cycles, drain_cycles; + real sdram_avg_lat, sdram_busy_pct, sdram_bytes_per_cycle; + integer kk, total_tiles_all; + begin + total_tiles_all = 0; + for (kk = 0; kk < N_SLOTS_CFG; kk = kk + 1) total_tiles_all = total_tiles_all + slot_tiles_delivered[kk]; + $display(" ---- STEP17 Part B: cycle decomposition ----"); + for (kk = 0; kk <= N_SLOTS_CFG; kk = kk + 1) begin + active_pct[kk] = (total_cycles > 0) ? (100.0*active_hist[kk]/total_cycles) : 0.0; + $display(" active_slots=%0d: %0d cycles (%0.2f%%)", kk, active_hist[kk], active_pct[kk]); + end + util_pct = (total_cycles > 0) ? (100.0*useful_mac_cycles/(total_cycles*1.0*N_SLOTS_CFG)) : 0.0; + $display(" useful_mac_cycles (slot-tile-delivery events, summed)=%0d (%0.2f%% of total_cycles*N_SLOTS)", useful_mac_cycles, util_pct); + startup_cycles = (first_tile_cyc >= 0) ? (1.0*first_tile_cyc) : 0.0; + drain_cycles = (last_tile_cyc >= 0) ? (1.0*(total_cycles - last_tile_cyc)) : 0.0; + $display(" startup (cycles before first tile delivered anywhere)=%0.0f", startup_cycles); + $display(" drain (cycles after last tile delivered, until job completion)=%0.0f", drain_cycles); + $display(" ---- STEP17 Part C: SDRAM effectiveness ----"); + sdram_avg_lat = (sdram_lat_n > 0) ? (1.0*sdram_lat_sum/sdram_lat_n) : 0.0; + sdram_busy_pct = (total_cycles > 0) ? (100.0*sdram_busy_cycles/total_cycles) : 0.0; + sdram_bytes_per_cycle = (total_cycles > 0) ? (8.0*sdram_ready_count/total_cycles) : 0.0; + $display(" sdram_req_count=%0d sdram_ready_count=%0d sdram_wr_count=%0d (real reads vs writes)", + sdram_req_count, sdram_ready_count, sdram_wr_count); + $display(" sdram_busy_cycles=%0d/%0d (%0.2f%%)", sdram_busy_cycles, total_cycles, sdram_busy_pct); + $display(" sdram_refresh_count=%0d (real AUTO REFRESH commands issued)", sdram_refresh_count); + $display(" sdram_request_latency: min=%0d max=%0d avg=%0.2f cycles (req-to-ready, single controller port)", + sdram_lat_min, sdram_lat_max, sdram_avg_lat); + $display(" sdram_avg_bytes_per_cycle (8 bytes/transaction * ready_count / total_cycles)=%0.4f", sdram_bytes_per_cycle); + end + endtask + + // ---- STEP11: weight-stall / prefetch-effectiveness instrumentation ---- + // slot_could_present_act: this slot's tile_idx is in-bounds and the + // activation operand for it is already resident -- i.e. everything + // EXCEPT the weight is ready. slot_weight_blocking: on top of that, + // the weight specifically is NOT yet ready (tile_idx>=wgt_ready_count) + // and the FSM is genuinely stalled on it (not mid-read-pipeline, not + // already holding a valid operand). + reg [N_SLOTS_CFG-1:0] slot_could_present_act; + reg [N_SLOTS_CFG-1:0] slot_weight_blocking; + reg [N_SLOTS_CFG-1:0] slot_stalled_this_tile; // sticky per current tile_idx + reg [31:0] prev_tile_idx [0:N_SLOTS_CFG-1]; + integer weight_stall_cycles [0:N_SLOTS_CFG-1]; + integer tiles_prefetched_clean [0:N_SLOTS_CFG-1]; // consumed w/ zero weight-blocking cycles + integer tiles_consumed_total [0:N_SLOTS_CFG-1]; + // plain (non-hierarchical) mirrors of each slot's tile_idx, populated + // combinationally inside the genvar-indexed generate block below -- + // a generate-block instance array (GEN_SLOT[.]) can only be indexed + // by a constant genvar, not a runtime `for` variable, so the + // sequential accumulation loop reads these plain arrays instead of + // reaching back into the hierarchy with a runtime index. + wire [31:0] slot_tile_idx_w [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_PF_MON + assign slot_tile_idx_w[gi] = {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx}; + always @(*) begin + slot_could_present_act[gi] = + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.n_tiles_reg}) && + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.usable_act}); + // nms_memory_manager_stream.v has no read_issued/ + // read_ready states (replaced by the rd_ptr/rd_pending + // read-ahead pipeline) -- the equivalent "blocked + // purely on weight readiness, nothing buffered yet" + // condition is simply: consumption pointer in bounds, + // activation ready, weight NOT ready, and no operand + // currently held in the skid buffer awaiting NP. + slot_weight_blocking[gi] = + slot_could_present_act[gi] && + !(u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx < + u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.wgt_ready_count) && + !u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.operand_valid; + end + end + endgenerate + + always @(posedge clk) begin + if (measure_en) begin + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (prev_tile_idx[ni] != slot_tile_idx_w[ni]) begin + // moved on to a new tile: clear the sticky flag for it + slot_stalled_this_tile[ni] <= 1'b0; + prev_tile_idx[ni] <= slot_tile_idx_w[ni]; + end else if (slot_weight_blocking[ni]) begin + slot_stalled_this_tile[ni] <= 1'b1; + weight_stall_cycles[ni] <= weight_stall_cycles[ni] + 1; + end + if (slot_tile_bit[ni]) begin + tiles_consumed_total[ni] <= tiles_consumed_total[ni] + 1; + if (!slot_stalled_this_tile[ni]) + tiles_prefetched_clean[ni] <= tiles_prefetched_clean[ni] + 1; + end + end + end + end + + // Director/dependency bookkeeping + integer jobs_allocated, jobs_completed, wakeups; + integer waiting_sum, ready_sum, dispatched_sum, sample_count; + + // Occupancy sampling is EXPENSIVE (a full N_NODES=512 scan) and is + // only needed for the small/structural workloads (A/B/E/F), not + // for the large neuron counts (C/D) where it would dominate + // simulation wall-time for no real benefit (per-slot/PSRAM/tile + // counters below are cheap and always collected). Gated by + // sample_occupancy, set per-workload. + reg sample_occupancy; + integer scan_i; + integer waiting_now, ready_now, dispatched_now; + + always @(posedge clk) begin + if (measure_en) begin + total_cycles <= total_cycles + 1; + if (u_nmp.u_arbiter.owner != 0) psram_busy_cycles <= psram_busy_cycles + 1; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (slot_busy_bit[ni]) slot_busy_cycles[ni] <= slot_busy_cycles[ni] + 1; + if (slot_tile_bit[ni]) slot_tiles_delivered[ni] <= slot_tiles_delivered[ni] + 1; + end + if (u_nmp.u_dataflow_core.dm_ready_valid && u_nmp.u_dataflow_core.dm_ready_ready) + jobs_allocated <= jobs_allocated + 1; + if (u_nmp.u_dataflow_core.dir_job_out_done) + jobs_completed <= jobs_completed + 1; + if (u_nmp.u_dataflow_core.dm_producer_done_valid) + wakeups <= wakeups + 1; + + if (sample_occupancy) begin + waiting_now = 0; ready_now = 0; dispatched_now = 0; + for (scan_i = 0; scan_i < N_NODES; scan_i = scan_i + 1) begin + case (u_nmp.u_dataflow_core.u_dep_mgr.node_state[scan_i]) + 2'd1: waiting_now = waiting_now + 1; + 2'd2: ready_now = ready_now + 1; + 2'd3: dispatched_now = dispatched_now + 1; + default: ; + endcase + end + waiting_sum <= waiting_sum + waiting_now; + ready_sum <= ready_sum + ready_now; + dispatched_sum <= dispatched_sum + dispatched_now; + sample_count <= sample_count + 1; + end + end + end + + task automatic reset_instrumentation(input do_sample_occupancy); + integer k; + begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + total_cycles = 0; psram_busy_cycles = 0; + jobs_allocated = 0; jobs_completed = 0; wakeups = 0; + waiting_sum = 0; ready_sum = 0; dispatched_sum = 0; sample_count = 0; + sample_occupancy = do_sample_occupancy; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + slot_busy_cycles[k] = 0; + slot_tiles_delivered[k] = 0; + weight_stall_cycles[k] = 0; + tiles_prefetched_clean[k] = 0; + tiles_consumed_total[k] = 0; + slot_stalled_this_tile[k] = 1'b0; + prev_tile_idx[k] = 32'hFFFFFFFF; + end + end + endtask + + task automatic report_instrumentation(input [255:0] label, input integer n_neurons_completed); + integer k, total_tiles; + integer total_weight_stall_cycles, total_tiles_consumed_all, total_tiles_prefetched_clean; + real avg_waiting, avg_ready, avg_dispatched; + real psram_util, sustained_mac_per_cycle, wallclock_us; + real processor_utilization, weight_stall_pct, prefetch_effectiveness_pct; + begin + total_tiles = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) total_tiles = total_tiles + slot_tiles_delivered[k]; + avg_waiting = (sample_count > 0) ? (1.0*waiting_sum/sample_count) : 0.0; + avg_ready = (sample_count > 0) ? (1.0*ready_sum/sample_count) : 0.0; + avg_dispatched = (sample_count > 0) ? (1.0*dispatched_sum/sample_count) : 0.0; + psram_util = (total_cycles > 0) ? (100.0*psram_busy_cycles/total_cycles) : 0.0; + sustained_mac_per_cycle = (total_cycles > 0) ? (1.0*total_tiles*P_IN/total_cycles) : 0.0; + wallclock_us = total_cycles * CLK_PERIOD / 1000.0; + $display("---- BENCHMARK REPORT: %0s ----", label); + $display(" total_cycles=%0d wallclock_us=%0.3f", total_cycles, wallclock_us); + $display(" neurons_completed=%0d tiles_delivered(real)=%0d", n_neurons_completed, total_tiles); + $display(" jobs_allocated=%0d jobs_completed=%0d dependency_wakeups=%0d", jobs_allocated, jobs_completed, wakeups); + $display(" shared AR (activation+result) arbiter-side utilization: %0.1f%% (%0d/%0d busy cycles)", psram_util, psram_busy_cycles, total_cycles); + for (k = 0; k < N_SLOTS_CFG; k = k + 1) + $display(" slot %0d: busy=%0d/%0d (%0.1f%%) tiles=%0d", k, slot_busy_cycles[k], total_cycles, + (total_cycles>0)?(100.0*slot_busy_cycles[k]/total_cycles):0.0, slot_tiles_delivered[k]); + if (sample_count > 0) + $display(" dependency_manager avg occupancy (sampled every measured cycle): waiting=%0.2f ready=%0.2f dispatched=%0.2f", avg_waiting, avg_ready, avg_dispatched); + else + $display(" dependency_manager occupancy: NOT SAMPLED for this workload (N_NODES scan skipped for large neuron counts to keep simulation time reasonable)"); + $display(" DERIVED: sustained end-to-end MAC/cycle = %0.4f (real tiles*%0d / real total_cycles)", sustained_mac_per_cycle, P_IN); + if (n_neurons_completed > 0) + $display(" DERIVED: cycles/neuron = %0.2f", 1.0*total_cycles/n_neurons_completed); + if (total_tiles > 0) + $display(" DERIVED: cycles/tile = %0.2f", 1.0*total_cycles/total_tiles); + + // ---- STEP11 metrics ---- + total_weight_stall_cycles = 0; total_tiles_consumed_all = 0; total_tiles_prefetched_clean = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + total_weight_stall_cycles = total_weight_stall_cycles + weight_stall_cycles[k]; + total_tiles_consumed_all = total_tiles_consumed_all + tiles_consumed_total[k]; + total_tiles_prefetched_clean = total_tiles_prefetched_clean + tiles_prefetched_clean[k]; + end + processor_utilization = (total_cycles > 0) ? (100.0*total_tiles/(total_cycles*1.0)) : 0.0; + weight_stall_pct = (total_cycles > 0) ? (100.0*total_weight_stall_cycles/(total_cycles*N_SLOTS_CFG*1.0)) : 0.0; + prefetch_effectiveness_pct = (total_tiles_consumed_all > 0) ? + (100.0*total_tiles_prefetched_clean/(total_tiles_consumed_all*1.0)) : 0.0; + $display(" [STEP11] PFD=%0d weight_stall_cycles(sum,all slots)=%0d (%0.2f%% of total_cycles*N_SLOTS)", + PFD_CFG, total_weight_stall_cycles, weight_stall_pct); + $display(" [STEP11] tiles_consumed=%0d tiles_prefetched_clean(zero weight-block before consumption)=%0d", + total_tiles_consumed_all, total_tiles_prefetched_clean); + $display(" [STEP11] DERIVED: prefetch_effectiveness = %0.2f%%", prefetch_effectiveness_pct); + $display(" [STEP11] DERIVED: processor_utilization (tiles*P_IN-equivalent proxy, see sustained MAC/cycle) reference sustained_mac_per_cycle=%0.4f", sustained_mac_per_cycle); + end + endtask + + // ============================================================ + // Workload generators + // ============================================================ + integer errors, tests; + integer wd; + + // A/B/C/D: shared-input dense layer. Generates the shared X + // vector, then N independent (neuron, weight-vector) jobs, each + // verified bit-exact against the golden model. + task automatic run_dense_layer( + input [255:0] label, + input integer n_neurons, + input integer n_tiles_count, + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, + input [ADDR_WIDTH-1:0] w_base, + input [ADDR_WIDTH-1:0] res_base, + input sample_occ + ); + integer n, t, k, len, acc; + reg signed [7:0] xv, wv, golden, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps; + integer completed, wd2; + begin + len = n_tiles_count * P_IN; + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + + // shared input vector + for (k = 0; k < len; k = k + 1) + poke_byte(x_base + k, ((k % 8) + 1)); + + reset_instrumentation(sample_occ); + measure_en = 1'b1; + + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) begin + for (k = 0; k < P_IN; k = k + 1) begin + xv = peek_byte(x_base + t*P_IN + k); + wv = (((n + t*P_IN + k) % 8) + 1); + poke_byte_weight(w_base + n*len + t*P_IN + k, wv); + acc = acc + xv*wv; + end + end + golden = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); // poison, must NOT still be 0 after completion (unless golden IS 0 -- checked separately) + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + x_base, w_base + n*len, n_tiles_count[15:0], res_base + n); + if ((n % 32) == 0) begin + $display(" [%0s] registered %0d/%0d", label, n+1, n_neurons); + $fflush; + end + end + $display(" [%0s] all %0d neurons registered, waiting for completion...", label, n_neurons); + $fflush; + + // wait for all n_neurons completions + completed = 0; wd2 = 0; + while (completed < n_neurons && wd2 < 2000000) begin + @(posedge clk); + wd2 = wd2 + 1; + completed = jobs_completed; + if ((wd2 % 20000) == 0) begin + $display(" [%0s] watchdog %0d: completed=%0d/%0d total_cycles=%0d", label, wd2, completed, n_neurons, total_cycles); +`ifdef STEP16_DEBUG_TRACE + $display(" slot0: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_ready); + $display(" slot1: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_ready); + $display(" sdram: req=%0d busy=%0d ready=%0d req_pending=%0d state=%0d | arb: owner=%0d pending=%0b wide_req=%0b wide_ready=%0b", + u_nmp.u_sdram_backend.u_sdram_ctrl.req, + u_nmp.u_sdram_backend.u_sdram_ctrl.busy, + u_nmp.u_sdram_backend.u_sdram_ctrl.ready, + u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.req_pending, + u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state, + u_nmp.u_arbiter_wide.owner, + u_nmp.u_arbiter_wide.pending, + u_nmp.wide_slot_mem_req, + u_nmp.wide_slot_mem_ready); +`endif + $fflush; + end + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + if (completed < n_neurons) begin + $display("FAIL %0s: only %0d/%0d neurons completed within watchdog", label, completed, n_neurons); + errors = errors + 1; + end else begin : check_block + integer local_errors; + local_errors = 0; + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) + for (k = 0; k < P_IN; k = k + 1) + acc = acc + peek_byte(x_base + t*P_IN + k) * peek_byte_weight(w_base + n*len + t*P_IN + k); + golden = relu_sat(acc); + real_y = peek_byte(res_base + n); + if (real_y !== golden) begin + $display("FAIL %0s neuron %0d: real=%0d golden=%0d", label, n, real_y, golden); + local_errors = local_errors + 1; + end + end + if (local_errors == 0) + $display("PASS %0s: all %0d neurons bit-exact vs golden", label, n_neurons); + else + errors = errors + 1; + end + report_instrumentation(label, n_neurons); + report_step17_instrumentation; + end + endtask + + // E: Multilayer (8 layer-1 random neurons -> shared hidden vector + // -> 2 layer-2 neurons consuming it, real dependency wake-up + + // real cross-node data forwarding through real PSRAM). + localparam L1_N = 8; + localparam L2_N = 2; + integer rand_seed; + + task automatic run_multilayer( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] l1x_base, input [ADDR_WIDTH-1:0] l1w_base, + input [ADDR_WIDTH-1:0] hidden_base, + input [ADDR_WIDTH-1:0] l2w_base, input [ADDR_WIDTH-1:0] l2res_base + ); + integer n, k, acc, completed, wd2; + reg signed [7:0] xv, wv, golden_l1 [0:L1_N-1], golden_l2, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps, l2_deps; + integer local_errors; + begin + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + l2_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + for (n = 0; n < L1_N; n = n + 1) + l2_deps[n*NODE_IDW +: NODE_IDW] = node_base + n[NODE_IDW-1:0]; + + rand_seed = 32'hC0FFEE01; + $display("RANDOM SEED (workload E, layer-1 data) = 32'h%08h", rand_seed); + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + for (n = 0; n < L1_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + xv = $random(rand_seed) % 9; // deterministic PRNG stream, range roughly [-8,8] + wv = $random(rand_seed) % 9; + poke_byte(l1x_base + n*P_IN + k, xv); + poke_byte(l1w_base + n*P_IN + k, wv); + acc = acc + xv*wv; + end + golden_l1[n] = relu_sat(acc); + poke_byte(hidden_base + n, 8'sd0); // poison hidden slot + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + l1x_base + n*P_IN, l1w_base + n*P_IN, 16'd1, hidden_base + n); + end + + for (n = 0; n < L2_N; n = n + 1) begin + for (k = 0; k < P_IN; k = k + 1) + poke_byte(l2w_base + n*P_IN + k, ((n + k) % 6) + 1); + register_node(node_base + L1_N[NODE_IDW-1:0] + n[NODE_IDW-1:0], L1_N[$clog2(MAX_DEPS+1)-1:0], l2_deps, + hidden_base, l2w_base + n*P_IN, 16'd1, l2res_base + n); + end + + completed = 0; wd2 = 0; + while (completed < (L1_N+L2_N) && wd2 < 2000000) begin + @(posedge clk); wd2 = wd2 + 1; completed = jobs_completed; + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < (L1_N+L2_N)) begin + $display("FAIL Multilayer: only %0d/%0d nodes completed", completed, L1_N+L2_N); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < L1_N; n = n + 1) begin + real_y = peek_byte(hidden_base + n); + if (real_y !== golden_l1[n]) begin + $display("FAIL Multilayer L1 neuron %0d: real=%0d golden=%0d", n, real_y, golden_l1[n]); + local_errors = local_errors + 1; + end + end + for (n = 0; n < L2_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) + acc = acc + golden_l1[k] * peek_byte(l2w_base + n*P_IN + k); + golden_l2 = relu_sat(acc); + real_y = peek_byte(l2res_base + n); + if (real_y !== golden_l2) begin + $display("FAIL Multilayer L2 neuron %0d: real=%0d golden=%0d (using REAL L1 hidden values)", n, real_y, golden_l2); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS Multilayer: 8 L1 (random) -> 2 L2 neurons, all bit-exact, real cross-node forwarding via real PSRAM"); + else errors = errors + 1; + report_instrumentation("E-Multilayer", L1_N+L2_N); + end + endtask + + // F: DAG diamond+fan-in (A,B indep; C dep-A; D dep-B; E dep-C&D + // [2-hop]; F dep-A,B,C [mixed, 3 producers]) + task automatic run_dag( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, input [ADDR_WIDTH-1:0] w_base, input [ADDR_WIDTH-1:0] res_base + ); + integer n, k, acc, completed, wd2, local_errors; + reg signed [7:0] golden [0:5]; + reg signed [7:0] real_y; + reg [MAX_DEPS*NODE_IDW-1:0] deps; + reg [NODE_IDW-1:0] idA, idB, idC, idD, idE, idF; + begin + idA = node_base+0; idB = node_base+1; idC = node_base+2; + idD = node_base+3; idE = node_base+4; idF = node_base+5; + + // Each of the 6 nodes: its own small independent 8-input + // job (deterministic, distinct per node) -- dependencies + // here are purely about SCHEDULING/wake-up order, not + // data forwarding (workload E already covers that). + for (n = 0; n < 6; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + poke_byte(x_base + n*P_IN + k, ((n+k)%4)+1); + poke_byte(w_base + n*P_IN + k, ((n+k)%5)+1); + acc = acc + peek_byte(x_base+n*P_IN+k)*peek_byte(w_base+n*P_IN+k); + end + golden[n] = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); + end + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + register_node(idA, 0, deps, x_base+0*P_IN, w_base+0*P_IN, 16'd1, res_base+0); + register_node(idB, 0, deps, x_base+1*P_IN, w_base+1*P_IN, 16'd1, res_base+1); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; + register_node(idC, 1, deps, x_base+2*P_IN, w_base+2*P_IN, 16'd1, res_base+2); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idB; + register_node(idD, 1, deps, x_base+3*P_IN, w_base+3*P_IN, 16'd1, res_base+3); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idC; deps[1*NODE_IDW+:NODE_IDW] = idD; + register_node(idE, 2, deps, x_base+4*P_IN, w_base+4*P_IN, 16'd1, res_base+4); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; deps[1*NODE_IDW+:NODE_IDW] = idB; deps[2*NODE_IDW+:NODE_IDW] = idC; + register_node(idF, 3, deps, x_base+5*P_IN, w_base+5*P_IN, 16'd1, res_base+5); + + completed = 0; wd2 = 0; + while (completed < 6 && wd2 < 2000000) begin @(posedge clk); wd2=wd2+1; completed = jobs_completed; end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < 6) begin + $display("FAIL DAG: only %0d/6 nodes completed", completed); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < 6; n = n + 1) begin + real_y = peek_byte(res_base+n); + if (real_y !== golden[n]) begin + $display("FAIL DAG node %0d: real=%0d golden=%0d", n, real_y, golden[n]); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS DAG: 6-node diamond+fan-in (2-hop transitive wake-up, 3-producer mixed-depth dependency), all bit-exact"); + else errors = errors + 1; + report_instrumentation("F-DAG", 6); + end + endtask + + initial begin + errors = 0; tests = 0; + rst = 1; rst_fast = 1; reg_valid = 0; reg_node_id = 0; reg_required = 0; reg_producer_ids = 0; + reg_x_base = 0; reg_w_base = 0; reg_n_tiles = 0; reg_result_addr = 0; + measure_en = 0; + repeat(5) @(posedge clk); + repeat(5) @(posedge clk_fast); + rst = 0; rst_fast = 0; + + $display("========================================"); + $display("NMS D-Stress benchmark (STEP19, SINGLE SDRAM (AS4C4M16SA-6TIN) for weights+activations+results, no PSRAM anywhere) -- N_SLOTS_CFG=%0d PFD_CFG=%0d", N_SLOTS_CFG, PFD_CFG); + $display("========================================"); + + wait (u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.state == u_nmp.u_sdram_backend.u_sdram_ctrl.u_sdram_ctrl.S_IDLE); + @(posedge clk); + + // Official V2 memory map (datasheet ch.5): weights @ 0x010000, + // activations @ 0x200000, results @ 0x300000 -- non-overlapping + // 1MB-aligned regions in the single SDRAM. + run_dense_layer("D-Stress", 256, 16, 16'd400, 26'h200000, 26'h010000, 26'h300000, 1'b0); + + // FPGA_DATA_READY check: the whole graph (256 nodes) just + // finished and no new work has been registered -- data_ready + // must be asserted (system-idle sticky flag, see + // nms_dataflow_core_sdram.v). A few idle cycles for the + // busy->idle edge to settle before sampling. + repeat (4) @(posedge clk); + if (u_nmp.data_ready !== 1'b1) begin + $display("FAIL data_ready: expected 1 after graph completion, got %b", u_nmp.data_ready); + errors = errors + 1; + end else begin + $display("PASS data_ready: correctly asserted after graph completion"); + end + + $display("========================================"); + if (errors == 0) + $display("ALL %0d WORKLOAD SUITES PASSED (N_SLOTS_CFG=%0d, PFD_CFG=%0d, SINGLE SDRAM for weights+activations+results, no PSRAM)", tests, N_SLOTS_CFG, PFD_CFG); + else + $display("FAILED: %0d/%0d workload suite(s) had errors -- see messages above", errors, tests); + $display("========================================"); + $finish; + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_nms_dstress_sdram_openrow.v b/hardware/v2/nms/sim/tb_nms_dstress_sdram_openrow.v new file mode 100644 index 0000000..3932653 --- /dev/null +++ b/hardware/v2/nms/sim/tb_nms_dstress_sdram_openrow.v @@ -0,0 +1,880 @@ +`timescale 1ns/1ps + +// ================================================================ +// FPGA-Neural V2 -- Final Benchmark Campaign (post-M10, real +// end-to-end characterization, docs/v2-description.md §22/§30/§32) +// +// One testbench, compiled once per N_SLOTS configuration (N_SLOTS_CFG +// parameter, overridden at Verilator invocation via -GN_SLOTS_CFG=N), +// running SIX representative workloads back-to-back through the REAL +// neural_multiprocessor.v (M8: dataflow_core + slot_mem_arbiter + the +// real, unmodified V1 PSRAM chain), with: +// - a software "golden" model replicating neural_processor.v's exact +// integer math (sum(x*w) over all tiles, ReLU + INT8 saturate -- +// dataflow_core.v hardcodes bias=0/ACT_RELU for every job, so the +// golden model only needs to replicate that one path) +// - bit-exact verification of EVERY neuron's real result against +// that golden model (peek_byte from the real psram_model backing +// array -- an oracle independent of the RTL under test) +// - real cycle-accounting instrumentation (testbench-only, no RTL +// touched): per-slot busy/idle cycles, shared PSRAM port busy/idle +// cycles, REAL tiles delivered per slot (operand_valid&& +// operand_ready pulses -- one pulse = one whole P_IN-wide tile +// consumed by neural_processor, NOT one byte), director/dependency +// bookkeeping (jobs allocated/completed, ready-queue occupancy, +// WAITING/READY/DISPATCHED node counts, producer-done wakeups) +// +// Workloads (node_id ranges are disjoint across all six so the WHOLE +// campaign runs in ONE continuous simulation -- only ONE real PSRAM +// power-up wait, no reset between phases, closer to real sustained +// operation than resetting between every workload): +// A) Small -- 16 independent neurons, 8 inputs each +// B) Medium -- 64 independent neurons, 32 inputs each +// C) Large -- 128 independent neurons, 128 inputs each +// D) Stress -- 256 independent neurons, 128 inputs each +// E) Multilayer -- 8 layer-1 neurons (RANDOM data, logged seed) feed +// a shared 8-byte hidden vector; 2 layer-2 neurons +// consume that vector (real cross-node data +// forwarding through real PSRAM, real dependency +// wake-up, "shared producer/multiple consumers") +// F) DAG -- 6-node diamond+fan-in graph (A,B independent; C +// dep on A; D dep on B; E dep on BOTH C and D +// [2-hop transitive wake-up]; F dep on A,B,C [mixed +// direct+1-hop, 3 producers]) +// +// All workloads A-D use a REALISTIC dense-layer shape: one shared +// input activation vector, N independent weight vectors (one per +// neuron) -- exactly how a real fully-connected layer's neurons share +// their layer's input. This is not an isolated synthetic microbench. +// +// Verified with Verilator (decisions.log DEC-0004). +// ================================================================ + +// ================================================================ +// STEP11 variant: identical D-Stress workload/golden-model/correctness +// criteria as tb_nms_dstress.v (STEP9's own official benchmark), but +// instantiating nms_neural_multiprocessor_pf (REAL weight prefetch +// engine, weight_prefetch_engine.v) instead of the baseline +// nms_neural_multiprocessor.v, with an added PFD_CFG (PREFETCH_DISTANCE) +// parameter, plus NEW instrumentation (testbench-only, no RTL touched) +// for the two STEP11-mandated metrics that cannot be derived from the +// STEP9 instrumentation alone: +// weight_stall_cycles = cycles a slot is otherwise ready to +// present a tile (activation resident, +// in bounds) but blocked purely because +// tile_idx >= wgt_ready_count +// prefetch_effectiveness = tiles consumed with ZERO such +// weight-blocking cycles beforehand +// (i.e. the weight was ALREADY resident +// the moment the tile became eligible) +// / total tiles consumed +// per STEP11's own explicit metric definitions. +// ================================================================ +module tb #( + parameter N_SLOTS_CFG = 2, + parameter PFD_CFG = 8 +); + + localparam ADDR_WIDTH = 26; // AS4C32M16SA: 25-bit word address + 1 byte-select bit + localparam DATA_WIDTH = 8; + localparam P_IN = 8; + localparam ACC_WIDTH = 32; + // N_NODES must exceed the HIGHEST node_id used by ANY workload + // (node_base + count - 1) -- workload D's own range alone + // (node_base=400, 256 neurons) reaches id 655. An earlier draft + // used N_NODES=512: D's ids silently wrapped (9-bit truncation) + // past id 511, colliding with workload A's already-DISPATCHED + // node 0 (dependency_manager never reclaims dispatched node slots, + // DEC-0008) and deadlocking register_node's reg_ready wait + // forever. A real consequence of DEC-0008's design choice, not an + // RTL bug -- fixed here by sizing N_NODES generously above the + // real id range used below (see decisions.log DEC-0008 and the + // final benchmark report's Limitations section). + localparam N_NODES = 1024; + localparam MAX_DEPS = 8; + localparam QUEUE_DEPTH = 8; + localparam NODE_IDW = $clog2(N_NODES); + localparam CLK_PERIOD = 12.5; // 80 MHz, matches psram_controller's CLK_FREQ_MHZ + + reg clk, rst; + initial begin clk = 1'b0; forever #(CLK_PERIOD/2.0) clk = ~clk; end + + reg reg_valid; + wire reg_ready; + reg [NODE_IDW-1:0] reg_node_id; + reg [$clog2(MAX_DEPS+1)-1:0] reg_required; + reg [MAX_DEPS*NODE_IDW-1:0] reg_producer_ids; + reg [ADDR_WIDTH-1:0] reg_x_base, reg_w_base, reg_result_addr; + reg [15:0] reg_n_tiles; + + // STEP19: ONE physical SDRAM interface. weights, activations, and + // results ALL share this single bus/chip now -- no PSRAM anywhere. + wire sdram_cke, sdram_cs_n, sdram_ras_n, sdram_cas_n, sdram_we_n; + wire [1:0] sdram_ba; + wire [12:0] sdram_a; + wire [15:0] sdram_dq; + wire [1:0] sdram_dqm; + + nms_neural_multiprocessor_sdram_openrow #( + .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH), + .N_SLOTS(N_SLOTS_CFG), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH), + .MAX_TILES(16), .PREFETCH_DISTANCE(PFD_CFG), .CLK_FREQ_MHZ(80) + ) u_nmp ( + .clk(clk), .rst(rst), + .reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id), + .reg_required(reg_required), .reg_producer_ids(reg_producer_ids), + .reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles), + .reg_result_addr(reg_result_addr), + .sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n), + .sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n), + .sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm) + ); + + sdram_model #(.CLK_FREQ_MHZ(80)) u_sdram ( + .clk(clk), .cke(sdram_cke), .cs_n(sdram_cs_n), .ras_n(sdram_ras_n), + .cas_n(sdram_cas_n), .we_n(sdram_we_n), .ba(sdram_ba), .a(sdram_a), + .dq(sdram_dq), .dqm(sdram_dqm) + ); + + // ============================================================ + // STEP19: byte-level backdoor access (test setup/verification + // only) -- weights, activations, AND results now ALL live on the + // single real SDRAM physical interface (u_sdram); there is no + // PSRAM anywhere in this system anymore. poke_byte/peek_byte (used + // by activation+result call sites) and poke_byte_weight/peek_byte + // _weight (used by weight call sites) are now identical in + // implementation -- kept as two names rather than merged, to avoid + // touching every one of their many existing call sites for a + // cosmetic rename; both correctly target the same u_sdram.mem + // backing array via the same byte_addr>>1 / byte_addr[0] pattern. + // ============================================================ + task automatic poke_byte(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // sdram_model.v's own `mem` array is flat-indexed by the 25-bit + // word address directly (bank*ROWS*COLS + row*COLS + col, which, + // given ROWS=8192/COLS=1024 are both powers of 2, is numerically + // IDENTICAL to treating the address as one flat 25-bit integer -- + // confirmed against sdram_model.v's own BANKS/ROWS/COLS localparams + // before writing this, not assumed) -- so this is the exact same + // byte_addr>>1 / byte_addr[0] pattern as the original single-chip + // poke_byte/peek_byte above, just against u_sdram.mem instead of + // u_psram.mem. + task automatic poke_byte_weight(input [ADDR_WIDTH-1:0] byte_addr, input signed [7:0] val); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + if (byte_addr[0] == 1'b0) u_sdram.mem[word_addr][7:0] = val; + else u_sdram.mem[word_addr][15:8] = val; + end + endtask + + function automatic signed [7:0] peek_byte_weight(input [ADDR_WIDTH-1:0] byte_addr); + reg [24:0] word_addr; + begin + word_addr = byte_addr[ADDR_WIDTH-1:1]; + peek_byte_weight = (byte_addr[0] == 1'b0) ? u_sdram.mem[word_addr][7:0] : u_sdram.mem[word_addr][15:8]; + end + endfunction + + // Golden model: exactly replicates neural_processor.v's real path + // through dataflow_core (bias=0, ACT_RELU always -- see + // dataflow_core.v's own hardcoded job_bias/job_activation). + function automatic signed [7:0] relu_sat(input integer acc); + begin + if (acc <= 0) relu_sat = 8'sd0; + else if (acc > 127) relu_sat = 8'sd127; + else relu_sat = acc[7:0]; + end + endfunction + + // ============================================================ + // Node registration (generalized to MAX_DEPS=8 producers, passed + // as a packed array; n_producers of them are meaningful, the rest + // ignored since reg_required gates how many entries the RTL + // actually reads). + // ============================================================ + task automatic register_node( + input [NODE_IDW-1:0] nid, + input [$clog2(MAX_DEPS+1)-1:0] required, + input [MAX_DEPS*NODE_IDW-1:0] producer_ids_packed, + input [ADDR_WIDTH-1:0] xb, input [ADDR_WIDTH-1:0] wb, + input [15:0] nt, input [ADDR_WIDTH-1:0] resaddr + ); + begin + @(posedge clk); + reg_node_id = nid; + reg_required = required; + reg_producer_ids = producer_ids_packed; + reg_x_base = xb; reg_w_base = wb; reg_n_tiles = nt; reg_result_addr = resaddr; + reg_valid = 1'b1; + while (!reg_ready) @(posedge clk); + @(posedge clk); + reg_valid = 1'b0; + end + endtask + + // ============================================================ + // M10+ real cycle-accounting instrumentation (testbench-only, no + // RTL touched -- same idiom as EXP-0013). + // ============================================================ + reg measure_en; + integer total_cycles; + integer psram_busy_cycles; + integer ni; // moved up from its original later declaration point + // (STEP20 tooling-compatibility fix, zero behavior + // change -- see nms_memory_manager_stream_wide.v's own + // header note on icarus 13.0's stricter declared- + // before-use rule for procedural blocks) + genvar gi; + + reg [N_SLOTS_CFG-1:0] slot_busy_bit; // memory_manager.state != MM_IDLE, this cycle + reg [N_SLOTS_CFG-1:0] slot_tile_bit; // operand_valid && operand_ready, this cycle + integer slot_busy_cycles [0:N_SLOTS_CFG-1]; + integer slot_tiles_delivered [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_MON + always @(*) begin + slot_busy_bit[gi] = (u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.state != 3'd0); + slot_tile_bit[gi] = u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_valid && + u_nmp.u_dataflow_core.GEN_SLOT[gi].mm_operand_ready; + end + end + endgenerate + + // ============================================================ + // STEP17 Part B/C: cycle-decomposition + SDRAM effectiveness + // instrumentation (testbench-only, no RTL touched). + // ============================================================ + integer active_count; // popcount(slot_busy_bit) this cycle + integer active_hist [0:4]; // cycles with exactly k active slots, k=0..4 + integer useful_mac_cycles; // sum over cycles of (#slots with slot_tile_bit this cycle) + integer first_tile_cyc; // total_cycles value at the first tile ever delivered (startup boundary) + integer last_tile_cyc; // total_cycles value at the most recent tile delivered (drain boundary) + integer any_tile_bit; + + // SDRAM controller-port instrumentation (real signals on the + // actual sdram_controller.v instance servicing all weight fetch) + integer sdram_req_count, sdram_ready_count, sdram_wr_count; + integer sdram_busy_cycles, sdram_refresh_count; + integer sdram_req_start_cyc, sdram_lat_sum, sdram_lat_min, sdram_lat_max, sdram_lat_n; + reg sdram_prev_state_is_refwait; + + initial begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + sdram_prev_state_is_refwait=1'b0; + end + + always @(posedge clk) begin + if (measure_en) begin + active_count = slot_busy_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) active_count = active_count + slot_busy_bit[ni]; + active_hist[active_count] <= active_hist[active_count] + 1; + + any_tile_bit = slot_tile_bit[0]; + for (ni = 1; ni < N_SLOTS_CFG; ni = ni + 1) any_tile_bit = any_tile_bit | slot_tile_bit[ni]; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) + if (slot_tile_bit[ni]) useful_mac_cycles <= useful_mac_cycles + 1; + if (any_tile_bit) begin + if (first_tile_cyc < 0) first_tile_cyc <= total_cycles; + last_tile_cyc <= total_cycles; + end + + // ---- real SDRAM controller port (single physical chip, + // all weight-fetch traffic funnels through this one + // instance) ---- + if (u_nmp.u_sdram_backend.u_sdram_ctrl.req) begin + sdram_req_count <= sdram_req_count + 1; + sdram_req_start_cyc <= total_cycles; + if (u_nmp.u_sdram_backend.u_sdram_ctrl.wr) sdram_wr_count <= sdram_wr_count + 1; + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.ready) begin + sdram_ready_count <= sdram_ready_count + 1; + sdram_lat_sum <= sdram_lat_sum + (total_cycles - sdram_req_start_cyc); + sdram_lat_n <= sdram_lat_n + 1; + if ((total_cycles - sdram_req_start_cyc) < sdram_lat_min) sdram_lat_min <= (total_cycles - sdram_req_start_cyc); + if ((total_cycles - sdram_req_start_cyc) > sdram_lat_max) sdram_lat_max <= (total_cycles - sdram_req_start_cyc); + end + if (u_nmp.u_sdram_backend.u_sdram_ctrl.busy) sdram_busy_cycles <= sdram_busy_cycles + 1; + sdram_prev_state_is_refwait <= (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9); + if (u_nmp.u_sdram_backend.u_sdram_ctrl.state == 5'd9 && !sdram_prev_state_is_refwait) + sdram_refresh_count <= sdram_refresh_count + 1; + end + end + + task automatic report_step17_instrumentation; + real active_pct [0:4]; + real util_pct, startup_cycles, drain_cycles; + real sdram_avg_lat, sdram_busy_pct, sdram_bytes_per_cycle; + integer kk, total_tiles_all; + begin + total_tiles_all = 0; + for (kk = 0; kk < N_SLOTS_CFG; kk = kk + 1) total_tiles_all = total_tiles_all + slot_tiles_delivered[kk]; + $display(" ---- STEP17 Part B: cycle decomposition ----"); + for (kk = 0; kk <= N_SLOTS_CFG; kk = kk + 1) begin + active_pct[kk] = (total_cycles > 0) ? (100.0*active_hist[kk]/total_cycles) : 0.0; + $display(" active_slots=%0d: %0d cycles (%0.2f%%)", kk, active_hist[kk], active_pct[kk]); + end + util_pct = (total_cycles > 0) ? (100.0*useful_mac_cycles/(total_cycles*1.0*N_SLOTS_CFG)) : 0.0; + $display(" useful_mac_cycles (slot-tile-delivery events, summed)=%0d (%0.2f%% of total_cycles*N_SLOTS)", useful_mac_cycles, util_pct); + startup_cycles = (first_tile_cyc >= 0) ? (1.0*first_tile_cyc) : 0.0; + drain_cycles = (last_tile_cyc >= 0) ? (1.0*(total_cycles - last_tile_cyc)) : 0.0; + $display(" startup (cycles before first tile delivered anywhere)=%0.0f", startup_cycles); + $display(" drain (cycles after last tile delivered, until job completion)=%0.0f", drain_cycles); + $display(" ---- STEP17 Part C: SDRAM effectiveness ----"); + sdram_avg_lat = (sdram_lat_n > 0) ? (1.0*sdram_lat_sum/sdram_lat_n) : 0.0; + sdram_busy_pct = (total_cycles > 0) ? (100.0*sdram_busy_cycles/total_cycles) : 0.0; + sdram_bytes_per_cycle = (total_cycles > 0) ? (8.0*sdram_ready_count/total_cycles) : 0.0; + $display(" sdram_req_count=%0d sdram_ready_count=%0d sdram_wr_count=%0d (real reads vs writes)", + sdram_req_count, sdram_ready_count, sdram_wr_count); + $display(" sdram_busy_cycles=%0d/%0d (%0.2f%%)", sdram_busy_cycles, total_cycles, sdram_busy_pct); + $display(" sdram_refresh_count=%0d (real AUTO REFRESH commands issued)", sdram_refresh_count); + $display(" sdram_request_latency: min=%0d max=%0d avg=%0.2f cycles (req-to-ready, single controller port)", + sdram_lat_min, sdram_lat_max, sdram_avg_lat); + $display(" sdram_avg_bytes_per_cycle (8 bytes/transaction * ready_count / total_cycles)=%0.4f", sdram_bytes_per_cycle); + end + endtask + + // ---- STEP11: weight-stall / prefetch-effectiveness instrumentation ---- + // slot_could_present_act: this slot's tile_idx is in-bounds and the + // activation operand for it is already resident -- i.e. everything + // EXCEPT the weight is ready. slot_weight_blocking: on top of that, + // the weight specifically is NOT yet ready (tile_idx>=wgt_ready_count) + // and the FSM is genuinely stalled on it (not mid-read-pipeline, not + // already holding a valid operand). + reg [N_SLOTS_CFG-1:0] slot_could_present_act; + reg [N_SLOTS_CFG-1:0] slot_weight_blocking; + reg [N_SLOTS_CFG-1:0] slot_stalled_this_tile; // sticky per current tile_idx + reg [31:0] prev_tile_idx [0:N_SLOTS_CFG-1]; + integer weight_stall_cycles [0:N_SLOTS_CFG-1]; + integer tiles_prefetched_clean [0:N_SLOTS_CFG-1]; // consumed w/ zero weight-blocking cycles + integer tiles_consumed_total [0:N_SLOTS_CFG-1]; + // plain (non-hierarchical) mirrors of each slot's tile_idx, populated + // combinationally inside the genvar-indexed generate block below -- + // a generate-block instance array (GEN_SLOT[.]) can only be indexed + // by a constant genvar, not a runtime `for` variable, so the + // sequential accumulation loop reads these plain arrays instead of + // reaching back into the hierarchy with a runtime index. + wire [31:0] slot_tile_idx_w [0:N_SLOTS_CFG-1]; + + generate + for (gi = 0; gi < N_SLOTS_CFG; gi = gi + 1) begin : GEN_SLOT_PF_MON + assign slot_tile_idx_w[gi] = {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx}; + always @(*) begin + slot_could_present_act[gi] = + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.n_tiles_reg}) && + ({{16{1'b0}}, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx} < + {16'b0, u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.usable_act}); + // nms_memory_manager_stream.v has no read_issued/ + // read_ready states (replaced by the rd_ptr/rd_pending + // read-ahead pipeline) -- the equivalent "blocked + // purely on weight readiness, nothing buffered yet" + // condition is simply: consumption pointer in bounds, + // activation ready, weight NOT ready, and no operand + // currently held in the skid buffer awaiting NP. + slot_weight_blocking[gi] = + slot_could_present_act[gi] && + !(u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.tile_idx < + u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.wgt_ready_count) && + !u_nmp.u_dataflow_core.GEN_SLOT[gi].u_mm.operand_valid; + end + end + endgenerate + + always @(posedge clk) begin + if (measure_en) begin + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (prev_tile_idx[ni] != slot_tile_idx_w[ni]) begin + // moved on to a new tile: clear the sticky flag for it + slot_stalled_this_tile[ni] <= 1'b0; + prev_tile_idx[ni] <= slot_tile_idx_w[ni]; + end else if (slot_weight_blocking[ni]) begin + slot_stalled_this_tile[ni] <= 1'b1; + weight_stall_cycles[ni] <= weight_stall_cycles[ni] + 1; + end + if (slot_tile_bit[ni]) begin + tiles_consumed_total[ni] <= tiles_consumed_total[ni] + 1; + if (!slot_stalled_this_tile[ni]) + tiles_prefetched_clean[ni] <= tiles_prefetched_clean[ni] + 1; + end + end + end + end + + // Director/dependency bookkeeping + integer jobs_allocated, jobs_completed, wakeups; + integer waiting_sum, ready_sum, dispatched_sum, sample_count; + + // Occupancy sampling is EXPENSIVE (a full N_NODES=512 scan) and is + // only needed for the small/structural workloads (A/B/E/F), not + // for the large neuron counts (C/D) where it would dominate + // simulation wall-time for no real benefit (per-slot/PSRAM/tile + // counters below are cheap and always collected). Gated by + // sample_occupancy, set per-workload. + reg sample_occupancy; + integer scan_i; + integer waiting_now, ready_now, dispatched_now; + + always @(posedge clk) begin + if (measure_en) begin + total_cycles <= total_cycles + 1; + if (u_nmp.u_arbiter.owner != 0) psram_busy_cycles <= psram_busy_cycles + 1; + for (ni = 0; ni < N_SLOTS_CFG; ni = ni + 1) begin + if (slot_busy_bit[ni]) slot_busy_cycles[ni] <= slot_busy_cycles[ni] + 1; + if (slot_tile_bit[ni]) slot_tiles_delivered[ni] <= slot_tiles_delivered[ni] + 1; + end + if (u_nmp.u_dataflow_core.dm_ready_valid && u_nmp.u_dataflow_core.dm_ready_ready) + jobs_allocated <= jobs_allocated + 1; + if (u_nmp.u_dataflow_core.dir_job_out_done) + jobs_completed <= jobs_completed + 1; + if (u_nmp.u_dataflow_core.dm_producer_done_valid) + wakeups <= wakeups + 1; + + if (sample_occupancy) begin + waiting_now = 0; ready_now = 0; dispatched_now = 0; + for (scan_i = 0; scan_i < N_NODES; scan_i = scan_i + 1) begin + case (u_nmp.u_dataflow_core.u_dep_mgr.node_state[scan_i]) + 2'd1: waiting_now = waiting_now + 1; + 2'd2: ready_now = ready_now + 1; + 2'd3: dispatched_now = dispatched_now + 1; + default: ; + endcase + end + waiting_sum <= waiting_sum + waiting_now; + ready_sum <= ready_sum + ready_now; + dispatched_sum <= dispatched_sum + dispatched_now; + sample_count <= sample_count + 1; + end + end + end + + task automatic reset_instrumentation(input do_sample_occupancy); + integer k; + begin + active_hist[0]=0; active_hist[1]=0; active_hist[2]=0; active_hist[3]=0; active_hist[4]=0; + useful_mac_cycles = 0; first_tile_cyc = -1; last_tile_cyc = -1; + sdram_req_count=0; sdram_ready_count=0; sdram_wr_count=0; + sdram_busy_cycles=0; sdram_refresh_count=0; + sdram_req_start_cyc=0; sdram_lat_sum=0; sdram_lat_min=999999; sdram_lat_max=0; sdram_lat_n=0; + total_cycles = 0; psram_busy_cycles = 0; + jobs_allocated = 0; jobs_completed = 0; wakeups = 0; + waiting_sum = 0; ready_sum = 0; dispatched_sum = 0; sample_count = 0; + sample_occupancy = do_sample_occupancy; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + slot_busy_cycles[k] = 0; + slot_tiles_delivered[k] = 0; + weight_stall_cycles[k] = 0; + tiles_prefetched_clean[k] = 0; + tiles_consumed_total[k] = 0; + slot_stalled_this_tile[k] = 1'b0; + prev_tile_idx[k] = 32'hFFFFFFFF; + end + end + endtask + + task automatic report_instrumentation(input [255:0] label, input integer n_neurons_completed); + integer k, total_tiles; + integer total_weight_stall_cycles, total_tiles_consumed_all, total_tiles_prefetched_clean; + real avg_waiting, avg_ready, avg_dispatched; + real psram_util, sustained_mac_per_cycle, wallclock_us; + real processor_utilization, weight_stall_pct, prefetch_effectiveness_pct; + begin + total_tiles = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) total_tiles = total_tiles + slot_tiles_delivered[k]; + avg_waiting = (sample_count > 0) ? (1.0*waiting_sum/sample_count) : 0.0; + avg_ready = (sample_count > 0) ? (1.0*ready_sum/sample_count) : 0.0; + avg_dispatched = (sample_count > 0) ? (1.0*dispatched_sum/sample_count) : 0.0; + psram_util = (total_cycles > 0) ? (100.0*psram_busy_cycles/total_cycles) : 0.0; + sustained_mac_per_cycle = (total_cycles > 0) ? (1.0*total_tiles*P_IN/total_cycles) : 0.0; + wallclock_us = total_cycles * CLK_PERIOD / 1000.0; + $display("---- BENCHMARK REPORT: %0s ----", label); + $display(" total_cycles=%0d wallclock_us=%0.3f", total_cycles, wallclock_us); + $display(" neurons_completed=%0d tiles_delivered(real)=%0d", n_neurons_completed, total_tiles); + $display(" jobs_allocated=%0d jobs_completed=%0d dependency_wakeups=%0d", jobs_allocated, jobs_completed, wakeups); + $display(" shared AR (activation+result) arbiter-side utilization: %0.1f%% (%0d/%0d busy cycles)", psram_util, psram_busy_cycles, total_cycles); + for (k = 0; k < N_SLOTS_CFG; k = k + 1) + $display(" slot %0d: busy=%0d/%0d (%0.1f%%) tiles=%0d", k, slot_busy_cycles[k], total_cycles, + (total_cycles>0)?(100.0*slot_busy_cycles[k]/total_cycles):0.0, slot_tiles_delivered[k]); + if (sample_count > 0) + $display(" dependency_manager avg occupancy (sampled every measured cycle): waiting=%0.2f ready=%0.2f dispatched=%0.2f", avg_waiting, avg_ready, avg_dispatched); + else + $display(" dependency_manager occupancy: NOT SAMPLED for this workload (N_NODES scan skipped for large neuron counts to keep simulation time reasonable)"); + $display(" DERIVED: sustained end-to-end MAC/cycle = %0.4f (real tiles*%0d / real total_cycles)", sustained_mac_per_cycle, P_IN); + if (n_neurons_completed > 0) + $display(" DERIVED: cycles/neuron = %0.2f", 1.0*total_cycles/n_neurons_completed); + if (total_tiles > 0) + $display(" DERIVED: cycles/tile = %0.2f", 1.0*total_cycles/total_tiles); + + // ---- STEP11 metrics ---- + total_weight_stall_cycles = 0; total_tiles_consumed_all = 0; total_tiles_prefetched_clean = 0; + for (k = 0; k < N_SLOTS_CFG; k = k + 1) begin + total_weight_stall_cycles = total_weight_stall_cycles + weight_stall_cycles[k]; + total_tiles_consumed_all = total_tiles_consumed_all + tiles_consumed_total[k]; + total_tiles_prefetched_clean = total_tiles_prefetched_clean + tiles_prefetched_clean[k]; + end + processor_utilization = (total_cycles > 0) ? (100.0*total_tiles/(total_cycles*1.0)) : 0.0; + weight_stall_pct = (total_cycles > 0) ? (100.0*total_weight_stall_cycles/(total_cycles*N_SLOTS_CFG*1.0)) : 0.0; + prefetch_effectiveness_pct = (total_tiles_consumed_all > 0) ? + (100.0*total_tiles_prefetched_clean/(total_tiles_consumed_all*1.0)) : 0.0; + $display(" [STEP11] PFD=%0d weight_stall_cycles(sum,all slots)=%0d (%0.2f%% of total_cycles*N_SLOTS)", + PFD_CFG, total_weight_stall_cycles, weight_stall_pct); + $display(" [STEP11] tiles_consumed=%0d tiles_prefetched_clean(zero weight-block before consumption)=%0d", + total_tiles_consumed_all, total_tiles_prefetched_clean); + $display(" [STEP11] DERIVED: prefetch_effectiveness = %0.2f%%", prefetch_effectiveness_pct); + $display(" [STEP11] DERIVED: processor_utilization (tiles*P_IN-equivalent proxy, see sustained MAC/cycle) reference sustained_mac_per_cycle=%0.4f", sustained_mac_per_cycle); + end + endtask + + // ============================================================ + // Workload generators + // ============================================================ + integer errors, tests; + integer wd; + + // A/B/C/D: shared-input dense layer. Generates the shared X + // vector, then N independent (neuron, weight-vector) jobs, each + // verified bit-exact against the golden model. + task automatic run_dense_layer( + input [255:0] label, + input integer n_neurons, + input integer n_tiles_count, + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, + input [ADDR_WIDTH-1:0] w_base, + input [ADDR_WIDTH-1:0] res_base, + input sample_occ + ); + integer n, t, k, len, acc; + reg signed [7:0] xv, wv, golden, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps; + integer completed, wd2; + begin + len = n_tiles_count * P_IN; + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + + // shared input vector + for (k = 0; k < len; k = k + 1) + poke_byte(x_base + k, ((k % 8) + 1)); + + reset_instrumentation(sample_occ); + measure_en = 1'b1; + + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) begin + for (k = 0; k < P_IN; k = k + 1) begin + xv = peek_byte(x_base + t*P_IN + k); + wv = (((n + t*P_IN + k) % 8) + 1); + poke_byte_weight(w_base + n*len + t*P_IN + k, wv); + acc = acc + xv*wv; + end + end + golden = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); // poison, must NOT still be 0 after completion (unless golden IS 0 -- checked separately) + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + x_base, w_base + n*len, n_tiles_count[15:0], res_base + n); + if ((n % 32) == 0) begin + $display(" [%0s] registered %0d/%0d", label, n+1, n_neurons); + $fflush; + end + end + $display(" [%0s] all %0d neurons registered, waiting for completion...", label, n_neurons); + $fflush; + + // wait for all n_neurons completions + completed = 0; wd2 = 0; + while (completed < n_neurons && wd2 < 2000000) begin + @(posedge clk); + wd2 = wd2 + 1; + completed = jobs_completed; + if ((wd2 % 20000) == 0) begin + $display(" [%0s] watchdog %0d: completed=%0d/%0d total_cycles=%0d", label, wd2, completed, n_neurons, total_cycles); +`ifdef STEP16_DEBUG_TRACE + $display(" slot0: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[0].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[0].mm_operand_ready); + $display(" slot1: mm.state=%0d tile_idx=%0d n_tiles_reg=%0d wgt_ready_count=%0d usable_act=%0d op_valid=%0d op_ready=%0d", + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.state, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.tile_idx, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.n_tiles_reg, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.wgt_ready_count, + u_nmp.u_dataflow_core.GEN_SLOT[1].u_mm.usable_act, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_valid, + u_nmp.u_dataflow_core.GEN_SLOT[1].mm_operand_ready); + $display(" sdram: req=%0d busy=%0d ready=%0d req_pending=%0d state=%0d | arb: owner=%0d pending=%0b wide_req=%0b wide_ready=%0b", + u_nmp.u_sdram_backend.u_sdram_ctrl.req, + u_nmp.u_sdram_backend.u_sdram_ctrl.busy, + u_nmp.u_sdram_backend.u_sdram_ctrl.ready, + u_nmp.u_sdram_backend.u_sdram_ctrl.req_pending, + u_nmp.u_sdram_backend.u_sdram_ctrl.state, + u_nmp.u_arbiter_wide.owner, + u_nmp.u_arbiter_wide.pending, + u_nmp.wide_slot_mem_req, + u_nmp.wide_slot_mem_ready); +`endif + $fflush; + end + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + if (completed < n_neurons) begin + $display("FAIL %0s: only %0d/%0d neurons completed within watchdog", label, completed, n_neurons); + errors = errors + 1; + end else begin : check_block + integer local_errors; + local_errors = 0; + for (n = 0; n < n_neurons; n = n + 1) begin + acc = 0; + for (t = 0; t < n_tiles_count; t = t + 1) + for (k = 0; k < P_IN; k = k + 1) + acc = acc + peek_byte(x_base + t*P_IN + k) * peek_byte_weight(w_base + n*len + t*P_IN + k); + golden = relu_sat(acc); + real_y = peek_byte(res_base + n); + if (real_y !== golden) begin + $display("FAIL %0s neuron %0d: real=%0d golden=%0d", label, n, real_y, golden); + local_errors = local_errors + 1; + end + end + if (local_errors == 0) + $display("PASS %0s: all %0d neurons bit-exact vs golden", label, n_neurons); + else + errors = errors + 1; + end + report_instrumentation(label, n_neurons); + report_step17_instrumentation; + end + endtask + + // E: Multilayer (8 layer-1 random neurons -> shared hidden vector + // -> 2 layer-2 neurons consuming it, real dependency wake-up + + // real cross-node data forwarding through real PSRAM). + localparam L1_N = 8; + localparam L2_N = 2; + integer rand_seed; + + task automatic run_multilayer( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] l1x_base, input [ADDR_WIDTH-1:0] l1w_base, + input [ADDR_WIDTH-1:0] hidden_base, + input [ADDR_WIDTH-1:0] l2w_base, input [ADDR_WIDTH-1:0] l2res_base + ); + integer n, k, acc, completed, wd2; + reg signed [7:0] xv, wv, golden_l1 [0:L1_N-1], golden_l2, real_y; + reg [MAX_DEPS*NODE_IDW-1:0] no_deps, l2_deps; + integer local_errors; + begin + no_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + l2_deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + for (n = 0; n < L1_N; n = n + 1) + l2_deps[n*NODE_IDW +: NODE_IDW] = node_base + n[NODE_IDW-1:0]; + + rand_seed = 32'hC0FFEE01; + $display("RANDOM SEED (workload E, layer-1 data) = 32'h%08h", rand_seed); + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + for (n = 0; n < L1_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + xv = $random(rand_seed) % 9; // deterministic PRNG stream, range roughly [-8,8] + wv = $random(rand_seed) % 9; + poke_byte(l1x_base + n*P_IN + k, xv); + poke_byte(l1w_base + n*P_IN + k, wv); + acc = acc + xv*wv; + end + golden_l1[n] = relu_sat(acc); + poke_byte(hidden_base + n, 8'sd0); // poison hidden slot + register_node(node_base + n[NODE_IDW-1:0], 0, no_deps, + l1x_base + n*P_IN, l1w_base + n*P_IN, 16'd1, hidden_base + n); + end + + for (n = 0; n < L2_N; n = n + 1) begin + for (k = 0; k < P_IN; k = k + 1) + poke_byte(l2w_base + n*P_IN + k, ((n + k) % 6) + 1); + register_node(node_base + L1_N[NODE_IDW-1:0] + n[NODE_IDW-1:0], L1_N[$clog2(MAX_DEPS+1)-1:0], l2_deps, + hidden_base, l2w_base + n*P_IN, 16'd1, l2res_base + n); + end + + completed = 0; wd2 = 0; + while (completed < (L1_N+L2_N) && wd2 < 2000000) begin + @(posedge clk); wd2 = wd2 + 1; completed = jobs_completed; + end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < (L1_N+L2_N)) begin + $display("FAIL Multilayer: only %0d/%0d nodes completed", completed, L1_N+L2_N); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < L1_N; n = n + 1) begin + real_y = peek_byte(hidden_base + n); + if (real_y !== golden_l1[n]) begin + $display("FAIL Multilayer L1 neuron %0d: real=%0d golden=%0d", n, real_y, golden_l1[n]); + local_errors = local_errors + 1; + end + end + for (n = 0; n < L2_N; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) + acc = acc + golden_l1[k] * peek_byte(l2w_base + n*P_IN + k); + golden_l2 = relu_sat(acc); + real_y = peek_byte(l2res_base + n); + if (real_y !== golden_l2) begin + $display("FAIL Multilayer L2 neuron %0d: real=%0d golden=%0d (using REAL L1 hidden values)", n, real_y, golden_l2); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS Multilayer: 8 L1 (random) -> 2 L2 neurons, all bit-exact, real cross-node forwarding via real PSRAM"); + else errors = errors + 1; + report_instrumentation("E-Multilayer", L1_N+L2_N); + end + endtask + + // F: DAG diamond+fan-in (A,B indep; C dep-A; D dep-B; E dep-C&D + // [2-hop]; F dep-A,B,C [mixed, 3 producers]) + task automatic run_dag( + input [NODE_IDW-1:0] node_base, + input [ADDR_WIDTH-1:0] x_base, input [ADDR_WIDTH-1:0] w_base, input [ADDR_WIDTH-1:0] res_base + ); + integer n, k, acc, completed, wd2, local_errors; + reg signed [7:0] golden [0:5]; + reg signed [7:0] real_y; + reg [MAX_DEPS*NODE_IDW-1:0] deps; + reg [NODE_IDW-1:0] idA, idB, idC, idD, idE, idF; + begin + idA = node_base+0; idB = node_base+1; idC = node_base+2; + idD = node_base+3; idE = node_base+4; idF = node_base+5; + + // Each of the 6 nodes: its own small independent 8-input + // job (deterministic, distinct per node) -- dependencies + // here are purely about SCHEDULING/wake-up order, not + // data forwarding (workload E already covers that). + for (n = 0; n < 6; n = n + 1) begin + acc = 0; + for (k = 0; k < P_IN; k = k + 1) begin + poke_byte(x_base + n*P_IN + k, ((n+k)%4)+1); + poke_byte(w_base + n*P_IN + k, ((n+k)%5)+1); + acc = acc + peek_byte(x_base+n*P_IN+k)*peek_byte(w_base+n*P_IN+k); + end + golden[n] = relu_sat(acc); + poke_byte(res_base + n, 8'sd0); + end + + reset_instrumentation(1'b1); + measure_en = 1'b1; + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; + register_node(idA, 0, deps, x_base+0*P_IN, w_base+0*P_IN, 16'd1, res_base+0); + register_node(idB, 0, deps, x_base+1*P_IN, w_base+1*P_IN, 16'd1, res_base+1); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; + register_node(idC, 1, deps, x_base+2*P_IN, w_base+2*P_IN, 16'd1, res_base+2); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idB; + register_node(idD, 1, deps, x_base+3*P_IN, w_base+3*P_IN, 16'd1, res_base+3); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idC; deps[1*NODE_IDW+:NODE_IDW] = idD; + register_node(idE, 2, deps, x_base+4*P_IN, w_base+4*P_IN, 16'd1, res_base+4); + + deps = {(MAX_DEPS*NODE_IDW){1'b0}}; deps[0*NODE_IDW+:NODE_IDW] = idA; deps[1*NODE_IDW+:NODE_IDW] = idB; deps[2*NODE_IDW+:NODE_IDW] = idC; + register_node(idF, 3, deps, x_base+5*P_IN, w_base+5*P_IN, 16'd1, res_base+5); + + completed = 0; wd2 = 0; + while (completed < 6 && wd2 < 2000000) begin @(posedge clk); wd2=wd2+1; completed = jobs_completed; end + repeat(5) @(posedge clk); + measure_en = 1'b0; + + tests = tests + 1; + local_errors = 0; + if (completed < 6) begin + $display("FAIL DAG: only %0d/6 nodes completed", completed); + local_errors = local_errors + 1; + end else begin + for (n = 0; n < 6; n = n + 1) begin + real_y = peek_byte(res_base+n); + if (real_y !== golden[n]) begin + $display("FAIL DAG node %0d: real=%0d golden=%0d", n, real_y, golden[n]); + local_errors = local_errors + 1; + end + end + end + if (local_errors == 0) $display("PASS DAG: 6-node diamond+fan-in (2-hop transitive wake-up, 3-producer mixed-depth dependency), all bit-exact"); + else errors = errors + 1; + report_instrumentation("F-DAG", 6); + end + endtask + + initial begin + errors = 0; tests = 0; + rst = 1; reg_valid = 0; reg_node_id = 0; reg_required = 0; reg_producer_ids = 0; + reg_x_base = 0; reg_w_base = 0; reg_n_tiles = 0; reg_result_addr = 0; + measure_en = 0; + repeat(5) @(posedge clk); + rst = 0; + + $display("========================================"); + $display("NMS D-Stress benchmark (STEP19, SINGLE SDRAM (AS4C4M16SA-6TIN) for weights+activations+results, no PSRAM anywhere) -- N_SLOTS_CFG=%0d PFD_CFG=%0d", N_SLOTS_CFG, PFD_CFG); + $display("========================================"); + + wait (u_nmp.u_sdram_backend.u_sdram_ctrl.state == u_nmp.u_sdram_backend.u_sdram_ctrl.S_IDLE); + @(posedge clk); + + // Official V2 memory map (datasheet ch.5): weights @ 0x010000, + // activations @ 0x200000, results @ 0x300000 -- non-overlapping + // 1MB-aligned regions in the single SDRAM. + run_dense_layer("D-Stress", 256, 16, 16'd400, 26'h200000, 26'h010000, 26'h300000, 1'b0); + + // FPGA_DATA_READY check: the whole graph (256 nodes) just + // finished and no new work has been registered -- data_ready + // must be asserted (system-idle sticky flag, see + // nms_dataflow_core_sdram.v). A few idle cycles for the + // busy->idle edge to settle before sampling. + repeat (4) @(posedge clk); + if (u_nmp.data_ready !== 1'b1) begin + $display("FAIL data_ready: expected 1 after graph completion, got %b", u_nmp.data_ready); + errors = errors + 1; + end else begin + $display("PASS data_ready: correctly asserted after graph completion"); + end + + $display("========================================"); + if (errors == 0) + $display("ALL %0d WORKLOAD SUITES PASSED (N_SLOTS_CFG=%0d, PFD_CFG=%0d, SINGLE SDRAM for weights+activations+results, no PSRAM)", tests, N_SLOTS_CFG, PFD_CFG); + else + $display("FAILED: %0d/%0d workload suite(s) had errors -- see messages above", errors, tests); + $display("========================================"); + $finish; + end + +endmodule diff --git a/hardware/v2/nms/sim/tb_sdram_cdc_bridge.v b/hardware/v2/nms/sim/tb_sdram_cdc_bridge.v new file mode 100644 index 0000000..6c7a38a --- /dev/null +++ b/hardware/v2/nms/sim/tb_sdram_cdc_bridge.v @@ -0,0 +1,287 @@ +`timescale 1ns/1ps + +// ============================================================ +// EXP-0053 -- isolated correctness + real measured speedup for +// sdram_cdc_bridge.v. Two independent DUTs share the exact same +// transaction sequence: +// +// dut_direct : sdram_controller.v driven directly at clk_slow +// (64MHz) -- today's real, unchanged baseline. +// dut_bridge : sdram_cdc_bridge.v, slow-domain interface at +// clk_slow (64MHz), internal sdram_controller.v +// running at clk_fast (115.2MHz, real; the RTL's own +// CLK_FREQ_MHZ_FAST=115 parameter is deliberately +// rounded DOWN -- see sdram_cdc_bridge.v header). +// +// clk_slow and clk_fast are free-running, independently generated, +// NON-integer-ratio (64 vs 115.2) -- deliberately the hardest case +// for a toggle-based CDC handshake (no lucky fixed phase alignment +// possible), to genuinely stress the synchronizers rather than test +// a convenient special case. +// +// Covers: +// 1) correctness battery (write->read, sequential/bank-sweep/ +// pseudo-random addresses) through the bridge, bit-exact vs the +// same golden pattern used by tb_sdram_controller.v's own idiom. +// 2) back-to-back stress: many transactions in a tight loop with NO +// idle gap between them, the maximum rate the existing busy/ +// ready protocol allows -- the toggle handshake must never drop, +// duplicate, or corrupt a transaction under sustained load. +// 3) REAL measured total-cycle comparison, direct vs bridged, over +// an identical transaction sequence -- the actual number this +// experiment exists to produce, not an estimate. +// ============================================================ +module tb; + localparam BURST_LEN = 8; + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS; + localparam ALIGN_BITS = (BURST_LEN<=1) ? 0 : $clog2(BURST_LEN); + + localparam CLK_FREQ_SLOW = 64; + localparam real CLK_FREQ_FAST_REAL = 115.2; + localparam SLOW_PERIOD_NS = 1000.0/CLK_FREQ_SLOW; + localparam real FAST_PERIOD_NS = 1000.0/CLK_FREQ_FAST_REAL; + + reg clk_slow = 0; + always #(SLOW_PERIOD_NS/2.0) clk_slow = ~clk_slow; + reg clk_fast = 0; + always #(FAST_PERIOD_NS/2.0) clk_fast = ~clk_fast; + + reg rst_slow, rst_fast; + + // ---- shared cycle counter (slow domain -- what actually matters + // for real system wall-clock, since every existing caller lives + // in the 64MHz compute domain) ---- + integer cyc; + always @(posedge clk_slow) if (!rst_slow) cyc <= cyc + 1; + + // ================= DUT A: direct, today's baseline ================= + reg reqA, wrA; + reg [ADDR_WIDTH-1:0] addrA; + reg [16*BURST_LEN-1:0] wdataA; + reg [2*BURST_LEN-1:0] wmaskA; + wire [16*BURST_LEN-1:0] rdataA; + wire readyA, busyA; + wire cke_A, cs_A, ras_A, cas_A, we_A; + wire [BANK_BITS-1:0] ba_A; + wire [ROW_BITS-1:0] a_A; + wire [15:0] dq_A; + wire [1:0] dqm_A; + + sdram_controller #( + .CLK_FREQ_MHZ(CLK_FREQ_SLOW), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut_direct ( + .clk(clk_slow), .rst(rst_slow), + .req(reqA), .wr(wrA), .addr(addrA), .wdata(wdataA), .wmask(wmaskA), + .rdata(rdataA), .ready(readyA), .busy(busyA), + .sdram_cke(cke_A), .sdram_cs_n(cs_A), .sdram_ras_n(ras_A), + .sdram_cas_n(cas_A), .sdram_we_n(we_A), + .sdram_ba(ba_A), .sdram_a(a_A), .sdram_dq(dq_A), .sdram_dqm(dqm_A) + ); + sdram_model #( + .CLK_FREQ_MHZ(CLK_FREQ_SLOW), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem_direct ( + .clk(clk_slow), .cke(cke_A), .cs_n(cs_A), .ras_n(ras_A), + .cas_n(cas_A), .we_n(we_A), .ba(ba_A), .a(a_A), .dq(dq_A), .dqm(dqm_A) + ); + + // ================= DUT B: bridged (64MHz iface, 115.2MHz memory) ==== + reg reqB, wrB; + reg [ADDR_WIDTH-1:0] addrB; + reg [16*BURST_LEN-1:0] wdataB; + reg [2*BURST_LEN-1:0] wmaskB; + wire [16*BURST_LEN-1:0] rdataB; + wire readyB, busyB; + wire cke_B, cs_B, ras_B, cas_B, we_B; + wire [BANK_BITS-1:0] ba_B; + wire [ROW_BITS-1:0] a_B; + wire [15:0] dq_B; + wire [1:0] dqm_B; + + sdram_cdc_bridge #( + .CLK_FREQ_MHZ_FAST(115), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut_bridge ( + .clk_slow(clk_slow), .rst_slow(rst_slow), + .clk_fast(clk_fast), .rst_fast(rst_fast), + .req(reqB), .wr(wrB), .addr(addrB), .wdata(wdataB), .wmask(wmaskB), + .rdata(rdataB), .ready(readyB), .busy(busyB), + .sdram_cke(cke_B), .sdram_cs_n(cs_B), .sdram_ras_n(ras_B), + .sdram_cas_n(cas_B), .sdram_we_n(we_B), + .sdram_ba(ba_B), .sdram_a(a_B), .sdram_dq(dq_B), .sdram_dqm(dqm_B) + ); + sdram_model #( + .CLK_FREQ_MHZ(115), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem_bridge ( + .clk(clk_fast), .cke(cke_B), .cs_n(cs_B), .ras_n(ras_B), + .cas_n(cas_B), .we_n(we_B), .ba(ba_B), .a(a_B), .dq(dq_B), .dqm(dqm_B) + ); + + integer errors, tests; + + task automatic do_txn_A( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk_slow); + while (busyA) @(posedge clk_slow); + t0 = cyc; + reqA = 1'b1; wrA = t_wr; addrA = t_addr; wdataA = t_wdata; wmaskA = {(2*BURST_LEN){1'b0}}; + @(posedge clk_slow); + reqA = 1'b0; + while (!readyA) @(posedge clk_slow); + t_rdata = rdataA; + t_cycles = cyc - t0; + end + endtask + + task automatic do_txn_B( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk_slow); + while (busyB) @(posedge clk_slow); + t0 = cyc; + reqB = 1'b1; wrB = t_wr; addrB = t_addr; wdataB = t_wdata; wmaskB = {(2*BURST_LEN){1'b0}}; + @(posedge clk_slow); + reqB = 1'b0; + while (!readyB) @(posedge clk_slow); + t_rdata = rdataB; + t_cycles = cyc - t0; + end + endtask + + reg [16*BURST_LEN-1:0] gotA, gotB, wpat; + integer elapsedA, elapsedB; + + task automatic check_word_both(input [ADDR_WIDTH-1:0] a, input [15:0] pattern); + integer k; + begin + for (k = 0; k < BURST_LEN; k = k + 1) + wpat[k*16 +: 16] = pattern + k[15:0]; + + do_txn_A(1'b1, a, wpat, gotA, elapsedA); + do_txn_A(1'b0, a, {(16*BURST_LEN){1'b0}}, gotA, elapsedA); + do_txn_B(1'b1, a, wpat, gotB, elapsedB); + do_txn_B(1'b0, a, {(16*BURST_LEN){1'b0}}, gotB, elapsedB); + + tests = tests + 1; + if (gotA !== wpat) begin + $display("FAIL (direct) addr=%0d: got=%h expected=%h", a, gotA, wpat); + errors = errors + 1; + end + if (gotB !== wpat) begin + $display("FAIL (bridge) addr=%0d: got=%h expected=%h", a, gotB, wpat); + errors = errors + 1; + end + if (gotA === wpat && gotB === wpat) begin + $display("PASS addr=%0d: both bit-exact (direct=%0d cyc, bridge=%0d cyc)", a, elapsedA, elapsedB); + end + end + endtask + + integer seed; + integer i; + reg [ADDR_WIDTH-1:0] rnd_addr; + + // ---- real measured total-cycle comparison over an identical, + // longer sequence (TEST 3) ---- + integer total_cyc_A, total_cyc_B, t0_seq; + + initial begin + errors = 0; tests = 0; cyc = 0; seed = 32'hFACADE; + rst_slow = 1; rst_fast = 1; + reqA = 0; wrA = 0; addrA = 0; wdataA = 0; wmaskA = 0; + reqB = 0; wrB = 0; addrB = 0; wdataB = 0; wmaskB = 0; + repeat(10) @(posedge clk_slow); + repeat(10) @(posedge clk_fast); + rst_slow = 0; rst_fast = 0; + @(posedge clk_slow); + while (busyA || busyB) @(posedge clk_slow); + + $display("=== TEST 1: correctness battery (direct vs bridge, same golden pattern) ==="); + check_word_both({ADDR_WIDTH{1'b0}}, 16'hA5A5); + for (i = 0; i < 8; i = i + 1) + check_word_both(i*BURST_LEN, 16'h1000 + i); + for (i = 0; i < 4; i = i + 1) + check_word_both((i << ALIGN_BITS) + (100 << (ALIGN_BITS+BANK_BITS)), 16'h2000 + i); + for (i = 0; i < 24; i = i + 1) begin + rnd_addr = ($random(seed) % ((1<read, sequential/bank-sweep/ +// pseudo-random), bit-exact vs the same golden pattern, on BOTH +// DUTs. +// 2) SAME-ROW consecutive access (the real production pattern: +// weight_prefetch_engine_wide.v's own strictly sequential tile +// stream) -- bit-exact AND real measured cycle savings vs +// baseline. +// 3) DIFFERENT-ROW access immediately after a row is open -- must +// stay bit-exact and NOT regress vs baseline (on-demand precharge +// pays the same total cost, just deferred). +// 4) explicit read-after-read / write-after-read / read-after-write +// / write-after-write SAME-ROW turnaround sequences -- the one +// hazard class sdram_model.v does NOT itself assert (no tCCD/ +// tRTW/tWTR check in that model -- see sdram_controller_ +// openrow.v's own header) -- checked here for DATA correctness, +// which is the strongest check available without a turnaround- +// timing-aware reference model. +// 5) refresh spanning while a row is open -- watches for ANY +// VIOLATION/WARNING from sdram_model.v (this project's own real +// command-sequence checker) across many iterations, specifically +// exercising the new precharge-before-refresh path. +// 6) REAL measured total-cycle comparison over a long, strictly +// sequential same-row run -- the actual production access +// pattern (weight_prefetch_engine_wide.v), the number this +// experiment exists to produce. +// ============================================================ +module tb; + localparam BURST_LEN = 8; + localparam ROW_BITS = 13; + localparam COL_BITS = 10; + localparam BANK_BITS = 2; + localparam ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS; + localparam ALIGN_BITS = (BURST_LEN<=1) ? 0 : $clog2(BURST_LEN); + localparam CLK_FREQ_MHZ = 64; + localparam CLK_PERIOD_NS = 1000.0/CLK_FREQ_MHZ; + + reg clk = 0; + always #(CLK_PERIOD_NS/2.0) clk = ~clk; + reg rst; + + integer cyc; + always @(posedge clk) if (!rst) cyc <= cyc + 1; + + function automatic [BANK_BITS-1:0] bank_of; + input [ADDR_WIDTH-1:0] a; + begin + bank_of = a[ALIGN_BITS +: BANK_BITS]; + end + endfunction + function automatic [ROW_BITS-1:0] row_of; + input [ADDR_WIDTH-1:0] a; + begin + row_of = a[ALIGN_BITS+BANK_BITS +: ROW_BITS]; + end + endfunction + + // ================= DUT BASE (today's real baseline) ================= + reg reqA, wrA; + reg [ADDR_WIDTH-1:0] addrA; + reg [16*BURST_LEN-1:0] wdataA; + reg [2*BURST_LEN-1:0] wmaskA; + wire [16*BURST_LEN-1:0] rdataA; + wire readyA, busyA; + wire cke_A, cs_A, ras_A, cas_A, we_A; + wire [BANK_BITS-1:0] ba_A; + wire [ROW_BITS-1:0] a_A; + wire [15:0] dq_A; + wire [1:0] dqm_A; + + sdram_controller #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut_base ( + .clk(clk), .rst(rst), + .req(reqA), .wr(wrA), .addr(addrA), .wdata(wdataA), .wmask(wmaskA), + .rdata(rdataA), .ready(readyA), .busy(busyA), + .sdram_cke(cke_A), .sdram_cs_n(cs_A), .sdram_ras_n(ras_A), + .sdram_cas_n(cas_A), .sdram_we_n(we_A), + .sdram_ba(ba_A), .sdram_a(a_A), .sdram_dq(dq_A), .sdram_dqm(dqm_A) + ); + sdram_model #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem_base ( + .clk(clk), .cke(cke_A), .cs_n(cs_A), .ras_n(ras_A), + .cas_n(cas_A), .we_n(we_A), .ba(ba_A), .a(a_A), .dq(dq_A), .dqm(dqm_A) + ); + + // ================= DUT OPENROW ================= + reg reqB, wrB; + reg [ADDR_WIDTH-1:0] addrB; + reg [16*BURST_LEN-1:0] wdataB; + reg [2*BURST_LEN-1:0] wmaskB; + wire [16*BURST_LEN-1:0] rdataB; + wire readyB, busyB; + wire cke_B, cs_B, ras_B, cas_B, we_B; + wire [BANK_BITS-1:0] ba_B; + wire [ROW_BITS-1:0] a_B; + wire [15:0] dq_B; + wire [1:0] dqm_B; + + sdram_controller_openrow #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(BURST_LEN), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) dut_openrow ( + .clk(clk), .rst(rst), + .req(reqB), .wr(wrB), .addr(addrB), .wdata(wdataB), .wmask(wmaskB), + .rdata(rdataB), .ready(readyB), .busy(busyB), + .sdram_cke(cke_B), .sdram_cs_n(cs_B), .sdram_ras_n(ras_B), + .sdram_cas_n(cas_B), .sdram_we_n(we_B), + .sdram_ba(ba_B), .sdram_a(a_B), .sdram_dq(dq_B), .sdram_dqm(dqm_B) + ); + sdram_model #( + .CLK_FREQ_MHZ(CLK_FREQ_MHZ), + .ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS) + ) mem_openrow ( + .clk(clk), .cke(cke_B), .cs_n(cs_B), .ras_n(ras_B), + .cas_n(cas_B), .we_n(we_B), .ba(ba_B), .a(a_B), .dq(dq_B), .dqm(dqm_B) + ); + + integer errors, tests; + + task automatic do_txn_A( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk); + while (busyA) @(posedge clk); + t0 = cyc; + reqA = 1'b1; wrA = t_wr; addrA = t_addr; wdataA = t_wdata; wmaskA = {(2*BURST_LEN){1'b0}}; + @(posedge clk); + reqA = 1'b0; + while (!readyA) @(posedge clk); + t_rdata = rdataA; + t_cycles = cyc - t0; + end + endtask + + task automatic do_txn_B( + input t_wr, + input [ADDR_WIDTH-1:0] t_addr, + input [16*BURST_LEN-1:0] t_wdata, + output [16*BURST_LEN-1:0] t_rdata, + output integer t_cycles + ); + integer t0; + begin + @(posedge clk); + while (busyB) @(posedge clk); + t0 = cyc; + reqB = 1'b1; wrB = t_wr; addrB = t_addr; wdataB = t_wdata; wmaskB = {(2*BURST_LEN){1'b0}}; + @(posedge clk); + reqB = 1'b0; + while (!readyB) @(posedge clk); + t_rdata = rdataB; + t_cycles = cyc - t0; + end + endtask + + reg [16*BURST_LEN-1:0] gotA, gotB, wpat; + integer elapsedA, elapsedB; + + task automatic check_word_both(input [ADDR_WIDTH-1:0] a, input [15:0] pattern); + integer k; + begin + for (k = 0; k < BURST_LEN; k = k + 1) + wpat[k*16 +: 16] = pattern + k[15:0]; + + do_txn_A(1'b1, a, wpat, gotA, elapsedA); + do_txn_A(1'b0, a, {(16*BURST_LEN){1'b0}}, gotA, elapsedA); + do_txn_B(1'b1, a, wpat, gotB, elapsedB); + do_txn_B(1'b0, a, {(16*BURST_LEN){1'b0}}, gotB, elapsedB); + + tests = tests + 1; + if (gotA !== wpat) begin + $display("FAIL (base) addr=%0d: got=%h expected=%h", a, gotA, wpat); + errors = errors + 1; + end + if (gotB !== wpat) begin + $display("FAIL (openrow) addr=%0d: got=%h expected=%h", a, gotB, wpat); + errors = errors + 1; + end + if (gotA === wpat && gotB === wpat) begin + $display("PASS addr=%0d bank=%0d row=%0d: both bit-exact (base=%0d cyc, openrow=%0d cyc)", + a, bank_of(a), row_of(a), elapsedA, elapsedB); + end + end + endtask + + integer seed; + integer i; + reg [ADDR_WIDTH-1:0] rnd_addr; + + initial begin + errors = 0; tests = 0; cyc = 0; seed = 32'hBADC0FFE; + rst = 1; + reqA = 0; wrA = 0; addrA = 0; wdataA = 0; wmaskA = 0; + reqB = 0; wrB = 0; addrB = 0; wdataB = 0; wmaskB = 0; + repeat(5) @(posedge clk); + rst = 0; + @(posedge clk); + while (busyA || busyB) @(posedge clk); + + $display("=== TEST 1: correctness battery (base vs openrow, same golden pattern) ==="); + check_word_both({ADDR_WIDTH{1'b0}}, 16'hA5A5); + for (i = 0; i < 8; i = i + 1) + check_word_both(i*BURST_LEN, 16'h1000 + i); + for (i = 0; i < 4; i = i + 1) + check_word_both((i << ALIGN_BITS) + (100 << (ALIGN_BITS+BANK_BITS)), 16'h2000 + i); + for (i = 0; i < 24; i = i + 1) begin + rnd_addr = ($random(seed) % ((1<