michele/FPGA-Neural
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

exp: SDRAM CDC bridge + open-row policy (EXP-0053/54/55) -- open-row is a real ~5% D-Stress win, CDC bridge measured net-negative once integrated

Browse Source 
EXP-0053: sdram_cdc_bridge.v decouples the SDRAM clock (115.2MHz, real
value derived from the board's own existing PLL VCO=576MHz, verified
via ecppll) from the 64MHz compute domain. Isolated: 137/137 tests, 0
errors, but real measured speedup is only 1.095x (not the naive 1.8x
clock-ratio estimate) -- the CDC handshake's own synchronizer
round-trip is a fixed per-transaction tax.

EXP-0054: sdram_controller_openrow.v implements the page-hit/
keep-row-open optimization sdram_controller.v's own header had always
deferred. weight_prefetch_engine_wide.v's real production traffic is
strictly sequential per job and mostly stays within one SDRAM row --
closing/reopening it every tile (today's fixed auto-precharge policy)
wastes tRP+tRCD for no reason. Isolated: 154/154 tests, 0 errors, 0
protocol violations (including the new refresh-while-row-open hazard,
fixed via an explicit precharge-before-refresh path). Real measured
speedup on the actual sequential access pattern: 1.141x.

EXP-0055: composed both, then integrated into the real D-Stress
benchmark (N=4/N=8, 256/256 bit-exact in every config). Result:
open-row ALONE gives a real, consistent ~5% cycle-count improvement
(47445/47468 vs baseline 49927/49909). CDC alone is a real ~8%
REGRESSION. Combined is still a ~4% regression -- the CDC's fixed tax
is paid on every transaction regardless of row-hit, and real D-Stress
traffic interleaves weight-fetch/activation-result access far more
than the isolated same-row test exercised, so open-row's real saving
doesn't offset it. Decision: do not adopt the CDC approach; open-row
alone is the disclosed, real win worth considering for production
next, pending an explicit go-ahead (not applied to the real board top
in this commit -- all additive, existing production RTL untouched).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUG92aM9m68TRc4rG55BcC
This commit is contained in:
michele
2026-09-16 07:07:57 +02:00
co-authored by Claude Sonnet 5
parent 03b5cbc25b
commit ee5a68f6e6
16 changed files with 6499 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_cdc.v
+173
View File
@@ -0,0 +1,173 @@
`timescale 1ns/1ps
// ================================================================
// Neural Memory System (NMS) -- STEP19 real hardware-facing top level.
//
// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_
// sdram_pack128.v (STEP18) with the ONE change this step's own
// governing spec mandates: the real hardware/v1/rtl/psram_controller.v
// + memory_interface.v pairing (activation-fill + result-writeback,
// 16-bit) is REMOVED from the V2 physical path entirely and replaced
// by sdram_unified_backend.v's own AR port, sharing the SAME single
// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_
// controller.v instance the weight-fetch path (W port) already uses.
//
// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_
// arbiter_wide.v (64-bit, weight) are BOTH reused completely
// UNCHANGED -- their own downstream ports now both terminate at
// sdram_unified_backend.v instead of two separate physical chains.
// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_
// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and
// neural_processor.v are ALL byte-for-byte unchanged -- this is a
// pure memory-side substitution, per the governing spec's own
// explicit instruction.
//
// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_
// interface.v simply are no longer INSTANTIATED by this top-level;
// neither file was modified, and V1's own golden-reference status is
// unaffected.
//
// Real pin count (weight+activation+result, ALL through ONE chip):
// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37
// pins total -- the SAME 37 pins the weight-only path already used in
// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/
// result traffic, since it shares the identical physical bus).
// ================================================================
module nms_neural_multiprocessor_sdram_cdc #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 2,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter CLK_FREQ_MHZ = 80
)(
input wire clk,
input wire rst,
input wire clk_fast,
input wire rst_fast,
input wire reg_valid,
output wire reg_ready,
input wire [$clog2(N_NODES)-1:0] reg_node_id,
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
input wire [ADDR_WIDTH-1:0] reg_x_base,
input wire [ADDR_WIDTH-1:0] reg_w_base,
input wire [15:0] reg_n_tiles,
input wire [ADDR_WIDTH-1:0] reg_result_addr,
// FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v
output wire data_ready,
// ---- STEP19: ONE physical SDRAM interface, ALL traffic
// (weights + activations + results) ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [1:0] sdram_ba,
output wire [12:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
wire [N_SLOTS:0] slot_mem_ready;
wire [N_SLOTS-1:0] wide_slot_mem_req;
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
wire [N_SLOTS-1:0] wide_slot_mem_ready;
nms_dataflow_core_sdram #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
) u_dataflow_core (
.clk(clk), .rst(rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.data_ready(data_ready),
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
);
// ---- AR: activation-fill (shared, 1 port) + per-slot result
// writeback (N_SLOTS ports), arbitrated exactly as before ----
wire arb_m_req, arb_m_wr;
wire [ADDR_WIDTH-1:0] arb_m_addr;
wire [15:0] arb_m_wdata;
wire arb_m_lb_n, arb_m_ub_n;
wire [15:0] arb_m_rdata;
wire arb_m_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
) u_arbiter (
.clk(clk), .rst(rst),
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
);
// ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as
// before -- weight fetch never writes, same tie-off convention
// as STEP16-18 ----
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
wire wide_arb_m_req, wide_arb_m_wr;
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
wire [63:0] wide_arb_m_wdata;
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
wire [63:0] wide_arb_m_rdata;
wire wide_arb_m_ready;
slot_mem_arbiter_wide #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
) u_arbiter_wide (
.clk(clk), .rst(rst),
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
);
// ---- STEP19: ONE physical SDRAM backend, both W and AR ports ----
sdram_unified_backend_cdc #(
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
) u_sdram_backend (
.clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast),
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
.ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata),
.ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n),
.ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_combined.v
+186
View File
@@ -0,0 +1,186 @@
`timescale 1ns/1ps
// ================================================================
// EXP-0055 -- combined fork of nms_neural_multiprocessor_sdram_
// unified.v (STEP19), same minimal-diff discipline as the pipelined
// fork (EXP-0052): the ONLY changes are two new clk_fast/rst_fast
// ports (forwarded straight through) and instantiating sdram_unified_
// backend_combined.v (EXP-0055's CDC+page-hit backend) instead of
// sdram_unified_backend.v. u_dataflow_core, both arbiters, are all
// byte-for-byte unchanged.
//
// Everything below is STEP19's own original header, preserved as-is:
//
// Neural Memory System (NMS) -- STEP19 real hardware-facing top level.
//
// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_
// sdram_pack128.v (STEP18) with the ONE change this step's own
// governing spec mandates: the real hardware/v1/rtl/psram_controller.v
// + memory_interface.v pairing (activation-fill + result-writeback,
// 16-bit) is REMOVED from the V2 physical path entirely and replaced
// by sdram_unified_backend.v's own AR port, sharing the SAME single
// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_
// controller.v instance the weight-fetch path (W port) already uses.
//
// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_
// arbiter_wide.v (64-bit, weight) are BOTH reused completely
// UNCHANGED -- their own downstream ports now both terminate at
// sdram_unified_backend.v instead of two separate physical chains.
// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_
// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and
// neural_processor.v are ALL byte-for-byte unchanged -- this is a
// pure memory-side substitution, per the governing spec's own
// explicit instruction.
//
// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_
// interface.v simply are no longer INSTANTIATED by this top-level;
// neither file was modified, and V1's own golden-reference status is
// unaffected.
//
// Real pin count (weight+activation+result, ALL through ONE chip):
// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37
// pins total -- the SAME 37 pins the weight-only path already used in
// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/
// result traffic, since it shares the identical physical bus).
// ================================================================
module nms_neural_multiprocessor_sdram_combined #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 2,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter CLK_FREQ_MHZ = 80
)(
input wire clk,
input wire rst,
// ---- EXP-0055: fast SDRAM clock domain, forwarded straight
// through to sdram_unified_backend_combined.v -- see that
// module's own header ----
input wire clk_fast,
input wire rst_fast,
input wire reg_valid,
output wire reg_ready,
input wire [$clog2(N_NODES)-1:0] reg_node_id,
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
input wire [ADDR_WIDTH-1:0] reg_x_base,
input wire [ADDR_WIDTH-1:0] reg_w_base,
input wire [15:0] reg_n_tiles,
input wire [ADDR_WIDTH-1:0] reg_result_addr,
// FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v
output wire data_ready,
// ---- STEP19: ONE physical SDRAM interface, ALL traffic
// (weights + activations + results) ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [1:0] sdram_ba,
output wire [12:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
wire [N_SLOTS:0] slot_mem_ready;
wire [N_SLOTS-1:0] wide_slot_mem_req;
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
wire [N_SLOTS-1:0] wide_slot_mem_ready;
nms_dataflow_core_sdram #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
) u_dataflow_core (
.clk(clk), .rst(rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.data_ready(data_ready),
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
);
// ---- AR: activation-fill (shared, 1 port) + per-slot result
// writeback (N_SLOTS ports), arbitrated exactly as before ----
wire arb_m_req, arb_m_wr;
wire [ADDR_WIDTH-1:0] arb_m_addr;
wire [15:0] arb_m_wdata;
wire arb_m_lb_n, arb_m_ub_n;
wire [15:0] arb_m_rdata;
wire arb_m_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
) u_arbiter (
.clk(clk), .rst(rst),
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
);
// ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as
// before -- weight fetch never writes, same tie-off convention
// as STEP16-18 ----
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
wire wide_arb_m_req, wide_arb_m_wr;
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
wire [63:0] wide_arb_m_wdata;
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
wire [63:0] wide_arb_m_rdata;
wire wide_arb_m_ready;
slot_mem_arbiter_wide #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
) u_arbiter_wide (
.clk(clk), .rst(rst),
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
);
// ---- STEP19: ONE physical SDRAM backend, both W and AR ports ----
sdram_unified_backend_combined #(
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
) u_sdram_backend (
.clk(clk), .rst(rst), .clk_fast(clk_fast), .rst_fast(rst_fast),
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
.ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata),
.ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n),
.ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/nms_neural_multiprocessor_sdram_openrow.v
+171
View File
@@ -0,0 +1,171 @@
`timescale 1ns/1ps
// ================================================================
// Neural Memory System (NMS) -- STEP19 real hardware-facing top level.
//
// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_
// sdram_pack128.v (STEP18) with the ONE change this step's own
// governing spec mandates: the real hardware/v1/rtl/psram_controller.v
// + memory_interface.v pairing (activation-fill + result-writeback,
// 16-bit) is REMOVED from the V2 physical path entirely and replaced
// by sdram_unified_backend.v's own AR port, sharing the SAME single
// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_
// controller.v instance the weight-fetch path (W port) already uses.
//
// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_
// arbiter_wide.v (64-bit, weight) are BOTH reused completely
// UNCHANGED -- their own downstream ports now both terminate at
// sdram_unified_backend.v instead of two separate physical chains.
// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_
// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and
// neural_processor.v are ALL byte-for-byte unchanged -- this is a
// pure memory-side substitution, per the governing spec's own
// explicit instruction.
//
// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_
// interface.v simply are no longer INSTANTIATED by this top-level;
// neither file was modified, and V1's own golden-reference status is
// unaffected.
//
// Real pin count (weight+activation+result, ALL through ONE chip):
// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37
// pins total -- the SAME 37 pins the weight-only path already used in
// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/
// result traffic, since it shares the identical physical bus).
// ================================================================
module nms_neural_multiprocessor_sdram_openrow #(
parameter DATA_WIDTH = 8,
parameter P_IN = 8,
parameter ACC_WIDTH = 32,
parameter ADDR_WIDTH = 26,
parameter N_SLOTS = 2,
parameter N_NODES = 16,
parameter MAX_DEPS = 4,
parameter QUEUE_DEPTH = 8,
parameter MAX_TILES = 16,
parameter PREFETCH_DISTANCE = 8,
parameter CLK_FREQ_MHZ = 80
)(
input wire clk,
input wire rst,
input wire reg_valid,
output wire reg_ready,
input wire [$clog2(N_NODES)-1:0] reg_node_id,
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
input wire [ADDR_WIDTH-1:0] reg_x_base,
input wire [ADDR_WIDTH-1:0] reg_w_base,
input wire [15:0] reg_n_tiles,
input wire [ADDR_WIDTH-1:0] reg_result_addr,
// FPGA_DATA_READY: system-idle sticky flag, see nms_dataflow_core_sdram.v
output wire data_ready,
// ---- STEP19: ONE physical SDRAM interface, ALL traffic
// (weights + activations + results) ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [1:0] sdram_ba,
output wire [12:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
wire [N_SLOTS:0] slot_mem_ready;
wire [N_SLOTS-1:0] wide_slot_mem_req;
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
wire [N_SLOTS-1:0] wide_slot_mem_ready;
nms_dataflow_core_sdram #(
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
) u_dataflow_core (
.clk(clk), .rst(rst),
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
.reg_result_addr(reg_result_addr),
.data_ready(data_ready),
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
);
// ---- AR: activation-fill (shared, 1 port) + per-slot result
// writeback (N_SLOTS ports), arbitrated exactly as before ----
wire arb_m_req, arb_m_wr;
wire [ADDR_WIDTH-1:0] arb_m_addr;
wire [15:0] arb_m_wdata;
wire arb_m_lb_n, arb_m_ub_n;
wire [15:0] arb_m_rdata;
wire arb_m_ready;
slot_mem_arbiter #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
) u_arbiter (
.clk(clk), .rst(rst),
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
);
// ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as
// before -- weight fetch never writes, same tie-off convention
// as STEP16-18 ----
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
wire wide_arb_m_req, wide_arb_m_wr;
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
wire [63:0] wide_arb_m_wdata;
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
wire [63:0] wide_arb_m_rdata;
wire wide_arb_m_ready;
slot_mem_arbiter_wide #(
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
) u_arbiter_wide (
.clk(clk), .rst(rst),
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
);
// ---- STEP19: ONE physical SDRAM backend, both W and AR ports ----
sdram_unified_backend_openrow #(
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
) u_sdram_backend (
.clk(clk), .rst(rst),
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
.ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata),
.ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n),
.ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/sdram_cdc_bridge.v
+253
View File
@@ -0,0 +1,253 @@
`timescale 1ns/1ps
// ============================================================
// EXP-0053 -- SDRAM clock-domain-crossing bridge.
//
// PURPOSE: let sdram_controller.v run on its OWN, faster clock
// (target: 115.2MHz, derived from the SAME PLL VCO as the existing
// 64MHz clk_sys -- see ecp5_pll_sys_clk_dualclk.v) while every
// existing caller (sdram_unified_backend.v's ctrl_req/ctrl_addr/...
// signals) stays on the unchanged 64MHz compute domain. Presents the
// EXACT same req/wr/addr/wdata/wmask -> rdata/ready/busy contract as
// sdram_controller.v itself, so it is a drop-in replacement for the
// direct sdram_controller instantiation at that one call site
// (verified by the isolated tb_sdram_cdc_bridge.v before any
// integration).
//
// WHY 115.2MHz and not the chip's rated 143MHz max (AS4C32M16SA-7,
// tCK=7ns min): the board's single PLL VCO is fixed at 576MHz by the
// existing, already-verified 64MHz CLKOP config (CLKFB_DIV=4,
// CLKOP_DIV=9 -- unchanged, not touched by this experiment). The only
// integer VCO/N divisors near the chip's ceiling are 576/4=144MHz
// (0.8% OVER the 143MHz max -- rejected, not "correctness first") and
// 576/5=115.2MHz (real ~19% margin under the rated max). 115.2MHz is
// therefore the fastest SAFE clock obtainable from this board's
// existing PLL without touching the verified 64MHz compute domain.
// Real measured gain vs the current 64MHz single-domain design is
// therefore 115.2/64 = 1.8x raw controller-clock speedup, NOT the 2.2x
// a naive 143MHz assumption would suggest -- this correction is
// intentional, verified against real ecppll output, not estimated.
//
// PROTOCOL: single-outstanding-request only (matches every existing
// caller's own req/busy/ready idiom exactly -- this bridge does NOT
// add multi-request pipelining; that is EXP-0052's explicitly
// deferred, larger, riskier follow-up, out of scope here). Because at
// most one transaction is ever in flight, a classic two-domain
// "toggle + last-seen" handshake is provably safe:
// - the requesting (slow) domain latches addr/wr/wdata/wmask and
// flips req_toggle_slow on the SAME clock edge, then holds ALL of
// those signals perfectly stable (no new request is ever issued
// while busy=1) until the response toggle comes back;
// - the fast domain double-flop-synchronizes req_toggle_slow (2 FF,
// standard metastability margin) and compares it against its own
// "last serviced" copy -- a mismatch means a new request is
// pending. Because addr/wr/wdata/wmask changed on the SAME edge
// that flipped the toggle, and never change again before the
// response, they are safe to sample directly (no per-bit
// synchronizer needed) once the synchronized toggle has visibly
// changed -- this is the standard "quasi-static bus + toggle"
// CDC idiom, not a shortcut.
// - the same reasoning applies in reverse for ack_toggle_fast/rdata
// going back to the slow domain.
// Reset: rst_slow and rst_fast are separate inputs, each assumed
// ALREADY synchronized to its own clock domain by the caller (this
// module does not itself synchronize an async reset -- matches this
// project's existing convention of a single, pre-synchronized `rst`
// per clock domain, see ecp5_pll_sys_clk.v's own reset handling).
// ============================================================
module sdram_cdc_bridge #(
parameter CLK_FREQ_MHZ_FAST = 115, // deliberately rounded DOWN from
// the real 115.2MHz (never over-
// count available ns/cycle --
// same "ceiling division" spirit
// as sdram_controller.v's own
// ns_to_cycles), so every derived
// timing constant (T_RCD/T_RP/...)
// gets AT LEAST as many cycles as
// the real, slightly-faster clock
// requires.
parameter BURST_LEN = 8,
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2,
parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS
)(
input wire clk_slow,
input wire rst_slow, // pre-synchronized to clk_slow
input wire clk_fast,
input wire rst_fast, // pre-synchronized to clk_fast
// ---- slow-domain caller interface (identical shape to
// sdram_controller.v's own ports) ----
input wire req,
input wire wr,
input wire [ADDR_WIDTH-1:0] addr,
input wire [16*BURST_LEN-1:0] wdata,
input wire [2*BURST_LEN-1:0] wmask,
output reg [16*BURST_LEN-1:0] rdata,
output reg ready,
output wire busy,
// ---- real SDRAM pins, driven directly by the fast-domain
// sdram_controller instance ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [BANK_BITS-1:0] sdram_ba,
output wire [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
// ============================================================
// Slow domain: capture request, drive toggle, wait for ack
// ============================================================
reg busy_slow;
reg req_toggle_slow;
reg wr_lat;
reg [ADDR_WIDTH-1:0] addr_lat;
reg [16*BURST_LEN-1:0] wdata_lat;
reg [2*BURST_LEN-1:0] wmask_lat;
assign busy = busy_slow;
// synchronize ack_toggle_fast (fast domain) into the slow domain
wire ack_toggle_fast;
reg ack_toggle_sync1, ack_toggle_sync2;
always @(posedge clk_slow) begin
if (rst_slow) begin
ack_toggle_sync1 <= 1'b0;
ack_toggle_sync2 <= 1'b0;
end else begin
ack_toggle_sync1 <= ack_toggle_fast;
ack_toggle_sync2 <= ack_toggle_sync1;
end
end
reg last_ack_toggle_seen_slow;
wire [16*BURST_LEN-1:0] rdata_fast_captured;
always @(posedge clk_slow) begin
if (rst_slow) begin
busy_slow <= 1'b0;
req_toggle_slow <= 1'b0;
last_ack_toggle_seen_slow <= 1'b0;
ready <= 1'b0;
rdata <= {(16*BURST_LEN){1'b0}};
wr_lat <= 1'b0;
addr_lat <= {ADDR_WIDTH{1'b0}};
wdata_lat <= {(16*BURST_LEN){1'b0}};
wmask_lat <= {(2*BURST_LEN){1'b0}};
end else begin
ready <= 1'b0;
if (req && !busy_slow) begin
wr_lat <= wr;
addr_lat <= addr;
wdata_lat <= wdata;
wmask_lat <= wmask;
req_toggle_slow <= ~req_toggle_slow;
busy_slow <= 1'b1;
end
if (busy_slow && (ack_toggle_sync2 != last_ack_toggle_seen_slow)) begin
last_ack_toggle_seen_slow <= ack_toggle_sync2;
rdata <= rdata_fast_captured;
ready <= 1'b1;
busy_slow <= 1'b0;
end
end
end
// ============================================================
// Fast domain: synchronize request toggle, drive the real
// sdram_controller, capture response, drive ack toggle back
// ============================================================
reg ctrl_req_f;
reg ctrl_wr_f;
reg [ADDR_WIDTH-1:0] ctrl_addr_f;
reg [16*BURST_LEN-1:0] ctrl_wdata_f;
reg [2*BURST_LEN-1:0] ctrl_wmask_f;
wire [16*BURST_LEN-1:0] ctrl_rdata_f;
wire ctrl_ready_f, ctrl_busy_f;
reg req_toggle_sync1, req_toggle_sync2;
always @(posedge clk_fast) begin
if (rst_fast) begin
req_toggle_sync1 <= 1'b0;
req_toggle_sync2 <= 1'b0;
end else begin
req_toggle_sync1 <= req_toggle_slow;
req_toggle_sync2 <= req_toggle_sync1;
end
end
localparam F_IDLE = 1'b0, F_WAIT = 1'b1;
reg f_state;
reg last_req_toggle_seen_fast;
reg ack_toggle_fast_r;
reg [16*BURST_LEN-1:0] rdata_fast_captured_r;
assign ack_toggle_fast = ack_toggle_fast_r;
assign rdata_fast_captured = rdata_fast_captured_r;
always @(posedge clk_fast) begin
if (rst_fast) begin
f_state <= F_IDLE;
last_req_toggle_seen_fast <= 1'b0;
ack_toggle_fast_r <= 1'b0;
rdata_fast_captured_r <= {(16*BURST_LEN){1'b0}};
ctrl_req_f <= 1'b0;
ctrl_wr_f <= 1'b0;
ctrl_addr_f <= {ADDR_WIDTH{1'b0}};
ctrl_wdata_f<= {(16*BURST_LEN){1'b0}};
ctrl_wmask_f<= {(2*BURST_LEN){1'b0}};
end else begin
ctrl_req_f <= 1'b0;
case (f_state)
F_IDLE: begin
if (req_toggle_sync2 != last_req_toggle_seen_fast) begin
// addr_lat/wr_lat/wdata_lat/wmask_lat (slow-
// domain regs) are quasi-static: they changed
// on the exact same slow-domain edge that
// flipped req_toggle_slow, and will not change
// again until busy_slow deasserts (long after
// this transaction completes) -- safe to
// sample directly, see module header.
ctrl_req_f <= 1'b1;
ctrl_wr_f <= wr_lat;
ctrl_addr_f <= addr_lat;
ctrl_wdata_f<= wdata_lat;
ctrl_wmask_f<= wmask_lat;
last_req_toggle_seen_fast <= req_toggle_sync2;
f_state <= F_WAIT;
end
end
F_WAIT: begin
if (ctrl_ready_f) begin
rdata_fast_captured_r <= ctrl_rdata_f;
ack_toggle_fast_r <= ~ack_toggle_fast_r;
f_state <= F_IDLE;
end
end
default: f_state <= F_IDLE;
endcase
end
end
sdram_controller #(
.CLK_FREQ_MHZ(CLK_FREQ_MHZ_FAST), .BURST_LEN(BURST_LEN),
.ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS)
) u_sdram_ctrl (
.clk(clk_fast), .rst(rst_fast),
.req(ctrl_req_f), .wr(ctrl_wr_f), .addr(ctrl_addr_f),
.wdata(ctrl_wdata_f), .wmask(ctrl_wmask_f),
.rdata(ctrl_rdata_f), .ready(ctrl_ready_f), .busy(ctrl_busy_f),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/sdram_cdc_bridge_openrow.v
+262
View File
@@ -0,0 +1,262 @@
`timescale 1ns/1ps
// ============================================================
// EXP-0055 -- combined fork of sdram_cdc_bridge.v (EXP-0053): the ONLY
// change is instantiating sdram_controller_openrow.v (EXP-0054,
// page-hit/keep-row-open policy) instead of plain sdram_controller.v
// as the fast-domain controller. The CDC handshake itself (toggle +
// last-seen, quasi-static bus sampling) is byte-for-byte unchanged --
// it treats whatever fast-domain controller it wraps as a black box
// behind the same req/wr/addr/wdata/wmask -> rdata/ready/busy
// contract, so this combination was expected to compose cleanly, and
// is verified as such by tb_sdram_cdc_bridge_openrow.v before any
// further integration.
//
// PURPOSE: let sdram_controller.v run on its OWN, faster clock
// (target: 115.2MHz, derived from the SAME PLL VCO as the existing
// 64MHz clk_sys -- see ecp5_pll_sys_clk_dualclk.v) while every
// existing caller (sdram_unified_backend.v's ctrl_req/ctrl_addr/...
// signals) stays on the unchanged 64MHz compute domain. Presents the
// EXACT same req/wr/addr/wdata/wmask -> rdata/ready/busy contract as
// sdram_controller.v itself, so it is a drop-in replacement for the
// direct sdram_controller instantiation at that one call site
// (verified by the isolated tb_sdram_cdc_bridge.v before any
// integration).
//
// WHY 115.2MHz and not the chip's rated 143MHz max (AS4C32M16SA-7,
// tCK=7ns min): the board's single PLL VCO is fixed at 576MHz by the
// existing, already-verified 64MHz CLKOP config (CLKFB_DIV=4,
// CLKOP_DIV=9 -- unchanged, not touched by this experiment). The only
// integer VCO/N divisors near the chip's ceiling are 576/4=144MHz
// (0.8% OVER the 143MHz max -- rejected, not "correctness first") and
// 576/5=115.2MHz (real ~19% margin under the rated max). 115.2MHz is
// therefore the fastest SAFE clock obtainable from this board's
// existing PLL without touching the verified 64MHz compute domain.
// Real measured gain vs the current 64MHz single-domain design is
// therefore 115.2/64 = 1.8x raw controller-clock speedup, NOT the 2.2x
// a naive 143MHz assumption would suggest -- this correction is
// intentional, verified against real ecppll output, not estimated.
//
// PROTOCOL: single-outstanding-request only (matches every existing
// caller's own req/busy/ready idiom exactly -- this bridge does NOT
// add multi-request pipelining; that is EXP-0052's explicitly
// deferred, larger, riskier follow-up, out of scope here). Because at
// most one transaction is ever in flight, a classic two-domain
// "toggle + last-seen" handshake is provably safe:
// - the requesting (slow) domain latches addr/wr/wdata/wmask and
// flips req_toggle_slow on the SAME clock edge, then holds ALL of
// those signals perfectly stable (no new request is ever issued
// while busy=1) until the response toggle comes back;
// - the fast domain double-flop-synchronizes req_toggle_slow (2 FF,
// standard metastability margin) and compares it against its own
// "last serviced" copy -- a mismatch means a new request is
// pending. Because addr/wr/wdata/wmask changed on the SAME edge
// that flipped the toggle, and never change again before the
// response, they are safe to sample directly (no per-bit
// synchronizer needed) once the synchronized toggle has visibly
// changed -- this is the standard "quasi-static bus + toggle"
// CDC idiom, not a shortcut.
// - the same reasoning applies in reverse for ack_toggle_fast/rdata
// going back to the slow domain.
// Reset: rst_slow and rst_fast are separate inputs, each assumed
// ALREADY synchronized to its own clock domain by the caller (this
// module does not itself synchronize an async reset -- matches this
// project's existing convention of a single, pre-synchronized `rst`
// per clock domain, see ecp5_pll_sys_clk.v's own reset handling).
// ============================================================
module sdram_cdc_bridge_openrow #(
parameter CLK_FREQ_MHZ_FAST = 115, // deliberately rounded DOWN from
// the real 115.2MHz (never over-
// count available ns/cycle --
// same "ceiling division" spirit
// as sdram_controller.v's own
// ns_to_cycles), so every derived
// timing constant (T_RCD/T_RP/...)
// gets AT LEAST as many cycles as
// the real, slightly-faster clock
// requires.
parameter BURST_LEN = 8,
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2,
parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS
)(
input wire clk_slow,
input wire rst_slow, // pre-synchronized to clk_slow
input wire clk_fast,
input wire rst_fast, // pre-synchronized to clk_fast
// ---- slow-domain caller interface (identical shape to
// sdram_controller.v's own ports) ----
input wire req,
input wire wr,
input wire [ADDR_WIDTH-1:0] addr,
input wire [16*BURST_LEN-1:0] wdata,
input wire [2*BURST_LEN-1:0] wmask,
output reg [16*BURST_LEN-1:0] rdata,
output reg ready,
output wire busy,
// ---- real SDRAM pins, driven directly by the fast-domain
// sdram_controller instance ----
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [BANK_BITS-1:0] sdram_ba,
output wire [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
// ============================================================
// Slow domain: capture request, drive toggle, wait for ack
// ============================================================
reg busy_slow;
reg req_toggle_slow;
reg wr_lat;
reg [ADDR_WIDTH-1:0] addr_lat;
reg [16*BURST_LEN-1:0] wdata_lat;
reg [2*BURST_LEN-1:0] wmask_lat;
assign busy = busy_slow;
// synchronize ack_toggle_fast (fast domain) into the slow domain
wire ack_toggle_fast;
reg ack_toggle_sync1, ack_toggle_sync2;
always @(posedge clk_slow) begin
if (rst_slow) begin
ack_toggle_sync1 <= 1'b0;
ack_toggle_sync2 <= 1'b0;
end else begin
ack_toggle_sync1 <= ack_toggle_fast;
ack_toggle_sync2 <= ack_toggle_sync1;
end
end
reg last_ack_toggle_seen_slow;
wire [16*BURST_LEN-1:0] rdata_fast_captured;
always @(posedge clk_slow) begin
if (rst_slow) begin
busy_slow <= 1'b0;
req_toggle_slow <= 1'b0;
last_ack_toggle_seen_slow <= 1'b0;
ready <= 1'b0;
rdata <= {(16*BURST_LEN){1'b0}};
wr_lat <= 1'b0;
addr_lat <= {ADDR_WIDTH{1'b0}};
wdata_lat <= {(16*BURST_LEN){1'b0}};
wmask_lat <= {(2*BURST_LEN){1'b0}};
end else begin
ready <= 1'b0;
if (req && !busy_slow) begin
wr_lat <= wr;
addr_lat <= addr;
wdata_lat <= wdata;
wmask_lat <= wmask;
req_toggle_slow <= ~req_toggle_slow;
busy_slow <= 1'b1;
end
if (busy_slow && (ack_toggle_sync2 != last_ack_toggle_seen_slow)) begin
last_ack_toggle_seen_slow <= ack_toggle_sync2;
rdata <= rdata_fast_captured;
ready <= 1'b1;
busy_slow <= 1'b0;
end
end
end
// ============================================================
// Fast domain: synchronize request toggle, drive the real
// sdram_controller, capture response, drive ack toggle back
// ============================================================
reg ctrl_req_f;
reg ctrl_wr_f;
reg [ADDR_WIDTH-1:0] ctrl_addr_f;
reg [16*BURST_LEN-1:0] ctrl_wdata_f;
reg [2*BURST_LEN-1:0] ctrl_wmask_f;
wire [16*BURST_LEN-1:0] ctrl_rdata_f;
wire ctrl_ready_f, ctrl_busy_f;
reg req_toggle_sync1, req_toggle_sync2;
always @(posedge clk_fast) begin
if (rst_fast) begin
req_toggle_sync1 <= 1'b0;
req_toggle_sync2 <= 1'b0;
end else begin
req_toggle_sync1 <= req_toggle_slow;
req_toggle_sync2 <= req_toggle_sync1;
end
end
localparam F_IDLE = 1'b0, F_WAIT = 1'b1;
reg f_state;
reg last_req_toggle_seen_fast;
reg ack_toggle_fast_r;
reg [16*BURST_LEN-1:0] rdata_fast_captured_r;
assign ack_toggle_fast = ack_toggle_fast_r;
assign rdata_fast_captured = rdata_fast_captured_r;
always @(posedge clk_fast) begin
if (rst_fast) begin
f_state <= F_IDLE;
last_req_toggle_seen_fast <= 1'b0;
ack_toggle_fast_r <= 1'b0;
rdata_fast_captured_r <= {(16*BURST_LEN){1'b0}};
ctrl_req_f <= 1'b0;
ctrl_wr_f <= 1'b0;
ctrl_addr_f <= {ADDR_WIDTH{1'b0}};
ctrl_wdata_f<= {(16*BURST_LEN){1'b0}};
ctrl_wmask_f<= {(2*BURST_LEN){1'b0}};
end else begin
ctrl_req_f <= 1'b0;
case (f_state)
F_IDLE: begin
if (req_toggle_sync2 != last_req_toggle_seen_fast) begin
// addr_lat/wr_lat/wdata_lat/wmask_lat (slow-
// domain regs) are quasi-static: they changed
// on the exact same slow-domain edge that
// flipped req_toggle_slow, and will not change
// again until busy_slow deasserts (long after
// this transaction completes) -- safe to
// sample directly, see module header.
ctrl_req_f <= 1'b1;
ctrl_wr_f <= wr_lat;
ctrl_addr_f <= addr_lat;
ctrl_wdata_f<= wdata_lat;
ctrl_wmask_f<= wmask_lat;
last_req_toggle_seen_fast <= req_toggle_sync2;
f_state <= F_WAIT;
end
end
F_WAIT: begin
if (ctrl_ready_f) begin
rdata_fast_captured_r <= ctrl_rdata_f;
ack_toggle_fast_r <= ~ack_toggle_fast_r;
f_state <= F_IDLE;
end
end
default: f_state <= F_IDLE;
endcase
end
end
sdram_controller_openrow #(
.CLK_FREQ_MHZ(CLK_FREQ_MHZ_FAST), .BURST_LEN(BURST_LEN),
.ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS)
) u_sdram_ctrl (
.clk(clk_fast), .rst(rst_fast),
.req(ctrl_req_f), .wr(ctrl_wr_f), .addr(ctrl_addr_f),
.wdata(ctrl_wdata_f), .wmask(ctrl_wmask_f),
.rdata(ctrl_rdata_f), .ready(ctrl_ready_f), .busy(ctrl_busy_f),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
endmodule
hardware/v2/nms/rtl/sdram_controller_openrow.v
+460
View File
@@ -0,0 +1,460 @@
`timescale 1ns/1ps
// ============================================================
// EXP-0054 -- open-row (page-hit) SDR SDRAM controller, forked from
// sdram_controller.v (STEP16). Implements the "page-hit/keep-row-open
// optimization" that sdram_controller.v's own header explicitly
// deferred:
// "ALWAYS uses auto-precharge... NOT the fastest possible design
// (no page-hit/keep-row-open optimization, unlike psram_
// controller.v's own real page-mode), but it is trivially correct"
//
// MOTIVATION: weight_prefetch_engine_wide.v (real production traffic,
// instantiated by nms_dataflow_core_sdram.v, PREFETCH_DISTANCE=8)
// already issues a stream of STRICTLY SEQUENTIAL tile addresses per
// job. With ROW_BITS=13/COL_BITS=10 (AS4C32M16SA: 1024 columns/row,
// 4 words/tile at 16-bit words -- see sdram_controller.v's own TILE
// comment), a single row holds 256 consecutive tiles before crossing
// a row boundary -- most real jobs' weight streams never leave the
// row they started in. Closing and reopening that row on EVERY single
// tile (today's fixed auto-precharge policy) pays tRP+tRCD twice per
// transaction for no reason when the next transaction is going to hit
// the SAME row anyway.
//
// POLICY: never auto-precharge (A10=0 on every READ/WRITE). Track the
// single currently-open bank+row (this controller has always modeled
// "one transaction in flight" -- this experiment keeps that same
// single-open-row scope, not per-bank tracking across multiple
// simultaneously-open banks, matching the project's own established
// risk posture). On the NEXT request (evaluated in S_IDLE, exactly
// where every prior request was already evaluated):
// - SAME bank+row as currently open ("row hit"): skip ACTIVATE
// entirely -- issue READ/WRITE directly, saving tRCD.
// - DIFFERENT bank+row while a row IS open ("row miss"): issue an
// explicit PRECHARGE first (this controller no longer gets that
// for free via auto-precharge), wait tRP, THEN activate the new
// row exactly as before -- same total cost as today's design,
// just paid on-demand instead of unconditionally after every
// transaction.
// - no row open (e.g. right after reset/refresh): activate directly,
// unchanged from today.
//
// REFRESH INTERACTION (the one real correctness hazard this policy
// introduces, absent from the original always-precharged design):
// JEDEC AUTO REFRESH requires ALL banks precharged first. The
// original S_IDLE refresh branch's own comment ("no row is ever left
// open between transactions... so we can refresh immediately") is no
// longer true under this policy -- fixed here by precharging first
// (S_PRE_THEN_REF_WAIT) whenever row_open is set at the moment
// refresh comes due, before issuing AUTO REFRESH exactly as before.
//
// WRITE RECOVERY (tWR): the original design folded tWR into its
// always-paid post-burst precharge wait ("T_RP + 1'b1 // tWR folded
// in conservatively"). This design no longer precharges after every
// write, so tWR is now paid explicitly and alone (T_WR=2 CLK, real
// AS4C32M16SA datasheet value, same explicit-CLK-units treatment as
// T_MRD) via a new S_WRITE_RECOVERY_WAIT state, before the row-open
// path returns to S_IDLE and can accept a same-row follow-on command.
//
// DISCLOSED, NOT INDEPENDENTLY VERIFIED: read-burst-end -> next
// command (read-to-read or read-to-write, same open row) has NO extra
// wait beyond the existing 1-cycle-minimum S_IDLE turnaround, on the
// reasoning that JEDEC SDR SDRAM page-mode reads support back-to-back
// column access with no additional bubble. sdram_model.v (this
// project's own real-command-sequence checker) does NOT itself assert
// tCCD/tRTW/tWTR -- it only checks ACTIVATE-while-active, tRP, tRAS
// (min), refresh spacing, and access-with-no-active-row (see its own
// VIOLATION messages). tb_sdram_controller_openrow.v exercises
// read-after-read and write-after-read same-row sequences explicitly
// and checks DATA correctness, but a genuine read-to-write DQ bus
// turnaround hazard would not be caught by sdram_model.v itself if
// present -- flagged here exactly as this project's own convention
// requires, not silently assumed safe.
//
// Every timing constant, the mrs_value encoding, the req_pending
// unconditional-latch fix, and the address decomposition are carried
// over UNCHANGED from sdram_controller.v -- only the state machine's
// precharge policy and the two new wait states are new.
// ============================================================
module sdram_controller_openrow #(
parameter CLK_FREQ_MHZ = 64,
parameter BURST_LEN = 4, // 1, 4, or 8 -- same real scope as sdram_controller.v (see its own mrs_value)
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2,
parameter ADDR_WIDTH = BANK_BITS + ROW_BITS + COL_BITS
)(
input wire clk,
input wire rst,
input wire req,
input wire wr,
input wire [ADDR_WIDTH-1:0] addr,
input wire [16*BURST_LEN-1:0] wdata,
input wire [2*BURST_LEN-1:0] wmask,
output reg [16*BURST_LEN-1:0] rdata,
output reg ready,
output reg busy,
output reg sdram_cke,
output reg sdram_cs_n,
output reg sdram_ras_n,
output reg sdram_cas_n,
output reg sdram_we_n,
output reg [1:0] sdram_ba,
output reg [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output reg [1:0] sdram_dqm
);
localparam BURST_IDXW = (BURST_LEN <= 1) ? 1 : $clog2(BURST_LEN);
initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS) begin
$display("FATAL sdram_controller_openrow: ADDR_WIDTH=%0d != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)=%0d",
ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS, BANK_BITS+ROW_BITS+COL_BITS);
$finish;
end
function integer ns_to_cycles;
input integer ns;
begin
ns_to_cycles = (ns * CLK_FREQ_MHZ + 999) / 1000;
end
endfunction
localparam T_RCD = ns_to_cycles(15);
localparam T_RP = ns_to_cycles(15);
localparam T_MRD = 2;
localparam T_WR = 2; // real AS4C32M16SA datasheet value, explicit CLK units (same treatment as T_MRD)
localparam T_INIT_US= 200;
localparam T_INIT = T_INIT_US * CLK_FREQ_MHZ;
localparam CAS_LATENCY = 3;
localparam T_REFI = ns_to_cycles(64000000 / (1 << ROW_BITS) + 1);
localparam CNTW = $clog2((T_INIT>T_REFI ? T_INIT : T_REFI) + 1);
function [CNTW-1:0] T_RC_MINUS1;
localparam integer T_RC = ns_to_cycles(65);
begin
T_RC_MINUS1 = T_RC[CNTW-1:0] - 1'b1;
end
endfunction
localparam
S_INIT_WAIT = 5'd0,
S_INIT_PRE_WAIT = 5'd2,
S_INIT_REF = 5'd3,
S_INIT_REF_WAIT = 5'd4,
S_INIT_MRS_WAIT = 5'd6,
S_IDLE = 5'd7,
S_REFRESH_WAIT = 5'd9,
S_ACTIVATE_WAIT = 5'd11,
S_CAS_WAIT = 5'd13,
S_BURST_READ = 5'd14,
S_BURST_WRITE = 5'd15,
S_PRE_THEN_ACT_WAIT = 5'd17,
S_PRE_THEN_REF_WAIT = 5'd18,
S_WRITE_RECOVERY_WAIT= 5'd19;
reg [4:0] state;
reg [CNTW-1:0] wait_cnt;
reg [3:0] init_ref_cnt;
reg [CNTW-1:0] refresh_timer;
reg [BURST_IDXW-1:0] burst_idx;
reg req_wr_reg;
reg [BANK_BITS-1:0] req_bank_reg;
reg [ROW_BITS-1:0] req_row_reg;
reg [COL_BITS-1:0] req_col_reg;
reg [16*BURST_LEN-1:0] wdata_reg;
reg [2*BURST_LEN-1:0] wmask_reg;
// ---- open-row tracking (new vs sdram_controller.v) ----
reg row_open;
reg [BANK_BITS-1:0] open_bank;
reg [ROW_BITS-1:0] open_row;
wire [BANK_BITS-1:0] addr_bank = addr[ADDR_WIDTH-1 -: BANK_BITS];
wire [ROW_BITS-1:0] addr_row = addr[ADDR_WIDTH-BANK_BITS-1 -: ROW_BITS];
wire [COL_BITS-1:0] addr_col = addr[COL_BITS-1:0];
reg req_pending;
wire eff_wr = req ? wr : req_wr_reg;
wire [BANK_BITS-1:0] eff_bank = req ? addr_bank : req_bank_reg;
wire [ROW_BITS-1:0] eff_row = req ? addr_row : req_row_reg;
wire [COL_BITS-1:0] eff_col = req ? addr_col : req_col_reg;
wire [16*BURST_LEN-1:0] eff_wdata = req ? wdata : wdata_reg;
wire [2*BURST_LEN-1:0] eff_wmask = req ? wmask : wmask_reg;
reg dq_out_en;
reg [15:0] dq_out;
assign sdram_dq = dq_out_en ? dq_out : 16'hzzzz;
function [ROW_BITS-1:0] mrs_value;
input integer burst_len;
reg [2:0] bl_code;
reg [ROW_BITS-1:0] v;
begin
bl_code = (burst_len==1) ? 3'b000 :
(burst_len==2) ? 3'b001 :
(burst_len==4) ? 3'b010 :
(burst_len==8) ? 3'b011 : 3'b111;
v = {ROW_BITS{1'b0}};
v[6:4] = 3'b011;
v[3] = 1'b0;
v[2:0] = bl_code;
mrs_value = v;
end
endfunction
always @(posedge clk) begin
if (rst) begin
state <= S_INIT_WAIT;
wait_cnt <= T_INIT[CNTW-1:0];
init_ref_cnt <= 4'd0;
refresh_timer <= T_REFI[CNTW-1:0];
sdram_cke <= 1'b1;
sdram_cs_n <= 1'b1;
sdram_ras_n <= 1'b1;
sdram_cas_n <= 1'b1;
sdram_we_n <= 1'b1;
sdram_ba <= 2'b00;
sdram_a <= {ROW_BITS{1'b0}};
sdram_dqm <= 2'b00;
dq_out_en <= 1'b0;
ready <= 1'b0;
busy <= 1'b1;
req_pending <= 1'b0;
row_open <= 1'b0;
open_bank <= {BANK_BITS{1'b0}};
open_row <= {ROW_BITS{1'b0}};
end else begin
sdram_cs_n <= 1'b0;
sdram_ras_n <= 1'b1;
sdram_cas_n <= 1'b1;
sdram_we_n <= 1'b1;
ready <= 1'b0;
dq_out_en <= 1'b0;
sdram_dqm <= 2'b00;
if (refresh_timer != 0) refresh_timer <= refresh_timer - 1'b1;
if (req) begin
req_wr_reg <= wr;
req_bank_reg <= addr_bank;
req_row_reg <= addr_row;
req_col_reg <= addr_col;
wdata_reg <= wdata;
wmask_reg <= wmask;
req_pending <= 1'b1;
end
case (state)
S_INIT_WAIT: begin
busy <= 1'b1;
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else begin
sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0;
sdram_a[10] <= 1'b1;
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
state <= S_INIT_PRE_WAIT;
end
end
S_INIT_PRE_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else state <= S_INIT_REF;
end
S_INIT_REF: begin
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0;
wait_cnt <= T_RC_MINUS1();
state <= S_INIT_REF_WAIT;
end
S_INIT_REF_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else if (init_ref_cnt < 4'd7) begin
init_ref_cnt <= init_ref_cnt + 1'b1;
state <= S_INIT_REF;
end else begin
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; sdram_we_n <= 1'b0;
sdram_ba <= 2'b00;
sdram_a <= mrs_value(BURST_LEN);
wait_cnt <= T_MRD[CNTW-1:0] - 1'b1;
state <= S_INIT_MRS_WAIT;
end
end
S_INIT_MRS_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else begin
busy <= 1'b0;
state <= S_IDLE;
end
end
S_IDLE: begin
busy <= 1'b0;
if (refresh_timer == 0) begin
busy <= 1'b1;
if (row_open) begin
// JEDEC: all banks must be precharged before
// AUTO REFRESH -- no longer free/automatic
// under the open-row policy (see header).
sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0;
sdram_ba <= open_bank;
sdram_a[10] <= 1'b1;
row_open <= 1'b0;
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
state <= S_PRE_THEN_REF_WAIT;
end else begin
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0;
wait_cnt <= T_RC_MINUS1();
refresh_timer <= T_REFI[CNTW-1:0];
state <= S_REFRESH_WAIT;
end
end else if (req || req_pending) begin
busy <= 1'b1;
req_wr_reg <= eff_wr;
req_bank_reg <= eff_bank;
req_row_reg <= eff_row;
req_col_reg <= eff_col;
wdata_reg <= eff_wdata;
wmask_reg <= eff_wmask;
req_pending <= 1'b0;
if (row_open && eff_bank == open_bank && eff_row == open_row) begin
// ROW HIT: skip ACTIVATE entirely, saves tRCD.
burst_idx <= {BURST_IDXW{1'b0}};
sdram_cas_n <= 1'b0;
sdram_we_n <= eff_wr ? 1'b0 : 1'b1;
sdram_ba <= eff_bank;
sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b0, {(10-COL_BITS){1'b0}}, eff_col}; // A10=0: no auto-precharge
if (eff_wr) begin
dq_out_en <= 1'b1;
dq_out <= eff_wdata[15:0];
sdram_dqm <= eff_wmask[1:0];
state <= S_BURST_WRITE;
end else begin
wait_cnt <= CAS_LATENCY[CNTW-1:0];
state <= S_CAS_WAIT;
end
end else if (row_open) begin
// ROW MISS, a different row is open: precharge
// it first (paid on-demand, same total cost as
// today's unconditional auto-precharge, just
// deferred until actually needed).
sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0;
sdram_ba <= open_bank;
sdram_a[10] <= 1'b1;
row_open <= 1'b0;
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
state <= S_PRE_THEN_ACT_WAIT;
end else begin
// no row open at all: activate directly.
sdram_ras_n <= 1'b0;
sdram_ba <= eff_bank;
sdram_a <= eff_row;
wait_cnt <= T_RCD[CNTW-1:0] - 1'b1;
state <= S_ACTIVATE_WAIT;
end
end
end
S_PRE_THEN_REF_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else begin
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0;
wait_cnt <= T_RC_MINUS1();
refresh_timer <= T_REFI[CNTW-1:0];
state <= S_REFRESH_WAIT;
end
end
S_PRE_THEN_ACT_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else begin
sdram_ras_n <= 1'b0;
sdram_ba <= req_bank_reg;
sdram_a <= req_row_reg;
wait_cnt <= T_RCD[CNTW-1:0] - 1'b1;
state <= S_ACTIVATE_WAIT;
end
end
S_REFRESH_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else state <= S_IDLE;
end
S_ACTIVATE_WAIT: begin
if (wait_cnt != 0) begin
wait_cnt <= wait_cnt - 1'b1;
end else begin
sdram_cas_n <= 1'b0;
sdram_we_n <= req_wr_reg ? 1'b0 : 1'b1;
sdram_ba <= req_bank_reg;
sdram_a <= {{(ROW_BITS-11){1'b0}}, 1'b0, {(10-COL_BITS){1'b0}}, req_col_reg}; // A10=0
burst_idx <= {BURST_IDXW{1'b0}};
row_open <= 1'b1;
open_bank <= req_bank_reg;
open_row <= req_row_reg;
if (req_wr_reg) begin
dq_out_en <= 1'b1;
dq_out <= wdata_reg[15:0];
sdram_dqm <= wmask_reg[1:0];
state <= S_BURST_WRITE;
end else begin
wait_cnt <= CAS_LATENCY[CNTW-1:0];
state <= S_CAS_WAIT;
end
end
end
S_CAS_WAIT: begin
if (wait_cnt != 0) begin
wait_cnt <= wait_cnt - 1'b1;
end else begin
rdata[0 +: 16] <= sdram_dq;
if (BURST_LEN == 1) begin
ready <= 1'b1;
state <= S_IDLE; // row stays open, no precharge
end else begin
burst_idx <= burst_idx + 1'b1;
state <= S_BURST_READ;
end
end
end
S_BURST_READ: begin
rdata[burst_idx*16 +: 16] <= sdram_dq;
if (burst_idx == BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin
ready <= 1'b1;
state <= S_IDLE; // row stays open, no precharge
end else begin
burst_idx <= burst_idx + 1'b1;
end
end
S_BURST_WRITE: begin
if (burst_idx < BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin
burst_idx <= burst_idx + 1'b1;
dq_out_en <= 1'b1;
dq_out <= wdata_reg[(burst_idx+1'b1)*16 +: 16];
sdram_dqm <= wmask_reg[(burst_idx+1'b1)*2 +: 2];
end else begin
ready <= 1'b1;
// tWR now paid alone (no longer folded with tRP,
// since we no longer precharge unconditionally --
// see header).
wait_cnt <= T_WR[CNTW-1:0] - 1'b1;
state <= S_WRITE_RECOVERY_WAIT;
end
end
S_WRITE_RECOVERY_WAIT: begin
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
else state <= S_IDLE; // row stays open, no precharge
end
default: state <= S_IDLE;
endcase
end
end
endmodule
hardware/v2/nms/rtl/sdram_unified_backend_cdc.v
+385
View File
@@ -0,0 +1,385 @@
`timescale 1ns/1ps
// ============================================================
// NMS STEP19 -- UNIFIED single-SDRAM memory backend.
//
// Replaces BOTH physical memory paths that existed through STEP18
// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/
// memory_interface.v + psram_controller.v for activation-fill/result-
// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE
// sdram_controller.v instance (BURST_LEN=8), serving THREE logical
// traffic classes through TWO external ports that exactly match what
// the existing, UNCHANGED consumers already drive:
//
// W port (64-bit): weight_prefetch_engine_wide.v's own real
// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external
// contract to STEP18's sdram_weight_backend_pack128.v (byte
// address in, 64-bit mem_rdata out), and internally reuses that
// module's own validated N_ENTRIES=4 "other half" cache
// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here).
//
// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's
// own activation reads AND every per-slot nms_memory_manager_
// stream_wide.v's own result writes, via slot_mem_arbiter.v --
// IDENTICAL external contract to the real V1 psram_controller.v
// port it replaces (word address in, 16-bit mem_wdata/mem_rdata,
// mem_lb_n/mem_ub_n byte-lane write masking). Neither
// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_
// wide.v needed ANY change -- they already produce a WORD
// address and already drive lb_n/ub_n exactly as the real V1
// PSRAM controller expected.
//
// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3.
// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed
// AT ALL for this step -- this is a pure memory-side substitution,
// per the governing spec's own explicit instruction.
//
// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE
// write mask (STEP19's own real, tested extension to sdram_
// controller.v's `wmask` port) -- this lets a single-BYTE result
// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by
// masking out every byte except the one/two the caller actually wants
// written, with NO read-modify-write needed at all (the real SDRAM
// chip itself leaves masked bytes untouched, by JEDEC definition).
// Activation reads need no such trick -- a full 128-bit block is
// fetched and the caller's own requested 16-bit word is extracted
// combinationally from it.
//
// Arbitration: simple, correctness-first 2-way priority (weight
// traffic strongly dominates real measured traffic -- STEP17 showed
// the activation/result path at <=7.2% of all external-memory
// activity -- so W is granted priority when both are pending, AR is
// never starved since W's own real traffic pattern always eventually
// idles between tiles/jobs). Exactly one physical SDRAM transaction
// in flight at a time (matches sdram_controller.v's own inherent
// single-transaction design, STEP18 Part E's own documented, accepted
// scope boundary -- not revisited here).
// ============================================================
module sdram_unified_backend_cdc #(
parameter ADDR_WIDTH = 26, // byte address width (W port convention)
parameter CLK_FREQ_MHZ = 64,
parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS
// physical SDRAM geometry, forwarded directly to sdram_controller.v
// (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9,
// 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 ==
// BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit),
// asserted at elaboration below.
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2
)(
input wire clk,
input wire rst,
input wire clk_fast,
input wire rst_fast,
// ---- W: weight fetch (64-bit, byte address, read-only) ----
input wire w_req,
input wire [ADDR_WIDTH-1:0] w_addr,
output reg [63:0] w_rdata,
output reg w_ready,
// ---- AR: activation-fill (read) + result-writeback (write),
// 16-bit, WORD address (matches the real V1 psram_controller.v
// convention this port replaces exactly) ----
input wire ar_req,
input wire ar_wr,
input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful
// (matches slot_mem_arbiter.v's own
// m_addr width convention exactly --
// that arbiter's real callers only ever
// drive a 22-bit-significant word
// address into an ADDR_WIDTH-wide bus)
input wire [15:0] ar_wdata,
input wire ar_lb_n,
input wire ar_ub_n,
output reg [15:0] ar_rdata,
output reg ar_ready,
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [BANK_BITS-1:0] sdram_ba,
output wire [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin
$display("FATAL sdram_unified_backend: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1",
ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS);
$finish;
end
// ============================================================
// W-port cache (identical logic to sdram_weight_backend_pack128.v
// -- an N_ENTRIES-deep, fully-associative "other half" cache,
// round-robin allocated; safe under any sizing, see that module's
// own header/ERR-0022 for the full rationale, not repeated here)
// ============================================================
localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES);
reg w_cache_valid [0:W_ENTRIES-1];
reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1];
reg [63:0] w_cache_data [0:W_ENTRIES-1];
reg [WEIDXW-1:0] w_alloc_ptr;
// ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R:
// worst seed1 total delay 17.909ns, 84% routing, dominant hop
// 2.5-2.8ns): the original RTL used a sequential for-loop that
// overwrites w_hit_idx_c on every match ("last valid+matching entry
// wins"), which Yosys/nextpnr synthesized as a serially-dependent
// cascade of PFUMX/OFX fast-mux primitives -- each entry's result
// depends on the previous one, forcing nextpnr to place the whole
// chain along one physical path with no freedom to shorten it. This
// is the same architectural fix class as ERR-0028 (activation_fill_
// ctrl's max-tree): replace the serial dependency chain with a flat
// one-hot compare (fully parallel, W_ENTRIES=4 comparators, no
// inter-entry dependency) followed by a single-level priority-encode
// casez, preserving the EXACT original "highest index wins" semantics
// bit-for-bit (verified: original loop always ends on the highest ei
// that matched, since ei counts up without break).
wire [W_ENTRIES-1:0] w_match_oh;
genvar wgi;
generate
for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH
assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr);
end
endgenerate
reg w_hit_found_c;
reg [WEIDXW-1:0] w_hit_idx_c;
integer ei;
generate
if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT
// real, measured configuration (see ERR-0029) -- flat,
// single-level priority encode over the parallel one-hot
// compare above, no serial inter-entry dependency.
always @(*) begin
w_hit_found_c = |w_match_oh;
casez (w_match_oh)
4'b1???: w_hit_idx_c = 2'd3;
4'b01??: w_hit_idx_c = 2'd2;
4'b001?: w_hit_idx_c = 2'd1;
4'b0001: w_hit_idx_c = 2'd0;
default: w_hit_idx_c = {WEIDXW{1'b0}};
endcase
end
end else begin : GEN_WHIT_FALLBACK
// any other W_ENTRIES value: fall back to the original,
// functionally-equivalent (but serially-dependent) scan --
// not the measured/optimized configuration this project
// actually builds, kept only for parametric safety.
always @(*) begin
w_hit_found_c = 1'b0;
w_hit_idx_c = {WEIDXW{1'b0}};
for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin
if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin
w_hit_found_c = 1'b1;
w_hit_idx_c = ei[WEIDXW-1:0];
end
end
end
end
endgenerate
wire w_cache_hit = w_hit_found_c && w_req;
// ============================================================
// Shared physical controller, BURST_LEN=8 (128-bit/16-byte real
// SDRAM transactions), reused UNCHANGED from STEP16-18.
// ============================================================
reg ctrl_req;
reg ctrl_wr;
reg [ADDR_WIDTH-2:0] ctrl_addr;
reg [127:0] ctrl_wdata;
reg [15:0] ctrl_wmask;
wire [127:0] ctrl_rdata;
wire ctrl_ready;
wire ctrl_busy;
sdram_cdc_bridge #(
.CLK_FREQ_MHZ_FAST(115), .BURST_LEN(8),
.ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS)
) u_sdram_ctrl (
.clk_slow(clk), .rst_slow(rst),
.clk_fast(clk_fast), .rst_fast(rst_fast),
.req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr),
.wdata(ctrl_wdata), .wmask(ctrl_wmask),
.rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
localparam S_IDLE = 3'd0,
S_W_WAIT = 3'd1,
S_AR_RD_WAIT = 3'd2,
S_AR_WR_WAIT = 3'd3;
reg [2:0] state;
reg w_pending_upper_half;
reg [ADDR_WIDTH-1:0] w_pending_addr;
reg [2:0] ar_pending_word;
// ---- req_pending latches (same fix class as sdram_controller.v's
// own ERR-0019/ERR-0020): this backend's own top-level S_IDLE
// arbitration can only START a new transaction when it is
// genuinely idle. A single-cycle w_req/ar_req pulse (this
// project's own established mem_req convention) arriving on a
// cycle this backend happens to be busy servicing the OTHER port
// would otherwise be silently dropped -- the caller has no idea,
// waits forever for a `ready` that never comes. Found the hard way
// (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked
// at 0/256 neurons, jobs_allocated stuck at 12, because the very
// first activation-fill read raced against weight-prefetch traffic
// and was lost exactly this way. Fix: latch EVERY req's own fields
// unconditionally, every cycle, regardless of current state (not
// just from S_IDLE), mirroring sdram_controller.v's own corrected
// fix exactly (ERR-0020: the FIRST attempt only latched from
// S_IDLE, which was still not enough -- latch unconditionally).
reg w_req_pending;
reg [ADDR_WIDTH-1:0] w_req_addr_lat;
reg ar_req_pending;
reg ar_req_wr_lat;
reg [ADDR_WIDTH-1:0] ar_req_addr_lat;
reg [15:0] ar_req_wdata_lat;
reg ar_req_lbn_lat, ar_req_ubn_lat;
wire w_eff_req = w_req || w_req_pending;
wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat;
wire ar_eff_req = ar_req || ar_req_pending;
wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat;
wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat;
wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat;
wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat;
wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat;
wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000};
wire w_eff_addr_is_upper_half = w_eff_addr[3];
wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000};
wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0];
integer ri;
always @(posedge clk) begin
if (rst) begin
state <= S_IDLE;
for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0;
w_alloc_ptr <= {WEIDXW{1'b0}};
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}};
ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF;
w_ready <= 1'b0; w_rdata <= 64'h0;
ar_ready <= 1'b0; ar_rdata <= 16'h0;
w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}};
ar_pending_word <= 3'h0;
w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}};
ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0;
ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0;
ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1;
end else begin
ctrl_req <= 1'b0;
w_ready <= 1'b0;
ar_ready <= 1'b0;
// latch fresh requests unconditionally, every cycle,
// regardless of state (see req_pending's own comment above)
if (w_req) begin
w_req_addr_lat <= w_addr;
w_req_pending <= 1'b1;
end
if (ar_req) begin
ar_req_wr_lat <= ar_wr;
ar_req_addr_lat <= ar_addr;
ar_req_wdata_lat <= ar_wdata;
ar_req_lbn_lat <= ar_lb_n;
ar_req_ubn_lat <= ar_ub_n;
ar_req_pending <= 1'b1;
end
case (state)
S_IDLE: begin
// W has priority when both are pending (real
// measured traffic: weight >>> activation+result,
// STEP17 EXP-0045 -- AR is never starved since W's
// own real access pattern idles between tiles).
if (w_cache_hit) begin
// fully serviced THIS cycle -- must also cancel
// the unconditional latch above, which just set
// w_req_pending<=1 for this SAME w_req pulse
// (real bug found via full regression, EXP-0048
// /ERR-0023: without this the latch survives
// uncontested, and next cycle w_eff_req reads
// true from STALE w_req_pending/w_req_addr_lat,
// issuing a bogus extra fetch that shifts every
// subsequent response by one).
w_rdata <= w_cache_data[w_hit_idx_c];
w_ready <= 1'b1;
w_cache_valid[w_hit_idx_c] <= 1'b0;
w_req_pending <= 1'b0;
end else if (w_eff_req) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= w_eff_aligned_word_addr;
ctrl_wmask <= 16'h0000;
w_pending_upper_half <= w_eff_addr_is_upper_half;
w_pending_addr <= w_eff_addr;
w_req_pending <= 1'b0;
state <= S_W_WAIT;
end else if (ar_eff_req && !ar_eff_wr) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= ar_eff_block_base;
ctrl_wmask <= 16'h0000;
ar_pending_word <= ar_eff_word_in_blk;
ar_req_pending <= 1'b0;
state <= S_AR_RD_WAIT;
end else if (ar_eff_req && ar_eff_wr) begin
// mask every word except the target one; within
// the target word, pass ar_lb_n/ar_ub_n through
// directly (same active-low "write this byte"
// polarity as real SDRAM DQM: lb_n=0 -> DQM=0
// -> byte written; lb_n=1 -> DQM=1 -> masked).
ctrl_req <= 1'b1;
ctrl_wr <= 1'b1;
ctrl_addr <= ar_eff_block_base;
ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter
ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2));
ar_req_pending <= 1'b0;
state <= S_AR_WR_WAIT;
end
end
S_W_WAIT: begin
if (ctrl_ready) begin
if (w_pending_upper_half) begin
w_rdata <= ctrl_rdata[127:64];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0];
w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end else begin
w_rdata <= ctrl_rdata[63:0];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64];
w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end
w_cache_valid[w_alloc_ptr] <= 1'b1;
w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1;
w_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_RD_WAIT: begin
if (ctrl_ready) begin
ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16];
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_WR_WAIT: begin
if (ctrl_ready) begin
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
default: state <= S_IDLE;
endcase
end
end
endmodule
hardware/v2/nms/rtl/sdram_unified_backend_combined.v
+404
View File
@@ -0,0 +1,404 @@
`timescale 1ns/1ps
// ============================================================
// EXP-0055 -- combined fork of sdram_unified_backend.v (STEP19) -- the
// ONLY changes are: (1) two new clk_fast/rst_fast ports, (2) the
// physical sdram_controller.v instance replaced by sdram_cdc_bridge_
// openrow.v (EXP-0053 CDC + EXP-0054 page-hit, composed together).
// W-port cache, arbitration, and the W/AR top-level FSM are ALL
// byte-for-byte unchanged, per the same fork discipline sdram_
// unified_backend_pipelined.v (EXP-0052) already established. See
// hardware/v2/logs/experiments.log (search "EXP-0055") for why this
// fork exists and its own isolated (tb_sdram_cdc_bridge_openrow.v)
// measured number before this integration step.
//
// Everything below this point is STEP19's own original header,
// preserved for the W/AR port contract description (still accurate --
// only the physical controller behind ctrl_req/.../ctrl_ready changed):
//
// NMS STEP19 -- UNIFIED single-SDRAM memory backend.
//
// Replaces BOTH physical memory paths that existed through STEP18
// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/
// memory_interface.v + psram_controller.v for activation-fill/result-
// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE
// sdram_controller.v instance (BURST_LEN=8), serving THREE logical
// traffic classes through TWO external ports that exactly match what
// the existing, UNCHANGED consumers already drive:
//
// W port (64-bit): weight_prefetch_engine_wide.v's own real
// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external
// contract to STEP18's sdram_weight_backend_pack128.v (byte
// address in, 64-bit mem_rdata out), and internally reuses that
// module's own validated N_ENTRIES=4 "other half" cache
// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here).
//
// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's
// own activation reads AND every per-slot nms_memory_manager_
// stream_wide.v's own result writes, via slot_mem_arbiter.v --
// IDENTICAL external contract to the real V1 psram_controller.v
// port it replaces (word address in, 16-bit mem_wdata/mem_rdata,
// mem_lb_n/mem_ub_n byte-lane write masking). Neither
// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_
// wide.v needed ANY change -- they already produce a WORD
// address and already drive lb_n/ub_n exactly as the real V1
// PSRAM controller expected.
//
// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3.
// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed
// AT ALL for this step -- this is a pure memory-side substitution,
// per the governing spec's own explicit instruction.
//
// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE
// write mask (STEP19's own real, tested extension to sdram_
// controller.v's `wmask` port) -- this lets a single-BYTE result
// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by
// masking out every byte except the one/two the caller actually wants
// written, with NO read-modify-write needed at all (the real SDRAM
// chip itself leaves masked bytes untouched, by JEDEC definition).
// Activation reads need no such trick -- a full 128-bit block is
// fetched and the caller's own requested 16-bit word is extracted
// combinationally from it.
//
// Arbitration: simple, correctness-first 2-way priority (weight
// traffic strongly dominates real measured traffic -- STEP17 showed
// the activation/result path at <=7.2% of all external-memory
// activity -- so W is granted priority when both are pending, AR is
// never starved since W's own real traffic pattern always eventually
// idles between tiles/jobs). Exactly one physical SDRAM transaction
// in flight at a time (matches sdram_controller.v's own inherent
// single-transaction design, STEP18 Part E's own documented, accepted
// scope boundary -- not revisited here).
// ============================================================
module sdram_unified_backend_combined #(
parameter ADDR_WIDTH = 26, // byte address width (W port convention)
parameter CLK_FREQ_MHZ = 64,
parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS
// physical SDRAM geometry, forwarded directly to sdram_controller.v
// (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9,
// 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 ==
// BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit),
// asserted at elaboration below.
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2
)(
input wire clk,
input wire rst,
// ---- EXP-0055: fast SDRAM clock domain (115.2MHz-class), see
// sdram_cdc_bridge_openrow.v -- everything else in this module
// (W/AR ports, W-cache, top-level FSM) stays on clk/rst exactly
// as before ----
input wire clk_fast,
input wire rst_fast,
// ---- W: weight fetch (64-bit, byte address, read-only) ----
input wire w_req,
input wire [ADDR_WIDTH-1:0] w_addr,
output reg [63:0] w_rdata,
output reg w_ready,
// ---- AR: activation-fill (read) + result-writeback (write),
// 16-bit, WORD address (matches the real V1 psram_controller.v
// convention this port replaces exactly) ----
input wire ar_req,
input wire ar_wr,
input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful
// (matches slot_mem_arbiter.v's own
// m_addr width convention exactly --
// that arbiter's real callers only ever
// drive a 22-bit-significant word
// address into an ADDR_WIDTH-wide bus)
input wire [15:0] ar_wdata,
input wire ar_lb_n,
input wire ar_ub_n,
output reg [15:0] ar_rdata,
output reg ar_ready,
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [BANK_BITS-1:0] sdram_ba,
output wire [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin
$display("FATAL sdram_unified_backend_combined: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1",
ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS);
$finish;
end
// ============================================================
// W-port cache (identical logic to sdram_weight_backend_pack128.v
// -- an N_ENTRIES-deep, fully-associative "other half" cache,
// round-robin allocated; safe under any sizing, see that module's
// own header/ERR-0022 for the full rationale, not repeated here)
// ============================================================
localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES);
reg w_cache_valid [0:W_ENTRIES-1];
reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1];
reg [63:0] w_cache_data [0:W_ENTRIES-1];
reg [WEIDXW-1:0] w_alloc_ptr;
// ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R:
// worst seed1 total delay 17.909ns, 84% routing, dominant hop
// 2.5-2.8ns): the original RTL used a sequential for-loop that
// overwrites w_hit_idx_c on every match ("last valid+matching entry
// wins"), which Yosys/nextpnr synthesized as a serially-dependent
// cascade of PFUMX/OFX fast-mux primitives -- each entry's result
// depends on the previous one, forcing nextpnr to place the whole
// chain along one physical path with no freedom to shorten it. This
// is the same architectural fix class as ERR-0028 (activation_fill_
// ctrl's max-tree): replace the serial dependency chain with a flat
// one-hot compare (fully parallel, W_ENTRIES=4 comparators, no
// inter-entry dependency) followed by a single-level priority-encode
// casez, preserving the EXACT original "highest index wins" semantics
// bit-for-bit (verified: original loop always ends on the highest ei
// that matched, since ei counts up without break).
wire [W_ENTRIES-1:0] w_match_oh;
genvar wgi;
generate
for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH
assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr);
end
endgenerate
reg w_hit_found_c;
reg [WEIDXW-1:0] w_hit_idx_c;
integer ei;
generate
if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT
// real, measured configuration (see ERR-0029) -- flat,
// single-level priority encode over the parallel one-hot
// compare above, no serial inter-entry dependency.
always @(*) begin
w_hit_found_c = |w_match_oh;
casez (w_match_oh)
4'b1???: w_hit_idx_c = 2'd3;
4'b01??: w_hit_idx_c = 2'd2;
4'b001?: w_hit_idx_c = 2'd1;
4'b0001: w_hit_idx_c = 2'd0;
default: w_hit_idx_c = {WEIDXW{1'b0}};
endcase
end
end else begin : GEN_WHIT_FALLBACK
// any other W_ENTRIES value: fall back to the original,
// functionally-equivalent (but serially-dependent) scan --
// not the measured/optimized configuration this project
// actually builds, kept only for parametric safety.
always @(*) begin
w_hit_found_c = 1'b0;
w_hit_idx_c = {WEIDXW{1'b0}};
for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin
if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin
w_hit_found_c = 1'b1;
w_hit_idx_c = ei[WEIDXW-1:0];
end
end
end
end
endgenerate
wire w_cache_hit = w_hit_found_c && w_req;
// ============================================================
// Shared physical controller, BURST_LEN=8 (128-bit/16-byte real
// SDRAM transactions), reused UNCHANGED from STEP16-18.
// ============================================================
reg ctrl_req;
reg ctrl_wr;
reg [ADDR_WIDTH-2:0] ctrl_addr;
reg [127:0] ctrl_wdata;
reg [15:0] ctrl_wmask;
wire [127:0] ctrl_rdata;
wire ctrl_ready;
wire ctrl_busy;
sdram_cdc_bridge_openrow #(
.CLK_FREQ_MHZ_FAST(115), .BURST_LEN(8),
.ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS)
) u_sdram_ctrl (
.clk_slow(clk), .rst_slow(rst),
.clk_fast(clk_fast), .rst_fast(rst_fast),
.req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr),
.wdata(ctrl_wdata), .wmask(ctrl_wmask),
.rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
localparam S_IDLE = 3'd0,
S_W_WAIT = 3'd1,
S_AR_RD_WAIT = 3'd2,
S_AR_WR_WAIT = 3'd3;
reg [2:0] state;
reg w_pending_upper_half;
reg [ADDR_WIDTH-1:0] w_pending_addr;
reg [2:0] ar_pending_word;
// ---- req_pending latches (same fix class as sdram_controller.v's
// own ERR-0019/ERR-0020): this backend's own top-level S_IDLE
// arbitration can only START a new transaction when it is
// genuinely idle. A single-cycle w_req/ar_req pulse (this
// project's own established mem_req convention) arriving on a
// cycle this backend happens to be busy servicing the OTHER port
// would otherwise be silently dropped -- the caller has no idea,
// waits forever for a `ready` that never comes. Found the hard way
// (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked
// at 0/256 neurons, jobs_allocated stuck at 12, because the very
// first activation-fill read raced against weight-prefetch traffic
// and was lost exactly this way. Fix: latch EVERY req's own fields
// unconditionally, every cycle, regardless of current state (not
// just from S_IDLE), mirroring sdram_controller.v's own corrected
// fix exactly (ERR-0020: the FIRST attempt only latched from
// S_IDLE, which was still not enough -- latch unconditionally).
reg w_req_pending;
reg [ADDR_WIDTH-1:0] w_req_addr_lat;
reg ar_req_pending;
reg ar_req_wr_lat;
reg [ADDR_WIDTH-1:0] ar_req_addr_lat;
reg [15:0] ar_req_wdata_lat;
reg ar_req_lbn_lat, ar_req_ubn_lat;
wire w_eff_req = w_req || w_req_pending;
wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat;
wire ar_eff_req = ar_req || ar_req_pending;
wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat;
wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat;
wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat;
wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat;
wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat;
wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000};
wire w_eff_addr_is_upper_half = w_eff_addr[3];
wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000};
wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0];
integer ri;
always @(posedge clk) begin
if (rst) begin
state <= S_IDLE;
for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0;
w_alloc_ptr <= {WEIDXW{1'b0}};
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}};
ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF;
w_ready <= 1'b0; w_rdata <= 64'h0;
ar_ready <= 1'b0; ar_rdata <= 16'h0;
w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}};
ar_pending_word <= 3'h0;
w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}};
ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0;
ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0;
ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1;
end else begin
ctrl_req <= 1'b0;
w_ready <= 1'b0;
ar_ready <= 1'b0;
// latch fresh requests unconditionally, every cycle,
// regardless of state (see req_pending's own comment above)
if (w_req) begin
w_req_addr_lat <= w_addr;
w_req_pending <= 1'b1;
end
if (ar_req) begin
ar_req_wr_lat <= ar_wr;
ar_req_addr_lat <= ar_addr;
ar_req_wdata_lat <= ar_wdata;
ar_req_lbn_lat <= ar_lb_n;
ar_req_ubn_lat <= ar_ub_n;
ar_req_pending <= 1'b1;
end
case (state)
S_IDLE: begin
// W has priority when both are pending (real
// measured traffic: weight >>> activation+result,
// STEP17 EXP-0045 -- AR is never starved since W's
// own real access pattern idles between tiles).
if (w_cache_hit) begin
// fully serviced THIS cycle -- must also cancel
// the unconditional latch above, which just set
// w_req_pending<=1 for this SAME w_req pulse
// (real bug found via full regression, EXP-0048
// /ERR-0023: without this the latch survives
// uncontested, and next cycle w_eff_req reads
// true from STALE w_req_pending/w_req_addr_lat,
// issuing a bogus extra fetch that shifts every
// subsequent response by one).
w_rdata <= w_cache_data[w_hit_idx_c];
w_ready <= 1'b1;
w_cache_valid[w_hit_idx_c] <= 1'b0;
w_req_pending <= 1'b0;
end else if (w_eff_req) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= w_eff_aligned_word_addr;
ctrl_wmask <= 16'h0000;
w_pending_upper_half <= w_eff_addr_is_upper_half;
w_pending_addr <= w_eff_addr;
w_req_pending <= 1'b0;
state <= S_W_WAIT;
end else if (ar_eff_req && !ar_eff_wr) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= ar_eff_block_base;
ctrl_wmask <= 16'h0000;
ar_pending_word <= ar_eff_word_in_blk;
ar_req_pending <= 1'b0;
state <= S_AR_RD_WAIT;
end else if (ar_eff_req && ar_eff_wr) begin
// mask every word except the target one; within
// the target word, pass ar_lb_n/ar_ub_n through
// directly (same active-low "write this byte"
// polarity as real SDRAM DQM: lb_n=0 -> DQM=0
// -> byte written; lb_n=1 -> DQM=1 -> masked).
ctrl_req <= 1'b1;
ctrl_wr <= 1'b1;
ctrl_addr <= ar_eff_block_base;
ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter
ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2));
ar_req_pending <= 1'b0;
state <= S_AR_WR_WAIT;
end
end
S_W_WAIT: begin
if (ctrl_ready) begin
if (w_pending_upper_half) begin
w_rdata <= ctrl_rdata[127:64];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0];
w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end else begin
w_rdata <= ctrl_rdata[63:0];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64];
w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end
w_cache_valid[w_alloc_ptr] <= 1'b1;
w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1;
w_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_RD_WAIT: begin
if (ctrl_ready) begin
ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16];
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_WR_WAIT: begin
if (ctrl_ready) begin
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
default: state <= S_IDLE;
endcase
end
end
endmodule
hardware/v2/nms/rtl/sdram_unified_backend_openrow.v
+382
View File
@@ -0,0 +1,382 @@
`timescale 1ns/1ps
// ============================================================
// NMS STEP19 -- UNIFIED single-SDRAM memory backend.
//
// Replaces BOTH physical memory paths that existed through STEP18
// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/
// memory_interface.v + psram_controller.v for activation-fill/result-
// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE
// sdram_controller.v instance (BURST_LEN=8), serving THREE logical
// traffic classes through TWO external ports that exactly match what
// the existing, UNCHANGED consumers already drive:
//
// W port (64-bit): weight_prefetch_engine_wide.v's own real
// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external
// contract to STEP18's sdram_weight_backend_pack128.v (byte
// address in, 64-bit mem_rdata out), and internally reuses that
// module's own validated N_ENTRIES=4 "other half" cache
// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here).
//
// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's
// own activation reads AND every per-slot nms_memory_manager_
// stream_wide.v's own result writes, via slot_mem_arbiter.v --
// IDENTICAL external contract to the real V1 psram_controller.v
// port it replaces (word address in, 16-bit mem_wdata/mem_rdata,
// mem_lb_n/mem_ub_n byte-lane write masking). Neither
// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_
// wide.v needed ANY change -- they already produce a WORD
// address and already drive lb_n/ub_n exactly as the real V1
// PSRAM controller expected.
//
// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3.
// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed
// AT ALL for this step -- this is a pure memory-side substitution,
// per the governing spec's own explicit instruction.
//
// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE
// write mask (STEP19's own real, tested extension to sdram_
// controller.v's `wmask` port) -- this lets a single-BYTE result
// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by
// masking out every byte except the one/two the caller actually wants
// written, with NO read-modify-write needed at all (the real SDRAM
// chip itself leaves masked bytes untouched, by JEDEC definition).
// Activation reads need no such trick -- a full 128-bit block is
// fetched and the caller's own requested 16-bit word is extracted
// combinationally from it.
//
// Arbitration: simple, correctness-first 2-way priority (weight
// traffic strongly dominates real measured traffic -- STEP17 showed
// the activation/result path at <=7.2% of all external-memory
// activity -- so W is granted priority when both are pending, AR is
// never starved since W's own real traffic pattern always eventually
// idles between tiles/jobs). Exactly one physical SDRAM transaction
// in flight at a time (matches sdram_controller.v's own inherent
// single-transaction design, STEP18 Part E's own documented, accepted
// scope boundary -- not revisited here).
// ============================================================
module sdram_unified_backend_openrow #(
parameter ADDR_WIDTH = 26, // byte address width (W port convention)
parameter CLK_FREQ_MHZ = 64,
parameter W_ENTRIES = 4, // weight-cache depth, >= real N_SLOTS
// physical SDRAM geometry, forwarded directly to sdram_controller.v
// (AS4C32M16SA defaults: 13 row bits/A0-A12, 10 col bits/A0-A9,
// 2 bank bits/BA0-BA1) -- must satisfy ADDR_WIDTH-1 ==
// BANK_BITS+ROW_BITS+COL_BITS (byte address = word address + 1 bit),
// asserted at elaboration below.
parameter ROW_BITS = 13,
parameter COL_BITS = 10,
parameter BANK_BITS = 2
)(
input wire clk,
input wire rst,
// ---- W: weight fetch (64-bit, byte address, read-only) ----
input wire w_req,
input wire [ADDR_WIDTH-1:0] w_addr,
output reg [63:0] w_rdata,
output reg w_ready,
// ---- AR: activation-fill (read) + result-writeback (write),
// 16-bit, WORD address (matches the real V1 psram_controller.v
// convention this port replaces exactly) ----
input wire ar_req,
input wire ar_wr,
input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful
// (matches slot_mem_arbiter.v's own
// m_addr width convention exactly --
// that arbiter's real callers only ever
// drive a 22-bit-significant word
// address into an ADDR_WIDTH-wide bus)
input wire [15:0] ar_wdata,
input wire ar_lb_n,
input wire ar_ub_n,
output reg [15:0] ar_rdata,
output reg ar_ready,
output wire sdram_cke,
output wire sdram_cs_n,
output wire sdram_ras_n,
output wire sdram_cas_n,
output wire sdram_we_n,
output wire [BANK_BITS-1:0] sdram_ba,
output wire [ROW_BITS-1:0] sdram_a,
inout wire [15:0] sdram_dq,
output wire [1:0] sdram_dqm
);
initial if (ADDR_WIDTH != BANK_BITS + ROW_BITS + COL_BITS + 1) begin
$display("FATAL sdram_unified_backend: ADDR_WIDTH(%0d) != BANK_BITS(%0d)+ROW_BITS(%0d)+COL_BITS(%0d)+1",
ADDR_WIDTH, BANK_BITS, ROW_BITS, COL_BITS);
$finish;
end
// ============================================================
// W-port cache (identical logic to sdram_weight_backend_pack128.v
// -- an N_ENTRIES-deep, fully-associative "other half" cache,
// round-robin allocated; safe under any sizing, see that module's
// own header/ERR-0022 for the full rationale, not repeated here)
// ============================================================
localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES);
reg w_cache_valid [0:W_ENTRIES-1];
reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1];
reg [63:0] w_cache_data [0:W_ENTRIES-1];
reg [WEIDXW-1:0] w_alloc_ptr;
// ERR-0029 fix (N=8 @64MHz critical-path, measured via real P&R:
// worst seed1 total delay 17.909ns, 84% routing, dominant hop
// 2.5-2.8ns): the original RTL used a sequential for-loop that
// overwrites w_hit_idx_c on every match ("last valid+matching entry
// wins"), which Yosys/nextpnr synthesized as a serially-dependent
// cascade of PFUMX/OFX fast-mux primitives -- each entry's result
// depends on the previous one, forcing nextpnr to place the whole
// chain along one physical path with no freedom to shorten it. This
// is the same architectural fix class as ERR-0028 (activation_fill_
// ctrl's max-tree): replace the serial dependency chain with a flat
// one-hot compare (fully parallel, W_ENTRIES=4 comparators, no
// inter-entry dependency) followed by a single-level priority-encode
// casez, preserving the EXACT original "highest index wins" semantics
// bit-for-bit (verified: original loop always ends on the highest ei
// that matched, since ei counts up without break).
wire [W_ENTRIES-1:0] w_match_oh;
genvar wgi;
generate
for (wgi = 0; wgi < W_ENTRIES; wgi = wgi + 1) begin : GEN_WMATCH
assign w_match_oh[wgi] = w_cache_valid[wgi] && (w_cache_addr[wgi] == w_addr);
end
endgenerate
reg w_hit_found_c;
reg [WEIDXW-1:0] w_hit_idx_c;
integer ei;
generate
if (W_ENTRIES == 4) begin : GEN_WHIT_FLAT
// real, measured configuration (see ERR-0029) -- flat,
// single-level priority encode over the parallel one-hot
// compare above, no serial inter-entry dependency.
always @(*) begin
w_hit_found_c = |w_match_oh;
casez (w_match_oh)
4'b1???: w_hit_idx_c = 2'd3;
4'b01??: w_hit_idx_c = 2'd2;
4'b001?: w_hit_idx_c = 2'd1;
4'b0001: w_hit_idx_c = 2'd0;
default: w_hit_idx_c = {WEIDXW{1'b0}};
endcase
end
end else begin : GEN_WHIT_FALLBACK
// any other W_ENTRIES value: fall back to the original,
// functionally-equivalent (but serially-dependent) scan --
// not the measured/optimized configuration this project
// actually builds, kept only for parametric safety.
always @(*) begin
w_hit_found_c = 1'b0;
w_hit_idx_c = {WEIDXW{1'b0}};
for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin
if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin
w_hit_found_c = 1'b1;
w_hit_idx_c = ei[WEIDXW-1:0];
end
end
end
end
endgenerate
wire w_cache_hit = w_hit_found_c && w_req;
// ============================================================
// Shared physical controller, BURST_LEN=8 (128-bit/16-byte real
// SDRAM transactions), reused UNCHANGED from STEP16-18.
// ============================================================
reg ctrl_req;
reg ctrl_wr;
reg [ADDR_WIDTH-2:0] ctrl_addr;
reg [127:0] ctrl_wdata;
reg [15:0] ctrl_wmask;
wire [127:0] ctrl_rdata;
wire ctrl_ready;
wire ctrl_busy;
sdram_controller_openrow #(
.CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8),
.ROW_BITS(ROW_BITS), .COL_BITS(COL_BITS), .BANK_BITS(BANK_BITS)
) u_sdram_ctrl (
.clk(clk), .rst(rst),
.req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr),
.wdata(ctrl_wdata), .wmask(ctrl_wmask),
.rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy),
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
);
localparam S_IDLE = 3'd0,
S_W_WAIT = 3'd1,
S_AR_RD_WAIT = 3'd2,
S_AR_WR_WAIT = 3'd3;
reg [2:0] state;
reg w_pending_upper_half;
reg [ADDR_WIDTH-1:0] w_pending_addr;
reg [2:0] ar_pending_word;
// ---- req_pending latches (same fix class as sdram_controller.v's
// own ERR-0019/ERR-0020): this backend's own top-level S_IDLE
// arbitration can only START a new transaction when it is
// genuinely idle. A single-cycle w_req/ar_req pulse (this
// project's own established mem_req convention) arriving on a
// cycle this backend happens to be busy servicing the OTHER port
// would otherwise be silently dropped -- the caller has no idea,
// waits forever for a `ready` that never comes. Found the hard way
// (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked
// at 0/256 neurons, jobs_allocated stuck at 12, because the very
// first activation-fill read raced against weight-prefetch traffic
// and was lost exactly this way. Fix: latch EVERY req's own fields
// unconditionally, every cycle, regardless of current state (not
// just from S_IDLE), mirroring sdram_controller.v's own corrected
// fix exactly (ERR-0020: the FIRST attempt only latched from
// S_IDLE, which was still not enough -- latch unconditionally).
reg w_req_pending;
reg [ADDR_WIDTH-1:0] w_req_addr_lat;
reg ar_req_pending;
reg ar_req_wr_lat;
reg [ADDR_WIDTH-1:0] ar_req_addr_lat;
reg [15:0] ar_req_wdata_lat;
reg ar_req_lbn_lat, ar_req_ubn_lat;
wire w_eff_req = w_req || w_req_pending;
wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat;
wire ar_eff_req = ar_req || ar_req_pending;
wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat;
wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat;
wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat;
wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat;
wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat;
wire [ADDR_WIDTH-2:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000};
wire w_eff_addr_is_upper_half = w_eff_addr[3];
wire [ADDR_WIDTH-2:0] ar_eff_block_base = {ar_eff_addr[ADDR_WIDTH-2:3], 3'b000};
wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0];
integer ri;
always @(posedge clk) begin
if (rst) begin
state <= S_IDLE;
for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0;
w_alloc_ptr <= {WEIDXW{1'b0}};
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {(ADDR_WIDTH-1){1'b0}};
ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF;
w_ready <= 1'b0; w_rdata <= 64'h0;
ar_ready <= 1'b0; ar_rdata <= 16'h0;
w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}};
ar_pending_word <= 3'h0;
w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}};
ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0;
ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0;
ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1;
end else begin
ctrl_req <= 1'b0;
w_ready <= 1'b0;
ar_ready <= 1'b0;
// latch fresh requests unconditionally, every cycle,
// regardless of state (see req_pending's own comment above)
if (w_req) begin
w_req_addr_lat <= w_addr;
w_req_pending <= 1'b1;
end
if (ar_req) begin
ar_req_wr_lat <= ar_wr;
ar_req_addr_lat <= ar_addr;
ar_req_wdata_lat <= ar_wdata;
ar_req_lbn_lat <= ar_lb_n;
ar_req_ubn_lat <= ar_ub_n;
ar_req_pending <= 1'b1;
end
case (state)
S_IDLE: begin
// W has priority when both are pending (real
// measured traffic: weight >>> activation+result,
// STEP17 EXP-0045 -- AR is never starved since W's
// own real access pattern idles between tiles).
if (w_cache_hit) begin
// fully serviced THIS cycle -- must also cancel
// the unconditional latch above, which just set
// w_req_pending<=1 for this SAME w_req pulse
// (real bug found via full regression, EXP-0048
// /ERR-0023: without this the latch survives
// uncontested, and next cycle w_eff_req reads
// true from STALE w_req_pending/w_req_addr_lat,
// issuing a bogus extra fetch that shifts every
// subsequent response by one).
w_rdata <= w_cache_data[w_hit_idx_c];
w_ready <= 1'b1;
w_cache_valid[w_hit_idx_c] <= 1'b0;
w_req_pending <= 1'b0;
end else if (w_eff_req) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= w_eff_aligned_word_addr;
ctrl_wmask <= 16'h0000;
w_pending_upper_half <= w_eff_addr_is_upper_half;
w_pending_addr <= w_eff_addr;
w_req_pending <= 1'b0;
state <= S_W_WAIT;
end else if (ar_eff_req && !ar_eff_wr) begin
ctrl_req <= 1'b1;
ctrl_wr <= 1'b0;
ctrl_addr <= ar_eff_block_base;
ctrl_wmask <= 16'h0000;
ar_pending_word <= ar_eff_word_in_blk;
ar_req_pending <= 1'b0;
state <= S_AR_RD_WAIT;
end else if (ar_eff_req && ar_eff_wr) begin
// mask every word except the target one; within
// the target word, pass ar_lb_n/ar_ub_n through
// directly (same active-low "write this byte"
// polarity as real SDRAM DQM: lb_n=0 -> DQM=0
// -> byte written; lb_n=1 -> DQM=1 -> masked).
ctrl_req <= 1'b1;
ctrl_wr <= 1'b1;
ctrl_addr <= ar_eff_block_base;
ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter
ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2));
ar_req_pending <= 1'b0;
state <= S_AR_WR_WAIT;
end
end
S_W_WAIT: begin
if (ctrl_ready) begin
if (w_pending_upper_half) begin
w_rdata <= ctrl_rdata[127:64];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0];
w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end else begin
w_rdata <= ctrl_rdata[63:0];
w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64];
w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
end
w_cache_valid[w_alloc_ptr] <= 1'b1;
w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1;
w_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_RD_WAIT: begin
if (ctrl_ready) begin
ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16];
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
S_AR_WR_WAIT: begin
if (ctrl_ready) begin
ar_ready <= 1'b1;
state <= S_IDLE;
end
end
default: state <= S_IDLE;
endcase
end
end
endmodule