V2.0.0 hardware freeze - single SDRAM
FASE #1 hardware freeze for FPGA-Neural V2, N4/P8, single external SDRAM (Alliance Memory AS4C4M16SA-6TIN) serving weights, activations, and results through one physical sdram_controller.v instance. Removes the PSRAM dependency (hardware/v1/rtl/psram_controller.v + memory_interface.v) from the V2 physical path entirely -- V1 itself remains fully unmodified, the golden reference. New RTL: sdram_unified_backend.v (2-way W/AR arbitration over one SDRAM controller, real per-byte DQM write masking added to sdram_controller.v for correct single-byte result writes with no read-modify-write), nms_neural_multiprocessor_sdram_unified.v (the frozen top-level). Two real bugs found and fixed via full-system testing before being accepted (ERR-0023): a deadlock and an off-by-one data-shift bug in the new arbitration logic. Real results: N=4 and N=2 D-Stress bit-exact (256/256 neurons), 40 real AUTO REFRESH events interleaved with zero corruption, real Yosys+nextpnr-ecp5 synthesis/P&R for LFE5U-45F-8CABGA381 (149/245 TRELLIS_IO, a real 45-pin reduction from the prior dual-memory design). Timing is MARGINAL (1/8 P&R seeds >=80MHz), reported honestly rather than masked by the best seed. Real, sourced ball-level pinout for the SDRAM bus + clk/rst (39/149 signals, P&R-verified) using the official Lattice ECP5U-45 pinout CSV found on disk during this step's own pre-commit review -- corrects an earlier draft that wrongly assumed no real pinout data was available. Chip readiness: NO. Real, disclosed blockers remain (no physical host interface exists yet -- the RTL's own reg_* ports are a 110-pin raw test-harness bus; clock source/PLL decision; power/configuration component selection) -- see hardware/v2/docs/{HARDWARE_FREEZE, CHIP_READINESS,OPEN_ITEMS}.md for the complete, itemized status. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013xXuuRUWZScuo1DeYJxs3v
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,69 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP 3: banked activation SRAM
|
||||
// contention model. SIMULATION-ONLY, NEVER SYNTHESIZED.
|
||||
//
|
||||
// Models a single shared activation vector striped across N_BANKS
|
||||
// banks by tile index (bank = tile_idx % N_BANKS). Each bank can serve
|
||||
// ONE DISTINCT address per cycle, but BROADCASTS that address's data
|
||||
// to every requester currently wanting it (a shared-producer, many-
|
||||
// consumer read costs exactly one read, not one per consumer -- see
|
||||
// hardware/v2/nms/rtl.. STEP2's own architecture.log note and the
|
||||
// user's own NMS spec §11). A cycle where two+ requesters mapped to
|
||||
// the same bank want DIFFERENT tile indices serves only one of them
|
||||
// (lowest-index requester wins this round); the others simply retry
|
||||
// next cycle (idempotent -- an ideal SRAM read has no state cost).
|
||||
//
|
||||
// Read latency is modeled as ZERO cycles (grant and data-available
|
||||
// are the same cycle) -- this file isolates the BANK CONTENTION
|
||||
// question specifically, decoupled from backing-store latency, which
|
||||
// STEP1 (hardware/v2/nms/rtl/ideal_memory_model.v, EXP-0017) already
|
||||
// characterized separately. Fill-from-PSRAM is out of scope here too
|
||||
// (the vector is assumed already resident, i.e. steady-state
|
||||
// consumption after prefetch -- the question this file answers is
|
||||
// "does the on-chip organization itself let N_SLOTS scale", not
|
||||
// "how do we hide PSRAM latency" (already answered by STEP1).
|
||||
// ============================================================
|
||||
module ideal_banked_activation #(
|
||||
parameter NREQ = 4,
|
||||
parameter N_BANKS = 2,
|
||||
parameter AW = 32
|
||||
)(
|
||||
input [NREQ-1:0] req_valid,
|
||||
input [NREQ*AW-1:0] req_addr_flat,
|
||||
output reg [NREQ-1:0] ack
|
||||
);
|
||||
integer b, i;
|
||||
reg [AW-1:0] req_addr [0:NREQ-1];
|
||||
reg [AW-1:0] served_addr;
|
||||
reg have_served;
|
||||
integer bank_of_i;
|
||||
|
||||
always @* begin
|
||||
for (i = 0; i < NREQ; i = i + 1)
|
||||
req_addr[i] = req_addr_flat[i*AW +: AW];
|
||||
|
||||
ack = {NREQ{1'b0}};
|
||||
for (b = 0; b < N_BANKS; b = b + 1) begin
|
||||
have_served = 1'b0;
|
||||
served_addr = {AW{1'b0}};
|
||||
// first pass: lowest-index valid requester in this bank sets
|
||||
// the address served this cycle
|
||||
for (i = 0; i < NREQ; i = i + 1) begin
|
||||
bank_of_i = req_addr[i] % N_BANKS;
|
||||
if (req_valid[i] && (bank_of_i == b) && !have_served) begin
|
||||
served_addr = req_addr[i];
|
||||
have_served = 1'b1;
|
||||
end
|
||||
end
|
||||
// second pass: broadcast ack to every requester in this bank
|
||||
// that wants the SAME address (free, one read serves all)
|
||||
if (have_served) begin
|
||||
for (i = 0; i < NREQ; i = i + 1) begin
|
||||
bank_of_i = req_addr[i] % N_BANKS;
|
||||
if (req_valid[i] && (bank_of_i == b) && (req_addr[i] == served_addr))
|
||||
ack[i] = 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
endmodule
|
||||
@@ -0,0 +1,195 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP 1 bandwidth requirement study.
|
||||
//
|
||||
// SIMULATION-ONLY, NEVER SYNTHESIZED, NOT A REAL MEMORY.
|
||||
//
|
||||
// Idealized single-port backing-store model: a shared pipe with two
|
||||
// independently configurable, RUNTIME (not just compile-time)
|
||||
// parameters:
|
||||
// cfg_latency -- fixed cycles from "bytes fully transferred" to
|
||||
// "response visible to the requester" (DRAM-style
|
||||
// round-trip latency, independent of throughput).
|
||||
// cfg_bw_bytes -- aggregate bytes/cycle the shared pipe can drain,
|
||||
// shared across however many TILE_BYTES-sized
|
||||
// transfers are queued (sustained bandwidth).
|
||||
//
|
||||
// Admission is effectively unconstrained (a deep FIFO, req_ready
|
||||
// combinationally follows req_valid) -- the real, modeled constraint
|
||||
// is entirely in the DRAIN stage: each cycle a shared byte budget of
|
||||
// cfg_bw_bytes is applied against the head of the queue, completing
|
||||
// as many whole TILE_BYTES-sized transfers as the budget allows
|
||||
// (zero, one, or several in the same cycle when cfg_bw_bytes is a
|
||||
// multiple of TILE_BYTES) before carrying any leftover partial
|
||||
// progress to the next cycle. This correctly separates BANDWIDTH
|
||||
// (how many bytes/cycle the shared pipe drains, aggregate across all
|
||||
// outstanding requests) from LATENCY (a fixed per-transfer delay
|
||||
// applied after draining, via a FIFO of pending completions -- see
|
||||
// hardware/v2/nms/logs, DEC for the first (broken) revision of this
|
||||
// module: EXP-0017 attempt 1 modeled only ONE transfer in service at
|
||||
// a time regardless of cfg_bw_bytes, which silently capped aggregate
|
||||
// system throughput at 1 transfer/cycle and made utilization collapse
|
||||
// to 1/N_SLOTS at every N_SLOTS>1 config -- a modeling bug, not a real
|
||||
// architectural finding; caught by the finding being suspiciously
|
||||
// exact (1/8, 1/4, 1/2 ...) instead of a physically-motivated curve).
|
||||
//
|
||||
// Requester ordering when several assert req_valid the same cycle is
|
||||
// a fixed, low-index-first tie-break -- not fairness-relevant here
|
||||
// since every requester in this study is symmetric.
|
||||
// ============================================================
|
||||
module ideal_memory_model #(
|
||||
parameter NREQ = 4,
|
||||
parameter TILE_BYTES = 16,
|
||||
parameter QDEPTH = 256, // max total outstanding across all requesters
|
||||
parameter MAX_ITERS = 16, // max queue pops/pushes processed in one cycle
|
||||
parameter TAGW = (NREQ <= 1) ? 1 : $clog2(NREQ)
|
||||
)(
|
||||
input clk,
|
||||
input rst,
|
||||
|
||||
input [15:0] cfg_latency, // cycles, runtime-configurable
|
||||
input [15:0] cfg_bw_bytes, // bytes/cycle, runtime-configurable, min 1
|
||||
|
||||
input [NREQ-1:0] req_valid,
|
||||
output wire [NREQ-1:0] req_ready, // combinational, essentially unconstrained
|
||||
output reg [NREQ-1:0] resp_valid // one-cycle completion pulse per requester
|
||||
);
|
||||
|
||||
// ---- global free-running cycle counter (absolute time base) ----
|
||||
reg [63:0] cycle_count;
|
||||
always @(posedge clk) begin
|
||||
if (rst) cycle_count <= 64'd0;
|
||||
else cycle_count <= cycle_count + 64'd1;
|
||||
end
|
||||
|
||||
wire [15:0] bw_eff = (cfg_bw_bytes == 16'd0) ? 16'd1 : cfg_bw_bytes;
|
||||
|
||||
// Admission is unconstrained (QDEPTH sized generously vs. the real
|
||||
// max outstanding this study ever drives: N_SLOTS*PREFETCH_DEPTH <= 64).
|
||||
assign req_ready = req_valid;
|
||||
|
||||
// ---- queue of admitted-but-not-yet-drained tags ----
|
||||
reg [TAGW-1:0] inq_tag [0:QDEPTH-1];
|
||||
reg [$clog2(QDEPTH+1)-1:0] inq_head, inq_count;
|
||||
|
||||
// ---- currently-draining head item (persists partial progress across cycles) ----
|
||||
reg head_active;
|
||||
reg [TAGW-1:0] head_tag;
|
||||
reg [31:0] head_remain;
|
||||
|
||||
// ---- latency tail FIFO (uniform latency -> strict FIFO completion order) ----
|
||||
reg [TAGW-1:0] lat_tag [0:QDEPTH-1];
|
||||
reg [63:0] lat_finish[0:QDEPTH-1];
|
||||
reg [$clog2(QDEPTH+1)-1:0] lat_head, lat_tail, lat_count;
|
||||
|
||||
integer i, k;
|
||||
|
||||
// ---- scratch (blocking-updated shadow state, committed via NBA at the end) ----
|
||||
reg [$clog2(QDEPTH+1)-1:0] s_inq_head, s_inq_count, s_lat_tail, s_lat_count;
|
||||
reg s_head_active;
|
||||
reg [TAGW-1:0] s_head_tag;
|
||||
reg [31:0] s_head_remain;
|
||||
integer s_budget;
|
||||
reg [TAGW-1:0] complete_tag [0:MAX_ITERS-1];
|
||||
integer n_complete;
|
||||
reg [$clog2(QDEPTH+1)-1:0] s_lat_head;
|
||||
integer n_popped;
|
||||
reg [TAGW-1:0] pop_tag [0:MAX_ITERS-1];
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
inq_head <= 0; inq_count <= 0;
|
||||
head_active <= 1'b0; head_tag <= {TAGW{1'b0}}; head_remain <= 32'd0;
|
||||
lat_head <= 0; lat_tail <= 0; lat_count <= 0;
|
||||
resp_valid <= {NREQ{1'b0}};
|
||||
end else begin
|
||||
// ================= 1. admission: append every asserted
|
||||
// requester this cycle to the queue tail, low-index-first =====
|
||||
s_inq_head = inq_head;
|
||||
s_inq_count = inq_count;
|
||||
// tail position for admission = (inq_head + inq_count) mod QDEPTH,
|
||||
// computed fresh per pushed item below.
|
||||
for (i = 0; i < NREQ; i = i + 1) begin
|
||||
if (req_valid[i]) begin
|
||||
inq_tag[(s_inq_head + s_inq_count) % QDEPTH] <= i[TAGW-1:0];
|
||||
s_inq_count = s_inq_count + 1;
|
||||
end
|
||||
end
|
||||
|
||||
// ================= 2. drain: shared byte budget consumes the
|
||||
// queue head (and the persisted in-progress head item), possibly
|
||||
// completing several TILE_BYTES-sized transfers in one cycle ====
|
||||
s_head_active = head_active;
|
||||
s_head_tag = head_tag;
|
||||
s_head_remain = head_remain;
|
||||
s_budget = bw_eff;
|
||||
n_complete = 0;
|
||||
|
||||
for (k = 0; k < MAX_ITERS; k = k + 1) begin
|
||||
if (!s_head_active && s_inq_count > 0) begin
|
||||
s_head_active = 1'b1;
|
||||
s_head_tag = inq_tag[s_inq_head];
|
||||
s_head_remain = TILE_BYTES;
|
||||
s_inq_head = (s_inq_head + 1) % QDEPTH;
|
||||
s_inq_count = s_inq_count - 1;
|
||||
end
|
||||
if (s_head_active && s_budget > 0) begin
|
||||
if (s_head_remain <= s_budget) begin
|
||||
s_budget = s_budget - s_head_remain;
|
||||
complete_tag[n_complete] = s_head_tag;
|
||||
n_complete = n_complete + 1;
|
||||
s_head_active = 1'b0;
|
||||
s_head_remain = 32'd0;
|
||||
end else begin
|
||||
s_head_remain = s_head_remain - s_budget;
|
||||
s_budget = 0;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
inq_head <= s_inq_head;
|
||||
inq_count <= s_inq_count;
|
||||
head_active <= s_head_active;
|
||||
head_tag <= s_head_tag;
|
||||
head_remain <= s_head_remain;
|
||||
|
||||
// push every completion from this cycle into the latency tail
|
||||
s_lat_tail = lat_tail;
|
||||
s_lat_count = lat_count;
|
||||
for (i = 0; i < MAX_ITERS; i = i + 1) begin
|
||||
if (i < n_complete) begin
|
||||
lat_tag[(s_lat_tail) % QDEPTH] <= complete_tag[i];
|
||||
lat_finish[(s_lat_tail) % QDEPTH] <= cycle_count + {48'd0, cfg_latency};
|
||||
s_lat_tail = s_lat_tail + 1;
|
||||
end
|
||||
end
|
||||
|
||||
// ================= 3. latency tail: pop every entry whose time
|
||||
// has come (uniform latency -> all due entries are contiguous
|
||||
// at the head, so a bounded scan suffices) =====================
|
||||
s_lat_head = lat_head;
|
||||
n_popped = 0;
|
||||
for (k = 0; k < MAX_ITERS; k = k + 1) begin
|
||||
if ((s_lat_count > 0) && (cycle_count >= lat_finish[s_lat_head])) begin
|
||||
pop_tag[n_popped] = lat_tag[s_lat_head];
|
||||
n_popped = n_popped + 1;
|
||||
s_lat_head = (s_lat_head + 1) % QDEPTH;
|
||||
s_lat_count = s_lat_count - 1;
|
||||
end
|
||||
end
|
||||
lat_head <= s_lat_head;
|
||||
lat_tail <= s_lat_tail % QDEPTH;
|
||||
// final count = old registered count + this cycle's new
|
||||
// completions (not yet poppable -- they only become visible
|
||||
// via lat_tag/lat_finish starting NEXT cycle, since those
|
||||
// array writes above are non-blocking) - this cycle's pops
|
||||
// (n_popped, drawn only from previously-registered entries).
|
||||
lat_count <= lat_count + n_complete - n_popped;
|
||||
|
||||
resp_valid <= {NREQ{1'b0}};
|
||||
for (i = 0; i < MAX_ITERS; i = i + 1) begin
|
||||
if (i < n_popped) resp_valid[pop_tag[i]] <= 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,164 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP4/5 candidate B: BANKED activation
|
||||
// memory with broadcast-on-same-address and round-robin arbitration.
|
||||
//
|
||||
// ONE logical copy of the shared activation vector, striped across
|
||||
// N_BANKS single-port BRAMs by tile index (bank = tile_idx % N_BANKS).
|
||||
// Real per-bank round-robin arbitration (same policy validated in
|
||||
// simulation, EXP-0018): the lowest-index requester currently pointed
|
||||
// to by that bank's own rotating pointer wins ties; every requester
|
||||
// wanting the SAME tile index as the winner is broadcast-acked for
|
||||
// free (one read serves them all).
|
||||
//
|
||||
// Two register stages (request -> arbitration decision -> BRAM
|
||||
// address; BRAM address -> BRAM data -> crossbar mux), register-to-
|
||||
// register throughout -- deliberately NOT the single-cycle
|
||||
// combinational hit-detection/broadcast structure that cost
|
||||
// activation_cache.v its Fmax margin at N_SLOTS=4 (DEC-0016). Request
|
||||
// accepted at cycle T; ack + data both become valid at cycle T+2.
|
||||
//
|
||||
// Real per-bank storage depth is MAX_TILES/N_BANKS -- N_BANKS times
|
||||
// LESS total on-chip storage than nms_activation_replicated.v's
|
||||
// N_SLOTS full copies, at the cost of the arbitration/crossbar logic
|
||||
// below. Both candidates are measured (EXP-0019/DEC-0019), not chosen
|
||||
// a priori.
|
||||
// ============================================================
|
||||
module nms_activation_banked #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_BANKS = 4,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter BDEPTH = (MAX_TILES + N_BANKS - 1) / N_BANKS,
|
||||
parameter BAW = (BDEPTH <= 1) ? 1 : $clog2(BDEPTH),
|
||||
parameter SLOTW = (N_SLOTS <= 1) ? 1 : $clog2(N_SLOTS),
|
||||
parameter BANKW = (N_BANKS <= 1) ? 1 : $clog2(N_BANKS)
|
||||
)(
|
||||
input clk,
|
||||
input rst,
|
||||
|
||||
// ---- fill port: one tile write, routed to its own bank ----
|
||||
input fill_we,
|
||||
input [TIW-1:0] fill_tile_idx,
|
||||
input [DATA_WIDTH*P_IN-1:0] fill_data,
|
||||
|
||||
// ---- per-slot request/response (2-cycle latency: ack+data valid
|
||||
// at T+2 for a request presented at T) ----
|
||||
input [N_SLOTS-1:0] req_valid,
|
||||
input [N_SLOTS*TIW-1:0] req_tile_idx_flat,
|
||||
output reg [N_SLOTS-1:0] ack,
|
||||
output [N_SLOTS*DATA_WIDTH*P_IN-1:0] rd_data_flat
|
||||
);
|
||||
|
||||
integer i, b;
|
||||
|
||||
// ---- bank storage (N_BANKS separate 1D arrays -> N_BANKS BRAMs) ----
|
||||
genvar gb;
|
||||
wire [DATA_WIDTH*P_IN-1:0] bank_rd_data_reg [0:N_BANKS-1];
|
||||
reg [BAW-1:0] bank_rd_addr_stage1 [0:N_BANKS-1];
|
||||
reg bank_rd_en_stage1 [0:N_BANKS-1];
|
||||
|
||||
generate
|
||||
for (gb = 0; gb < N_BANKS; gb = gb + 1) begin : GEN_BANK
|
||||
reg [DATA_WIDTH*P_IN-1:0] mem [0:BDEPTH-1];
|
||||
reg [DATA_WIDTH*P_IN-1:0] rd_data_r;
|
||||
wire this_bank_we = fill_we && ((fill_tile_idx % N_BANKS) == gb);
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (this_bank_we)
|
||||
mem[fill_tile_idx / N_BANKS] <= fill_data;
|
||||
if (bank_rd_en_stage1[gb])
|
||||
rd_data_r <= mem[bank_rd_addr_stage1[gb]];
|
||||
end
|
||||
assign bank_rd_data_reg[gb] = rd_data_r;
|
||||
end
|
||||
endgenerate
|
||||
|
||||
// ---- stage 0 (combinational): per-bank round-robin arbitration +
|
||||
// broadcast-ack decision, using THIS cycle's req_valid/req_tile_idx ----
|
||||
reg [TIW-1:0] req_tile_idx [0:N_SLOTS-1];
|
||||
reg [BANKW-1:0] req_bank [0:N_SLOTS-1];
|
||||
reg [SLOTW-1:0] rr_ptr [0:N_BANKS-1];
|
||||
|
||||
reg win_valid [0:N_BANKS-1];
|
||||
reg [TIW-1:0] win_tile_idx [0:N_BANKS-1];
|
||||
reg [SLOTW-1:0] win_first [0:N_BANKS-1];
|
||||
reg [N_SLOTS-1:0] ack_comb;
|
||||
reg [BANKW-1:0] slot_bank_comb [0:N_SLOTS-1];
|
||||
integer scan_i, cand;
|
||||
|
||||
always @* begin
|
||||
for (i = 0; i < N_SLOTS; i = i + 1) begin
|
||||
req_tile_idx[i] = req_tile_idx_flat[i*TIW +: TIW];
|
||||
req_bank[i] = req_tile_idx[i] % N_BANKS;
|
||||
slot_bank_comb[i] = req_bank[i];
|
||||
end
|
||||
ack_comb = {N_SLOTS{1'b0}};
|
||||
for (b = 0; b < N_BANKS; b = b + 1) begin
|
||||
win_valid[b] = 1'b0;
|
||||
win_tile_idx[b] = {TIW{1'b0}};
|
||||
win_first[b] = {SLOTW{1'b0}};
|
||||
for (scan_i = 0; scan_i < N_SLOTS; scan_i = scan_i + 1) begin
|
||||
cand = (rr_ptr[b] + scan_i) % N_SLOTS;
|
||||
if (req_valid[cand] && (req_bank[cand] == b) && !win_valid[b]) begin
|
||||
win_valid[b] = 1'b1;
|
||||
win_tile_idx[b] = req_tile_idx[cand];
|
||||
win_first[b] = cand[SLOTW-1:0];
|
||||
end
|
||||
end
|
||||
if (win_valid[b]) begin
|
||||
for (i = 0; i < N_SLOTS; i = i + 1) begin
|
||||
if (req_valid[i] && (req_bank[i] == b) && (req_tile_idx[i] == win_tile_idx[b]))
|
||||
ack_comb[i] = 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ---- stage 1 registers: arbitration decision -> BRAM address,
|
||||
// plus the per-slot bookkeeping needed to route data back 1 cycle
|
||||
// later (stage 2) ----
|
||||
reg [N_SLOTS-1:0] ack_stage1;
|
||||
reg [BANKW-1:0] slot_bank_stage1 [0:N_SLOTS-1];
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
for (b = 0; b < N_BANKS; b = b + 1) begin
|
||||
rr_ptr[b] <= {SLOTW{1'b0}};
|
||||
bank_rd_en_stage1[b] <= 1'b0;
|
||||
bank_rd_addr_stage1[b] <= {BAW{1'b0}};
|
||||
end
|
||||
ack_stage1 <= {N_SLOTS{1'b0}};
|
||||
ack <= {N_SLOTS{1'b0}};
|
||||
for (i = 0; i < N_SLOTS; i = i + 1) begin
|
||||
slot_bank_stage1[i] <= {BANKW{1'b0}};
|
||||
end
|
||||
end else begin
|
||||
for (b = 0; b < N_BANKS; b = b + 1) begin
|
||||
bank_rd_en_stage1[b] <= win_valid[b];
|
||||
bank_rd_addr_stage1[b] <= win_tile_idx[b] / N_BANKS;
|
||||
if (win_valid[b]) rr_ptr[b] <= win_first[b] + 1'b1;
|
||||
end
|
||||
ack_stage1 <= ack_comb;
|
||||
for (i = 0; i < N_SLOTS; i = i + 1)
|
||||
slot_bank_stage1[i] <= slot_bank_comb[i];
|
||||
|
||||
// stage 2: ack becomes valid the cycle the BRAM's own
|
||||
// registered read output (bank_rd_data_reg) is valid
|
||||
ack <= ack_stage1;
|
||||
end
|
||||
end
|
||||
|
||||
// ---- stage 2 (combinational crossbar): route each bank's
|
||||
// registered read output to whichever slot(s) it was serving,
|
||||
// using the stage1-registered bank assignment ----
|
||||
genvar gs;
|
||||
generate
|
||||
for (gs = 0; gs < N_SLOTS; gs = gs + 1) begin : GEN_XBAR
|
||||
assign rd_data_flat[gs*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN] =
|
||||
bank_rd_data_reg[slot_bank_stage1[gs]];
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,184 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- shared Activation fill controller.
|
||||
//
|
||||
// One instance per neural_memory_system (shared across all N_SLOTS),
|
||||
// backing nms_activation_replicated.v's single broadcast-write fill
|
||||
// port. Owns ONE prefetch_engine.v instance (real word-level PSRAM
|
||||
// fetch, DEC-0015 convention, reused verbatim -- it is generic
|
||||
// P_IN-byte-tile fetch logic, not weight-specific despite its name).
|
||||
//
|
||||
// Single-tag design (same honest limitation as the superseded
|
||||
// hardware/v2/rtl/activation_cache.v, DEC-0016): tracks ONE resident
|
||||
// x_base at a time. Refills (resident_count resets to 0, restarts
|
||||
// fetching from tile 0) whenever the lowest-indexed currently-active
|
||||
// slot's own x_base differs from what is resident -- correct always,
|
||||
// but can thrash under interleaved, genuinely-different-x_base
|
||||
// concurrent traffic; not exercised by this project's own realistic
|
||||
// dense-layer workloads (shared-producer dispatch, many slots given
|
||||
// the SAME x_base together).
|
||||
//
|
||||
// resident_count extends to the MAX n_tiles needed by any currently
|
||||
// active slot that shares resident_tag (not just the reference slot
|
||||
// that triggered the refill), so a later-joining slot with a deeper
|
||||
// need is served without a second refill.
|
||||
// ============================================================
|
||||
module nms_activation_fill_ctrl #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
// TIW indexes the SRAM fill address (0..MAX_TILES-1); CNTW is for
|
||||
// resident_count, which must represent the VALUE MAX_TILES itself
|
||||
// (e.g. a fully-resident 16-tile vector with MAX_TILES=16) -- one
|
||||
// bit wider than TIW, same distinction/bug as
|
||||
// nms_memory_manager.v's own tile_idx/wgt_fetched (see that file's
|
||||
// header for the real deadlock this caused before the fix).
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- per-slot job status (levels, held while that slot's job is active) ----
|
||||
input wire [N_SLOTS-1:0] job_active,
|
||||
input wire [N_SLOTS*ADDR_WIDTH-1:0] x_base_flat,
|
||||
input wire [N_SLOTS*16-1:0] n_tiles_flat,
|
||||
|
||||
// ---- broadcast status (every slot compares this against its own x_base) ----
|
||||
output reg [ADDR_WIDTH-1:0] resident_tag,
|
||||
output reg [CNTW-1:0] resident_count,
|
||||
|
||||
// ---- fill port into nms_activation_replicated.v ----
|
||||
output wire fill_we,
|
||||
output wire [TIW-1:0] fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] fill_data,
|
||||
|
||||
// ---- real word-level PSRAM backend (arbitrated externally) ----
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
integer i;
|
||||
|
||||
// ---- desired x_base: lowest-indexed currently-active slot (fixed
|
||||
// priority -- simple, not fairness-critical here since this only
|
||||
// decides which TAG to chase, not who gets bandwidth) ----
|
||||
reg desired_valid;
|
||||
reg [ADDR_WIDTH-1:0] desired_x_base;
|
||||
reg [15:0] max_n_tiles;
|
||||
|
||||
always @* begin
|
||||
desired_valid = 1'b0;
|
||||
desired_x_base = {ADDR_WIDTH{1'b0}};
|
||||
max_n_tiles = 16'h0;
|
||||
for (i = N_SLOTS-1; i >= 0; i = i - 1) begin
|
||||
if (job_active[i]) begin
|
||||
desired_valid = 1'b1;
|
||||
desired_x_base = x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH];
|
||||
end
|
||||
end
|
||||
// max n_tiles across every ACTIVE slot that shares resident_tag
|
||||
// (recomputed every cycle so a later-joining deeper-need slot
|
||||
// extends the fill target without a second refill)
|
||||
for (i = 0; i < N_SLOTS; i = i + 1) begin
|
||||
if (job_active[i] && (x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH] == resident_tag)) begin
|
||||
if (n_tiles_flat[i*16 +: 16] > max_n_tiles)
|
||||
max_n_tiles = n_tiles_flat[i*16 +: 16];
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
localparam ST_IDLE = 1'd0;
|
||||
localparam ST_FETCH = 1'd1;
|
||||
reg state;
|
||||
|
||||
reg pf_start;
|
||||
reg [ADDR_WIDTH-1:0] pf_addr;
|
||||
wire pf_busy, pf_done;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] pf_tile;
|
||||
|
||||
prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_pf (
|
||||
.clk(clk), .rst(rst),
|
||||
.fetch_start(pf_start), .w_addr(pf_addr),
|
||||
.fetch_busy(pf_busy), .fetch_done(pf_done), .tile_w(pf_tile),
|
||||
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(mem_addr), .mem_wdata(mem_wdata),
|
||||
.mem_lb_n(mem_lb_n), .mem_ub_n(mem_ub_n),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
|
||||
reg fill_we_reg;
|
||||
reg [TIW-1:0] fill_addr_reg;
|
||||
reg [DATA_WIDTH*P_IN-1:0] fill_data_reg;
|
||||
assign fill_we = fill_we_reg;
|
||||
assign fill_addr = fill_addr_reg;
|
||||
assign fill_data = fill_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
resident_tag <= {ADDR_WIDTH{1'b1}}; // sentinel: matches no real x_base at reset
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
state <= ST_IDLE;
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
end else begin
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
|
||||
// latch a completed fetch into the replicated activation
|
||||
// memory's broadcast fill port
|
||||
if (pf_done) begin
|
||||
fill_we_reg <= 1'b1;
|
||||
fill_addr_reg <= resident_count[TIW-1:0]; // valid: gated < max_n_tiles <= MAX_TILES
|
||||
fill_data_reg <= pf_tile;
|
||||
resident_count <= resident_count + 1'b1;
|
||||
end
|
||||
|
||||
// refill trigger: the reference slot wants a DIFFERENT tag,
|
||||
// and the fetch engine is genuinely idle (never interrupt an
|
||||
// in-flight fetch -- same discipline as memory_manager.v's
|
||||
// own pf_pending guard, ERR-0006). Stays in ST_IDLE (not
|
||||
// ST_FETCH): only updates resident_tag/resident_count here;
|
||||
// the ST_IDLE case below is what actually issues pf_start,
|
||||
// reading the NEW resident_tag starting next cycle -- this
|
||||
// path must NOT itself jump to ST_FETCH without a matching
|
||||
// pf_start, or the engine would sit in ST_FETCH forever
|
||||
// waiting for a pf_done that was never triggered.
|
||||
if (desired_valid && (desired_x_base != resident_tag) && !pf_busy && (state == ST_IDLE)) begin
|
||||
resident_tag <= desired_x_base;
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
end
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
// stay idle: fetching further tiles for the CURRENT
|
||||
// tag (if any active slot still needs more) is
|
||||
// handled below, symmetric to the refill case.
|
||||
if (!pf_busy && !pf_start && (resident_count < max_n_tiles) &&
|
||||
desired_valid && (desired_x_base == resident_tag)) begin
|
||||
pf_start <= 1'b1;
|
||||
pf_addr <= resident_tag + (resident_count * P_IN[ADDR_WIDTH-1:0]);
|
||||
state <= ST_FETCH;
|
||||
end
|
||||
end
|
||||
ST_FETCH: begin
|
||||
if (pf_done) begin
|
||||
// resident_count already bumped above this cycle;
|
||||
// decide whether more remain once back in IDLE.
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,205 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- shared Activation fill controller.
|
||||
//
|
||||
// One instance per neural_memory_system (shared across all N_SLOTS),
|
||||
// backing nms_activation_replicated.v's single broadcast-write fill
|
||||
// port. Owns ONE prefetch_engine.v instance (real word-level PSRAM
|
||||
// fetch, DEC-0015 convention, reused verbatim -- it is generic
|
||||
// P_IN-byte-tile fetch logic, not weight-specific despite its name).
|
||||
//
|
||||
// Single-tag design (same honest limitation as the superseded
|
||||
// hardware/v2/rtl/activation_cache.v, DEC-0016): tracks ONE resident
|
||||
// x_base at a time. Refills (resident_count resets to 0, restarts
|
||||
// fetching from tile 0) whenever the lowest-indexed currently-active
|
||||
// slot's own x_base differs from what is resident -- correct always,
|
||||
// but can thrash under interleaved, genuinely-different-x_base
|
||||
// concurrent traffic; not exercised by this project's own realistic
|
||||
// dense-layer workloads (shared-producer dispatch, many slots given
|
||||
// the SAME x_base together).
|
||||
//
|
||||
// resident_count extends to the MAX n_tiles needed by any currently
|
||||
// active slot that shares resident_tag (not just the reference slot
|
||||
// that triggered the refill), so a later-joining slot with a deeper
|
||||
// need is served without a second refill.
|
||||
// ============================================================
|
||||
module nms_activation_fill_ctrl_v2 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
// TIW indexes the SRAM fill address (0..MAX_TILES-1); CNTW is for
|
||||
// resident_count, which must represent the VALUE MAX_TILES itself
|
||||
// (e.g. a fully-resident 16-tile vector with MAX_TILES=16) -- one
|
||||
// bit wider than TIW, same distinction/bug as
|
||||
// nms_memory_manager.v's own tile_idx/wgt_fetched (see that file's
|
||||
// header for the real deadlock this caused before the fix).
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- per-slot job status (levels, held while that slot's job is active) ----
|
||||
input wire [N_SLOTS-1:0] job_active,
|
||||
input wire [N_SLOTS*ADDR_WIDTH-1:0] x_base_flat,
|
||||
input wire [N_SLOTS*16-1:0] n_tiles_flat,
|
||||
|
||||
// ---- broadcast status (every slot compares this against its own x_base) ----
|
||||
output reg [ADDR_WIDTH-1:0] resident_tag,
|
||||
output reg [CNTW-1:0] resident_count,
|
||||
|
||||
// ---- fill port into nms_activation_replicated.v ----
|
||||
output wire fill_we,
|
||||
output wire [TIW-1:0] fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] fill_data,
|
||||
|
||||
// ---- real word-level PSRAM backend (arbitrated externally) ----
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
integer i;
|
||||
|
||||
// ---- desired x_base: lowest-indexed currently-active slot (fixed
|
||||
// priority -- simple, not fairness-critical here since this only
|
||||
// decides which TAG to chase, not who gets bandwidth) ----
|
||||
reg desired_valid;
|
||||
reg [ADDR_WIDTH-1:0] desired_x_base;
|
||||
reg [15:0] max_n_tiles;
|
||||
|
||||
always @* begin
|
||||
desired_valid = 1'b0;
|
||||
desired_x_base = {ADDR_WIDTH{1'b0}};
|
||||
max_n_tiles = 16'h0;
|
||||
for (i = N_SLOTS-1; i >= 0; i = i - 1) begin
|
||||
if (job_active[i]) begin
|
||||
desired_valid = 1'b1;
|
||||
desired_x_base = x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH];
|
||||
end
|
||||
end
|
||||
// max n_tiles across every ACTIVE slot that shares resident_tag
|
||||
// (recomputed every cycle so a later-joining deeper-need slot
|
||||
// extends the fill target without a second refill)
|
||||
for (i = 0; i < N_SLOTS; i = i + 1) begin
|
||||
if (job_active[i] && (x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH] == resident_tag)) begin
|
||||
if (n_tiles_flat[i*16 +: 16] > max_n_tiles)
|
||||
max_n_tiles = n_tiles_flat[i*16 +: 16];
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ---- STEP14/EXP-0029/DEC-0026: max_n_tiles registered ONE cycle
|
||||
// before use in the resident_count comparison below. Real post-
|
||||
// P&R critical path trace (EXP-0029, N_SLOTS=4, Fmax=55.22MHz
|
||||
// FAIL@80MHz) found max_n_tiles's own computation (a full 16-bit
|
||||
// running-max carry chain, above) feeding DIRECTLY, same-cycle,
|
||||
// into the resident_count < max_n_tiles comparison further below
|
||||
// -- two chained 16-bit magnitude comparisons in ONE combinational
|
||||
// cone, 6.25ns logic + 11.85ns routing, 18.11ns total. Breaking
|
||||
// them into separate clock edges is the minimum fix: this
|
||||
// refill-decision path is evaluated only at tile-fill-trigger
|
||||
// boundaries (gated by pf_busy/state==ST_IDLE below), never on
|
||||
// the real-time-critical per-tile-consumption path (already fully
|
||||
// decoupled by nms_memory_manager_stream.v's own read-ahead
|
||||
// pipeline, STEP13) -- one added cycle of latency here is
|
||||
// functionally free for steady-state throughput. ----
|
||||
reg [15:0] max_n_tiles_reg;
|
||||
always @(posedge clk) begin
|
||||
if (rst) max_n_tiles_reg <= 16'h0;
|
||||
else max_n_tiles_reg <= max_n_tiles;
|
||||
end
|
||||
|
||||
localparam ST_IDLE = 1'd0;
|
||||
localparam ST_FETCH = 1'd1;
|
||||
reg state;
|
||||
|
||||
reg pf_start;
|
||||
reg [ADDR_WIDTH-1:0] pf_addr;
|
||||
wire pf_busy, pf_done;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] pf_tile;
|
||||
|
||||
prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_pf (
|
||||
.clk(clk), .rst(rst),
|
||||
.fetch_start(pf_start), .w_addr(pf_addr),
|
||||
.fetch_busy(pf_busy), .fetch_done(pf_done), .tile_w(pf_tile),
|
||||
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(mem_addr), .mem_wdata(mem_wdata),
|
||||
.mem_lb_n(mem_lb_n), .mem_ub_n(mem_ub_n),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
|
||||
reg fill_we_reg;
|
||||
reg [TIW-1:0] fill_addr_reg;
|
||||
reg [DATA_WIDTH*P_IN-1:0] fill_data_reg;
|
||||
assign fill_we = fill_we_reg;
|
||||
assign fill_addr = fill_addr_reg;
|
||||
assign fill_data = fill_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
resident_tag <= {ADDR_WIDTH{1'b1}}; // sentinel: matches no real x_base at reset
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
state <= ST_IDLE;
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
end else begin
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
|
||||
// latch a completed fetch into the replicated activation
|
||||
// memory's broadcast fill port
|
||||
if (pf_done) begin
|
||||
fill_we_reg <= 1'b1;
|
||||
fill_addr_reg <= resident_count[TIW-1:0]; // valid: gated < max_n_tiles <= MAX_TILES
|
||||
fill_data_reg <= pf_tile;
|
||||
resident_count <= resident_count + 1'b1;
|
||||
end
|
||||
|
||||
// refill trigger: the reference slot wants a DIFFERENT tag,
|
||||
// and the fetch engine is genuinely idle (never interrupt an
|
||||
// in-flight fetch -- same discipline as memory_manager.v's
|
||||
// own pf_pending guard, ERR-0006). Stays in ST_IDLE (not
|
||||
// ST_FETCH): only updates resident_tag/resident_count here;
|
||||
// the ST_IDLE case below is what actually issues pf_start,
|
||||
// reading the NEW resident_tag starting next cycle -- this
|
||||
// path must NOT itself jump to ST_FETCH without a matching
|
||||
// pf_start, or the engine would sit in ST_FETCH forever
|
||||
// waiting for a pf_done that was never triggered.
|
||||
if (desired_valid && (desired_x_base != resident_tag) && !pf_busy && (state == ST_IDLE)) begin
|
||||
resident_tag <= desired_x_base;
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
end
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
// stay idle: fetching further tiles for the CURRENT
|
||||
// tag (if any active slot still needs more) is
|
||||
// handled below, symmetric to the refill case.
|
||||
if (!pf_busy && !pf_start && (resident_count < max_n_tiles_reg) &&
|
||||
desired_valid && (desired_x_base == resident_tag)) begin
|
||||
pf_start <= 1'b1;
|
||||
pf_addr <= resident_tag + (resident_count * P_IN[ADDR_WIDTH-1:0]);
|
||||
state <= ST_FETCH;
|
||||
end
|
||||
end
|
||||
ST_FETCH: begin
|
||||
if (pf_done) begin
|
||||
// resident_count already bumped above this cycle;
|
||||
// decide whether more remain once back in IDLE.
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,223 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- shared Activation fill controller.
|
||||
//
|
||||
// One instance per neural_memory_system (shared across all N_SLOTS),
|
||||
// backing nms_activation_replicated.v's single broadcast-write fill
|
||||
// port. Owns ONE prefetch_engine.v instance (real word-level PSRAM
|
||||
// fetch, DEC-0015 convention, reused verbatim -- it is generic
|
||||
// P_IN-byte-tile fetch logic, not weight-specific despite its name).
|
||||
//
|
||||
// Single-tag design (same honest limitation as the superseded
|
||||
// hardware/v2/rtl/activation_cache.v, DEC-0016): tracks ONE resident
|
||||
// x_base at a time. Refills (resident_count resets to 0, restarts
|
||||
// fetching from tile 0) whenever the lowest-indexed currently-active
|
||||
// slot's own x_base differs from what is resident -- correct always,
|
||||
// but can thrash under interleaved, genuinely-different-x_base
|
||||
// concurrent traffic; not exercised by this project's own realistic
|
||||
// dense-layer workloads (shared-producer dispatch, many slots given
|
||||
// the SAME x_base together).
|
||||
//
|
||||
// resident_count extends to the MAX n_tiles needed by any currently
|
||||
// active slot that shares resident_tag (not just the reference slot
|
||||
// that triggered the refill), so a later-joining slot with a deeper
|
||||
// need is served without a second refill.
|
||||
// ============================================================
|
||||
module nms_activation_fill_ctrl_v3 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
// TIW indexes the SRAM fill address (0..MAX_TILES-1); CNTW is for
|
||||
// resident_count, which must represent the VALUE MAX_TILES itself
|
||||
// (e.g. a fully-resident 16-tile vector with MAX_TILES=16) -- one
|
||||
// bit wider than TIW, same distinction/bug as
|
||||
// nms_memory_manager.v's own tile_idx/wgt_fetched (see that file's
|
||||
// header for the real deadlock this caused before the fix).
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- per-slot job status (levels, held while that slot's job is active) ----
|
||||
input wire [N_SLOTS-1:0] job_active,
|
||||
input wire [N_SLOTS*ADDR_WIDTH-1:0] x_base_flat,
|
||||
input wire [N_SLOTS*16-1:0] n_tiles_flat,
|
||||
|
||||
// ---- broadcast status (every slot compares this against its own x_base) ----
|
||||
output reg [ADDR_WIDTH-1:0] resident_tag,
|
||||
output reg [CNTW-1:0] resident_count,
|
||||
|
||||
// ---- fill port into nms_activation_replicated.v ----
|
||||
output wire fill_we,
|
||||
output wire [TIW-1:0] fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] fill_data,
|
||||
|
||||
// ---- real word-level PSRAM backend (arbitrated externally) ----
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
integer i;
|
||||
|
||||
// ---- desired x_base: lowest-indexed currently-active slot (fixed
|
||||
// priority -- simple, not fairness-critical here since this only
|
||||
// decides which TAG to chase, not who gets bandwidth) ----
|
||||
reg desired_valid;
|
||||
reg [ADDR_WIDTH-1:0] desired_x_base;
|
||||
|
||||
always @* begin
|
||||
desired_valid = 1'b0;
|
||||
desired_x_base = {ADDR_WIDTH{1'b0}};
|
||||
for (i = N_SLOTS-1; i >= 0; i = i - 1) begin
|
||||
if (job_active[i]) begin
|
||||
desired_valid = 1'b1;
|
||||
desired_x_base = x_base_flat[i*ADDR_WIDTH +: ADDR_WIDTH];
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ---- STEP14/EXP-0029->EXP-0030: max_n_tiles computation split
|
||||
// into TWO pipeline stages, since registering ONLY its final use
|
||||
// (v2, DEC-0026) left the computation ITSELF as the new critical
|
||||
// path (EXP-0030, N=4 Fmax=72.78MHz, still FAIL@80MHz): the
|
||||
// original single-cycle logic mixed, PER SLOT, a 23-bit tag
|
||||
// equality check (x_base_flat[i]==resident_tag) together with an
|
||||
// N_SLOTS-wide SEQUENTIALLY-CHAINED 16-bit running-max fold (each
|
||||
// iteration's update depends on the previous one) -- both
|
||||
// combinational, both in the same cycle as the register that
|
||||
// captures the result.
|
||||
//
|
||||
// Stage 1 (independent per-slot work, no chain dependency between
|
||||
// slots): register a per-slot "counts toward this refill" mask
|
||||
// (job_active[i] && tag-match) and, gated by that mask, each
|
||||
// slot's own n_tiles value (0 if it doesn't count) -- N_SLOTS
|
||||
// independent 23-bit equality checks, no data dependency between
|
||||
// slots, so their combined depth does not grow with N_SLOTS the
|
||||
// way a sequential fold does.
|
||||
// Stage 2 (the actual reduction): fold the REGISTERED, already-
|
||||
// masked per-slot values into max_n_tiles_reg -- still an
|
||||
// N_SLOTS-wide sequential chain (same fold as before), but now
|
||||
// operating alone, without the equality check sharing the same
|
||||
// cycle.
|
||||
reg [15:0] n_tiles_masked [0:N_SLOTS-1];
|
||||
genvar gsi;
|
||||
generate
|
||||
for (gsi = 0; gsi < N_SLOTS; gsi = gsi + 1) begin : GEN_MASK
|
||||
wire slot_counts = job_active[gsi] &&
|
||||
(x_base_flat[gsi*ADDR_WIDTH +: ADDR_WIDTH] == resident_tag);
|
||||
always @(posedge clk) begin
|
||||
if (rst) n_tiles_masked[gsi] <= 16'h0;
|
||||
else n_tiles_masked[gsi] <= slot_counts ? n_tiles_flat[gsi*16 +: 16] : 16'h0;
|
||||
end
|
||||
end
|
||||
endgenerate
|
||||
|
||||
reg [15:0] max_n_tiles_reg;
|
||||
integer j;
|
||||
reg [15:0] max_n_tiles_comb;
|
||||
always @* begin
|
||||
max_n_tiles_comb = 16'h0;
|
||||
for (j = 0; j < N_SLOTS; j = j + 1)
|
||||
if (n_tiles_masked[j] > max_n_tiles_comb)
|
||||
max_n_tiles_comb = n_tiles_masked[j];
|
||||
end
|
||||
always @(posedge clk) begin
|
||||
if (rst) max_n_tiles_reg <= 16'h0;
|
||||
else max_n_tiles_reg <= max_n_tiles_comb;
|
||||
end
|
||||
|
||||
localparam ST_IDLE = 1'd0;
|
||||
localparam ST_FETCH = 1'd1;
|
||||
reg state;
|
||||
|
||||
reg pf_start;
|
||||
reg [ADDR_WIDTH-1:0] pf_addr;
|
||||
wire pf_busy, pf_done;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] pf_tile;
|
||||
|
||||
prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_pf (
|
||||
.clk(clk), .rst(rst),
|
||||
.fetch_start(pf_start), .w_addr(pf_addr),
|
||||
.fetch_busy(pf_busy), .fetch_done(pf_done), .tile_w(pf_tile),
|
||||
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(mem_addr), .mem_wdata(mem_wdata),
|
||||
.mem_lb_n(mem_lb_n), .mem_ub_n(mem_ub_n),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
|
||||
reg fill_we_reg;
|
||||
reg [TIW-1:0] fill_addr_reg;
|
||||
reg [DATA_WIDTH*P_IN-1:0] fill_data_reg;
|
||||
assign fill_we = fill_we_reg;
|
||||
assign fill_addr = fill_addr_reg;
|
||||
assign fill_data = fill_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
resident_tag <= {ADDR_WIDTH{1'b1}}; // sentinel: matches no real x_base at reset
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
state <= ST_IDLE;
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
end else begin
|
||||
pf_start <= 1'b0;
|
||||
fill_we_reg <= 1'b0;
|
||||
|
||||
// latch a completed fetch into the replicated activation
|
||||
// memory's broadcast fill port
|
||||
if (pf_done) begin
|
||||
fill_we_reg <= 1'b1;
|
||||
fill_addr_reg <= resident_count[TIW-1:0]; // valid: gated < max_n_tiles <= MAX_TILES
|
||||
fill_data_reg <= pf_tile;
|
||||
resident_count <= resident_count + 1'b1;
|
||||
end
|
||||
|
||||
// refill trigger: the reference slot wants a DIFFERENT tag,
|
||||
// and the fetch engine is genuinely idle (never interrupt an
|
||||
// in-flight fetch -- same discipline as memory_manager.v's
|
||||
// own pf_pending guard, ERR-0006). Stays in ST_IDLE (not
|
||||
// ST_FETCH): only updates resident_tag/resident_count here;
|
||||
// the ST_IDLE case below is what actually issues pf_start,
|
||||
// reading the NEW resident_tag starting next cycle -- this
|
||||
// path must NOT itself jump to ST_FETCH without a matching
|
||||
// pf_start, or the engine would sit in ST_FETCH forever
|
||||
// waiting for a pf_done that was never triggered.
|
||||
if (desired_valid && (desired_x_base != resident_tag) && !pf_busy && (state == ST_IDLE)) begin
|
||||
resident_tag <= desired_x_base;
|
||||
resident_count <= {CNTW{1'b0}};
|
||||
end
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
// stay idle: fetching further tiles for the CURRENT
|
||||
// tag (if any active slot still needs more) is
|
||||
// handled below, symmetric to the refill case.
|
||||
if (!pf_busy && !pf_start && (resident_count < max_n_tiles_reg) &&
|
||||
desired_valid && (desired_x_base == resident_tag)) begin
|
||||
pf_start <= 1'b1;
|
||||
pf_addr <= resident_tag + (resident_count * P_IN[ADDR_WIDTH-1:0]);
|
||||
state <= ST_FETCH;
|
||||
end
|
||||
end
|
||||
ST_FETCH: begin
|
||||
if (pf_done) begin
|
||||
// resident_count already bumped above this cycle;
|
||||
// decide whether more remain once back in IDLE.
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,54 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP4/5 candidate A: REPLICATED
|
||||
// activation memory.
|
||||
//
|
||||
// One full copy of the shared activation vector's tile storage per
|
||||
// slot (N_SLOTS independent single-write/single-read BRAMs). A shared
|
||||
// fill engine broadcasts each filled tile to EVERY copy on the same
|
||||
// cycle (one PSRAM-side write, N_SLOTS on-chip writes) -- after fill,
|
||||
// every slot's own read port is completely private: zero contention,
|
||||
// ever, by construction (no arbitration logic at all on the read
|
||||
// side). Real cost is N_SLOTS x the single-copy storage; this file
|
||||
// exists to MEASURE that real DP16KD/LUT/Fmax cost against Candidate
|
||||
// B (nms_activation_banked.v) rather than assume replication is too
|
||||
// expensive a priori (EXP-0019/DEC-0019).
|
||||
// ============================================================
|
||||
module nms_activation_replicated #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES)
|
||||
)(
|
||||
input clk,
|
||||
input rst,
|
||||
|
||||
// ---- fill port: one write, broadcast to every copy ----
|
||||
input fill_we,
|
||||
input [TIW-1:0] fill_addr,
|
||||
input [DATA_WIDTH*P_IN-1:0] fill_data,
|
||||
|
||||
// ---- per-slot private read port ----
|
||||
input [N_SLOTS-1:0] rd_en,
|
||||
input [N_SLOTS*TIW-1:0] rd_addr_flat,
|
||||
output [N_SLOTS*DATA_WIDTH*P_IN-1:0] rd_data_flat
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_COPY
|
||||
reg [DATA_WIDTH*P_IN-1:0] mem [0:MAX_TILES-1];
|
||||
reg [DATA_WIDTH*P_IN-1:0] rd_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (fill_we)
|
||||
mem[fill_addr] <= fill_data;
|
||||
if (rd_en[g])
|
||||
rd_data_reg <= mem[rd_addr_flat[g*TIW +: TIW]];
|
||||
end
|
||||
|
||||
assign rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN] = rd_data_reg;
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,260 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,262 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_actfix #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl_v2 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_stream #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,262 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_actfix2 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl_v3 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_stream #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,277 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_dual32 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready,
|
||||
|
||||
// ---- STEP15 (continuation): NEW, separate wide (32-bit logical)
|
||||
// weight-fetch backend interface, per slot -- N_SLOTS wide (NOT
|
||||
// N_SLOTS+1: the shared activation-fill controller stays on the
|
||||
// ORIGINAL 16-bit port above, unchanged). Arbitrated one level up
|
||||
// (slot_mem_arbiter_wide.v) down to the real dual-chip 32-bit
|
||||
// physical interface (psram_controller_dual32.v). ----
|
||||
output wire [N_SLOTS-1:0] wide_slot_mem_req,
|
||||
output wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr,
|
||||
input wire [32*N_SLOTS-1:0] wide_slot_mem_rdata,
|
||||
input wire [N_SLOTS-1:0] wide_slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl_v3 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_stream_wide #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE), .MEM_DATA_WIDTH(32)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g]),
|
||||
.wide_mem_req(wide_slot_mem_req[g]),
|
||||
.wide_mem_addr(wide_slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.wide_mem_rdata(wide_slot_mem_rdata[g*32 +: 32]),
|
||||
.wide_mem_ready(wide_slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,262 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_pf #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_pf #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,290 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
//
|
||||
// STEP16 Phase 5: forked from nms_dataflow_core_dual32.v (STEP15's
|
||||
// own dual-chip-32-bit variant) with ONLY the wide weight-fetch port
|
||||
// widened from 32 to 64 bits (MEM_DATA_WIDTH=64 in the per-slot
|
||||
// nms_memory_manager_stream_wide instance below) -- the natural
|
||||
// P_IN*DATA_WIDTH/16=4-word SDRAM burst identified in Phase 1 means
|
||||
// ONE mem_req now fetches exactly one whole tile (WORDS_PER_TILE=1),
|
||||
// same as the dual32 case fetched one whole tile per 32-bit request.
|
||||
// Everything else (activation path, dependency manager, neural
|
||||
// director, per-slot neural_processor) is BYTE-FOR-BYTE UNCHANGED --
|
||||
// per the governing spec's own "do not create an artificial
|
||||
// benchmark" instruction, only the piece under test (weight-fetch
|
||||
// physical memory) differs from the validated dual32 baseline.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_sdram #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready,
|
||||
|
||||
// ---- STEP16 Phase 5: wide (64-bit logical) weight-fetch backend
|
||||
// interface, per slot -- N_SLOTS wide (NOT N_SLOTS+1: the shared
|
||||
// activation-fill controller stays on the ORIGINAL 16-bit port
|
||||
// above, unchanged). Arbitrated one level up (slot_mem_arbiter_
|
||||
// wide.v, reused unchanged at DATA_WIDTH=64) down to the real
|
||||
// SDRAM physical interface (sdram_weight_backend.v). ----
|
||||
output wire [N_SLOTS-1:0] wide_slot_mem_req,
|
||||
output wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr,
|
||||
input wire [64*N_SLOTS-1:0] wide_slot_mem_rdata,
|
||||
input wire [N_SLOTS-1:0] wide_slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl_v3 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_stream_wide #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE), .MEM_DATA_WIDTH(64)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g]),
|
||||
.wide_mem_req(wide_slot_mem_req[g]),
|
||||
.wide_mem_addr(wide_slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.wide_mem_rdata(wide_slot_mem_rdata[g*64 +: 64]),
|
||||
.wide_mem_ready(wide_slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,262 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP8 full integration, mirrors
|
||||
// hardware/v2/rtl/dataflow_core.v's own scope exactly (M6 Dependency
|
||||
// Manager -> M5 Neural Director -> N_SLOTS x (memory manager + neural
|
||||
// processor)), but replaces the M4 memory_manager.v +
|
||||
// activation_cache.v cluster with the NMS's own decided pieces
|
||||
// (DEC-0019/DEC-0020):
|
||||
// - nms_activation_replicated.v: N_SLOTS private full-vector
|
||||
// activation copies, broadcast-filled by...
|
||||
// - nms_activation_fill_ctrl.v: the shared dedup/fetch controller
|
||||
// (single logical tag, same honest thrash-under-interleaved-
|
||||
// different-x_base limitation as the superseded activation_cache.v)
|
||||
// - nms_weight_packed.v: N_SLOTS private, per-MAC-lane packed weight
|
||||
// copies (never shared, no arbitration needed)
|
||||
// - nms_memory_manager.v: per-slot job FSM, reads directly from the
|
||||
// two SRAMs above instead of double-buffering 2 banks (the whole
|
||||
// vector is resident, not just 2 tiles worth)
|
||||
//
|
||||
// hardware/v2/rtl/dependency_manager.v and neural_director.v are
|
||||
// REUSED VERBATIM, unmodified -- the node-registration and slot-
|
||||
// dispatch protocol did not change at all; only what happens between
|
||||
// "job dispatched to a slot" and "job_done" changed.
|
||||
//
|
||||
// Memory Backend Interface: exposed N_SLOTS+1 wide exactly like
|
||||
// dataflow_core.v (indices [0,N_SLOTS) = per-slot memory managers'
|
||||
// own weight-fetch+result-write port, index [N_SLOTS] = the shared
|
||||
// activation fill controller's own port) -- arbitrated one level up,
|
||||
// reusing hardware/v2/rtl/slot_mem_arbiter.v unchanged.
|
||||
// ================================================================
|
||||
|
||||
module nms_dataflow_core_stream #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// Must match nms_memory_manager.v's/nms_activation_fill_ctrl.v's
|
||||
// own CNTW exactly -- this top-level wire connecting the two was
|
||||
// left at the narrower TIW after those modules were widened,
|
||||
// silently truncating resident_count's real value (16) back down
|
||||
// to 0 right when it should have reached MAX_TILES, deadlocking
|
||||
// the very last tile of any n_tiles==MAX_TILES job forever (found
|
||||
// via simulation: D-Stress's real 16-tile neurons hung 1 tile
|
||||
// short, act_resident_count visibly reset to 0 the exact cycle it
|
||||
// should have become 16).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [N_SLOTS:0] slot_mem_req,
|
||||
output wire [N_SLOTS:0] slot_mem_wr,
|
||||
output wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr,
|
||||
output wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata,
|
||||
output wire [N_SLOTS:0] slot_mem_lb_n,
|
||||
output wire [N_SLOTS:0] slot_mem_ub_n,
|
||||
input wire [16*(N_SLOTS+1)-1:0] slot_mem_rdata,
|
||||
input wire [N_SLOTS:0] slot_mem_ready
|
||||
);
|
||||
|
||||
localparam NODE_IDW = $clog2(N_NODES);
|
||||
|
||||
wire dm_ready_valid;
|
||||
wire dm_ready_ready;
|
||||
wire [NODE_IDW-1:0] dm_ready_node_id;
|
||||
wire [ADDR_WIDTH-1:0] dm_ready_x_base, dm_ready_w_base, dm_ready_result_addr;
|
||||
wire [15:0] dm_ready_n_tiles;
|
||||
|
||||
wire dm_producer_done_valid;
|
||||
wire [NODE_IDW-1:0] dm_producer_done_node_id;
|
||||
|
||||
dependency_manager #(
|
||||
.N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_dep_mgr (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.producer_done_valid(dm_producer_done_valid), .producer_done_node_id(dm_producer_done_node_id),
|
||||
.ready_valid(dm_ready_valid), .ready_ready(dm_ready_ready), .ready_node_id(dm_ready_node_id),
|
||||
.ready_x_base(dm_ready_x_base), .ready_w_base(dm_ready_w_base),
|
||||
.ready_n_tiles(dm_ready_n_tiles), .ready_result_addr(dm_ready_result_addr)
|
||||
);
|
||||
|
||||
wire [15:0] dm_ready_node_id_ext = {{(16-NODE_IDW){1'b0}}, dm_ready_node_id};
|
||||
|
||||
wire [N_SLOTS-1:0] dir_slot_job_start;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] dir_slot_x_base, dir_slot_w_base, dir_slot_result_addr;
|
||||
wire [16*N_SLOTS-1:0] dir_slot_n_tiles, dir_slot_node_id;
|
||||
wire [N_SLOTS-1:0] dir_slot_job_done;
|
||||
wire dir_job_out_done;
|
||||
wire [$clog2(N_SLOTS)-1:0] dir_job_out_slot;
|
||||
wire [3:0] dir_state;
|
||||
wire dir_error;
|
||||
|
||||
neural_director #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_SLOTS(N_SLOTS), .QUEUE_DEPTH(QUEUE_DEPTH)
|
||||
) u_director (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_in_valid(dm_ready_valid), .job_in_ready(dm_ready_ready),
|
||||
.job_in_x_base(dm_ready_x_base), .job_in_w_base(dm_ready_w_base),
|
||||
.job_in_n_tiles(dm_ready_n_tiles), .job_in_result_addr(dm_ready_result_addr),
|
||||
.job_in_node_id(dm_ready_node_id_ext),
|
||||
.slot_job_start(dir_slot_job_start), .slot_x_base(dir_slot_x_base), .slot_w_base(dir_slot_w_base),
|
||||
.slot_n_tiles(dir_slot_n_tiles), .slot_result_addr(dir_slot_result_addr),
|
||||
.slot_node_id(dir_slot_node_id), .slot_job_done(dir_slot_job_done),
|
||||
.job_out_done(dir_job_out_done), .job_out_slot(dir_job_out_slot),
|
||||
.dir_state(dir_state), .dir_error(dir_error)
|
||||
);
|
||||
|
||||
wire [15:0] completed_node_id_16 = dir_slot_node_id[dir_job_out_slot*16 +: 16];
|
||||
assign dm_producer_done_valid = dir_job_out_done;
|
||||
assign dm_producer_done_node_id = completed_node_id_16[NODE_IDW-1:0];
|
||||
|
||||
// ---- NMS memory: shared Activation SRAM (replicated) + private
|
||||
// Weight SRAM (packed), per DEC-0019/DEC-0020 ----
|
||||
wire fill_we;
|
||||
wire [TIW-1:0] fill_addr;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] fill_data;
|
||||
wire [ADDR_WIDTH-1:0] act_resident_tag;
|
||||
wire [CNTW-1:0] act_resident_count;
|
||||
|
||||
wire [N_SLOTS-1:0] act_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] act_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] act_rd_data_flat;
|
||||
|
||||
nms_activation_replicated #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_act_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.rd_en(act_rd_en), .rd_addr_flat(act_rd_addr_flat), .rd_data_flat(act_rd_data_flat)
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] job_active;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] job_x_base_flat;
|
||||
wire [16*N_SLOTS-1:0] job_n_tiles_flat;
|
||||
|
||||
nms_activation_fill_ctrl #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES)
|
||||
) u_act_fill (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active), .x_base_flat(job_x_base_flat), .n_tiles_flat(job_n_tiles_flat),
|
||||
.resident_tag(act_resident_tag), .resident_count(act_resident_count),
|
||||
.fill_we(fill_we), .fill_addr(fill_addr), .fill_data(fill_data),
|
||||
.mem_req(slot_mem_req[N_SLOTS]), .mem_wr(slot_mem_wr[N_SLOTS]),
|
||||
.mem_addr(slot_mem_addr[N_SLOTS*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[N_SLOTS*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[N_SLOTS]), .mem_ub_n(slot_mem_ub_n[N_SLOTS]),
|
||||
.mem_rdata(slot_mem_rdata[N_SLOTS*16 +: 16]), .mem_ready(slot_mem_ready[N_SLOTS])
|
||||
);
|
||||
|
||||
wire [N_SLOTS-1:0] wgt_fill_we;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_fill_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_fill_data_flat;
|
||||
wire [N_SLOTS-1:0] wgt_rd_en;
|
||||
wire [N_SLOTS*TIW-1:0] wgt_rd_addr_flat;
|
||||
wire signed [DATA_WIDTH*P_IN*N_SLOTS-1:0] wgt_rd_data_flat;
|
||||
|
||||
nms_weight_packed #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .N_SLOTS(N_SLOTS), .MAX_TILES(MAX_TILES)
|
||||
) u_wgt_mem (
|
||||
.clk(clk), .rst(rst),
|
||||
.fill_we(wgt_fill_we), .fill_addr_flat(wgt_fill_addr_flat), .fill_data_flat(wgt_fill_data_flat),
|
||||
.rd_en(wgt_rd_en), .rd_addr_flat(wgt_rd_addr_flat), .rd_data_flat(wgt_rd_data_flat)
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
|
||||
wire mm_operand_valid, mm_operand_ready;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] mm_input_data, mm_weight_data;
|
||||
wire mm_tile_last;
|
||||
wire mm_result_valid, mm_result_ready;
|
||||
wire signed [DATA_WIDTH-1:0] mm_result_data;
|
||||
|
||||
nms_memory_manager_stream #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH), .MAX_TILES(MAX_TILES),
|
||||
.PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_mm (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_start(dir_slot_job_start[g]),
|
||||
.x_base(dir_slot_x_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.w_base(dir_slot_w_base[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.n_tiles(dir_slot_n_tiles[g*16 +: 16]),
|
||||
.result_addr(dir_slot_result_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_done(dir_slot_job_done[g]),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(mm_result_valid), .result_ready(mm_result_ready), .result_data(mm_result_data),
|
||||
.job_active(job_active[g]),
|
||||
.job_x_base(job_x_base_flat[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.job_n_tiles(job_n_tiles_flat[g*16 +: 16]),
|
||||
.act_resident_tag(act_resident_tag), .act_resident_count(act_resident_count),
|
||||
.act_rd_en(act_rd_en[g]),
|
||||
.act_rd_addr(act_rd_addr_flat[g*TIW +: TIW]),
|
||||
.act_rd_data(act_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_fill_we(wgt_fill_we[g]),
|
||||
.wgt_fill_addr(wgt_fill_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_fill_data(wgt_fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.wgt_rd_en(wgt_rd_en[g]),
|
||||
.wgt_rd_addr(wgt_rd_addr_flat[g*TIW +: TIW]),
|
||||
.wgt_rd_data(wgt_rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]),
|
||||
.mem_req(slot_mem_req[g]), .mem_wr(slot_mem_wr[g]),
|
||||
.mem_addr(slot_mem_addr[g*ADDR_WIDTH +: ADDR_WIDTH]),
|
||||
.mem_wdata(slot_mem_wdata[g*16 +: 16]),
|
||||
.mem_lb_n(slot_mem_lb_n[g]), .mem_ub_n(slot_mem_ub_n[g]),
|
||||
.mem_rdata(slot_mem_rdata[g*16 +: 16]), .mem_ready(slot_mem_ready[g])
|
||||
);
|
||||
|
||||
reg job_valid_np;
|
||||
wire job_ready_np;
|
||||
wire result_valid_np;
|
||||
wire signed [DATA_WIDTH-1:0] result_data_np;
|
||||
wire [3:0] np_state;
|
||||
wire np_error;
|
||||
|
||||
neural_processor #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH)
|
||||
) u_np (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_valid(job_valid_np), .job_ready(job_ready_np),
|
||||
.job_node_id(16'h0), .job_bias(8'sd0), .job_activation(2'd1),
|
||||
.operand_valid(mm_operand_valid), .operand_ready(mm_operand_ready),
|
||||
.input_data(mm_input_data), .weight_data(mm_weight_data), .tile_last(mm_tile_last),
|
||||
.result_valid(result_valid_np), .result_ready(mm_result_ready),
|
||||
.result_data(result_data_np), .result_node_id(),
|
||||
.np_state(np_state), .np_error(np_error)
|
||||
);
|
||||
assign mm_result_valid = result_valid_np;
|
||||
assign mm_result_data = result_data_np;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) job_valid_np <= 1'b0;
|
||||
else if (dir_slot_job_start[g]) job_valid_np <= 1'b1;
|
||||
else if (job_valid_np && job_ready_np) job_valid_np <= 1'b0;
|
||||
end
|
||||
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,332 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- per-slot memory manager.
|
||||
//
|
||||
// Same EXTERNAL job/operand/result interface as
|
||||
// hardware/v2/rtl/memory_manager.v (drop-in for the Neural Director/
|
||||
// Dependency Manager side -- neither needs to change). Internally
|
||||
// simpler: since the on-chip Activation/Weight SRAMs
|
||||
// (nms_activation_replicated.v, nms_weight_packed.v) each hold the
|
||||
// ENTIRE shared/private vector (sized MAX_TILES deep), there is no
|
||||
// more double-buffering/bank-swap logic at all -- this module just
|
||||
// tracks a private WEIGHT fetch progress counter (own prefetch_engine
|
||||
// instance, exactly memory_manager.v's own proven pattern, writing
|
||||
// into this slot's own private SRAM lane instead of a bank register)
|
||||
// and reads the shared Activation fill controller's own
|
||||
// resident_tag/resident_count to know how many tiles of ITS x_base are
|
||||
// currently usable.
|
||||
//
|
||||
// A tile is presentable once tile_idx is below BOTH: this slot's own
|
||||
// weight-fetch progress, and (resident_tag==x_base_reg) ?
|
||||
// resident_count : 0. SRAM reads have 1-cycle registered latency
|
||||
// (matches nms_activation_replicated.v/nms_weight_packed.v exactly).
|
||||
// ============================================================
|
||||
module nms_memory_manager #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
// TIW indexes SRAM addresses (0..MAX_TILES-1) -- matches
|
||||
// nms_activation_replicated.v/nms_weight_packed.v's own address
|
||||
// port width exactly, must stay in sync with them.
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
// CNTW is for COUNTERS (tile_idx, wgt_fetched) and the resident-
|
||||
// count status they compare against -- these must be able to
|
||||
// represent the VALUE MAX_TILES itself (e.g. n_tiles=16 with
|
||||
// MAX_TILES=16), one bit wider than an address index needs. Using
|
||||
// TIW for these counters was a real bug: a 4-bit wgt_fetched
|
||||
// (MAX_TILES=16) can count 0..15 but overflows 15->0 right when it
|
||||
// should reach 16, so "wgt_fetched < n_tiles" was NEVER false once
|
||||
// truly done -- the weight fetch looped forever, and separately
|
||||
// resident_count never advanced past its own analogous ceiling
|
||||
// (found via simulation: any n_tiles==MAX_TILES job -- exactly
|
||||
// D-Stress's real 16-tile neurons -- hung forever, while every
|
||||
// earlier n_tiles<MAX_TILES test in this project passed).
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- job control (from Neural Director, unchanged interface) ----
|
||||
input wire job_start,
|
||||
input wire [ADDR_WIDTH-1:0] x_base,
|
||||
input wire [ADDR_WIDTH-1:0] w_base,
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] result_addr,
|
||||
output reg job_done,
|
||||
|
||||
// ---- Neural Processor-facing operand stream (unchanged) ----
|
||||
output reg operand_valid,
|
||||
input wire operand_ready,
|
||||
output reg signed [DATA_WIDTH*P_IN-1:0] input_data,
|
||||
output reg signed [DATA_WIDTH*P_IN-1:0] weight_data,
|
||||
output reg tile_last,
|
||||
|
||||
// ---- Neural Processor-facing result consumption (unchanged) ----
|
||||
input wire result_valid,
|
||||
output reg result_ready,
|
||||
input wire signed [DATA_WIDTH-1:0] result_data,
|
||||
|
||||
// ---- job status broadcast to the shared activation fill controller ----
|
||||
output wire job_active,
|
||||
output wire [ADDR_WIDTH-1:0] job_x_base,
|
||||
output wire [15:0] job_n_tiles,
|
||||
|
||||
// ---- shared activation fill controller status (broadcast, same for every slot) ----
|
||||
input wire [ADDR_WIDTH-1:0] act_resident_tag,
|
||||
input wire [CNTW-1:0] act_resident_count,
|
||||
|
||||
// ---- this slot's own private lane into nms_activation_replicated.v (READ only -- fill is owned by the shared controller) ----
|
||||
output reg act_rd_en,
|
||||
output reg [TIW-1:0] act_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] act_rd_data,
|
||||
|
||||
// ---- this slot's own private lane into nms_weight_packed.v (fill AND read -- private) ----
|
||||
output reg wgt_fill_we,
|
||||
output reg [TIW-1:0] wgt_fill_addr,
|
||||
output reg [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
output reg wgt_rd_en,
|
||||
output reg [TIW-1:0] wgt_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] wgt_rd_data,
|
||||
|
||||
// ---- real word-level PSRAM backend for THIS slot's own weight
|
||||
// fetch + result write-back (arbitrated externally, exactly
|
||||
// memory_manager.v's own mem_* port) ----
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
localparam ST_IDLE = 3'd0;
|
||||
localparam ST_RUN = 3'd1;
|
||||
localparam ST_WAIT_RESULT = 3'd2;
|
||||
localparam ST_WRITE_RES = 3'd3;
|
||||
localparam ST_DONE = 3'd4;
|
||||
|
||||
reg [2:0] state;
|
||||
reg job_active_reg;
|
||||
assign job_active = job_active_reg;
|
||||
assign job_x_base = x_base_reg;
|
||||
assign job_n_tiles = n_tiles_reg;
|
||||
|
||||
reg [ADDR_WIDTH-1:0] x_base_reg, w_base_reg, result_addr_reg;
|
||||
reg [15:0] n_tiles_reg;
|
||||
reg [CNTW-1:0] tile_idx; // consumption pointer (0..MAX_TILES)
|
||||
reg [CNTW-1:0] wgt_fetched; // this slot's own weight fetch progress (0..MAX_TILES)
|
||||
// Two-stage read pipeline, NOT one: both nms_activation_replicated.v
|
||||
// and nms_weight_packed.v register rd_en THEN register the memory
|
||||
// read off THAT (rd_data_reg <= mem[addr]) -- i.e. asserting rd_en
|
||||
// at cycle T makes the SRAM's own always block see it at cycle T+1
|
||||
// (scheduling the read for T+2), so rd_data is only valid starting
|
||||
// T+2, not T+1. A single "read_issued" flag capturing rd_data one
|
||||
// cycle after issuing it (T+1) grabbed the SRAM's PRE-read (stale)
|
||||
// output -- found via simulation: node0's own first tile computed
|
||||
// 0 instead of 48 (2*3*8) because input_data/weight_data were still
|
||||
// read AS ZERO the very cycle operand_valid first asserted (the
|
||||
// real act_rd_data/wgt_rd_data were correct by then, but the NBA
|
||||
// capture into input_data/weight_data was one cycle too early to
|
||||
// use them). Fixed with a genuine 2-stage pipeline: read_issued
|
||||
// (SRAM now computing) -> read_ready (SRAM output now valid,
|
||||
// capture NOW).
|
||||
reg read_issued;
|
||||
reg read_ready;
|
||||
|
||||
wire usable_act_count_valid = (act_resident_tag == x_base_reg);
|
||||
wire [CNTW-1:0] usable_act = usable_act_count_valid ? act_resident_count : {CNTW{1'b0}};
|
||||
// tile_idx/wgt_fetched/usable_act are CNTW-bit (able to represent
|
||||
// the value MAX_TILES itself, not just index it). First term
|
||||
// compares against the full 16-bit n_tiles_reg (Verilog zero-
|
||||
// extends automatically since CNTW<16 for any real MAX_TILES);
|
||||
// the other two compare tile_idx (what's NEEDED now) against
|
||||
// wgt_fetched/usable_act (what's actually AVAILABLE) -- NOT
|
||||
// against n_tiles_reg, which says nothing about availability. An
|
||||
// earlier revision of this fix mistakenly compared wgt_fetched
|
||||
// against n_tiles_reg here instead of against tile_idx: once
|
||||
// wgt_fetched legitimately reached n_tiles (fetch complete, no
|
||||
// more needed), that comparison went permanently false and
|
||||
// deadlocked consumption forever even though every tile was
|
||||
// genuinely ready -- caught because act_resident_count kept
|
||||
// climbing normally while operand_valid never once asserted.
|
||||
wire can_present = ({{(16-CNTW){1'b0}}, tile_idx} < n_tiles_reg) &&
|
||||
(tile_idx < wgt_fetched) &&
|
||||
(tile_idx < usable_act);
|
||||
|
||||
// ---- private weight prefetch (own prefetch_engine instance,
|
||||
// exactly memory_manager.v's own proven pattern -- fetch AS FAST AS
|
||||
// POSSIBLE up to n_tiles, no lookahead throttling needed since the
|
||||
// SRAM holds the whole vector, not just 2 double-buffered banks) ----
|
||||
reg pf_start;
|
||||
reg [ADDR_WIDTH-1:0] pf_w_addr;
|
||||
wire pf_busy, pf_done;
|
||||
wire signed [DATA_WIDTH*P_IN-1:0] pf_tile_w;
|
||||
|
||||
prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH)
|
||||
) u_prefetch (
|
||||
.clk(clk), .rst(rst),
|
||||
.fetch_start(pf_start), .w_addr(pf_w_addr),
|
||||
.fetch_busy(pf_busy), .fetch_done(pf_done), .tile_w(pf_tile_w),
|
||||
.mem_req(pf_mem_req), .mem_wr(pf_mem_wr), .mem_addr(pf_mem_addr), .mem_wdata(pf_mem_wdata),
|
||||
.mem_lb_n(pf_mem_lb_n), .mem_ub_n(pf_mem_ub_n),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
wire pf_mem_req, pf_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pf_mem_addr;
|
||||
wire [15:0] pf_mem_wdata;
|
||||
wire pf_mem_lb_n, pf_mem_ub_n;
|
||||
|
||||
reg wr_mem_req;
|
||||
reg [ADDR_WIDTH-1:0] wr_mem_addr;
|
||||
reg [15:0] wr_mem_wdata;
|
||||
reg wr_mem_lb_n, wr_mem_ub_n;
|
||||
|
||||
wire wr_active = (state == ST_WRITE_RES) || (state == ST_DONE);
|
||||
assign mem_req = wr_active ? wr_mem_req : pf_mem_req;
|
||||
assign mem_wr = wr_active ? 1'b1 : pf_mem_wr;
|
||||
assign mem_addr = wr_active ? wr_mem_addr : pf_mem_addr;
|
||||
assign mem_wdata = wr_active ? wr_mem_wdata : pf_mem_wdata;
|
||||
assign mem_lb_n = wr_active ? wr_mem_lb_n : pf_mem_lb_n;
|
||||
assign mem_ub_n = wr_active ? wr_mem_ub_n : pf_mem_ub_n;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= ST_IDLE;
|
||||
job_active_reg <= 1'b0;
|
||||
job_done <= 1'b0;
|
||||
operand_valid <= 1'b0;
|
||||
tile_last <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
wgt_fetched <= {CNTW{1'b0}};
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b0;
|
||||
pf_start <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
wr_mem_req <= 1'b0;
|
||||
wr_mem_lb_n <= 1'b1;
|
||||
wr_mem_ub_n <= 1'b1;
|
||||
end else begin
|
||||
job_done <= 1'b0;
|
||||
pf_start <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
|
||||
// latch a completed private weight fetch into this slot's
|
||||
// own SRAM lane
|
||||
if (pf_done) begin
|
||||
wgt_fill_we <= 1'b1;
|
||||
wgt_fill_addr <= wgt_fetched[TIW-1:0]; // valid: gated < n_tiles <= MAX_TILES
|
||||
wgt_fill_data <= pf_tile_w;
|
||||
wgt_fetched <= wgt_fetched + 1'b1;
|
||||
end
|
||||
|
||||
// keep fetching weight tiles as fast as the (single, private)
|
||||
// prefetch engine allows, up to n_tiles. The `!pf_done` guard
|
||||
// is required, not cosmetic: pf_done and the "wgt_fetched<=
|
||||
// wgt_fetched+1" increment above happen the SAME cycle
|
||||
// pf_busy also drops back to 0 (prefetch_engine.v's own
|
||||
// ST_DONE clears fetch_busy the same cycle it pulses
|
||||
// fetch_done) -- without this guard, THIS SAME cycle would
|
||||
// read the OLD (pre-increment) wgt_fetched to compute
|
||||
// pf_w_addr, re-issuing a fetch for the tile that JUST
|
||||
// completed instead of the next one, and that duplicate
|
||||
// fetch's own completion would then write into the NEXT
|
||||
// tile's SRAM slot using the WRONG (duplicated) source data
|
||||
// -- silently corrupting every other tile for any n_tiles>1
|
||||
// job (found via simulation once a >1-tile test was run;
|
||||
// every n_tiles=1 test in this file's own first pass never
|
||||
// exercised this path). Same bug class as ERR-0006's own
|
||||
// "don't gate solely on a signal with its own same-cycle
|
||||
// side effect" lesson.
|
||||
if (job_active_reg && !pf_busy && !pf_start && !pf_done &&
|
||||
({{(16-CNTW){1'b0}}, wgt_fetched} < n_tiles_reg)) begin
|
||||
pf_start <= 1'b1;
|
||||
pf_w_addr <= w_base_reg + (wgt_fetched * P_IN[ADDR_WIDTH-1:0]);
|
||||
end
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
if (job_start) begin
|
||||
x_base_reg <= x_base;
|
||||
w_base_reg <= w_base;
|
||||
n_tiles_reg <= n_tiles;
|
||||
result_addr_reg <= result_addr;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
wgt_fetched <= {CNTW{1'b0}};
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b0;
|
||||
operand_valid <= 1'b0;
|
||||
job_active_reg <= 1'b1;
|
||||
state <= ST_RUN;
|
||||
end
|
||||
end
|
||||
|
||||
ST_RUN: begin
|
||||
if (!operand_valid && !read_issued && !read_ready && can_present) begin
|
||||
act_rd_en <= 1'b1;
|
||||
act_rd_addr <= tile_idx[TIW-1:0]; // valid: gated < n_tiles <= MAX_TILES
|
||||
wgt_rd_en <= 1'b1;
|
||||
wgt_rd_addr <= tile_idx[TIW-1:0];
|
||||
read_issued <= 1'b1;
|
||||
end else if (read_issued) begin
|
||||
// SRAM's own always block has now seen rd_en
|
||||
// (this cycle) and scheduled rd_data_reg<=mem[addr]
|
||||
// for the NEXT cycle -- wait one more cycle before
|
||||
// trusting act_rd_data/wgt_rd_data.
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b1;
|
||||
end else if (read_ready) begin
|
||||
operand_valid <= 1'b1;
|
||||
input_data <= act_rd_data;
|
||||
weight_data <= wgt_rd_data;
|
||||
tile_last <= ({{(16-CNTW){1'b0}}, tile_idx} == n_tiles_reg - 16'd1);
|
||||
read_ready <= 1'b0;
|
||||
end else if (operand_valid && operand_ready) begin
|
||||
operand_valid <= 1'b0;
|
||||
if ({{(16-CNTW){1'b0}}, tile_idx} + 16'd1 == n_tiles_reg) begin
|
||||
job_active_reg <= 1'b0;
|
||||
state <= ST_WAIT_RESULT;
|
||||
end else begin
|
||||
tile_idx <= tile_idx + 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
ST_WAIT_RESULT: begin
|
||||
result_ready <= 1'b1;
|
||||
if (result_valid && result_ready) begin
|
||||
wr_mem_wdata <= result_addr_reg[0] ? {result_data, 8'h00} : {8'h00, result_data};
|
||||
wr_mem_lb_n <= result_addr_reg[0] ? 1'b1 : 1'b0;
|
||||
wr_mem_ub_n <= result_addr_reg[0] ? 1'b0 : 1'b1;
|
||||
state <= ST_WRITE_RES;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WRITE_RES: begin
|
||||
wr_mem_req <= 1'b1;
|
||||
wr_mem_addr <= result_addr_reg[ADDR_WIDTH-1:1];
|
||||
state <= ST_DONE;
|
||||
end
|
||||
|
||||
ST_DONE: begin
|
||||
wr_mem_req <= 1'b0;
|
||||
if (mem_ready) begin
|
||||
job_done <= 1'b1;
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,232 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP11: per-slot memory manager, REAL
|
||||
// WEIGHT PREFETCH variant ("_pf").
|
||||
//
|
||||
// Identical external interface to nms_memory_manager.v (drop-in, same
|
||||
// Director/Dependency-Manager side, same Neural-Processor-facing
|
||||
// operand/result streams) -- the ONLY change is the private weight
|
||||
// path: prefetch_engine.v (single-shot, one tile in flight, real
|
||||
// per-tile control-restart overhead, ERR-0013/STEP11's own analysis)
|
||||
// is replaced by weight_prefetch_engine.v (continuous multi-tile
|
||||
// fetch stream, configurable PREFETCH_DISTANCE lookahead window).
|
||||
//
|
||||
// nms_memory_manager.v itself is UNTOUCHED -- this file exists
|
||||
// alongside it specifically so "Current NMS" (baseline) and "NMS +
|
||||
// weight prefetch" remain independently reproducible for the A/B
|
||||
// comparison STEP11 explicitly requires.
|
||||
// ============================================================
|
||||
module nms_memory_manager_pf #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire job_start,
|
||||
input wire [ADDR_WIDTH-1:0] x_base,
|
||||
input wire [ADDR_WIDTH-1:0] w_base,
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] result_addr,
|
||||
output reg job_done,
|
||||
|
||||
output reg operand_valid,
|
||||
input wire operand_ready,
|
||||
output reg signed [DATA_WIDTH*P_IN-1:0] input_data,
|
||||
output reg signed [DATA_WIDTH*P_IN-1:0] weight_data,
|
||||
output reg tile_last,
|
||||
|
||||
input wire result_valid,
|
||||
output reg result_ready,
|
||||
input wire signed [DATA_WIDTH-1:0] result_data,
|
||||
|
||||
output wire job_active,
|
||||
output wire [ADDR_WIDTH-1:0] job_x_base,
|
||||
output wire [15:0] job_n_tiles,
|
||||
|
||||
input wire [ADDR_WIDTH-1:0] act_resident_tag,
|
||||
input wire [CNTW-1:0] act_resident_count,
|
||||
|
||||
output reg act_rd_en,
|
||||
output reg [TIW-1:0] act_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] act_rd_data,
|
||||
|
||||
// ---- weight SRAM fill port now driven by weight_prefetch_engine
|
||||
// (below), NOT by this FSM directly -- read port unchanged ----
|
||||
output wire wgt_fill_we,
|
||||
output wire [TIW-1:0] wgt_fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
output reg wgt_rd_en,
|
||||
output reg [TIW-1:0] wgt_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] wgt_rd_data,
|
||||
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
localparam ST_IDLE = 3'd0;
|
||||
localparam ST_RUN = 3'd1;
|
||||
localparam ST_WAIT_RESULT = 3'd2;
|
||||
localparam ST_WRITE_RES = 3'd3;
|
||||
localparam ST_DONE = 3'd4;
|
||||
|
||||
reg [2:0] state;
|
||||
reg job_active_reg;
|
||||
assign job_active = job_active_reg;
|
||||
assign job_x_base = x_base_reg;
|
||||
assign job_n_tiles = n_tiles_reg;
|
||||
|
||||
reg [ADDR_WIDTH-1:0] x_base_reg, w_base_reg, result_addr_reg;
|
||||
reg [15:0] n_tiles_reg;
|
||||
reg [CNTW-1:0] tile_idx;
|
||||
|
||||
reg read_issued;
|
||||
reg read_ready;
|
||||
|
||||
wire [CNTW-1:0] wgt_ready_count;
|
||||
|
||||
wire usable_act_count_valid = (act_resident_tag == x_base_reg);
|
||||
wire [CNTW-1:0] usable_act = usable_act_count_valid ? act_resident_count : {CNTW{1'b0}};
|
||||
wire can_present = ({{(16-CNTW){1'b0}}, tile_idx} < n_tiles_reg) &&
|
||||
(tile_idx < wgt_ready_count) &&
|
||||
(tile_idx < usable_act);
|
||||
|
||||
// ---- REAL weight prefetch engine (STEP11): continuous multi-tile
|
||||
// fetch stream, PREFETCH_DISTANCE-bounded lookahead ahead of
|
||||
// tile_idx (this module's own consumption pointer) ----
|
||||
weight_prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_wpf (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active_reg), .w_base(w_base_reg), .n_tiles(n_tiles_reg),
|
||||
.consumed_count(tile_idx),
|
||||
.wgt_fill_we(wgt_fill_we), .wgt_fill_addr(wgt_fill_addr), .wgt_fill_data(wgt_fill_data),
|
||||
.ready_count(wgt_ready_count),
|
||||
.mem_req(mem_req_wpf), .mem_wr(mem_wr_wpf), .mem_addr(mem_addr_wpf), .mem_wdata(mem_wdata_wpf),
|
||||
.mem_lb_n(mem_lb_n_wpf), .mem_ub_n(mem_ub_n_wpf),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
wire mem_req_wpf, mem_wr_wpf;
|
||||
wire [ADDR_WIDTH-1:0] mem_addr_wpf;
|
||||
wire [15:0] mem_wdata_wpf;
|
||||
wire mem_lb_n_wpf, mem_ub_n_wpf;
|
||||
|
||||
reg wr_mem_req;
|
||||
reg [ADDR_WIDTH-1:0] wr_mem_addr;
|
||||
reg [15:0] wr_mem_wdata;
|
||||
reg wr_mem_lb_n, wr_mem_ub_n;
|
||||
|
||||
wire wr_active = (state == ST_WRITE_RES) || (state == ST_DONE);
|
||||
assign mem_req = wr_active ? wr_mem_req : mem_req_wpf;
|
||||
assign mem_wr = wr_active ? 1'b1 : mem_wr_wpf;
|
||||
assign mem_addr = wr_active ? wr_mem_addr : mem_addr_wpf;
|
||||
assign mem_wdata = wr_active ? wr_mem_wdata : mem_wdata_wpf;
|
||||
assign mem_lb_n = wr_active ? wr_mem_lb_n : mem_lb_n_wpf;
|
||||
assign mem_ub_n = wr_active ? wr_mem_ub_n : mem_ub_n_wpf;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= ST_IDLE;
|
||||
job_active_reg <= 1'b0;
|
||||
job_done <= 1'b0;
|
||||
operand_valid <= 1'b0;
|
||||
tile_last <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
wr_mem_req <= 1'b0;
|
||||
wr_mem_lb_n <= 1'b1;
|
||||
wr_mem_ub_n <= 1'b1;
|
||||
end else begin
|
||||
job_done <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
if (job_start) begin
|
||||
x_base_reg <= x_base;
|
||||
w_base_reg <= w_base;
|
||||
n_tiles_reg <= n_tiles;
|
||||
result_addr_reg <= result_addr;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b0;
|
||||
operand_valid <= 1'b0;
|
||||
job_active_reg <= 1'b1;
|
||||
state <= ST_RUN;
|
||||
end
|
||||
end
|
||||
|
||||
ST_RUN: begin
|
||||
if (!operand_valid && !read_issued && !read_ready && can_present) begin
|
||||
act_rd_en <= 1'b1;
|
||||
act_rd_addr <= tile_idx[TIW-1:0];
|
||||
wgt_rd_en <= 1'b1;
|
||||
wgt_rd_addr <= tile_idx[TIW-1:0];
|
||||
read_issued <= 1'b1;
|
||||
end else if (read_issued) begin
|
||||
read_issued <= 1'b0;
|
||||
read_ready <= 1'b1;
|
||||
end else if (read_ready) begin
|
||||
operand_valid <= 1'b1;
|
||||
input_data <= act_rd_data;
|
||||
weight_data <= wgt_rd_data;
|
||||
tile_last <= ({{(16-CNTW){1'b0}}, tile_idx} == n_tiles_reg - 16'd1);
|
||||
read_ready <= 1'b0;
|
||||
end else if (operand_valid && operand_ready) begin
|
||||
operand_valid <= 1'b0;
|
||||
if ({{(16-CNTW){1'b0}}, tile_idx} + 16'd1 == n_tiles_reg) begin
|
||||
job_active_reg <= 1'b0;
|
||||
state <= ST_WAIT_RESULT;
|
||||
end else begin
|
||||
tile_idx <= tile_idx + 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
ST_WAIT_RESULT: begin
|
||||
result_ready <= 1'b1;
|
||||
if (result_valid && result_ready) begin
|
||||
wr_mem_wdata <= result_addr_reg[0] ? {result_data, 8'h00} : {8'h00, result_data};
|
||||
wr_mem_lb_n <= result_addr_reg[0] ? 1'b1 : 1'b0;
|
||||
wr_mem_ub_n <= result_addr_reg[0] ? 1'b0 : 1'b1;
|
||||
state <= ST_WRITE_RES;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WRITE_RES: begin
|
||||
wr_mem_req <= 1'b1;
|
||||
wr_mem_addr <= result_addr_reg[ADDR_WIDTH-1:1];
|
||||
state <= ST_DONE;
|
||||
end
|
||||
|
||||
ST_DONE: begin
|
||||
wr_mem_req <= 1'b0;
|
||||
if (mem_ready) begin
|
||||
job_done <= 1'b1;
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,310 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP13: per-slot memory manager,
|
||||
// PIPELINED CONTINUOUS TILE STREAM variant ("_stream").
|
||||
//
|
||||
// Identical external interface to nms_memory_manager_pf.v (drop-in,
|
||||
// same Director/Dependency-Manager side, same Neural-Processor-facing
|
||||
// operand/result streams, same weight_prefetch_engine.v instance) --
|
||||
// the ONLY change is INSIDE ST_RUN: the operand-delivery pipeline that
|
||||
// feeds neural_processor.v.
|
||||
//
|
||||
// EXP-0025 traced nms_memory_manager_pf.v's own ST_RUN state with a
|
||||
// zero-real-memory-latency configuration and found it costs EXACTLY 4
|
||||
// cycles/tile in steady state (read_issued -> read_ready -> present
|
||||
// -> consumed, a strictly sequential if/else-if chain with ZERO
|
||||
// overlap between consecutive tiles), even though neither side of the
|
||||
// interface requires it: the local activation/weight SRAMs
|
||||
// (nms_activation_replicated.v / nms_weight_packed.v) have only a
|
||||
// 1-cycle rd_en-to-data latency, and neural_processor.v's own
|
||||
// operand_ready is held continuously high through the whole
|
||||
// NP_WAIT_OPERANDS phase (its datapath is explicitly designed to
|
||||
// accept a new tile every cycle). That 4-cycles/tile serialization
|
||||
// was found to account for 93.4% of EXP-0024's real, measured
|
||||
// "non-memory" cycle floor (DEC-0024) -- the dominant real bottleneck,
|
||||
// NOT per-job dispatch overhead.
|
||||
//
|
||||
// This variant replaces ST_RUN's 4-state chain with a pipelined
|
||||
// read-ahead design:
|
||||
// - `rd_ptr` (CNTW bits): the tile index whose SRAM read has been
|
||||
// (or is about to be) ISSUED -- independent of, and normally one
|
||||
// tile AHEAD of, `tile_idx` (the CONSUMPTION pointer, i.e. how
|
||||
// many tiles neural_processor.v has actually accepted).
|
||||
// - a 1-deep skid buffer (`buf_valid`/`buf_input`/`buf_weight`/
|
||||
// `buf_last`) holds one tile's fully-read SRAM data, presented to
|
||||
// NP as `operand_valid`/`input_data`/`weight_data`/`tile_last`.
|
||||
// - every cycle: if a read was issued last cycle (`rd_pending`), its
|
||||
// data is now valid (1-cycle SRAM latency) and is captured into
|
||||
// the skid buffer; independently, a NEW read is issued for
|
||||
// `rd_ptr` whenever it is legal to do so (in bounds, weight+
|
||||
// activation ready) AND the skid buffer will not overflow (it is
|
||||
// empty, or being drained -- consumed -- this very same cycle).
|
||||
// Since NP's own operand_ready is continuously high through the tile-
|
||||
// loading phase, the skid buffer is drained every cycle it is full,
|
||||
// so a new read can be issued every cycle too: sustained ~1 cycle/
|
||||
// tile, down from 4 -- a real ~4x reduction in the dominant component
|
||||
// of EXP-0024's measured floor.
|
||||
//
|
||||
// `tile_idx` (the CONSUMPTION pointer) is still what is fed to
|
||||
// weight_prefetch_engine.v's own `consumed_count` port -- its
|
||||
// external contract (bound the lookahead window against how far the
|
||||
// CONSUMER has progressed) is unchanged; only the local SRAM
|
||||
// read-issue pointer (`rd_ptr`) is new, and it can run up to ONE tile
|
||||
// ahead of `tile_idx` (the skid buffer's own depth), same as before
|
||||
// conceptually (read_issued/read_ready already implied a similar
|
||||
// small lookahead, just serialized rather than pipelined).
|
||||
//
|
||||
// nms_memory_manager_pf.v itself is UNTOUCHED -- this file exists
|
||||
// alongside it (and alongside the original nms_memory_manager.v) so
|
||||
// all three ("Current NMS", "NMS + weight prefetch",
|
||||
// "NMS + weight prefetch + continuous tile stream") remain
|
||||
// independently reproducible for A/B/C comparison.
|
||||
// ============================================================
|
||||
module nms_memory_manager_stream #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire job_start,
|
||||
input wire [ADDR_WIDTH-1:0] x_base,
|
||||
input wire [ADDR_WIDTH-1:0] w_base,
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] result_addr,
|
||||
output reg job_done,
|
||||
|
||||
output wire operand_valid,
|
||||
input wire operand_ready,
|
||||
output wire signed [DATA_WIDTH*P_IN-1:0] input_data,
|
||||
output wire signed [DATA_WIDTH*P_IN-1:0] weight_data,
|
||||
output wire tile_last,
|
||||
|
||||
input wire result_valid,
|
||||
output reg result_ready,
|
||||
input wire signed [DATA_WIDTH-1:0] result_data,
|
||||
|
||||
output wire job_active,
|
||||
output wire [ADDR_WIDTH-1:0] job_x_base,
|
||||
output wire [15:0] job_n_tiles,
|
||||
|
||||
input wire [ADDR_WIDTH-1:0] act_resident_tag,
|
||||
input wire [CNTW-1:0] act_resident_count,
|
||||
|
||||
output reg act_rd_en,
|
||||
output reg [TIW-1:0] act_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] act_rd_data,
|
||||
|
||||
// ---- weight SRAM fill port driven by weight_prefetch_engine
|
||||
// (below), NOT by this FSM directly -- read port unchanged ----
|
||||
output wire wgt_fill_we,
|
||||
output wire [TIW-1:0] wgt_fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
output reg wgt_rd_en,
|
||||
output reg [TIW-1:0] wgt_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] wgt_rd_data,
|
||||
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
localparam ST_IDLE = 3'd0;
|
||||
localparam ST_RUN = 3'd1;
|
||||
localparam ST_WAIT_RESULT = 3'd2;
|
||||
localparam ST_WRITE_RES = 3'd3;
|
||||
localparam ST_DONE = 3'd4;
|
||||
|
||||
reg [2:0] state;
|
||||
reg job_active_reg;
|
||||
assign job_active = job_active_reg;
|
||||
assign job_x_base = x_base_reg;
|
||||
assign job_n_tiles = n_tiles_reg;
|
||||
|
||||
reg [ADDR_WIDTH-1:0] x_base_reg, w_base_reg, result_addr_reg;
|
||||
reg [15:0] n_tiles_reg;
|
||||
reg [CNTW-1:0] tile_idx; // CONSUMPTION pointer (tiles handed to NP so far)
|
||||
reg [CNTW-1:0] rd_ptr; // READ-ISSUE pointer (tiles whose SRAM read has been issued)
|
||||
|
||||
wire [CNTW-1:0] wgt_ready_count;
|
||||
|
||||
wire usable_act_count_valid = (act_resident_tag == x_base_reg);
|
||||
wire [CNTW-1:0] usable_act = usable_act_count_valid ? act_resident_count : {CNTW{1'b0}};
|
||||
|
||||
// Gating for the READ-ISSUE pointer (rd_ptr), same semantics as
|
||||
// the old can_present but evaluated against rd_ptr instead of
|
||||
// tile_idx, since reads may now run ahead of consumption.
|
||||
wire can_issue_rd = ({{(16-CNTW){1'b0}}, rd_ptr} < n_tiles_reg) &&
|
||||
(rd_ptr < wgt_ready_count) &&
|
||||
(rd_ptr < usable_act);
|
||||
|
||||
// ---- pipelined read-ahead + 1-deep skid buffer ----
|
||||
reg rd_pending; // a read issued last cycle; its data is valid THIS cycle
|
||||
reg [CNTW-1:0] rd_pending_tile;
|
||||
reg rd_pending_last;
|
||||
reg buf_valid;
|
||||
reg signed [DATA_WIDTH*P_IN-1:0] buf_input, buf_weight;
|
||||
reg buf_last;
|
||||
|
||||
assign operand_valid = buf_valid;
|
||||
assign input_data = buf_input;
|
||||
assign weight_data = buf_weight;
|
||||
assign tile_last = buf_last;
|
||||
|
||||
// May issue a new read this cycle iff the skid buffer will not
|
||||
// overflow: it's currently empty, or it is being drained
|
||||
// (consumed) THIS cycle.
|
||||
wire buf_will_be_free = !buf_valid || (operand_valid && operand_ready);
|
||||
wire issue_rd_now = (state == ST_RUN) && can_issue_rd && buf_will_be_free;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= ST_IDLE;
|
||||
job_active_reg <= 1'b0;
|
||||
job_done <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
rd_ptr <= {CNTW{1'b0}};
|
||||
rd_pending <= 1'b0;
|
||||
buf_valid <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
wr_mem_req <= 1'b0;
|
||||
wr_mem_lb_n <= 1'b1;
|
||||
wr_mem_ub_n <= 1'b1;
|
||||
end else begin
|
||||
job_done <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
if (job_start) begin
|
||||
x_base_reg <= x_base;
|
||||
w_base_reg <= w_base;
|
||||
n_tiles_reg <= n_tiles;
|
||||
result_addr_reg <= result_addr;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
rd_ptr <= {CNTW{1'b0}};
|
||||
rd_pending <= 1'b0;
|
||||
buf_valid <= 1'b0;
|
||||
job_active_reg <= 1'b1;
|
||||
state <= ST_RUN;
|
||||
end
|
||||
end
|
||||
|
||||
ST_RUN: begin
|
||||
// ---- Step 1: a read issued LAST cycle lands now ----
|
||||
if (rd_pending) begin
|
||||
buf_valid <= 1'b1;
|
||||
buf_input <= act_rd_data;
|
||||
buf_weight <= wgt_rd_data;
|
||||
buf_last <= rd_pending_last;
|
||||
end else if (operand_valid && operand_ready) begin
|
||||
// no new data arriving this cycle -- if the
|
||||
// buffer is being drained and nothing refills
|
||||
// it, it goes empty.
|
||||
buf_valid <= 1'b0;
|
||||
end
|
||||
|
||||
// ---- Step 2: consumption bookkeeping ----
|
||||
if (operand_valid && operand_ready) begin
|
||||
if ({{(16-CNTW){1'b0}}, tile_idx} + 16'd1 == n_tiles_reg) begin
|
||||
job_active_reg <= 1'b0;
|
||||
state <= ST_WAIT_RESULT;
|
||||
end else begin
|
||||
tile_idx <= tile_idx + 1'b1;
|
||||
end
|
||||
end
|
||||
|
||||
// ---- Step 3: issue the NEXT read, if legal ----
|
||||
if (issue_rd_now) begin
|
||||
act_rd_en <= 1'b1;
|
||||
act_rd_addr <= rd_ptr[TIW-1:0];
|
||||
wgt_rd_en <= 1'b1;
|
||||
wgt_rd_addr <= rd_ptr[TIW-1:0];
|
||||
rd_pending <= 1'b1;
|
||||
rd_pending_tile <= rd_ptr;
|
||||
rd_pending_last <= ({{(16-CNTW){1'b0}}, rd_ptr} == n_tiles_reg - 16'd1);
|
||||
rd_ptr <= rd_ptr + 1'b1;
|
||||
end else begin
|
||||
rd_pending <= 1'b0;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WAIT_RESULT: begin
|
||||
result_ready <= 1'b1;
|
||||
if (result_valid && result_ready) begin
|
||||
wr_mem_wdata <= result_addr_reg[0] ? {result_data, 8'h00} : {8'h00, result_data};
|
||||
wr_mem_lb_n <= result_addr_reg[0] ? 1'b1 : 1'b0;
|
||||
wr_mem_ub_n <= result_addr_reg[0] ? 1'b0 : 1'b1;
|
||||
state <= ST_WRITE_RES;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WRITE_RES: begin
|
||||
wr_mem_req <= 1'b1;
|
||||
wr_mem_addr <= result_addr_reg[ADDR_WIDTH-1:1];
|
||||
state <= ST_DONE;
|
||||
end
|
||||
|
||||
ST_DONE: begin
|
||||
wr_mem_req <= 1'b0;
|
||||
if (mem_ready) begin
|
||||
job_done <= 1'b1;
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
// ---- REAL weight prefetch engine (STEP11): unchanged from
|
||||
// nms_memory_manager_pf.v -- consumed_count is still the
|
||||
// CONSUMPTION pointer (tile_idx), not the read-issue pointer
|
||||
// (rd_ptr): the engine's own lookahead window is bounded by how
|
||||
// far the CONSUMER has progressed, exactly as before. ----
|
||||
weight_prefetch_engine #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_wpf (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active_reg), .w_base(w_base_reg), .n_tiles(n_tiles_reg),
|
||||
.consumed_count(tile_idx),
|
||||
.wgt_fill_we(wgt_fill_we), .wgt_fill_addr(wgt_fill_addr), .wgt_fill_data(wgt_fill_data),
|
||||
.ready_count(wgt_ready_count),
|
||||
.mem_req(mem_req_wpf), .mem_wr(mem_wr_wpf), .mem_addr(mem_addr_wpf), .mem_wdata(mem_wdata_wpf),
|
||||
.mem_lb_n(mem_lb_n_wpf), .mem_ub_n(mem_ub_n_wpf),
|
||||
.mem_rdata(mem_rdata), .mem_ready(mem_ready)
|
||||
);
|
||||
wire mem_req_wpf, mem_wr_wpf;
|
||||
wire [ADDR_WIDTH-1:0] mem_addr_wpf;
|
||||
wire [15:0] mem_wdata_wpf;
|
||||
wire mem_lb_n_wpf, mem_ub_n_wpf;
|
||||
|
||||
reg wr_mem_req;
|
||||
reg [ADDR_WIDTH-1:0] wr_mem_addr;
|
||||
reg [15:0] wr_mem_wdata;
|
||||
reg wr_mem_lb_n, wr_mem_ub_n;
|
||||
|
||||
wire wr_active = (state == ST_WRITE_RES) || (state == ST_DONE);
|
||||
assign mem_req = wr_active ? wr_mem_req : mem_req_wpf;
|
||||
assign mem_wr = wr_active ? 1'b1 : mem_wr_wpf;
|
||||
assign mem_addr = wr_active ? wr_mem_addr : mem_addr_wpf;
|
||||
assign mem_wdata = wr_active ? wr_mem_wdata : mem_wdata_wpf;
|
||||
assign mem_lb_n = wr_active ? wr_mem_lb_n : mem_lb_n_wpf;
|
||||
assign mem_ub_n = wr_active ? wr_mem_ub_n : mem_ub_n_wpf;
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,321 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP13: per-slot memory manager,
|
||||
// PIPELINED CONTINUOUS TILE STREAM variant ("_stream").
|
||||
//
|
||||
// Identical external interface to nms_memory_manager_pf.v (drop-in,
|
||||
// same Director/Dependency-Manager side, same Neural-Processor-facing
|
||||
// operand/result streams, same weight_prefetch_engine.v instance) --
|
||||
// the ONLY change is INSIDE ST_RUN: the operand-delivery pipeline that
|
||||
// feeds neural_processor.v.
|
||||
//
|
||||
// EXP-0025 traced nms_memory_manager_pf.v's own ST_RUN state with a
|
||||
// zero-real-memory-latency configuration and found it costs EXACTLY 4
|
||||
// cycles/tile in steady state (read_issued -> read_ready -> present
|
||||
// -> consumed, a strictly sequential if/else-if chain with ZERO
|
||||
// overlap between consecutive tiles), even though neither side of the
|
||||
// interface requires it: the local activation/weight SRAMs
|
||||
// (nms_activation_replicated.v / nms_weight_packed.v) have only a
|
||||
// 1-cycle rd_en-to-data latency, and neural_processor.v's own
|
||||
// operand_ready is held continuously high through the whole
|
||||
// NP_WAIT_OPERANDS phase (its datapath is explicitly designed to
|
||||
// accept a new tile every cycle). That 4-cycles/tile serialization
|
||||
// was found to account for 93.4% of EXP-0024's real, measured
|
||||
// "non-memory" cycle floor (DEC-0024) -- the dominant real bottleneck,
|
||||
// NOT per-job dispatch overhead.
|
||||
//
|
||||
// This variant replaces ST_RUN's 4-state chain with a pipelined
|
||||
// read-ahead design:
|
||||
// - `rd_ptr` (CNTW bits): the tile index whose SRAM read has been
|
||||
// (or is about to be) ISSUED -- independent of, and normally one
|
||||
// tile AHEAD of, `tile_idx` (the CONSUMPTION pointer, i.e. how
|
||||
// many tiles neural_processor.v has actually accepted).
|
||||
// - a 1-deep skid buffer (`buf_valid`/`buf_input`/`buf_weight`/
|
||||
// `buf_last`) holds one tile's fully-read SRAM data, presented to
|
||||
// NP as `operand_valid`/`input_data`/`weight_data`/`tile_last`.
|
||||
// - every cycle: if a read was issued last cycle (`rd_pending`), its
|
||||
// data is now valid (1-cycle SRAM latency) and is captured into
|
||||
// the skid buffer; independently, a NEW read is issued for
|
||||
// `rd_ptr` whenever it is legal to do so (in bounds, weight+
|
||||
// activation ready) AND the skid buffer will not overflow (it is
|
||||
// empty, or being drained -- consumed -- this very same cycle).
|
||||
// Since NP's own operand_ready is continuously high through the tile-
|
||||
// loading phase, the skid buffer is drained every cycle it is full,
|
||||
// so a new read can be issued every cycle too: sustained ~1 cycle/
|
||||
// tile, down from 4 -- a real ~4x reduction in the dominant component
|
||||
// of EXP-0024's measured floor.
|
||||
//
|
||||
// `tile_idx` (the CONSUMPTION pointer) is still what is fed to
|
||||
// weight_prefetch_engine.v's own `consumed_count` port -- its
|
||||
// external contract (bound the lookahead window against how far the
|
||||
// CONSUMER has progressed) is unchanged; only the local SRAM
|
||||
// read-issue pointer (`rd_ptr`) is new, and it can run up to ONE tile
|
||||
// ahead of `tile_idx` (the skid buffer's own depth), same as before
|
||||
// conceptually (read_issued/read_ready already implied a similar
|
||||
// small lookahead, just serialized rather than pipelined).
|
||||
//
|
||||
// nms_memory_manager_pf.v itself is UNTOUCHED -- this file exists
|
||||
// alongside it (and alongside the original nms_memory_manager.v) so
|
||||
// all three ("Current NMS", "NMS + weight prefetch",
|
||||
// "NMS + weight prefetch + continuous tile stream") remain
|
||||
// independently reproducible for A/B/C comparison.
|
||||
// ============================================================
|
||||
module nms_memory_manager_stream_wide #(
|
||||
parameter MEM_DATA_WIDTH = 64,
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire job_start,
|
||||
input wire [ADDR_WIDTH-1:0] x_base,
|
||||
input wire [ADDR_WIDTH-1:0] w_base,
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] result_addr,
|
||||
output reg job_done,
|
||||
|
||||
output wire operand_valid,
|
||||
input wire operand_ready,
|
||||
output wire signed [DATA_WIDTH*P_IN-1:0] input_data,
|
||||
output wire signed [DATA_WIDTH*P_IN-1:0] weight_data,
|
||||
output wire tile_last,
|
||||
|
||||
input wire result_valid,
|
||||
output reg result_ready,
|
||||
input wire signed [DATA_WIDTH-1:0] result_data,
|
||||
|
||||
output wire job_active,
|
||||
output wire [ADDR_WIDTH-1:0] job_x_base,
|
||||
output wire [15:0] job_n_tiles,
|
||||
|
||||
input wire [ADDR_WIDTH-1:0] act_resident_tag,
|
||||
input wire [CNTW-1:0] act_resident_count,
|
||||
|
||||
output reg act_rd_en,
|
||||
output reg [TIW-1:0] act_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] act_rd_data,
|
||||
|
||||
// ---- weight SRAM fill port driven by weight_prefetch_engine
|
||||
// (below), NOT by this FSM directly -- read port unchanged ----
|
||||
output wire wgt_fill_we,
|
||||
output wire [TIW-1:0] wgt_fill_addr,
|
||||
output wire [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
output reg wgt_rd_en,
|
||||
output reg [TIW-1:0] wgt_rd_addr,
|
||||
input wire signed [DATA_WIDTH*P_IN-1:0] wgt_rd_data,
|
||||
|
||||
// ---- result write-back only in this experimental variant (real
|
||||
// 16-bit-word protocol, matches memory_interface.v exactly) --
|
||||
// weight fetch uses the separate wide logical port below instead
|
||||
// of sharing this one, since this variant exists purely to
|
||||
// explore logical weight-path width in isolation (STEP14 Part A).
|
||||
output wire mem_req,
|
||||
output wire mem_wr,
|
||||
output wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready,
|
||||
|
||||
// ---- separate wide logical weight-fetch port (ideal_memory_
|
||||
// model_wide.v or a real packing adapter, STEP14 Part A/A5) ----
|
||||
output wire wide_mem_req,
|
||||
output wire [ADDR_WIDTH-1:0] wide_mem_addr,
|
||||
input wire [MEM_DATA_WIDTH-1:0] wide_mem_rdata,
|
||||
input wire wide_mem_ready
|
||||
);
|
||||
|
||||
localparam ST_IDLE = 3'd0;
|
||||
localparam ST_RUN = 3'd1;
|
||||
localparam ST_WAIT_RESULT = 3'd2;
|
||||
localparam ST_WRITE_RES = 3'd3;
|
||||
localparam ST_DONE = 3'd4;
|
||||
|
||||
reg [2:0] state;
|
||||
reg job_active_reg;
|
||||
assign job_active = job_active_reg;
|
||||
assign job_x_base = x_base_reg;
|
||||
assign job_n_tiles = n_tiles_reg;
|
||||
|
||||
reg [ADDR_WIDTH-1:0] x_base_reg, w_base_reg, result_addr_reg;
|
||||
reg [15:0] n_tiles_reg;
|
||||
reg [CNTW-1:0] tile_idx; // CONSUMPTION pointer (tiles handed to NP so far)
|
||||
reg [CNTW-1:0] rd_ptr; // READ-ISSUE pointer (tiles whose SRAM read has been issued)
|
||||
|
||||
wire [CNTW-1:0] wgt_ready_count;
|
||||
|
||||
wire usable_act_count_valid = (act_resident_tag == x_base_reg);
|
||||
wire [CNTW-1:0] usable_act = usable_act_count_valid ? act_resident_count : {CNTW{1'b0}};
|
||||
|
||||
// Gating for the READ-ISSUE pointer (rd_ptr), same semantics as
|
||||
// the old can_present but evaluated against rd_ptr instead of
|
||||
// tile_idx, since reads may now run ahead of consumption.
|
||||
wire can_issue_rd = ({{(16-CNTW){1'b0}}, rd_ptr} < n_tiles_reg) &&
|
||||
(rd_ptr < wgt_ready_count) &&
|
||||
(rd_ptr < usable_act);
|
||||
|
||||
// ---- pipelined read-ahead + 1-deep skid buffer ----
|
||||
reg rd_pending; // a read issued last cycle; its data is valid THIS cycle
|
||||
reg [CNTW-1:0] rd_pending_tile;
|
||||
reg rd_pending_last;
|
||||
reg buf_valid;
|
||||
reg signed [DATA_WIDTH*P_IN-1:0] buf_input, buf_weight;
|
||||
reg buf_last;
|
||||
|
||||
assign operand_valid = buf_valid;
|
||||
assign input_data = buf_input;
|
||||
assign weight_data = buf_weight;
|
||||
assign tile_last = buf_last;
|
||||
|
||||
// May issue a new read this cycle iff the skid buffer will not
|
||||
// overflow: it's currently empty, or it is being drained
|
||||
// (consumed) THIS cycle.
|
||||
wire buf_will_be_free = !buf_valid || (operand_valid && operand_ready);
|
||||
wire issue_rd_now = (state == ST_RUN) && can_issue_rd && buf_will_be_free;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= ST_IDLE;
|
||||
job_active_reg <= 1'b0;
|
||||
job_done <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
rd_ptr <= {CNTW{1'b0}};
|
||||
rd_pending <= 1'b0;
|
||||
buf_valid <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
wr_mem_req <= 1'b0;
|
||||
wr_mem_lb_n <= 1'b1;
|
||||
wr_mem_ub_n <= 1'b1;
|
||||
end else begin
|
||||
job_done <= 1'b0;
|
||||
act_rd_en <= 1'b0;
|
||||
wgt_rd_en <= 1'b0;
|
||||
result_ready <= 1'b0;
|
||||
|
||||
case (state)
|
||||
ST_IDLE: begin
|
||||
if (job_start) begin
|
||||
x_base_reg <= x_base;
|
||||
w_base_reg <= w_base;
|
||||
n_tiles_reg <= n_tiles;
|
||||
result_addr_reg <= result_addr;
|
||||
tile_idx <= {CNTW{1'b0}};
|
||||
rd_ptr <= {CNTW{1'b0}};
|
||||
rd_pending <= 1'b0;
|
||||
buf_valid <= 1'b0;
|
||||
job_active_reg <= 1'b1;
|
||||
state <= ST_RUN;
|
||||
end
|
||||
end
|
||||
|
||||
ST_RUN: begin
|
||||
// ---- Step 1: a read issued LAST cycle lands now ----
|
||||
if (rd_pending) begin
|
||||
buf_valid <= 1'b1;
|
||||
buf_input <= act_rd_data;
|
||||
buf_weight <= wgt_rd_data;
|
||||
buf_last <= rd_pending_last;
|
||||
end else if (operand_valid && operand_ready) begin
|
||||
// no new data arriving this cycle -- if the
|
||||
// buffer is being drained and nothing refills
|
||||
// it, it goes empty.
|
||||
buf_valid <= 1'b0;
|
||||
end
|
||||
|
||||
// ---- Step 2: consumption bookkeeping ----
|
||||
if (operand_valid && operand_ready) begin
|
||||
if ({{(16-CNTW){1'b0}}, tile_idx} + 16'd1 == n_tiles_reg) begin
|
||||
job_active_reg <= 1'b0;
|
||||
state <= ST_WAIT_RESULT;
|
||||
end else begin
|
||||
tile_idx <= tile_idx + 1'b1;
|
||||
end
|
||||
end
|
||||
|
||||
// ---- Step 3: issue the NEXT read, if legal ----
|
||||
if (issue_rd_now) begin
|
||||
act_rd_en <= 1'b1;
|
||||
act_rd_addr <= rd_ptr[TIW-1:0];
|
||||
wgt_rd_en <= 1'b1;
|
||||
wgt_rd_addr <= rd_ptr[TIW-1:0];
|
||||
rd_pending <= 1'b1;
|
||||
rd_pending_tile <= rd_ptr;
|
||||
rd_pending_last <= ({{(16-CNTW){1'b0}}, rd_ptr} == n_tiles_reg - 16'd1);
|
||||
rd_ptr <= rd_ptr + 1'b1;
|
||||
end else begin
|
||||
rd_pending <= 1'b0;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WAIT_RESULT: begin
|
||||
result_ready <= 1'b1;
|
||||
if (result_valid && result_ready) begin
|
||||
wr_mem_wdata <= result_addr_reg[0] ? {result_data, 8'h00} : {8'h00, result_data};
|
||||
wr_mem_lb_n <= result_addr_reg[0] ? 1'b1 : 1'b0;
|
||||
wr_mem_ub_n <= result_addr_reg[0] ? 1'b0 : 1'b1;
|
||||
state <= ST_WRITE_RES;
|
||||
end
|
||||
end
|
||||
|
||||
ST_WRITE_RES: begin
|
||||
wr_mem_req <= 1'b1;
|
||||
wr_mem_addr <= result_addr_reg[ADDR_WIDTH-1:1];
|
||||
state <= ST_DONE;
|
||||
end
|
||||
|
||||
ST_DONE: begin
|
||||
wr_mem_req <= 1'b0;
|
||||
if (mem_ready) begin
|
||||
job_done <= 1'b1;
|
||||
state <= ST_IDLE;
|
||||
end
|
||||
end
|
||||
|
||||
default: state <= ST_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
// ---- REAL weight prefetch engine (STEP11): unchanged from
|
||||
// nms_memory_manager_pf.v -- consumed_count is still the
|
||||
// CONSUMPTION pointer (tile_idx), not the read-issue pointer
|
||||
// (rd_ptr): the engine's own lookahead window is bounded by how
|
||||
// far the CONSUMER has progressed, exactly as before. ----
|
||||
weight_prefetch_engine_wide #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE),
|
||||
.MEM_DATA_WIDTH(MEM_DATA_WIDTH)
|
||||
) u_wpf (
|
||||
.clk(clk), .rst(rst),
|
||||
.job_active(job_active_reg), .w_base(w_base_reg), .n_tiles(n_tiles_reg),
|
||||
.consumed_count(tile_idx),
|
||||
.wgt_fill_we(wgt_fill_we), .wgt_fill_addr(wgt_fill_addr), .wgt_fill_data(wgt_fill_data),
|
||||
.ready_count(wgt_ready_count),
|
||||
.mem_req(wide_mem_req), .mem_addr(wide_mem_addr),
|
||||
.mem_rdata(wide_mem_rdata), .mem_ready(wide_mem_ready)
|
||||
);
|
||||
|
||||
// Result write-back has the real 16-bit port entirely to itself
|
||||
// in this variant (no mux needed -- weight fetch lives on the
|
||||
// separate wide port above).
|
||||
reg wr_mem_req;
|
||||
reg [ADDR_WIDTH-1:0] wr_mem_addr;
|
||||
reg [15:0] wr_mem_wdata;
|
||||
reg wr_mem_lb_n, wr_mem_ub_n;
|
||||
|
||||
assign mem_req = wr_mem_req;
|
||||
assign mem_wr = 1'b1;
|
||||
assign mem_addr = wr_mem_addr;
|
||||
assign mem_wdata = wr_mem_wdata;
|
||||
assign mem_lb_n = wr_mem_lb_n;
|
||||
assign mem_ub_n = wr_mem_ub_n;
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,122 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
nms_dataflow_core #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,123 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_actfix #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_actfix #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,123 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_actfix2 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_actfix2 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,233 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_dual32 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n,
|
||||
|
||||
// ---- STEP15 (continuation): real dual-chip 32-bit physical
|
||||
// weight-fetch interface, TWO independent 16-bit PSRAM chips.
|
||||
//
|
||||
// Address and control (CE#/OE#/WE#/LB#/UB#/ZZ#) are SHARED, ONE
|
||||
// set of real FPGA pins, not duplicated per chip: both
|
||||
// psram_controller.v instances inside psram_controller_dual32.v
|
||||
// are fed byte-for-byte IDENTICAL mem_req/mem_wr/mem_addr/
|
||||
// mem_lb_n/mem_ub_n every cycle (that IS the whole synchronization
|
||||
// mechanism, EXP-0037/DEC-0029), so their own address/control
|
||||
// OUTPUTS are, by construction, always identical too -- a real
|
||||
// PCB ties ONE FPGA pin's own net to BOTH chips' corresponding
|
||||
// input pin (a simple fan-out trace, not a bus-contention
|
||||
// concern, since these are FPGA OUTPUTS driving PASSIVE chip
|
||||
// inputs). Only DQ (bidirectional, chip-specific data) genuinely
|
||||
// needs independent pins per chip.
|
||||
//
|
||||
// An earlier draft exposed FULLY separate psram0_*/psram1_*
|
||||
// address+control pins (90 total pins for this interface) --
|
||||
// real synthesis+P&R (EXP-0039) found this EXCEEDS the real
|
||||
// package's own I/O budget by exactly 2 pins (245 total
|
||||
// TRELLIS_IO on the LFE5U-45F-8CABGA381; 157 already committed by
|
||||
// the existing registration+single-chip-PSRAM interface, leaving
|
||||
// 88 free; 90 needed). Sharing address/control (a real, valid
|
||||
// PCB technique, not a synthesis trick) drops the requirement to
|
||||
// 23(addr)+6(ctrl)+16(chip0 dq)+16(chip1 dq) = 61 pins, which
|
||||
// fits comfortably (61 < 88).
|
||||
output wire [ADDR_WIDTH-1:0] psram01_a,
|
||||
output wire psram01_ce_n,
|
||||
output wire psram01_oe_n,
|
||||
output wire psram01_we_n,
|
||||
output wire psram01_lb_n,
|
||||
output wire psram01_ub_n,
|
||||
output wire psram01_zz_n,
|
||||
inout wire [15:0] psram0_dq,
|
||||
inout wire [15:0] psram1_dq
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_req;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
|
||||
wire [32*N_SLOTS-1:0] wide_slot_mem_rdata;
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_dual32 #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
|
||||
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
|
||||
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
// ================================================================
|
||||
// STEP15 (continuation): real dual-chip 32-bit weight-fetch
|
||||
// backend -- a SEPARATE arbiter (slot_mem_arbiter_wide.v, N_PORTS=
|
||||
// N_SLOTS, no activation-fill port sharing this one) feeding the
|
||||
// real psram_controller_dual32.v (two independent, real,
|
||||
// UNMODIFIED psram_controller.v instances, DEC-0029's own
|
||||
// selected architecture). Weight fetch never writes: mem_wr/
|
||||
// mem_wdata tied to 0 at the arbiter's own per-port inputs, and
|
||||
// the dual32 controller's own 2-bit lane-enables tied to
|
||||
// "always both bytes of both chips" (2'b00), matching
|
||||
// weight_prefetch_engine_wide.v's own always-full-word read
|
||||
// behavior exactly (same convention as its real 16-bit
|
||||
// counterpart, weight_prefetch_engine.v).
|
||||
// ================================================================
|
||||
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
|
||||
wire [32*N_SLOTS-1:0] wide_s_wdata = {(32*N_SLOTS){1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
|
||||
|
||||
wire wide_arb_m_req, wide_arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
|
||||
wire [31:0] wide_arb_m_wdata;
|
||||
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
|
||||
wire [31:0] wide_arb_m_rdata;
|
||||
wire wide_arb_m_ready;
|
||||
|
||||
slot_mem_arbiter_wide #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(32)
|
||||
) u_arbiter_wide (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
|
||||
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
|
||||
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
|
||||
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
|
||||
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
|
||||
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
|
||||
);
|
||||
|
||||
wire dual32_lane_sync_error;
|
||||
// chip1's own address/control outputs (identical in value to
|
||||
// chip0's, per the shared-input synchronization argument above) --
|
||||
// deliberately left unconnected to any top-level pin.
|
||||
wire [ADDR_WIDTH-1:0] unused_psram1_a;
|
||||
wire unused_psram1_ce_n, unused_psram1_oe_n, unused_psram1_we_n;
|
||||
wire unused_psram1_lb_n, unused_psram1_ub_n, unused_psram1_zz_n;
|
||||
|
||||
psram_controller_dual32 #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_dual32 (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(wide_arb_m_req), .mem_wr(1'b0), .mem_addr(wide_arb_m_addr), .mem_wdata(32'h0),
|
||||
.mem_lb_n(2'b00), .mem_ub_n(2'b00),
|
||||
.mem_rdata(wide_arb_m_rdata), .mem_ready(wide_arb_m_ready),
|
||||
.lane_sync_error(dual32_lane_sync_error),
|
||||
.psram0_a(psram01_a), .psram0_dq(psram0_dq),
|
||||
.psram0_ce_n(psram01_ce_n), .psram0_oe_n(psram01_oe_n), .psram0_we_n(psram01_we_n),
|
||||
.psram0_lb_n(psram01_lb_n), .psram0_ub_n(psram01_ub_n), .psram0_zz_n(psram01_zz_n),
|
||||
// chip1's own address/control outputs are byte-for-byte
|
||||
// identical to chip0's (EXP-0037/DEC-0029) -- left as
|
||||
// internal-only wires here, not exposed as separate top-level
|
||||
// pins; a real PCB fans the SAME psram01_* net out to both
|
||||
// chips' corresponding input pin instead.
|
||||
.psram1_a(unused_psram1_a), .psram1_dq(psram1_dq),
|
||||
.psram1_ce_n(unused_psram1_ce_n), .psram1_oe_n(unused_psram1_oe_n), .psram1_we_n(unused_psram1_we_n),
|
||||
.psram1_lb_n(unused_psram1_lb_n), .psram1_ub_n(unused_psram1_ub_n), .psram1_zz_n(unused_psram1_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,123 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_pf #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_pf #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,205 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
//
|
||||
// STEP16 Phase 5: forked from nms_neural_multiprocessor_dual32.v
|
||||
// (STEP15's real dual-chip-32-bit weight-fetch variant). The ORIGINAL
|
||||
// single-chip 16-bit PSRAM activation+writeback path (memory_
|
||||
// interface.v -> psram_controller.v) is BYTE-FOR-BYTE UNCHANGED --
|
||||
// STEP15 itself established this path sits at only 4.4% utilization
|
||||
// and was never the bandwidth bottleneck under test. ONLY the wide
|
||||
// weight-fetch backend changes: nms_dataflow_core_sdram.v (64-bit
|
||||
// wide port) -> slot_mem_arbiter_wide.v (reused unchanged at
|
||||
// DATA_WIDTH=64) -> sdram_weight_backend.v (the real, isolated-and-
|
||||
// validated AS4C4M16SA-6TIN SDRAM controller, BURST_LEN=4), replacing
|
||||
// psram_controller_dual32.v exactly at this one point in the design.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_sdram #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n,
|
||||
|
||||
// ---- STEP16 Phase 5: real single-chip SDRAM physical weight-
|
||||
// fetch interface (AS4C4M16SA-6TIN, x16, real JEDEC SDR SDRAM
|
||||
// command pins). Real pin count: 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)
|
||||
// +1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37 pins, confirmed by real P&R
|
||||
// in Phase 6 (see the STEP16 report's own I/O analysis section) --
|
||||
// far fewer than the dual32 baseline's own 61-pin weight-fetch
|
||||
// interface, since this is a single chip, not two.
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_req;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
|
||||
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_sdram #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
|
||||
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
|
||||
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
// ================================================================
|
||||
// STEP16 Phase 5: real single-chip SDRAM weight-fetch backend --
|
||||
// a SEPARATE arbiter (slot_mem_arbiter_wide.v, N_PORTS=N_SLOTS, no
|
||||
// activation-fill port sharing this one, reused UNCHANGED from the
|
||||
// dual32 baseline at DATA_WIDTH=64) feeding sdram_weight_backend.v
|
||||
// (the real, isolated-and-validated AS4C4M16SA-6TIN controller).
|
||||
// Weight fetch never writes: mem_wr/mem_wdata tied to 0 at the
|
||||
// arbiter's own per-port inputs, matching weight_prefetch_engine_
|
||||
// wide.v's own always-full-word read behavior exactly (same
|
||||
// convention as the dual32 baseline).
|
||||
// ================================================================
|
||||
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
|
||||
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
|
||||
|
||||
wire wide_arb_m_req, wide_arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
|
||||
wire [63:0] wide_arb_m_wdata;
|
||||
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
|
||||
wire [63:0] wide_arb_m_rdata;
|
||||
wire wide_arb_m_ready;
|
||||
|
||||
slot_mem_arbiter_wide #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
|
||||
) u_arbiter_wide (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
|
||||
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
|
||||
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
|
||||
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
|
||||
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
|
||||
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
|
||||
);
|
||||
|
||||
sdram_weight_backend #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_sdram_backend (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(wide_arb_m_req), .mem_wr(1'b0), .mem_addr(wide_arb_m_addr), .mem_wdata(64'h0),
|
||||
.mem_rdata(wide_arb_m_rdata), .mem_ready(wide_arb_m_ready),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,205 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
//
|
||||
// STEP16 Phase 5: forked from nms_neural_multiprocessor_dual32.v
|
||||
// (STEP15's real dual-chip-32-bit weight-fetch variant). The ORIGINAL
|
||||
// single-chip 16-bit PSRAM activation+writeback path (memory_
|
||||
// interface.v -> psram_controller.v) is BYTE-FOR-BYTE UNCHANGED --
|
||||
// STEP15 itself established this path sits at only 4.4% utilization
|
||||
// and was never the bandwidth bottleneck under test. ONLY the wide
|
||||
// weight-fetch backend changes: nms_dataflow_core_sdram.v (64-bit
|
||||
// wide port) -> slot_mem_arbiter_wide.v (reused unchanged at
|
||||
// DATA_WIDTH=64) -> sdram_weight_backend.v (the real, isolated-and-
|
||||
// validated AS4C4M16SA-6TIN SDRAM controller, BURST_LEN=4), replacing
|
||||
// psram_controller_dual32.v exactly at this one point in the design.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_sdram_pack128 #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n,
|
||||
|
||||
// ---- STEP16 Phase 5: real single-chip SDRAM physical weight-
|
||||
// fetch interface (AS4C4M16SA-6TIN, x16, real JEDEC SDR SDRAM
|
||||
// command pins). Real pin count: 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)
|
||||
// +1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37 pins, confirmed by real P&R
|
||||
// in Phase 6 (see the STEP16 report's own I/O analysis section) --
|
||||
// far fewer than the dual32 baseline's own 61-pin weight-fetch
|
||||
// interface, since this is a single chip, not two.
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_req;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
|
||||
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_sdram #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
|
||||
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
|
||||
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
// ================================================================
|
||||
// STEP16 Phase 5: real single-chip SDRAM weight-fetch backend --
|
||||
// a SEPARATE arbiter (slot_mem_arbiter_wide.v, N_PORTS=N_SLOTS, no
|
||||
// activation-fill port sharing this one, reused UNCHANGED from the
|
||||
// dual32 baseline at DATA_WIDTH=64) feeding sdram_weight_backend.v
|
||||
// (the real, isolated-and-validated AS4C4M16SA-6TIN controller).
|
||||
// Weight fetch never writes: mem_wr/mem_wdata tied to 0 at the
|
||||
// arbiter's own per-port inputs, matching weight_prefetch_engine_
|
||||
// wide.v's own always-full-word read behavior exactly (same
|
||||
// convention as the dual32 baseline).
|
||||
// ================================================================
|
||||
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
|
||||
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
|
||||
|
||||
wire wide_arb_m_req, wide_arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
|
||||
wire [63:0] wide_arb_m_wdata;
|
||||
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
|
||||
wire [63:0] wide_arb_m_rdata;
|
||||
wire wide_arb_m_ready;
|
||||
|
||||
slot_mem_arbiter_wide #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
|
||||
) u_arbiter_wide (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
|
||||
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
|
||||
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
|
||||
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
|
||||
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
|
||||
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
|
||||
);
|
||||
|
||||
sdram_weight_backend_pack128 #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_sdram_backend (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(wide_arb_m_req), .mem_wr(1'b0), .mem_addr(wide_arb_m_addr), .mem_wdata(64'h0),
|
||||
.mem_rdata(wide_arb_m_rdata), .mem_ready(wide_arb_m_ready),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,167 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP19 real hardware-facing top level.
|
||||
//
|
||||
// SINGLE EXTERNAL SDRAM ONLY. Forked from nms_neural_multiprocessor_
|
||||
// sdram_pack128.v (STEP18) with the ONE change this step's own
|
||||
// governing spec mandates: the real hardware/v1/rtl/psram_controller.v
|
||||
// + memory_interface.v pairing (activation-fill + result-writeback,
|
||||
// 16-bit) is REMOVED from the V2 physical path entirely and replaced
|
||||
// by sdram_unified_backend.v's own AR port, sharing the SAME single
|
||||
// physical AS4C4M16SA-6TIN SDRAM chip and the SAME single sdram_
|
||||
// controller.v instance the weight-fetch path (W port) already uses.
|
||||
//
|
||||
// slot_mem_arbiter.v (16-bit, activation+result) and slot_mem_
|
||||
// arbiter_wide.v (64-bit, weight) are BOTH reused completely
|
||||
// UNCHANGED -- their own downstream ports now both terminate at
|
||||
// sdram_unified_backend.v instead of two separate physical chains.
|
||||
// nms_dataflow_core_sdram.v, nms_activation_fill_ctrl_v3.v, nms_
|
||||
// memory_manager_stream_wide.v, weight_prefetch_engine_wide.v, and
|
||||
// neural_processor.v are ALL byte-for-byte unchanged -- this is a
|
||||
// pure memory-side substitution, per the governing spec's own
|
||||
// explicit instruction.
|
||||
//
|
||||
// V1 (hardware/v1/**) is untouched -- psram_controller.v and memory_
|
||||
// interface.v simply are no longer INSTANTIATED by this top-level;
|
||||
// neither file was modified, and V1's own golden-reference status is
|
||||
// unaffected.
|
||||
//
|
||||
// Real pin count (weight+activation+result, ALL through ONE chip):
|
||||
// 2(BA)+12(A)+1(CKE)+1(CS#)+1(RAS#)+1(CAS#)+1(WE#)+2(DQM)+16(DQ) = 37
|
||||
// pins total -- the SAME 37 pins the weight-only path already used in
|
||||
// STEP16-18 (no NEW physical SDRAM pins are needed to add activation/
|
||||
// result traffic, since it shares the identical physical bus).
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_sdram_unified #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
// ---- STEP19: ONE physical SDRAM interface, ALL traffic
|
||||
// (weights + activations + results) ----
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_req;
|
||||
wire [ADDR_WIDTH*N_SLOTS-1:0] wide_slot_mem_addr;
|
||||
wire [64*N_SLOTS-1:0] wide_slot_mem_rdata;
|
||||
wire [N_SLOTS-1:0] wide_slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_sdram #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready),
|
||||
.wide_slot_mem_req(wide_slot_mem_req), .wide_slot_mem_addr(wide_slot_mem_addr),
|
||||
.wide_slot_mem_rdata(wide_slot_mem_rdata), .wide_slot_mem_ready(wide_slot_mem_ready)
|
||||
);
|
||||
|
||||
// ---- AR: activation-fill (shared, 1 port) + per-slot result
|
||||
// writeback (N_SLOTS ports), arbitrated exactly as before ----
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
// ---- W: weight fetch (N_SLOTS ports), arbitrated exactly as
|
||||
// before -- weight fetch never writes, same tie-off convention
|
||||
// as STEP16-18 ----
|
||||
wire [N_SLOTS-1:0] wide_s_wr = {N_SLOTS{1'b0}};
|
||||
wire [64*N_SLOTS-1:0] wide_s_wdata = {(64*N_SLOTS){1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_lb_n = {N_SLOTS{1'b0}};
|
||||
wire [N_SLOTS-1:0] wide_s_ub_n = {N_SLOTS{1'b0}};
|
||||
|
||||
wire wide_arb_m_req, wide_arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] wide_arb_m_addr;
|
||||
wire [63:0] wide_arb_m_wdata;
|
||||
wire wide_arb_m_lb_n, wide_arb_m_ub_n;
|
||||
wire [63:0] wide_arb_m_rdata;
|
||||
wire wide_arb_m_ready;
|
||||
|
||||
slot_mem_arbiter_wide #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS), .DATA_WIDTH(64)
|
||||
) u_arbiter_wide (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(wide_slot_mem_req), .s_wr(wide_s_wr), .s_addr(wide_slot_mem_addr),
|
||||
.s_wdata(wide_s_wdata), .s_lb_n(wide_s_lb_n), .s_ub_n(wide_s_ub_n),
|
||||
.s_rdata(wide_slot_mem_rdata), .s_ready(wide_slot_mem_ready),
|
||||
.m_req(wide_arb_m_req), .m_wr(wide_arb_m_wr), .m_addr(wide_arb_m_addr), .m_wdata(wide_arb_m_wdata),
|
||||
.m_lb_n(wide_arb_m_lb_n), .m_ub_n(wide_arb_m_ub_n),
|
||||
.m_rdata(wide_arb_m_rdata), .m_ready(wide_arb_m_ready)
|
||||
);
|
||||
|
||||
// ---- STEP19: ONE physical SDRAM backend, both W and AR ports ----
|
||||
sdram_unified_backend #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_sdram_backend (
|
||||
.clk(clk), .rst(rst),
|
||||
.w_req(wide_arb_m_req), .w_addr(wide_arb_m_addr),
|
||||
.w_rdata(wide_arb_m_rdata), .w_ready(wide_arb_m_ready),
|
||||
.ar_req(arb_m_req), .ar_wr(arb_m_wr), .ar_addr(arb_m_addr), .ar_wdata(arb_m_wdata),
|
||||
.ar_lb_n(arb_m_lb_n), .ar_ub_n(arb_m_ub_n),
|
||||
.ar_rdata(arb_m_rdata), .ar_ready(arb_m_ready),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,123 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ================================================================
|
||||
// Neural Memory System (NMS) -- STEP9 real hardware-facing top level.
|
||||
// Mirrors hardware/v2/rtl/neural_multiprocessor.v's own scope exactly
|
||||
// (M8): nms_dataflow_core.v's N_SLOTS+1 independent Memory Backend
|
||||
// Interface ports funneled through the SAME, UNMODIFIED
|
||||
// slot_mem_arbiter.v down to the SAME, UNMODIFIED real V1 PSRAM
|
||||
// backend chain (memory_interface.v -> psram_controller.v).
|
||||
//
|
||||
// Nothing about the arbiter or the real PSRAM chain changes for the
|
||||
// NMS -- only nms_dataflow_core.v's own internals (the memory
|
||||
// organization DEC-0019/DEC-0020 decided) differ from the frozen V2
|
||||
// dataflow_core.v this module's own structure is copied from.
|
||||
// ================================================================
|
||||
|
||||
module nms_neural_multiprocessor_stream #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ACC_WIDTH = 32,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter N_SLOTS = 2,
|
||||
parameter N_NODES = 16,
|
||||
parameter MAX_DEPS = 4,
|
||||
parameter QUEUE_DEPTH = 8,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter PSRAM_DATA_WIDTH = 16,
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire reg_valid,
|
||||
output wire reg_ready,
|
||||
input wire [$clog2(N_NODES)-1:0] reg_node_id,
|
||||
input wire [$clog2(MAX_DEPS+1)-1:0] reg_required,
|
||||
input wire [MAX_DEPS*$clog2(N_NODES)-1:0] reg_producer_ids,
|
||||
input wire [ADDR_WIDTH-1:0] reg_x_base,
|
||||
input wire [ADDR_WIDTH-1:0] reg_w_base,
|
||||
input wire [15:0] reg_n_tiles,
|
||||
input wire [ADDR_WIDTH-1:0] reg_result_addr,
|
||||
|
||||
output wire [ADDR_WIDTH-1:0] psram_a,
|
||||
inout wire [PSRAM_DATA_WIDTH-1:0] psram_dq,
|
||||
output wire psram_ce_n,
|
||||
output wire psram_oe_n,
|
||||
output wire psram_we_n,
|
||||
output wire psram_lb_n,
|
||||
output wire psram_ub_n,
|
||||
output wire psram_zz_n
|
||||
);
|
||||
|
||||
wire [N_SLOTS:0] slot_mem_req, slot_mem_wr;
|
||||
wire [ADDR_WIDTH*(N_SLOTS+1)-1:0] slot_mem_addr;
|
||||
wire [16*(N_SLOTS+1)-1:0] slot_mem_wdata, slot_mem_rdata;
|
||||
wire [N_SLOTS:0] slot_mem_lb_n, slot_mem_ub_n;
|
||||
wire [N_SLOTS:0] slot_mem_ready;
|
||||
|
||||
nms_dataflow_core_stream #(
|
||||
.DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ACC_WIDTH(ACC_WIDTH), .ADDR_WIDTH(ADDR_WIDTH),
|
||||
.N_SLOTS(N_SLOTS), .N_NODES(N_NODES), .MAX_DEPS(MAX_DEPS), .QUEUE_DEPTH(QUEUE_DEPTH),
|
||||
.MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
|
||||
) u_dataflow_core (
|
||||
.clk(clk), .rst(rst),
|
||||
.reg_valid(reg_valid), .reg_ready(reg_ready), .reg_node_id(reg_node_id),
|
||||
.reg_required(reg_required), .reg_producer_ids(reg_producer_ids),
|
||||
.reg_x_base(reg_x_base), .reg_w_base(reg_w_base), .reg_n_tiles(reg_n_tiles),
|
||||
.reg_result_addr(reg_result_addr),
|
||||
.slot_mem_req(slot_mem_req), .slot_mem_wr(slot_mem_wr), .slot_mem_addr(slot_mem_addr),
|
||||
.slot_mem_wdata(slot_mem_wdata), .slot_mem_lb_n(slot_mem_lb_n), .slot_mem_ub_n(slot_mem_ub_n),
|
||||
.slot_mem_rdata(slot_mem_rdata), .slot_mem_ready(slot_mem_ready)
|
||||
);
|
||||
|
||||
wire arb_m_req, arb_m_wr;
|
||||
wire [ADDR_WIDTH-1:0] arb_m_addr;
|
||||
wire [15:0] arb_m_wdata;
|
||||
wire arb_m_lb_n, arb_m_ub_n;
|
||||
wire [15:0] arb_m_rdata;
|
||||
wire arb_m_ready;
|
||||
|
||||
slot_mem_arbiter #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .N_PORTS(N_SLOTS+1)
|
||||
) u_arbiter (
|
||||
.clk(clk), .rst(rst),
|
||||
.s_req(slot_mem_req), .s_wr(slot_mem_wr), .s_addr(slot_mem_addr),
|
||||
.s_wdata(slot_mem_wdata), .s_lb_n(slot_mem_lb_n), .s_ub_n(slot_mem_ub_n),
|
||||
.s_rdata(slot_mem_rdata), .s_ready(slot_mem_ready),
|
||||
.m_req(arb_m_req), .m_wr(arb_m_wr), .m_addr(arb_m_addr), .m_wdata(arb_m_wdata),
|
||||
.m_lb_n(arb_m_lb_n), .m_ub_n(arb_m_ub_n),
|
||||
.m_rdata(arb_m_rdata), .m_ready(arb_m_ready)
|
||||
);
|
||||
|
||||
wire pc_mem_req, pc_mem_wr;
|
||||
wire [ADDR_WIDTH-1:0] pc_mem_addr;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_wdata;
|
||||
wire pc_mem_lb_n, pc_mem_ub_n;
|
||||
wire [PSRAM_DATA_WIDTH-1:0] pc_mem_rdata;
|
||||
wire pc_mem_ready;
|
||||
|
||||
memory_interface #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH)) u_memif (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(arb_m_req), .wr(arb_m_wr), .addr(arb_m_addr), .wdata(arb_m_wdata),
|
||||
.lb_n(arb_m_lb_n), .ub_n(arb_m_ub_n),
|
||||
.rdata(arb_m_rdata), .ready(arb_m_ready),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready)
|
||||
);
|
||||
|
||||
psram_controller #(
|
||||
.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(PSRAM_DATA_WIDTH), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)
|
||||
) u_psram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(pc_mem_req), .mem_wr(pc_mem_wr), .mem_addr(pc_mem_addr), .mem_wdata(pc_mem_wdata),
|
||||
.mem_lb_n(pc_mem_lb_n), .mem_ub_n(pc_mem_ub_n),
|
||||
.mem_rdata(pc_mem_rdata), .mem_ready(pc_mem_ready),
|
||||
.psram_a(psram_a), .psram_dq(psram_dq),
|
||||
.psram_ce_n(psram_ce_n), .psram_oe_n(psram_oe_n), .psram_we_n(psram_we_n),
|
||||
.psram_lb_n(psram_lb_n), .psram_ub_n(psram_ub_n), .psram_zz_n(psram_zz_n)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,48 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- Weight SRAM candidate W1 "direct":
|
||||
// N_SLOTS private, single-port, natively P_IN*DATA_WIDTH-wide memories
|
||||
// (one per slot). Weights are NEVER shared across neurons (STEP2's own
|
||||
// analytical conclusion), so there is no arbitration to design at all
|
||||
// here -- every slot's own fill+read port is fully private. This
|
||||
// candidate mirrors hardware/v2/rtl/weight_buffer.v's own original
|
||||
// width/depth structure (EXP-0004/M3) exactly, replicated N_SLOTS
|
||||
// times, to measure the REAL total DP16KD cost of that replication
|
||||
// rather than assume it from the single-copy number alone.
|
||||
// ============================================================
|
||||
module nms_weight_direct #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES)
|
||||
)(
|
||||
input clk,
|
||||
input rst,
|
||||
|
||||
input [N_SLOTS-1:0] fill_we,
|
||||
input [N_SLOTS*TIW-1:0] fill_addr_flat,
|
||||
input [N_SLOTS*DATA_WIDTH*P_IN-1:0] fill_data_flat,
|
||||
|
||||
input [N_SLOTS-1:0] rd_en,
|
||||
input [N_SLOTS*TIW-1:0] rd_addr_flat,
|
||||
output [N_SLOTS*DATA_WIDTH*P_IN-1:0] rd_data_flat
|
||||
);
|
||||
|
||||
genvar g;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
reg [DATA_WIDTH*P_IN-1:0] mem [0:MAX_TILES-1];
|
||||
reg [DATA_WIDTH*P_IN-1:0] rd_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (fill_we[g])
|
||||
mem[fill_addr_flat[g*TIW +: TIW]] <= fill_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN];
|
||||
if (rd_en[g])
|
||||
rd_data_reg <= mem[rd_addr_flat[g*TIW +: TIW]];
|
||||
end
|
||||
|
||||
assign rd_data_flat[g*DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN] = rd_data_reg;
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,53 @@
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- Weight SRAM candidate W2 "packed":
|
||||
// same private-per-slot semantics as nms_weight_direct.v, but each
|
||||
// slot's P_IN*DATA_WIDTH-wide tile storage is decomposed into P_IN
|
||||
// separate, narrow (DATA_WIDTH=8-bit-wide) single-port memories (one
|
||||
// per MAC lane) instead of one wide 64-bit memory. Reassembly into the
|
||||
// full tile word is a static concatenation of P_IN registered
|
||||
// per-lane outputs -- no runtime mux, no real LUT cost expected there.
|
||||
// Exists to measure whether narrower-but-more-numerous memories pack
|
||||
// into fewer total DP16KD blocks than nms_weight_direct.v's wider-but-
|
||||
// fewer instances, per the user's own explicit ask (NMS spec S6) to
|
||||
// measure width/depth/packing real DP16KD cost rather than assume it.
|
||||
// ============================================================
|
||||
module nms_weight_packed #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter N_SLOTS = 4,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES)
|
||||
)(
|
||||
input clk,
|
||||
input rst,
|
||||
|
||||
input [N_SLOTS-1:0] fill_we,
|
||||
input [N_SLOTS*TIW-1:0] fill_addr_flat,
|
||||
input [N_SLOTS*DATA_WIDTH*P_IN-1:0] fill_data_flat,
|
||||
|
||||
input [N_SLOTS-1:0] rd_en,
|
||||
input [N_SLOTS*TIW-1:0] rd_addr_flat,
|
||||
output [N_SLOTS*DATA_WIDTH*P_IN-1:0] rd_data_flat
|
||||
);
|
||||
|
||||
genvar g, p;
|
||||
generate
|
||||
for (g = 0; g < N_SLOTS; g = g + 1) begin : GEN_SLOT
|
||||
for (p = 0; p < P_IN; p = p + 1) begin : GEN_LANE
|
||||
reg [DATA_WIDTH-1:0] mem [0:MAX_TILES-1];
|
||||
reg [DATA_WIDTH-1:0] rd_data_reg;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (fill_we[g])
|
||||
mem[fill_addr_flat[g*TIW +: TIW]] <=
|
||||
fill_data_flat[g*DATA_WIDTH*P_IN + p*DATA_WIDTH +: DATA_WIDTH];
|
||||
if (rd_en[g])
|
||||
rd_data_reg <= mem[rd_addr_flat[g*TIW +: TIW]];
|
||||
end
|
||||
|
||||
assign rd_data_flat[g*DATA_WIDTH*P_IN + p*DATA_WIDTH +: DATA_WIDTH] = rd_data_reg;
|
||||
end
|
||||
end
|
||||
endgenerate
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,182 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// NMS STEP15 (continuation) -- REAL 32-bit physical memory
|
||||
// interface: two independent, real, UNMODIFIED
|
||||
// hardware/v1/rtl/psram_controller.v instances (each driving its
|
||||
// own physical 16-bit ISSI IS66WVE4M16EBLL-70BLI chip), presenting
|
||||
// a single, coherent 32-bit mem_req/mem_addr/mem_wdata/mem_rdata/
|
||||
// mem_ready interface to the rest of the NMS.
|
||||
//
|
||||
// ARCHITECTURE DECISION (not assumed -- evaluated against the
|
||||
// alternatives STEP15's own governing spec explicitly listed):
|
||||
// - "one widened controller": REJECTED. psram_controller.v's own
|
||||
// internal state machine (STATE_INIT/CR_INIT/IDLE/READ/
|
||||
// PAGE_OPEN/PAGE_CLOSE/PAGE_REOPEN/WRITE/WRITE_WAIT) is written
|
||||
// around a DATA_WIDTH-parametrized single physical DQ bus
|
||||
// (psram_dq is `inout [DATA_WIDTH-1:0]`) -- widening DATA_WIDTH
|
||||
// to 32 in a SINGLE instance would drive ONE 32-bit inout bus,
|
||||
// which does not correspond to two SEPARATE physical chips each
|
||||
// with their OWN independent DQ pins, CE#, OE#, WE# etc. Two
|
||||
// real, separate physical chips cannot share one inout bus.
|
||||
// - "interleaved controllers" (alternate words to alternate
|
||||
// chips): REJECTED. This would double the ADDRESSABLE space
|
||||
// seen by the LOGICAL 32-bit interface without doubling the
|
||||
// WIDTH of a single transaction -- it solves a different
|
||||
// problem (more capacity at the same per-transfer width) than
|
||||
// what STEP15 asked for (wider per-transfer width at the same
|
||||
// tile-fetch granularity).
|
||||
// - "duplicated controller instances, shared control, duplicated
|
||||
// data path" -- SELECTED. Two full, real, byte-for-byte
|
||||
// UNMODIFIED psram_controller.v instances, each wired to its own
|
||||
// physical chip. Both instances receive IDENTICAL clk/rst/
|
||||
// mem_req/mem_wr/mem_addr every cycle (broadcast) -- since both
|
||||
// instances are the exact same RTL executing the exact same real
|
||||
// timing FSM against the exact same inputs, they are
|
||||
// STRUCTURALLY, CYCLE-EXACT synchronized by construction, not by
|
||||
// any added synchronization logic. This preserves
|
||||
// psram_controller.v's own real page/open/close timing
|
||||
// semantics EXACTLY (DEC requirement: "preserve existing memory
|
||||
// timing behavior") -- neither instance's own internal state
|
||||
// machine is touched at all.
|
||||
//
|
||||
// Address mapping: this module's own external mem_addr is the BYTE
|
||||
// address of the 32-bit transaction (matching
|
||||
// weight_prefetch_engine_wide.v's own established byte-address
|
||||
// convention, STEP14 -- so it plugs in without modifying that
|
||||
// already-validated module). Internally, mem_addr[ADDR_WIDTH-1:2]
|
||||
// (a right-shift by 2, i.e. divide by 4 bytes/32-bit-word) is fed
|
||||
// IDENTICALLY to both real psram_controller.v instances as THEIR own
|
||||
// required per-chip WORD address (2 bytes/word each) -- i.e. logical
|
||||
// 32-bit word W is stored as chip0's own word W (bits [15:0]) and
|
||||
// chip1's own word W (bits [31:16]). This is the standard byte-lane
|
||||
// "bus-widening" mapping (NOT address interleaving): the combined
|
||||
// interface holds the SAME NUMBER of 32-bit words as either chip
|
||||
// alone holds 16-bit words (matching real DEC-0029's own "2 parallel
|
||||
// chips, shared address bus" architecture). An earlier draft fed
|
||||
// mem_addr to both instances UNSHIFTED (treating a byte address as if
|
||||
// it were already a word address) -- a real bug, caught by the bit-
|
||||
// exact regression (tb_psram_dual32.v): every byte beyond the very
|
||||
// first tile's own lane 0 read back as 0 (uninitialized), because the
|
||||
// real controllers were being addressed ~4x further out than
|
||||
// intended. Fixed by adding the explicit >>2 conversion here.
|
||||
//
|
||||
// mem_ready: both instances' own mem_ready are asserted the SAME
|
||||
// cycle by construction (see above) -- output is chip0's own
|
||||
// mem_ready, with a real, synthesizable assertion checking chip1's
|
||||
// mem_ready never diverges (STEP15's own explicit "verify
|
||||
// simultaneous read/write timing, latency matching" requirement).
|
||||
// ============================================================
|
||||
module psram_controller_dual32 #(
|
||||
parameter ADDR_WIDTH = 23, // word address width, PER CHIP (same value fed to both)
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- 32-bit logical interface ----
|
||||
input wire mem_req,
|
||||
input wire mem_wr,
|
||||
input wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
input wire [31:0] mem_wdata,
|
||||
input wire [1:0] mem_lb_n, // [0]=chip0 low-byte enable#, [1]=chip1 low-byte enable#
|
||||
input wire [1:0] mem_ub_n, // [0]=chip0 high-byte enable#, [1]=chip1 high-byte enable#
|
||||
|
||||
output wire [31:0] mem_rdata,
|
||||
output wire mem_ready,
|
||||
output reg lane_sync_error, // latched, real assertion: should NEVER go high
|
||||
|
||||
// ---- physical interface, chip 0 (bits [15:0]) ----
|
||||
output wire [ADDR_WIDTH-1:0] psram0_a,
|
||||
inout wire [15:0] psram0_dq,
|
||||
output wire psram0_ce_n,
|
||||
output wire psram0_oe_n,
|
||||
output wire psram0_we_n,
|
||||
output wire psram0_lb_n,
|
||||
output wire psram0_ub_n,
|
||||
output wire psram0_zz_n,
|
||||
|
||||
// ---- physical interface, chip 1 (bits [31:16]) ----
|
||||
output wire [ADDR_WIDTH-1:0] psram1_a,
|
||||
inout wire [15:0] psram1_dq,
|
||||
output wire psram1_ce_n,
|
||||
output wire psram1_oe_n,
|
||||
output wire psram1_we_n,
|
||||
output wire psram1_lb_n,
|
||||
output wire psram1_ub_n,
|
||||
output wire psram1_zz_n
|
||||
);
|
||||
|
||||
wire [15:0] rdata0, rdata1;
|
||||
wire ready0, ready1;
|
||||
|
||||
// mem_addr (this module's own external contract) is the BYTE
|
||||
// address of the 32-bit transaction -- matching
|
||||
// weight_prefetch_engine_wide.v's own established convention
|
||||
// (STEP14), so it plugs in without modifying that already-
|
||||
// validated module. The REAL psram_controller.v instances, in
|
||||
// contrast, each expect a per-chip WORD address (2 bytes/word) --
|
||||
// exactly like weight_prefetch_engine.v's own real, original
|
||||
// 16-bit usage, which explicitly converts via `w_base[ADDR_WIDTH-
|
||||
// 1:1]` before using it as mem_addr. For this 32-bit interface
|
||||
// (4 bytes/logical word, mapped straight across both 16-bit
|
||||
// chips at the SAME per-chip word index), the equivalent
|
||||
// conversion is a right-shift by 2, not 1 -- byte address bits
|
||||
// [1:0] select which of the 4 bytes within the 32-bit word (not
|
||||
// meaningful to the per-chip word address itself, only to
|
||||
// mem_lb_n/mem_ub_n lane selection, already handled separately
|
||||
// by the caller).
|
||||
wire [ADDR_WIDTH-1:0] chip_word_addr = mem_addr[ADDR_WIDTH-1:2];
|
||||
|
||||
psram_controller #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(16), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)) u_ctrl0 (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(chip_word_addr),
|
||||
.mem_wdata(mem_wdata[15:0]), .mem_lb_n(mem_lb_n[0]), .mem_ub_n(mem_ub_n[0]),
|
||||
.mem_rdata(rdata0), .mem_ready(ready0),
|
||||
.psram_a(psram0_a), .psram_dq(psram0_dq),
|
||||
.psram_ce_n(psram0_ce_n), .psram_oe_n(psram0_oe_n), .psram_we_n(psram0_we_n),
|
||||
.psram_lb_n(psram0_lb_n), .psram_ub_n(psram0_ub_n), .psram_zz_n(psram0_zz_n)
|
||||
);
|
||||
|
||||
psram_controller #(.ADDR_WIDTH(ADDR_WIDTH), .DATA_WIDTH(16), .CLK_FREQ_MHZ(CLK_FREQ_MHZ)) u_ctrl1 (
|
||||
.clk(clk), .rst(rst),
|
||||
.mem_req(mem_req), .mem_wr(mem_wr), .mem_addr(chip_word_addr),
|
||||
.mem_wdata(mem_wdata[31:16]), .mem_lb_n(mem_lb_n[1]), .mem_ub_n(mem_ub_n[1]),
|
||||
.mem_rdata(rdata1), .mem_ready(ready1),
|
||||
.psram_a(psram1_a), .psram_dq(psram1_dq),
|
||||
.psram_ce_n(psram1_ce_n), .psram_oe_n(psram1_oe_n), .psram_we_n(psram1_we_n),
|
||||
.psram_lb_n(psram1_lb_n), .psram_ub_n(psram1_ub_n), .psram_zz_n(psram1_zz_n)
|
||||
);
|
||||
|
||||
assign mem_rdata = {rdata1, rdata0};
|
||||
|
||||
// mem_ready must be COMBINATIONAL, passed straight through from
|
||||
// ready0 -- NOT registered. An earlier draft registered it
|
||||
// (`mem_ready <= ready0`), adding one cycle of spurious latency
|
||||
// relative to mem_rdata (which reflects the controllers' own
|
||||
// CURRENT output combinationally) -- a real timing misalignment
|
||||
// caught by the bit-exact regression (every byte beyond tile 0's
|
||||
// own coincidental zero read back wrong): the caller sampled
|
||||
// mem_rdata one cycle before the (delayed) ready pulse told it to,
|
||||
// capturing stale/settling data. Fixed by making mem_ready a
|
||||
// simple continuous assignment, matching the real, single-chip
|
||||
// psram_controller.v's own timing exactly (which this module must
|
||||
// preserve, per its own design goal).
|
||||
assign mem_ready = ready0;
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
lane_sync_error <= 1'b0;
|
||||
end else begin
|
||||
// Real, synthesizable cross-check: both instances are fed
|
||||
// byte-for-byte identical control/address every cycle, so
|
||||
// their own real timing FSMs MUST assert ready the same
|
||||
// cycle. This is not expected to ever fire; if it does,
|
||||
// the two physical chips have gone out of lockstep (e.g.
|
||||
// a real hardware fault or a genuine RTL bug), and
|
||||
// lane_sync_error latches permanently until reset.
|
||||
if (ready0 !== ready1) lane_sync_error <= 1'b1;
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,445 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// NMS STEP16 -- minimal, CORRECT-FIRST SDR SDRAM controller for
|
||||
// Alliance Memory AS4C4M16SA-6TIN (64Mbit/8MB, x16, -6 speed grade:
|
||||
// tCK=6ns/166MHz max, CAS latency 3).
|
||||
//
|
||||
// Design priority explicitly stated by the governing spec:
|
||||
// correctness > performance > elegance. This controller therefore:
|
||||
// - ALWAYS uses auto-precharge (A10=1 on every READ/WRITE) --
|
||||
// every transaction activates a row, bursts BURST_LEN words, and
|
||||
// closes the row again before the next transaction. This is NOT
|
||||
// the fastest possible design (no page-hit/keep-row-open
|
||||
// optimization, unlike psram_controller.v's own real page-mode),
|
||||
// but it is trivially correct: no per-row state to track, no
|
||||
// risk of a stale-open-row bug, exactly one code path for every
|
||||
// transaction regardless of address history.
|
||||
// - Real JEDEC SDR SDRAM command encoding (CS#/RAS#/CAS#/WE#),
|
||||
// real power-up sequence (200us wait, PRECHARGE ALL, 8x AUTO
|
||||
// REFRESH, LOAD MODE REGISTER), real periodic AUTO REFRESH
|
||||
// insertion between transactions (tREFI = 4096 rows / 64ms).
|
||||
// - Real, standard -6-speed-grade SDR SDRAM timing (datasheet-
|
||||
// standard values, not vendor-specific tuning): tRCD=3cyc,
|
||||
// tRP=3cyc, tRAS(min)=7cyc, tRC=10cyc, tMRD=2cyc @166MHz -- all
|
||||
// re-derived per CLK_FREQ_MHZ so the same RTL is reused across
|
||||
// the Phase 4 100/133/166MHz sweep (STEP16's own explicit
|
||||
// "measure, do not estimate" requirement).
|
||||
//
|
||||
// Address format: word address (16-bit words), decomposed as
|
||||
// {bank[1:0], row[11:0], col[7:0]} -- matches the REAL AS4C4M16SA's
|
||||
// own 4-bank x 4096-row x 256-column x16 organization (4*4096*256 =
|
||||
// 4M words = 8MB, confirmed against the real datasheet capacity).
|
||||
//
|
||||
// External protocol matches this project's own established
|
||||
// mem_req/mem_wr/mem_addr/mem_wdata/mem_rdata/mem_ready convention
|
||||
// (same idiom as psram_controller.v), generalized to a BURST: one
|
||||
// req initiates a full BURST_LEN-word transaction (the natural unit
|
||||
// for this workload -- one weight TILE = P_IN*DATA_WIDTH/16 = 4
|
||||
// words at BURST_LEN=4, an exact match, not a coincidence chosen
|
||||
// after the fact -- STEP16 Phase 1 identified this exact byte count
|
||||
// per tile before any RTL was written).
|
||||
// ============================================================
|
||||
module sdram_controller #(
|
||||
parameter CLK_FREQ_MHZ = 166,
|
||||
parameter BURST_LEN = 4, // 1, 4, or 8 -- Phase 4 sweep parameter
|
||||
parameter ADDR_WIDTH = 22 // word address: 2 bank + 12 row + 8 col
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire req,
|
||||
input wire wr,
|
||||
input wire [ADDR_WIDTH-1:0] addr, // burst-aligned word address
|
||||
input wire [16*BURST_LEN-1:0] wdata, // BURST_LEN words, word0 first
|
||||
// STEP19: per-burst-word DQM write mask, 2 bits/word (bit0=low
|
||||
// byte, bit1=high byte, real SDR SDRAM DQM polarity: 1=masked/
|
||||
// NOT written, memory array retains its old value for that byte;
|
||||
// 0=written). Ties to {2*BURST_LEN{1'b0}} (never mask, i.e.
|
||||
// "always write full word") reproduces this module's own STEP16
|
||||
// behavior exactly -- every existing caller (sdram_weight_
|
||||
// backend.v, sdram_weight_backend_pack128.v, tb_sdram_controller.v)
|
||||
// was updated to pass that literal tie-off, so read/weight-fetch
|
||||
// behavior is byte-for-byte unchanged. Only meaningful for `wr`
|
||||
// transactions; ignored for reads (dqm is forced 0 during reads
|
||||
// regardless, since real SDR SDRAM masks READ OUTPUT with DQM too,
|
||||
// and this controller always wants valid read data back).
|
||||
input wire [2*BURST_LEN-1:0] wmask,
|
||||
output reg [16*BURST_LEN-1:0] rdata, // valid the same cycle `ready` pulses
|
||||
output reg ready, // pulses once, whole burst transaction done
|
||||
output reg busy,
|
||||
|
||||
// ---- real SDRAM physical pins ----
|
||||
output reg sdram_cke,
|
||||
output reg sdram_cs_n,
|
||||
output reg sdram_ras_n,
|
||||
output reg sdram_cas_n,
|
||||
output reg sdram_we_n,
|
||||
output reg [1:0] sdram_ba,
|
||||
output reg [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output reg [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
localparam BURST_IDXW = (BURST_LEN <= 1) ? 1 : $clog2(BURST_LEN);
|
||||
|
||||
// ---- real, standard -6-speed-grade timing, re-derived per
|
||||
// CLK_FREQ_MHZ (ceiling division: never UNDER-count a real ns
|
||||
// requirement) ----
|
||||
function integer ns_to_cycles;
|
||||
input integer ns;
|
||||
begin
|
||||
ns_to_cycles = (ns * CLK_FREQ_MHZ + 999) / 1000;
|
||||
end
|
||||
endfunction
|
||||
localparam T_RCD = ns_to_cycles(18); // ACTIVE -> READ/WRITE
|
||||
localparam T_RP = ns_to_cycles(18); // PRECHARGE -> ACTIVE
|
||||
// ACTIVE->PRECHARGE minimum (tRAS=42ns=7cyc@166MHz) is not
|
||||
// separately waited on: this design's own fixed sequencing
|
||||
// (tRCD + CAS_LATENCY + BURST_LEN data cycles, always >= 3+3+1=7
|
||||
// even at the narrowest BURST_LEN=1) already comfortably exceeds
|
||||
// it by construction before auto-precharge can begin internally.
|
||||
localparam T_MRD = ns_to_cycles(12); // LOAD MODE REGISTER -> any command
|
||||
localparam T_INIT_US= 200; // power-up wait, real datasheet value
|
||||
localparam T_INIT = T_INIT_US * CLK_FREQ_MHZ;
|
||||
localparam CAS_LATENCY = 3; // fixed for this part/speed grade
|
||||
// real refresh interval: 4096 rows must each be refreshed within
|
||||
// 64ms -> one AUTO REFRESH at least every 64e6ns/4096 = 15625ns
|
||||
localparam T_REFI = ns_to_cycles(15625);
|
||||
|
||||
localparam CNTW = $clog2((T_INIT>T_REFI ? T_INIT : T_REFI) + 1);
|
||||
|
||||
// JEDEC SDR SDRAM commands are encoded directly in the FSM below
|
||||
// via named signal drives (cs_n/ras_n/cas_n/we_n), not a lookup
|
||||
// table -- clearer to review against the real datasheet's own
|
||||
// command truth table line by line.
|
||||
|
||||
// tRC (ACTIVATE-to-ACTIVATE minimum, same bank), used by both the
|
||||
// init-refresh and steady-state refresh wait.
|
||||
function [CNTW-1:0] T_RC_MINUS1;
|
||||
localparam integer T_RC = ns_to_cycles(60);
|
||||
begin
|
||||
T_RC_MINUS1 = T_RC[CNTW-1:0] - 1'b1;
|
||||
end
|
||||
endfunction
|
||||
|
||||
localparam
|
||||
S_INIT_WAIT = 5'd0,
|
||||
S_INIT_PRE_WAIT = 5'd2,
|
||||
S_INIT_REF = 5'd3,
|
||||
S_INIT_REF_WAIT = 5'd4,
|
||||
S_INIT_MRS_WAIT = 5'd6,
|
||||
S_IDLE = 5'd7,
|
||||
S_REFRESH_WAIT = 5'd9,
|
||||
S_ACTIVATE_WAIT = 5'd11,
|
||||
S_CAS_WAIT = 5'd13,
|
||||
S_BURST_READ = 5'd14,
|
||||
S_BURST_WRITE = 5'd15,
|
||||
S_PRECHARGE_WAIT = 5'd16;
|
||||
|
||||
reg [4:0] state;
|
||||
reg [CNTW-1:0] wait_cnt;
|
||||
reg [3:0] init_ref_cnt;
|
||||
reg [CNTW-1:0] refresh_timer;
|
||||
reg [BURST_IDXW-1:0] burst_idx;
|
||||
reg req_wr_reg;
|
||||
reg [1:0] req_bank_reg;
|
||||
reg [11:0] req_row_reg;
|
||||
reg [7:0] req_col_reg;
|
||||
reg [16*BURST_LEN-1:0] wdata_reg;
|
||||
reg [2*BURST_LEN-1:0] wmask_reg;
|
||||
|
||||
wire [1:0] addr_bank = addr[ADDR_WIDTH-1:ADDR_WIDTH-2];
|
||||
wire [11:0] addr_row = addr[ADDR_WIDTH-3:8];
|
||||
wire [7:0] addr_col = addr[7:0];
|
||||
|
||||
// req_pending: latches a req that arrives in S_IDLE on the SAME
|
||||
// cycle a periodic AUTO REFRESH is also due. Without this, a
|
||||
// single-cycle req pulse (this project's own established
|
||||
// mem_req convention -- see weight_prefetch_engine.v's own header
|
||||
// comment) would be silently dropped whenever refresh wins
|
||||
// arbitration that cycle: the caller only holds req high for one
|
||||
// cycle, has no idea refresh was chosen instead, and then waits
|
||||
// forever for a `ready` that will never come -- a real,
|
||||
// frequency/burst-alignment-dependent deadlock found by STEP16's
|
||||
// own Phase 4 100/133/166MHz sweep (reproduced at BURST_LEN=1,
|
||||
// CLK_FREQ_MHZ=133, but the race is general, not specific to that
|
||||
// combination -- it is a matter of which absolute cycle each test
|
||||
// vector's req happens to land on).
|
||||
reg req_pending;
|
||||
wire eff_wr = req ? wr : req_wr_reg;
|
||||
wire [1:0] eff_bank = req ? addr_bank : req_bank_reg;
|
||||
wire [11:0] eff_row = req ? addr_row : req_row_reg;
|
||||
wire [7:0] eff_col = req ? addr_col : req_col_reg;
|
||||
wire [16*BURST_LEN-1:0] eff_wdata = req ? wdata : wdata_reg;
|
||||
wire [2*BURST_LEN-1:0] eff_wmask = req ? wmask : wmask_reg;
|
||||
|
||||
// tri-state DQ: driven only during a write burst
|
||||
reg dq_out_en;
|
||||
reg [15:0] dq_out;
|
||||
assign sdram_dq = dq_out_en ? dq_out : 16'hzzzz;
|
||||
|
||||
// Mode register value: burst length code + sequential burst type
|
||||
// (A3=0) + CAS latency 3 (A6:4=011) + standard write burst (A9=0).
|
||||
function [11:0] mrs_value;
|
||||
input integer burst_len;
|
||||
reg [2:0] bl_code;
|
||||
begin
|
||||
bl_code = (burst_len==1) ? 3'b000 :
|
||||
(burst_len==2) ? 3'b001 :
|
||||
(burst_len==4) ? 3'b010 :
|
||||
(burst_len==8) ? 3'b011 : 3'b111; // 111 = full page, unused here
|
||||
mrs_value = {3'b000, 1'b0, 3'b011, 1'b0, bl_code};
|
||||
end
|
||||
endfunction
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= S_INIT_WAIT;
|
||||
wait_cnt <= T_INIT[CNTW-1:0];
|
||||
init_ref_cnt <= 4'd0;
|
||||
refresh_timer <= T_REFI[CNTW-1:0];
|
||||
sdram_cke <= 1'b1; // held high throughout, real part supports CKE-always-high operation
|
||||
sdram_cs_n <= 1'b1;
|
||||
sdram_ras_n <= 1'b1;
|
||||
sdram_cas_n <= 1'b1;
|
||||
sdram_we_n <= 1'b1;
|
||||
sdram_ba <= 2'b00;
|
||||
sdram_a <= 12'h000;
|
||||
sdram_dqm <= 2'b00; // both byte lanes always enabled (weight/tile fetch always full-word)
|
||||
dq_out_en <= 1'b0;
|
||||
ready <= 1'b0;
|
||||
busy <= 1'b1;
|
||||
req_pending <= 1'b0;
|
||||
end else begin
|
||||
// default: NOP every cycle unless a state below overrides it
|
||||
sdram_cs_n <= 1'b0;
|
||||
sdram_ras_n <= 1'b1;
|
||||
sdram_cas_n <= 1'b1;
|
||||
sdram_we_n <= 1'b1;
|
||||
ready <= 1'b0;
|
||||
dq_out_en <= 1'b0;
|
||||
sdram_dqm <= 2'b00; // default: no mask (reads always want valid data; writes override below per-word)
|
||||
|
||||
if (refresh_timer != 0) refresh_timer <= refresh_timer - 1'b1;
|
||||
|
||||
// Latch a fresh req's fields UNCONDITIONALLY, every cycle,
|
||||
// regardless of what state the controller is currently in
|
||||
// -- not just while in S_IDLE. ERR-0019's own fix only
|
||||
// covered "refresh wins arbitration the SAME cycle S_IDLE
|
||||
// sees req" -- but a real caller (e.g. slot_mem_arbiter_
|
||||
// wide.v) can pulse req for exactly one cycle at ANY time,
|
||||
// including a cycle where the controller is mid-refresh
|
||||
// (S_REFRESH_WAIT) or finishing a PREVIOUS transaction's
|
||||
// own PRECHARGE_WAIT tail -- i.e. NOT in S_IDLE at all that
|
||||
// cycle. The old S_IDLE-only latch silently missed those,
|
||||
// permanently starving whichever requester's pulse landed
|
||||
// there (found via the real N=2 D-Stress integration
|
||||
// benchmark, EXP-0042: both slots' memory managers hung
|
||||
// forever at tile_idx=0 while the arbiter's own `owner`
|
||||
// stayed locked on a grant the controller had already
|
||||
// forgotten -- a real, reproducible full-system deadlock,
|
||||
// not merely a slower run).
|
||||
if (req) begin
|
||||
req_wr_reg <= wr;
|
||||
req_bank_reg <= addr_bank;
|
||||
req_row_reg <= addr_row;
|
||||
req_col_reg <= addr_col;
|
||||
wdata_reg <= wdata;
|
||||
wmask_reg <= wmask;
|
||||
req_pending <= 1'b1;
|
||||
end
|
||||
|
||||
case (state)
|
||||
S_INIT_WAIT: begin
|
||||
busy <= 1'b1;
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else begin
|
||||
// PRECHARGE ALL: RAS#=0,CAS#=1,WE#=0, A10=1
|
||||
sdram_ras_n <= 1'b0; sdram_we_n <= 1'b0;
|
||||
sdram_a[10] <= 1'b1;
|
||||
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
|
||||
state <= S_INIT_PRE_WAIT;
|
||||
end
|
||||
end
|
||||
S_INIT_PRE_WAIT: begin
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else begin
|
||||
state <= S_INIT_REF;
|
||||
end
|
||||
end
|
||||
S_INIT_REF: begin
|
||||
// AUTO REFRESH: RAS#=0,CAS#=0,WE#=1
|
||||
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0;
|
||||
wait_cnt <= T_RC_MINUS1();
|
||||
state <= S_INIT_REF_WAIT;
|
||||
end
|
||||
S_INIT_REF_WAIT: begin
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else if (init_ref_cnt < 4'd7) begin
|
||||
init_ref_cnt <= init_ref_cnt + 1'b1;
|
||||
state <= S_INIT_REF;
|
||||
end else begin
|
||||
// LOAD MODE REGISTER: RAS#=0,CAS#=0,WE#=0, addr=mode value
|
||||
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0; sdram_we_n <= 1'b0;
|
||||
sdram_ba <= 2'b00;
|
||||
sdram_a <= mrs_value(BURST_LEN);
|
||||
wait_cnt <= T_MRD[CNTW-1:0] - 1'b1;
|
||||
state <= S_INIT_MRS_WAIT;
|
||||
end
|
||||
end
|
||||
S_INIT_MRS_WAIT: begin
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else begin
|
||||
busy <= 1'b0;
|
||||
state <= S_IDLE;
|
||||
end
|
||||
end
|
||||
|
||||
S_IDLE: begin
|
||||
busy <= 1'b0;
|
||||
// req (if any) was already latched into req_pending
|
||||
// unconditionally above, regardless of state -- see
|
||||
// that latch's own comment for why it must not be
|
||||
// scoped to only this state.
|
||||
if (refresh_timer == 0) begin
|
||||
// periodic AUTO REFRESH -- no row is ever left
|
||||
// open between transactions (auto-precharge
|
||||
// always used), so we can refresh immediately,
|
||||
// no PRECHARGE-ALL needed here.
|
||||
busy <= 1'b1;
|
||||
sdram_ras_n <= 1'b0; sdram_cas_n <= 1'b0;
|
||||
wait_cnt <= T_RC_MINUS1();
|
||||
refresh_timer <= T_REFI[CNTW-1:0];
|
||||
state <= S_REFRESH_WAIT;
|
||||
end else if (req || req_pending) begin
|
||||
busy <= 1'b1;
|
||||
req_wr_reg <= eff_wr;
|
||||
req_bank_reg <= eff_bank;
|
||||
req_row_reg <= eff_row;
|
||||
req_col_reg <= eff_col;
|
||||
wdata_reg <= eff_wdata;
|
||||
wmask_reg <= eff_wmask;
|
||||
req_pending <= 1'b0;
|
||||
// ACTIVATE: RAS#=0,CAS#=1,WE#=1, ba=bank, a=row
|
||||
sdram_ras_n <= 1'b0;
|
||||
sdram_ba <= eff_bank;
|
||||
sdram_a <= eff_row;
|
||||
wait_cnt <= T_RCD[CNTW-1:0] - 1'b1;
|
||||
state <= S_ACTIVATE_WAIT;
|
||||
end
|
||||
end
|
||||
|
||||
S_REFRESH_WAIT: begin
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else state <= S_IDLE;
|
||||
end
|
||||
|
||||
S_ACTIVATE_WAIT: begin
|
||||
if (wait_cnt != 0) begin
|
||||
wait_cnt <= wait_cnt - 1'b1;
|
||||
end else begin
|
||||
// READ or WRITE with auto-precharge (A10=1):
|
||||
// CAS#=0, WE#=(0 for write /1 for read), ba=bank,
|
||||
// a[7:0]=col, a[10]=1
|
||||
sdram_cas_n <= 1'b0;
|
||||
sdram_we_n <= req_wr_reg ? 1'b0 : 1'b1;
|
||||
sdram_ba <= req_bank_reg;
|
||||
sdram_a <= {4'b0100, req_col_reg}; // a[11]=0,a[10]=1(auto-precharge),a[9:8]=0
|
||||
burst_idx <= {BURST_IDXW{1'b0}};
|
||||
if (req_wr_reg) begin
|
||||
dq_out_en <= 1'b1;
|
||||
dq_out <= wdata_reg[15:0];
|
||||
sdram_dqm <= wmask_reg[1:0];
|
||||
state <= S_BURST_WRITE;
|
||||
end else begin
|
||||
// Cycle-exact derivation (not assumed --
|
||||
// see the module's own design log /
|
||||
// EXP-0040 for the full walkthrough):
|
||||
// cas_n=0 becomes VISIBLE to the real chip
|
||||
// one cycle after this NBA (call that
|
||||
// cycle "C"). Entering S_CAS_WAIT also
|
||||
// takes effect at cycle C, with wait_cnt
|
||||
// set here. The state's own "wait_cnt==0"
|
||||
// capture branch first fires at cycle
|
||||
// C + wait_cnt_initial. We want that to be
|
||||
// C + CAS_LATENCY (data must be valid
|
||||
// exactly CAS_LATENCY real clocks after
|
||||
// the command is sampled) -- so
|
||||
// wait_cnt_initial = CAS_LATENCY exactly,
|
||||
// no adjustment.
|
||||
wait_cnt <= CAS_LATENCY[CNTW-1:0];
|
||||
state <= S_CAS_WAIT;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
S_CAS_WAIT: begin
|
||||
if (wait_cnt != 0) begin
|
||||
wait_cnt <= wait_cnt - 1'b1;
|
||||
end else begin
|
||||
rdata[0 +: 16] <= sdram_dq;
|
||||
// BURST_LEN==1 is a real, distinct edge case:
|
||||
// word0 IS the whole (only) burst -- go
|
||||
// straight to precharge-wait. Routing it
|
||||
// through S_BURST_READ instead (burst_idx
|
||||
// already at 1, one past the only valid
|
||||
// index) was a real deadlock, found and fixed
|
||||
// via the Phase 3 burst=1 test (EXP-0040):
|
||||
// S_BURST_READ's own "burst_idx==BURST_LEN-1"
|
||||
// exit check (==0) can never be true again
|
||||
// once burst_idx has already advanced to 1.
|
||||
if (BURST_LEN == 1) begin
|
||||
ready <= 1'b1;
|
||||
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
|
||||
state <= S_PRECHARGE_WAIT;
|
||||
end else begin
|
||||
burst_idx <= burst_idx + 1'b1;
|
||||
state <= S_BURST_READ;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
S_BURST_READ: begin
|
||||
// burst_idx's own width (BURST_IDXW=clog2(BURST_
|
||||
// LEN)) can only ever represent 0..BURST_LEN-1 --
|
||||
// capture therefore happens unconditionally every
|
||||
// cycle spent in this state (an explicit "<
|
||||
// BURST_LEN" guard here would always be true by
|
||||
// construction and was removed as dead logic).
|
||||
rdata[burst_idx*16 +: 16] <= sdram_dq;
|
||||
if (burst_idx == BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin
|
||||
ready <= 1'b1;
|
||||
// auto-precharge already running internally;
|
||||
// enforce tRP before the next ACTIVATE.
|
||||
wait_cnt <= T_RP[CNTW-1:0] - 1'b1;
|
||||
state <= S_PRECHARGE_WAIT;
|
||||
end else begin
|
||||
burst_idx <= burst_idx + 1'b1;
|
||||
end
|
||||
end
|
||||
|
||||
S_BURST_WRITE: begin
|
||||
if (burst_idx < BURST_LEN[BURST_IDXW-1:0] - 1'b1) begin
|
||||
burst_idx <= burst_idx + 1'b1;
|
||||
dq_out_en <= 1'b1;
|
||||
dq_out <= wdata_reg[(burst_idx+1'b1)*16 +: 16];
|
||||
sdram_dqm <= wmask_reg[(burst_idx+1'b1)*2 +: 2];
|
||||
end else begin
|
||||
ready <= 1'b1;
|
||||
wait_cnt <= T_RP[CNTW-1:0] + 1'b1; // tWR folded in conservatively
|
||||
state <= S_PRECHARGE_WAIT;
|
||||
end
|
||||
end
|
||||
|
||||
S_PRECHARGE_WAIT: begin
|
||||
if (wait_cnt != 0) wait_cnt <= wait_cnt - 1'b1;
|
||||
else state <= S_IDLE;
|
||||
end
|
||||
|
||||
default: state <= S_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,322 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// NMS STEP19 -- UNIFIED single-SDRAM memory backend.
|
||||
//
|
||||
// Replaces BOTH physical memory paths that existed through STEP18
|
||||
// (sdram_weight_backend_pack128.v for weights, and hardware/v1/rtl/
|
||||
// memory_interface.v + psram_controller.v for activation-fill/result-
|
||||
// writeback) with ONE physical AS4C4M16SA-6TIN SDRAM chip, ONE
|
||||
// sdram_controller.v instance (BURST_LEN=8), serving THREE logical
|
||||
// traffic classes through TWO external ports that exactly match what
|
||||
// the existing, UNCHANGED consumers already drive:
|
||||
//
|
||||
// W port (64-bit): weight_prefetch_engine_wide.v's own real
|
||||
// traffic, via slot_mem_arbiter_wide.v -- IDENTICAL external
|
||||
// contract to STEP18's sdram_weight_backend_pack128.v (byte
|
||||
// address in, 64-bit mem_rdata out), and internally reuses that
|
||||
// module's own validated N_ENTRIES=4 "other half" cache
|
||||
// unchanged (EXP-0046/ERR-0022's own fix, not re-derived here).
|
||||
//
|
||||
// AR port (16-bit, byte-maskable): nms_activation_fill_ctrl_v3.v's
|
||||
// own activation reads AND every per-slot nms_memory_manager_
|
||||
// stream_wide.v's own result writes, via slot_mem_arbiter.v --
|
||||
// IDENTICAL external contract to the real V1 psram_controller.v
|
||||
// port it replaces (word address in, 16-bit mem_wdata/mem_rdata,
|
||||
// mem_lb_n/mem_ub_n byte-lane write masking). Neither
|
||||
// nms_activation_fill_ctrl_v3.v nor nms_memory_manager_stream_
|
||||
// wide.v needed ANY change -- they already produce a WORD
|
||||
// address and already drive lb_n/ub_n exactly as the real V1
|
||||
// PSRAM controller expected.
|
||||
//
|
||||
// Neither weight_prefetch_engine_wide.v, nms_activation_fill_ctrl_v3.
|
||||
// v, nms_memory_manager_stream_wide.v, nor neural_processor.v changed
|
||||
// AT ALL for this step -- this is a pure memory-side substitution,
|
||||
// per the governing spec's own explicit instruction.
|
||||
//
|
||||
// KEY ENABLING FACT: real SDR SDRAM's own DQM pins are a per-BYTE
|
||||
// write mask (STEP19's own real, tested extension to sdram_
|
||||
// controller.v's `wmask` port) -- this lets a single-BYTE result
|
||||
// write happen INSIDE a shared BURST_LEN=8 (128-bit) transaction by
|
||||
// masking out every byte except the one/two the caller actually wants
|
||||
// written, with NO read-modify-write needed at all (the real SDRAM
|
||||
// chip itself leaves masked bytes untouched, by JEDEC definition).
|
||||
// Activation reads need no such trick -- a full 128-bit block is
|
||||
// fetched and the caller's own requested 16-bit word is extracted
|
||||
// combinationally from it.
|
||||
//
|
||||
// Arbitration: simple, correctness-first 2-way priority (weight
|
||||
// traffic strongly dominates real measured traffic -- STEP17 showed
|
||||
// the activation/result path at <=7.2% of all external-memory
|
||||
// activity -- so W is granted priority when both are pending, AR is
|
||||
// never starved since W's own real traffic pattern always eventually
|
||||
// idles between tiles/jobs). Exactly one physical SDRAM transaction
|
||||
// in flight at a time (matches sdram_controller.v's own inherent
|
||||
// single-transaction design, STEP18 Part E's own documented, accepted
|
||||
// scope boundary -- not revisited here).
|
||||
// ============================================================
|
||||
module sdram_unified_backend #(
|
||||
parameter ADDR_WIDTH = 23, // byte address width (W port convention)
|
||||
parameter CLK_FREQ_MHZ = 80,
|
||||
parameter W_ENTRIES = 4 // weight-cache depth, >= real N_SLOTS
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- W: weight fetch (64-bit, byte address, read-only) ----
|
||||
input wire w_req,
|
||||
input wire [ADDR_WIDTH-1:0] w_addr,
|
||||
output reg [63:0] w_rdata,
|
||||
output reg w_ready,
|
||||
|
||||
// ---- AR: activation-fill (read) + result-writeback (write),
|
||||
// 16-bit, WORD address (matches the real V1 psram_controller.v
|
||||
// convention this port replaces exactly) ----
|
||||
input wire ar_req,
|
||||
input wire ar_wr,
|
||||
input wire [ADDR_WIDTH-1:0] ar_addr, // word address, low 22 bits meaningful
|
||||
// (matches slot_mem_arbiter.v's own
|
||||
// m_addr width convention exactly --
|
||||
// that arbiter's real callers only ever
|
||||
// drive a 22-bit-significant word
|
||||
// address into an ADDR_WIDTH-wide bus)
|
||||
input wire [15:0] ar_wdata,
|
||||
input wire ar_lb_n,
|
||||
input wire ar_ub_n,
|
||||
output reg [15:0] ar_rdata,
|
||||
output reg ar_ready,
|
||||
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
// ============================================================
|
||||
// W-port cache (identical logic to sdram_weight_backend_pack128.v
|
||||
// -- an N_ENTRIES-deep, fully-associative "other half" cache,
|
||||
// round-robin allocated; safe under any sizing, see that module's
|
||||
// own header/ERR-0022 for the full rationale, not repeated here)
|
||||
// ============================================================
|
||||
localparam WEIDXW = (W_ENTRIES <= 1) ? 1 : $clog2(W_ENTRIES);
|
||||
reg w_cache_valid [0:W_ENTRIES-1];
|
||||
reg [ADDR_WIDTH-1:0] w_cache_addr [0:W_ENTRIES-1];
|
||||
reg [63:0] w_cache_data [0:W_ENTRIES-1];
|
||||
reg [WEIDXW-1:0] w_alloc_ptr;
|
||||
|
||||
reg w_hit_found_c;
|
||||
reg [WEIDXW-1:0] w_hit_idx_c;
|
||||
integer ei;
|
||||
always @(*) begin
|
||||
w_hit_found_c = 1'b0;
|
||||
w_hit_idx_c = {WEIDXW{1'b0}};
|
||||
for (ei = 0; ei < W_ENTRIES; ei = ei + 1) begin
|
||||
if (w_cache_valid[ei] && w_cache_addr[ei] == w_addr) begin
|
||||
w_hit_found_c = 1'b1;
|
||||
w_hit_idx_c = ei[WEIDXW-1:0];
|
||||
end
|
||||
end
|
||||
end
|
||||
wire w_cache_hit = w_hit_found_c && w_req;
|
||||
|
||||
// ============================================================
|
||||
// Shared physical controller, BURST_LEN=8 (128-bit/16-byte real
|
||||
// SDRAM transactions), reused UNCHANGED from STEP16-18.
|
||||
// ============================================================
|
||||
reg ctrl_req;
|
||||
reg ctrl_wr;
|
||||
reg [21:0] ctrl_addr;
|
||||
reg [127:0] ctrl_wdata;
|
||||
reg [15:0] ctrl_wmask;
|
||||
wire [127:0] ctrl_rdata;
|
||||
wire ctrl_ready;
|
||||
wire ctrl_busy;
|
||||
|
||||
sdram_controller #(
|
||||
.CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8), .ADDR_WIDTH(22)
|
||||
) u_sdram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr),
|
||||
.wdata(ctrl_wdata), .wmask(ctrl_wmask),
|
||||
.rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
localparam S_IDLE = 3'd0,
|
||||
S_W_WAIT = 3'd1,
|
||||
S_AR_RD_WAIT = 3'd2,
|
||||
S_AR_WR_WAIT = 3'd3;
|
||||
reg [2:0] state;
|
||||
reg w_pending_upper_half;
|
||||
reg [ADDR_WIDTH-1:0] w_pending_addr;
|
||||
reg [2:0] ar_pending_word;
|
||||
|
||||
// ---- req_pending latches (same fix class as sdram_controller.v's
|
||||
// own ERR-0019/ERR-0020): this backend's own top-level S_IDLE
|
||||
// arbitration can only START a new transaction when it is
|
||||
// genuinely idle. A single-cycle w_req/ar_req pulse (this
|
||||
// project's own established mem_req convention) arriving on a
|
||||
// cycle this backend happens to be busy servicing the OTHER port
|
||||
// would otherwise be silently dropped -- the caller has no idea,
|
||||
// waits forever for a `ready` that never comes. Found the hard way
|
||||
// (STEP19, EXP-0048): the first real N=4 D-Stress run deadlocked
|
||||
// at 0/256 neurons, jobs_allocated stuck at 12, because the very
|
||||
// first activation-fill read raced against weight-prefetch traffic
|
||||
// and was lost exactly this way. Fix: latch EVERY req's own fields
|
||||
// unconditionally, every cycle, regardless of current state (not
|
||||
// just from S_IDLE), mirroring sdram_controller.v's own corrected
|
||||
// fix exactly (ERR-0020: the FIRST attempt only latched from
|
||||
// S_IDLE, which was still not enough -- latch unconditionally).
|
||||
reg w_req_pending;
|
||||
reg [ADDR_WIDTH-1:0] w_req_addr_lat;
|
||||
reg ar_req_pending;
|
||||
reg ar_req_wr_lat;
|
||||
reg [ADDR_WIDTH-1:0] ar_req_addr_lat;
|
||||
reg [15:0] ar_req_wdata_lat;
|
||||
reg ar_req_lbn_lat, ar_req_ubn_lat;
|
||||
|
||||
wire w_eff_req = w_req || w_req_pending;
|
||||
wire [ADDR_WIDTH-1:0] w_eff_addr = w_req ? w_addr : w_req_addr_lat;
|
||||
wire ar_eff_req = ar_req || ar_req_pending;
|
||||
wire ar_eff_wr = ar_req ? ar_wr : ar_req_wr_lat;
|
||||
wire [ADDR_WIDTH-1:0] ar_eff_addr = ar_req ? ar_addr : ar_req_addr_lat;
|
||||
wire [15:0] ar_eff_wdata= ar_req ? ar_wdata : ar_req_wdata_lat;
|
||||
wire ar_eff_lbn = ar_req ? ar_lb_n : ar_req_lbn_lat;
|
||||
wire ar_eff_ubn = ar_req ? ar_ub_n : ar_req_ubn_lat;
|
||||
|
||||
wire [21:0] w_eff_aligned_word_addr = {w_eff_addr[ADDR_WIDTH-1:4], 3'b000};
|
||||
wire w_eff_addr_is_upper_half = w_eff_addr[3];
|
||||
wire [21:0] ar_eff_block_base = {ar_eff_addr[21:3], 3'b000};
|
||||
wire [2:0] ar_eff_word_in_blk = ar_eff_addr[2:0];
|
||||
|
||||
integer ri;
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= S_IDLE;
|
||||
for (ri = 0; ri < W_ENTRIES; ri = ri + 1) w_cache_valid[ri] <= 1'b0;
|
||||
w_alloc_ptr <= {WEIDXW{1'b0}};
|
||||
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= 22'h0;
|
||||
ctrl_wdata <= 128'h0; ctrl_wmask <= 16'hFFFF;
|
||||
w_ready <= 1'b0; w_rdata <= 64'h0;
|
||||
ar_ready <= 1'b0; ar_rdata <= 16'h0;
|
||||
w_pending_upper_half <= 1'b0; w_pending_addr <= {ADDR_WIDTH{1'b0}};
|
||||
ar_pending_word <= 3'h0;
|
||||
w_req_pending <= 1'b0; w_req_addr_lat <= {ADDR_WIDTH{1'b0}};
|
||||
ar_req_pending <= 1'b0; ar_req_wr_lat <= 1'b0;
|
||||
ar_req_addr_lat <= {ADDR_WIDTH{1'b0}}; ar_req_wdata_lat <= 16'h0;
|
||||
ar_req_lbn_lat <= 1'b1; ar_req_ubn_lat <= 1'b1;
|
||||
end else begin
|
||||
ctrl_req <= 1'b0;
|
||||
w_ready <= 1'b0;
|
||||
ar_ready <= 1'b0;
|
||||
|
||||
// latch fresh requests unconditionally, every cycle,
|
||||
// regardless of state (see req_pending's own comment above)
|
||||
if (w_req) begin
|
||||
w_req_addr_lat <= w_addr;
|
||||
w_req_pending <= 1'b1;
|
||||
end
|
||||
if (ar_req) begin
|
||||
ar_req_wr_lat <= ar_wr;
|
||||
ar_req_addr_lat <= ar_addr;
|
||||
ar_req_wdata_lat <= ar_wdata;
|
||||
ar_req_lbn_lat <= ar_lb_n;
|
||||
ar_req_ubn_lat <= ar_ub_n;
|
||||
ar_req_pending <= 1'b1;
|
||||
end
|
||||
|
||||
case (state)
|
||||
S_IDLE: begin
|
||||
// W has priority when both are pending (real
|
||||
// measured traffic: weight >>> activation+result,
|
||||
// STEP17 EXP-0045 -- AR is never starved since W's
|
||||
// own real access pattern idles between tiles).
|
||||
if (w_cache_hit) begin
|
||||
// fully serviced THIS cycle -- must also cancel
|
||||
// the unconditional latch above, which just set
|
||||
// w_req_pending<=1 for this SAME w_req pulse
|
||||
// (real bug found via full regression, EXP-0048
|
||||
// /ERR-0023: without this the latch survives
|
||||
// uncontested, and next cycle w_eff_req reads
|
||||
// true from STALE w_req_pending/w_req_addr_lat,
|
||||
// issuing a bogus extra fetch that shifts every
|
||||
// subsequent response by one).
|
||||
w_rdata <= w_cache_data[w_hit_idx_c];
|
||||
w_ready <= 1'b1;
|
||||
w_cache_valid[w_hit_idx_c] <= 1'b0;
|
||||
w_req_pending <= 1'b0;
|
||||
end else if (w_eff_req) begin
|
||||
ctrl_req <= 1'b1;
|
||||
ctrl_wr <= 1'b0;
|
||||
ctrl_addr <= w_eff_aligned_word_addr;
|
||||
ctrl_wmask <= 16'h0000;
|
||||
w_pending_upper_half <= w_eff_addr_is_upper_half;
|
||||
w_pending_addr <= w_eff_addr;
|
||||
w_req_pending <= 1'b0;
|
||||
state <= S_W_WAIT;
|
||||
end else if (ar_eff_req && !ar_eff_wr) begin
|
||||
ctrl_req <= 1'b1;
|
||||
ctrl_wr <= 1'b0;
|
||||
ctrl_addr <= ar_eff_block_base;
|
||||
ctrl_wmask <= 16'h0000;
|
||||
ar_pending_word <= ar_eff_word_in_blk;
|
||||
ar_req_pending <= 1'b0;
|
||||
state <= S_AR_RD_WAIT;
|
||||
end else if (ar_eff_req && ar_eff_wr) begin
|
||||
// mask every word except the target one; within
|
||||
// the target word, pass ar_lb_n/ar_ub_n through
|
||||
// directly (same active-low "write this byte"
|
||||
// polarity as real SDRAM DQM: lb_n=0 -> DQM=0
|
||||
// -> byte written; lb_n=1 -> DQM=1 -> masked).
|
||||
ctrl_req <= 1'b1;
|
||||
ctrl_wr <= 1'b1;
|
||||
ctrl_addr <= ar_eff_block_base;
|
||||
ctrl_wdata <= {8{ar_eff_wdata}}; // replicate; only the target word's mask bits matter
|
||||
ctrl_wmask <= {16{1'b1}} & ~(16'h0003 << (ar_eff_word_in_blk*2)) | ({14'b0, ar_eff_ubn, ar_eff_lbn} << (ar_eff_word_in_blk*2));
|
||||
ar_req_pending <= 1'b0;
|
||||
state <= S_AR_WR_WAIT;
|
||||
end
|
||||
end
|
||||
S_W_WAIT: begin
|
||||
if (ctrl_ready) begin
|
||||
if (w_pending_upper_half) begin
|
||||
w_rdata <= ctrl_rdata[127:64];
|
||||
w_cache_data[w_alloc_ptr] <= ctrl_rdata[63:0];
|
||||
w_cache_addr[w_alloc_ptr] <= w_pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
|
||||
end else begin
|
||||
w_rdata <= ctrl_rdata[63:0];
|
||||
w_cache_data[w_alloc_ptr] <= ctrl_rdata[127:64];
|
||||
w_cache_addr[w_alloc_ptr] <= w_pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
|
||||
end
|
||||
w_cache_valid[w_alloc_ptr] <= 1'b1;
|
||||
w_alloc_ptr <= (w_alloc_ptr == W_ENTRIES[WEIDXW-1:0]-1'b1) ? {WEIDXW{1'b0}} : w_alloc_ptr + 1'b1;
|
||||
w_ready <= 1'b1;
|
||||
state <= S_IDLE;
|
||||
end
|
||||
end
|
||||
S_AR_RD_WAIT: begin
|
||||
if (ctrl_ready) begin
|
||||
ar_rdata <= ctrl_rdata[ar_pending_word*16 +: 16];
|
||||
ar_ready <= 1'b1;
|
||||
state <= S_IDLE;
|
||||
end
|
||||
end
|
||||
S_AR_WR_WAIT: begin
|
||||
if (ctrl_ready) begin
|
||||
ar_ready <= 1'b1;
|
||||
state <= S_IDLE;
|
||||
end
|
||||
end
|
||||
default: state <= S_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,68 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// NMS STEP16 Phase 5 -- SDRAM weight-fetch backend. Wraps the real,
|
||||
// isolated-and-validated sdram_controller.v (Phase 1-4, EXP-0040/0041,
|
||||
// ERR-0016..0019) with BURST_LEN=4 (P_IN*DATA_WIDTH/16 = 4 words/tile,
|
||||
// the exact natural match identified in Phase 1) and presents the
|
||||
// same mem_req/mem_wr/mem_addr/mem_wdata/mem_rdata/mem_ready
|
||||
// convention as psram_controller_dual32.v, so it drops into
|
||||
// nms_neural_multiprocessor_sdram.v's wide weight-fetch port exactly
|
||||
// where psram_controller_dual32.v sits in the dual32 baseline --
|
||||
// same external contract, different physical memory underneath, for
|
||||
// a direct, apples-to-apples STEP16 comparison.
|
||||
//
|
||||
// Address conversion: `mem_addr` is a BYTE address (this project's
|
||||
// own established convention, matching weight_prefetch_engine_wide.v
|
||||
// and psram_controller_dual32.v's own external contract). The real
|
||||
// AS4C4M16SA-6TIN is x16 (2 bytes/word), so the SDRAM controller's
|
||||
// own word address is `mem_addr >> 1` -- exactly analogous to
|
||||
// STEP15's own real ">>2" byte-to-32-bit-word fix (EXP-0037 bug #1),
|
||||
// here ">>1" for a 16-bit-word device.
|
||||
//
|
||||
// Weight fetch never writes (same convention as psram_controller_
|
||||
// dual32.v's own top-level instantiation, which ties mem_wr=0):
|
||||
// `mem_wr` is exposed here for interface symmetry but the underlying
|
||||
// sdram_controller is only ever driven with wr=0 by this wrapper's
|
||||
// own real usage in nms_neural_multiprocessor_sdram.v.
|
||||
// ============================================================
|
||||
module sdram_weight_backend #(
|
||||
parameter ADDR_WIDTH = 23, // byte address width (project convention)
|
||||
parameter CLK_FREQ_MHZ = 80
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire mem_req,
|
||||
input wire mem_wr,
|
||||
input wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
input wire [63:0] mem_wdata,
|
||||
output wire [63:0] mem_rdata,
|
||||
output wire mem_ready,
|
||||
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
wire busy;
|
||||
wire [21:0] sdram_word_addr = mem_addr[ADDR_WIDTH-1:1]; // byte -> 16-bit-word address
|
||||
|
||||
sdram_controller #(
|
||||
.CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(4), .ADDR_WIDTH(22)
|
||||
) u_sdram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(mem_req), .wr(mem_wr), .addr(sdram_word_addr),
|
||||
.wdata(mem_wdata), .wmask(8'h00), .rdata(mem_rdata), .ready(mem_ready), .busy(busy),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,171 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// NMS STEP18 Part C/D/F -- SDRAM weight-fetch backend, PACKED variant.
|
||||
//
|
||||
// Same external contract as sdram_weight_backend.v (STEP16): byte
|
||||
// address in, 64-bit mem_rdata out, mem_req/mem_wr/mem_ready --
|
||||
// weight_prefetch_engine_wide.v (MEM_DATA_WIDTH=64) plugs in UNCHANGED,
|
||||
// and neural_processor.v is never touched. ONLY the physical transfer
|
||||
// underneath changes: BURST_LEN=8 (128 bits/16 bytes per real SDRAM
|
||||
// transaction, 2 P8 tiles' worth) instead of STEP16's BURST_LEN=4 (64
|
||||
// bits/8 bytes, 1 tile).
|
||||
//
|
||||
// Design: an N_ENTRIES-deep, fully-associative "other half" cache
|
||||
// holds the not-yet-requested half of each real 128-bit fetch still
|
||||
// issued, tagged by its own byte address. On mem_req(addr):
|
||||
// - cache HIT (some entry's addr matches): serve instantly, no new
|
||||
// SDRAM transaction, invalidate that entry.
|
||||
// - cache MISS: issue a real BURST_LEN=8 fetch at the 16-byte-
|
||||
// aligned base covering addr, serve the HALF the caller actually
|
||||
// asked for, and cache the OTHER half (round-robin-allocated
|
||||
// entry) for a possible future hit.
|
||||
//
|
||||
// WHY N_ENTRIES>1 IS REQUIRED (found the hard way, EXP-0046): a first
|
||||
// draft used a single-entry cache, correct in isolation (see
|
||||
// tb_sdram_weight_backend_pack128.v, 20/20 PASS for one requester) but
|
||||
// a REAL regression in the full N=4 system (74004 cycles vs the
|
||||
// STEP16 baseline's 49430 -- WORSE, not better). Root cause: the real
|
||||
// system's N_SLOTS independent memory managers all share ONE physical
|
||||
// backend through slot_mem_arbiter_wide.v, which interleaves their
|
||||
// requests round-robin. A single cache entry gets overwritten by
|
||||
// ANOTHER slot's own "other half" before the ORIGINAL slot's own next
|
||||
// (paired) request ever arrives, so nearly every access became a real
|
||||
// 16-cycle miss instead of the intended ~50% instant-hit rate --
|
||||
// WORSE than the STEP16 baseline's 10-cycle BURST_LEN=4 transactions.
|
||||
// Fix: size the cache to N_ENTRIES (>= N_SLOTS, the real worst-case
|
||||
// number of simultaneously-pending "other halves" -- each slot has at
|
||||
// most ONE outstanding request at a time, by the existing memory
|
||||
// manager's own design, so N_SLOTS entries can never be exceeded in
|
||||
// real traffic). Round-robin eviction (not LRU) is used for
|
||||
// simplicity; it is SAFE regardless of sizing accuracy, since an
|
||||
// evicted-too-early entry only costs an extra real fetch (a
|
||||
// performance effect), never incorrect data (a cache MISS always
|
||||
// falls back to a real, address-exact fetch).
|
||||
// ============================================================
|
||||
module sdram_weight_backend_pack128 #(
|
||||
parameter ADDR_WIDTH = 23, // byte address width (project convention)
|
||||
parameter CLK_FREQ_MHZ = 80,
|
||||
parameter N_ENTRIES = 4 // >= real N_SLOTS in the system using this backend
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire mem_req,
|
||||
input wire mem_wr,
|
||||
input wire [ADDR_WIDTH-1:0] mem_addr,
|
||||
input wire [63:0] mem_wdata,
|
||||
output reg [63:0] mem_rdata,
|
||||
output reg mem_ready,
|
||||
|
||||
output wire sdram_cke,
|
||||
output wire sdram_cs_n,
|
||||
output wire sdram_ras_n,
|
||||
output wire sdram_cas_n,
|
||||
output wire sdram_we_n,
|
||||
output wire [1:0] sdram_ba,
|
||||
output wire [11:0] sdram_a,
|
||||
inout wire [15:0] sdram_dq,
|
||||
output wire [1:0] sdram_dqm
|
||||
);
|
||||
|
||||
localparam EIDXW = (N_ENTRIES <= 1) ? 1 : $clog2(N_ENTRIES);
|
||||
|
||||
reg cache_valid [0:N_ENTRIES-1];
|
||||
reg [ADDR_WIDTH-1:0] cache_addr [0:N_ENTRIES-1];
|
||||
reg [63:0] cache_data [0:N_ENTRIES-1];
|
||||
reg [EIDXW-1:0] alloc_ptr;
|
||||
|
||||
reg hit_found_c;
|
||||
reg [EIDXW-1:0] hit_idx_c;
|
||||
integer ei;
|
||||
always @(*) begin
|
||||
hit_found_c = 1'b0;
|
||||
hit_idx_c = {EIDXW{1'b0}};
|
||||
for (ei = 0; ei < N_ENTRIES; ei = ei + 1) begin
|
||||
if (cache_valid[ei] && cache_addr[ei] == mem_addr) begin
|
||||
hit_found_c = 1'b1;
|
||||
hit_idx_c = ei[EIDXW-1:0];
|
||||
end
|
||||
end
|
||||
end
|
||||
wire cache_hit = hit_found_c && mem_req && !mem_wr;
|
||||
|
||||
// ---- real SDRAM controller, BURST_LEN=8 (128-bit/16-byte txns) ----
|
||||
reg ctrl_req;
|
||||
reg ctrl_wr;
|
||||
reg [21:0] ctrl_addr; // 22-bit word address (16-bit words)
|
||||
wire [127:0] ctrl_rdata;
|
||||
wire ctrl_ready;
|
||||
wire ctrl_busy;
|
||||
|
||||
wire [21:0] aligned_word_addr = {mem_addr[ADDR_WIDTH-1:4], 3'b000}; // 16-byte-aligned word address
|
||||
wire addr_is_upper_half = mem_addr[3]; // 1 = caller wants bytes [aligned+8 .. aligned+15]
|
||||
|
||||
sdram_controller #(
|
||||
.CLK_FREQ_MHZ(CLK_FREQ_MHZ), .BURST_LEN(8), .ADDR_WIDTH(22)
|
||||
) u_sdram_ctrl (
|
||||
.clk(clk), .rst(rst),
|
||||
.req(ctrl_req), .wr(ctrl_wr), .addr(ctrl_addr),
|
||||
.wdata(64'h0), .wmask(16'h0000), .rdata(ctrl_rdata), .ready(ctrl_ready), .busy(ctrl_busy),
|
||||
.sdram_cke(sdram_cke), .sdram_cs_n(sdram_cs_n), .sdram_ras_n(sdram_ras_n),
|
||||
.sdram_cas_n(sdram_cas_n), .sdram_we_n(sdram_we_n),
|
||||
.sdram_ba(sdram_ba), .sdram_a(sdram_a), .sdram_dq(sdram_dq), .sdram_dqm(sdram_dqm)
|
||||
);
|
||||
|
||||
localparam S_IDLE = 2'd0, S_WAIT = 2'd1;
|
||||
reg [1:0] state;
|
||||
reg pending_upper_half;
|
||||
reg [ADDR_WIDTH-1:0] pending_addr;
|
||||
|
||||
integer ri;
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
state <= S_IDLE;
|
||||
for (ri = 0; ri < N_ENTRIES; ri = ri + 1) cache_valid[ri] <= 1'b0;
|
||||
alloc_ptr <= {EIDXW{1'b0}};
|
||||
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= 22'h0;
|
||||
mem_ready <= 1'b0; mem_rdata <= 64'h0;
|
||||
pending_upper_half <= 1'b0; pending_addr <= {ADDR_WIDTH{1'b0}};
|
||||
end else begin
|
||||
ctrl_req <= 1'b0;
|
||||
mem_ready <= 1'b0;
|
||||
|
||||
case (state)
|
||||
S_IDLE: begin
|
||||
if (cache_hit) begin
|
||||
mem_rdata <= cache_data[hit_idx_c];
|
||||
mem_ready <= 1'b1;
|
||||
cache_valid[hit_idx_c] <= 1'b0;
|
||||
end else if (mem_req && !mem_wr) begin
|
||||
ctrl_req <= 1'b1;
|
||||
ctrl_wr <= 1'b0;
|
||||
ctrl_addr <= aligned_word_addr;
|
||||
pending_upper_half <= addr_is_upper_half;
|
||||
pending_addr <= mem_addr;
|
||||
state <= S_WAIT;
|
||||
end
|
||||
end
|
||||
S_WAIT: begin
|
||||
if (ctrl_ready) begin
|
||||
if (pending_upper_half) begin
|
||||
mem_rdata <= ctrl_rdata[127:64];
|
||||
cache_data[alloc_ptr] <= ctrl_rdata[63:0];
|
||||
cache_addr[alloc_ptr] <= pending_addr - {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
|
||||
end else begin
|
||||
mem_rdata <= ctrl_rdata[63:0];
|
||||
cache_data[alloc_ptr] <= ctrl_rdata[127:64];
|
||||
cache_addr[alloc_ptr] <= pending_addr + {{(ADDR_WIDTH-4){1'b0}}, 4'd8};
|
||||
end
|
||||
cache_valid[alloc_ptr] <= 1'b1;
|
||||
alloc_ptr <= (alloc_ptr == N_ENTRIES[EIDXW-1:0]-1'b1) ? {EIDXW{1'b0}} : alloc_ptr + 1'b1;
|
||||
mem_ready <= 1'b1;
|
||||
state <= S_IDLE;
|
||||
end
|
||||
end
|
||||
default: state <= S_IDLE;
|
||||
endcase
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,181 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP11: real weight prefetch engine.
|
||||
//
|
||||
// Replaces prefetch_engine.v's per-tile single-shot usage inside
|
||||
// nms_memory_manager.v with a CONTINUOUS, multi-tile fetch stream.
|
||||
//
|
||||
// Real analysis (see docs/architecture/nms_weight_prefetch.md and
|
||||
// hardware/v2/logs/development.log): the real backend
|
||||
// (hardware/v1/rtl/memory_interface.v -> psram_controller.v) is a
|
||||
// fire-and-forget, ONE-transaction-in-flight-at-a-time protocol (a
|
||||
// single mem_req pulse, wait for mem_ready, that IS the transaction --
|
||||
// no wire-level pipelining is physically possible against this real
|
||||
// backend, matching the real PSRAM's own single physical port). So
|
||||
// "multiple outstanding requests" cannot mean multiple simultaneous
|
||||
// WORD transactions -- it means eliminating the CONTROL-PLANE
|
||||
// overhead the old design paid at every tile boundary (prefetch_engine
|
||||
// return-to-IDLE, fetch_done pulse, nms_memory_manager's own
|
||||
// !pf_done-gated restart, ERR-0013) and letting the fetch stream run
|
||||
// CONTINUOUSLY across tile boundaries, queueing up to
|
||||
// PREFETCH_DISTANCE tiles' worth of lookahead ahead of consumption
|
||||
// instead of restarting control state once per tile.
|
||||
//
|
||||
// Tiles are always fetched in strict sequential order (0..n_tiles-1,
|
||||
// never reordered, never re-fetched) -- so no per-tile state array is
|
||||
// needed; two monotonic counters (fetch progress, consumption
|
||||
// progress) fully describe the system, exactly like the superseded
|
||||
// design, but the FETCH counter now advances continuously instead of
|
||||
// stalling at each tile boundary.
|
||||
// ============================================================
|
||||
module weight_prefetch_engine #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1),
|
||||
parameter WORDS_PER_TILE = P_IN/2,
|
||||
parameter WIW = $clog2(WORDS_PER_TILE+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
// ---- job control (level-held while a job is running; the
|
||||
// consumer -- nms_memory_manager.v -- resets its OWN tile_idx to 0
|
||||
// on job_start, this engine mirrors that via job_active falling/
|
||||
// rising) ----
|
||||
input wire job_active,
|
||||
input wire [ADDR_WIDTH-1:0] w_base, // byte address, word-aligned
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [CNTW-1:0] consumed_count, // consumer's own tile_idx, bounds lookahead
|
||||
|
||||
// ---- fill port into nms_weight_packed.v (this slot's own private lane) ----
|
||||
output reg wgt_fill_we,
|
||||
output reg [TIW-1:0] wgt_fill_addr,
|
||||
output reg [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
|
||||
// ---- status to consumer: tiles 0..ready_count-1 are fully resident ----
|
||||
output reg [CNTW-1:0] ready_count,
|
||||
|
||||
// ---- real word-level PSRAM backend (matches
|
||||
// hardware/v1/rtl/memory_interface.v's contract exactly, same as
|
||||
// prefetch_engine.v's own real, proven usage) ----
|
||||
output reg mem_req,
|
||||
output wire mem_wr,
|
||||
output reg [ADDR_WIDTH-1:0] mem_addr,
|
||||
output wire [15:0] mem_wdata,
|
||||
output wire mem_lb_n,
|
||||
output wire mem_ub_n,
|
||||
input wire [15:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
assign mem_wr = 1'b0;
|
||||
assign mem_wdata = 16'h0000;
|
||||
assign mem_lb_n = 1'b0; // fetch the whole word, both byte lanes
|
||||
assign mem_ub_n = 1'b0;
|
||||
|
||||
// fetch_tile/fetch_word: the NEXT word to be requested (or, while
|
||||
// req_outstanding, the word CURRENTLY in flight).
|
||||
reg [CNTW-1:0] fetch_tile;
|
||||
reg [WIW-1:0] fetch_word;
|
||||
reg req_outstanding;
|
||||
reg [DATA_WIDTH*P_IN-1:0] tile_buf;
|
||||
|
||||
wire [ADDR_WIDTH-1:0] w_word_base = w_base[ADDR_WIDTH-1:1];
|
||||
|
||||
// Don't fetch past n_tiles, and don't get more than
|
||||
// PREFETCH_DISTANCE tiles ahead of the consumer's own progress --
|
||||
// the real, configurable lookahead window (STEP11's own
|
||||
// requirement). consumed_count is the consumer's tile_idx
|
||||
// (registered, one cycle old at most -- fine, this only bounds a
|
||||
// SOFT bandwidth-shaping window, not a correctness-critical value:
|
||||
// over-fetching by one extra tile due to a one-cycle-stale compare
|
||||
// is harmless, the SRAM has room for the whole vector regardless).
|
||||
//
|
||||
// window_limit/the comparison below are computed in a FIXED 32-bit
|
||||
// width, wide enough to hold PREFETCH_DISTANCE undamaged for any
|
||||
// realistic parameter value -- an earlier version truncated
|
||||
// PREFETCH_DISTANCE down to CNTW bits before adding it
|
||||
// (PREFETCH_DISTANCE[CNTW-1:0]), which silently wrapped PFD=32 to 0
|
||||
// at MAX_TILES=16 (CNTW=5 bits), making window_limit==consumed_count
|
||||
// and more_to_fetch permanently false -- a full, real deadlock (see
|
||||
// errors.log ERR-0015, same TIW/CNTW-truncation bug class as
|
||||
// ERR-0014, this time on the newly-introduced PREFETCH_DISTANCE
|
||||
// parameter itself rather than a tile counter).
|
||||
wire [31:0] window_limit = {{(32-CNTW){1'b0}}, consumed_count} + PREFETCH_DISTANCE;
|
||||
wire more_to_fetch = job_active &&
|
||||
({{(16-CNTW){1'b0}}, fetch_tile} < n_tiles) &&
|
||||
({{(32-CNTW){1'b0}}, fetch_tile} < window_limit);
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
fetch_tile <= {CNTW{1'b0}};
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
ready_count <= {CNTW{1'b0}};
|
||||
req_outstanding <= 1'b0;
|
||||
mem_req <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
end else begin
|
||||
mem_req <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
|
||||
if (!job_active) begin
|
||||
// mirrors the consumer's own job_start reset (nms_
|
||||
// memory_manager.v resets its tile_idx the same way)
|
||||
fetch_tile <= {CNTW{1'b0}};
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
ready_count <= {CNTW{1'b0}};
|
||||
req_outstanding <= 1'b0;
|
||||
end else if (mem_ready && req_outstanding) begin
|
||||
// A word just completed. Commit it, THEN -- same
|
||||
// cycle, not next -- decide the very next request
|
||||
// (same tile's next word, or the following tile's
|
||||
// first word): this is what makes the fetch stream
|
||||
// genuinely continuous across tile boundaries, not
|
||||
// just within one tile the way prefetch_engine.v's own
|
||||
// design already was. Computed with blocking-style
|
||||
// "next state" locals so both the commit and the next
|
||||
// request land in a single, unambiguous set of NBAs.
|
||||
req_outstanding <= 1'b0;
|
||||
tile_buf[fetch_word*16 +: 16] <= mem_rdata;
|
||||
|
||||
if (fetch_word == WORDS_PER_TILE[WIW-1:0] - 1'b1) begin
|
||||
wgt_fill_we <= 1'b1;
|
||||
wgt_fill_addr <= fetch_tile[TIW-1:0];
|
||||
wgt_fill_data <= {mem_rdata, tile_buf[DATA_WIDTH*P_IN-17:0]};
|
||||
ready_count <= ready_count + 1'b1;
|
||||
fetch_tile <= fetch_tile + 1'b1;
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
// start the NEXT tile's first word immediately if
|
||||
// the (post-increment) tile is still within bounds
|
||||
if (({{(16-CNTW){1'b0}}, fetch_tile + 1'b1} < n_tiles) &&
|
||||
({{(32-CNTW){1'b0}}, fetch_tile + 1'b1} < window_limit)) begin
|
||||
mem_req <= 1'b1;
|
||||
mem_addr <= w_word_base + (fetch_tile + 1'b1) * WORDS_PER_TILE[CNTW-1:0];
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end else begin
|
||||
fetch_word <= fetch_word + 1'b1;
|
||||
mem_req <= 1'b1;
|
||||
mem_addr <= w_word_base + fetch_tile*WORDS_PER_TILE[CNTW-1:0] + {{(ADDR_WIDTH-WIW){1'b0}}, fetch_word} + 1'b1;
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end else if (!req_outstanding && more_to_fetch) begin
|
||||
// reached only when nothing has ever been requested
|
||||
// yet for this job (the very first word) -- every
|
||||
// subsequent request is issued from the branch above,
|
||||
// in the same cycle its predecessor's mem_ready
|
||||
// arrives, with zero gap, including across tile
|
||||
// boundaries.
|
||||
mem_req <= 1'b1;
|
||||
mem_addr <= w_word_base + fetch_tile*WORDS_PER_TILE[CNTW-1:0] + {{(ADDR_WIDTH-WIW){1'b0}}, fetch_word};
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -0,0 +1,162 @@
|
||||
`timescale 1ns/1ps
|
||||
|
||||
// ============================================================
|
||||
// Neural Memory System (NMS) -- STEP14 Part A: parameterized-width
|
||||
// experimental weight prefetch engine.
|
||||
//
|
||||
// SIMULATION-ONLY / EXPLORATORY (same status as ideal_memory_model.v,
|
||||
// STEP11's own EXP-0017/23): establishes the ARCHITECTURAL requirement
|
||||
// (what logical weight-path width removes the fetch-rate bottleneck
|
||||
// EXP-0026/0027 identified) BEFORE committing to any specific real
|
||||
// hardware implementation. Generalizes weight_prefetch_engine.v's own
|
||||
// continuous cross-tile-boundary streaming design (STEP11, unchanged
|
||||
// in spirit) to an arbitrary MEM_DATA_WIDTH instead of the real V1
|
||||
// PSRAM's fixed 16 bits. Byte-lane enables (mem_lb_n/mem_ub_n) are
|
||||
// dropped at this level of abstraction -- not meaningful for a
|
||||
// logical bus wider than 16 bits; the real 16-bit interface (with
|
||||
// lane enables) is reintroduced separately by the A5 packing adapter
|
||||
// (weight_fetch_pack_adapter.v) that connects this engine's logical
|
||||
// wide requests to the REAL, unmodified V1 PSRAM chain.
|
||||
//
|
||||
// WORDS_PER_TILE generalizes to
|
||||
// ceil(DATA_WIDTH*P_IN / MEM_DATA_WIDTH), clamped to a minimum of 1
|
||||
// (a bus wider than one full tile still costs exactly 1 transaction,
|
||||
// with the surplus bits simply unused -- this experiment does not
|
||||
// attempt multi-tile-per-transaction bursting).
|
||||
// ============================================================
|
||||
module weight_prefetch_engine_wide #(
|
||||
parameter DATA_WIDTH = 8,
|
||||
parameter P_IN = 8,
|
||||
parameter ADDR_WIDTH = 23,
|
||||
parameter MAX_TILES = 16,
|
||||
parameter PREFETCH_DISTANCE = 8,
|
||||
parameter MEM_DATA_WIDTH = 64, // 16, 32, 64, 128 -- the STEP14 Part A sweep parameter
|
||||
parameter TIW = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
|
||||
parameter CNTW = $clog2(MAX_TILES+1),
|
||||
// ceil(TILE_BITS / MEM_DATA_WIDTH), minimum 1
|
||||
parameter TILE_BITS = DATA_WIDTH*P_IN,
|
||||
parameter WORDS_PER_TILE = (TILE_BITS + MEM_DATA_WIDTH - 1) / MEM_DATA_WIDTH,
|
||||
parameter WIW = $clog2(WORDS_PER_TILE+1)
|
||||
)(
|
||||
input wire clk,
|
||||
input wire rst,
|
||||
|
||||
input wire job_active,
|
||||
input wire [ADDR_WIDTH-1:0] w_base, // byte address
|
||||
input wire [15:0] n_tiles,
|
||||
input wire [CNTW-1:0] consumed_count,
|
||||
|
||||
output reg wgt_fill_we,
|
||||
output reg [TIW-1:0] wgt_fill_addr,
|
||||
output reg [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
|
||||
|
||||
output reg [CNTW-1:0] ready_count,
|
||||
|
||||
// ---- logical wide memory port (ideal_memory_model_wide.v) ----
|
||||
output reg mem_req,
|
||||
output reg [ADDR_WIDTH-1:0] mem_addr, // byte address of this transaction's first byte
|
||||
input wire [MEM_DATA_WIDTH-1:0] mem_rdata,
|
||||
input wire mem_ready
|
||||
);
|
||||
|
||||
localparam BYTES_PER_WORD = MEM_DATA_WIDTH/8;
|
||||
// The backing store is packed at the tile's OWN natural byte size
|
||||
// (TILE_BITS/8 = P_IN*DATA_WIDTH/8, e.g. 8 bytes for P_IN=8/
|
||||
// DATA_WIDTH=8), regardless of MEM_DATA_WIDTH. This equals
|
||||
// WORDS_PER_TILE*BYTES_PER_WORD whenever MEM_DATA_WIDTH<=TILE_BITS
|
||||
// (no waste, e.g. 16/32/64-bit busses), but NOT when
|
||||
// MEM_DATA_WIDTH>TILE_BITS (e.g. a 128-bit bus fetching a 64-bit
|
||||
// tile in one transaction, using only its low half) -- using
|
||||
// WORDS_PER_TILE*BYTES_PER_WORD as the inter-tile address stride
|
||||
// in that case would double-count the unused surplus bits as real
|
||||
// address space and skip over the next tile's actual data in the
|
||||
// packed backing store. TILE_BYTES is the correct stride always.
|
||||
localparam TILE_BYTES = TILE_BITS/8;
|
||||
|
||||
reg [CNTW-1:0] fetch_tile;
|
||||
reg [WIW-1:0] fetch_word;
|
||||
reg req_outstanding;
|
||||
reg [WORDS_PER_TILE*MEM_DATA_WIDTH-1:0] tile_buf; // oversized scratch, only low TILE_BITS used
|
||||
|
||||
// Final-word tile assembly, selected at ELABORATION time
|
||||
// (WORDS_PER_TILE is a parameter) via generate -- avoids an
|
||||
// invalid zero/negative-width part-select on tile_buf when
|
||||
// WORDS_PER_TILE==1 (bus wider than one full tile: no "earlier
|
||||
// words" exist at all, the ternary-operator alternative would
|
||||
// still be elaborated structurally by most tools even though
|
||||
// never selected at runtime).
|
||||
wire [TILE_BITS-1:0] final_word_tile_data;
|
||||
generate
|
||||
if (WORDS_PER_TILE == 1) begin : GEN_ASSEMBLE_SINGLE
|
||||
assign final_word_tile_data = mem_rdata[TILE_BITS-1:0];
|
||||
end else begin : GEN_ASSEMBLE_MULTI
|
||||
// Yosys' Verilog frontend rejects a part-select applied
|
||||
// directly to a concatenation ({a,b}[msb:lsb]); Verilator
|
||||
// accepts it, but real synthesis requires an intermediate
|
||||
// signal instead.
|
||||
wire [WORDS_PER_TILE*MEM_DATA_WIDTH-1:0] assembled_full;
|
||||
assign assembled_full = {mem_rdata, tile_buf[(WORDS_PER_TILE-1)*MEM_DATA_WIDTH-1:0]};
|
||||
assign final_word_tile_data = assembled_full[TILE_BITS-1:0];
|
||||
end
|
||||
endgenerate
|
||||
|
||||
wire [31:0] window_limit = {{(32-CNTW){1'b0}}, consumed_count} + PREFETCH_DISTANCE;
|
||||
wire more_to_fetch = job_active &&
|
||||
({{(16-CNTW){1'b0}}, fetch_tile} < n_tiles) &&
|
||||
({{(32-CNTW){1'b0}}, fetch_tile} < window_limit);
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (rst) begin
|
||||
fetch_tile <= {CNTW{1'b0}};
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
ready_count <= {CNTW{1'b0}};
|
||||
req_outstanding <= 1'b0;
|
||||
mem_req <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
end else begin
|
||||
mem_req <= 1'b0;
|
||||
wgt_fill_we <= 1'b0;
|
||||
|
||||
if (!job_active) begin
|
||||
fetch_tile <= {CNTW{1'b0}};
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
ready_count <= {CNTW{1'b0}};
|
||||
req_outstanding <= 1'b0;
|
||||
end else if (mem_ready && req_outstanding) begin
|
||||
req_outstanding <= 1'b0;
|
||||
tile_buf[fetch_word*MEM_DATA_WIDTH +: MEM_DATA_WIDTH] <= mem_rdata;
|
||||
|
||||
if (fetch_word == WORDS_PER_TILE[WIW-1:0] - 1'b1) begin
|
||||
wgt_fill_we <= 1'b1;
|
||||
wgt_fill_addr <= fetch_tile[TIW-1:0];
|
||||
wgt_fill_data <= final_word_tile_data;
|
||||
ready_count <= ready_count + 1'b1;
|
||||
fetch_tile <= fetch_tile + 1'b1;
|
||||
fetch_word <= {WIW{1'b0}};
|
||||
if (({{(16-CNTW){1'b0}}, fetch_tile + 1'b1} < n_tiles) &&
|
||||
({{(32-CNTW){1'b0}}, fetch_tile + 1'b1} < window_limit)) begin
|
||||
mem_req <= 1'b1;
|
||||
mem_addr <= w_base + (fetch_tile + 1'b1) * TILE_BYTES[CNTW-1:0];
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end else begin
|
||||
fetch_word <= fetch_word + 1'b1;
|
||||
mem_req <= 1'b1;
|
||||
// next word within the SAME tile (only reached when
|
||||
// WORDS_PER_TILE>1, i.e. MEM_DATA_WIDTH<=TILE_BITS,
|
||||
// where WORDS_PER_TILE*BYTES_PER_WORD==TILE_BYTES
|
||||
// exactly -- no surplus/waste in that regime):
|
||||
// byte offset = fetch_tile*TILE_BYTES + (fetch_word+1)*BYTES_PER_WORD
|
||||
mem_addr <= w_base + fetch_tile*TILE_BYTES[CNTW-1:0] +
|
||||
({{(CNTW-WIW){1'b0}}, fetch_word} + 1'b1) * BYTES_PER_WORD[CNTW-1:0];
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end else if (!req_outstanding && more_to_fetch) begin
|
||||
mem_req <= 1'b1;
|
||||
mem_addr <= w_base + fetch_tile*TILE_BYTES[CNTW-1:0];
|
||||
req_outstanding <= 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
Reference in New Issue
Block a user