`timescale 1ns/1ps // ============================================================ // V3 -- real, hierarchical 2-level arbiter for n16_system_ddr3_top.v // (EXP-0094), built to fix EXP-0093's own real, measured P&R timing // failure (WNS=-0.913ns on the real 155.039MHz clk_pll_i domain). // // REAL ROOT CAUSE THIS FIXES (traced via EXP-0093's own worst violated // path, not guessed): sdram_arbiter_n.v's own flat req_wdata/req_addr/ // etc mux grew from a 3-way select at N=2 to a real 20/21-way select at // N=16 over a 256-bit-wide bus -- EXP-0093's own real post-route report // showed the delay dominated by ROUTE (73%), not logic (27%): this is a // real PHYSICAL problem (20 separately-placed PE/weight-fetch sources // all converging on one central mux at mig_native_adapter.v), not just // a logic-depth one. // // REAL FIX: reuse `sdram_arbiter_n.v` UNMODIFIED, twice, hierarchically // -- NOT a new arbitration algorithm, the SAME already-proven // combinational-first-grant/locked semantics, just applied at two // levels instead of one: // - LEAF level (4x, one per systolic_group.v): a real // `sdram_arbiter_n` instance with NUM_REQ=5 (1 weight-fetch + 4 PE // activation/writeback requesters of THAT group), physically local // to its own group's own real PEs -- short wires, small mux. // - TOP level (1x): a real `sdram_arbiter_n` instance with // NUM_REQ=5 (4 groups' own pipelined output + 1 host_mem_bridge.v, // unpipelined -- see below), deciding which GROUP gets the real // shared ctrl_* port this cycle. // - ONE real pipeline register stage, BOTH directions, between the // two levels (leaf's own ctrl_req/wr/addr/wdata/wmask -> top's own // req_*, AND top's own req_ready/req_busy/req_rdata -> leaf's own // ctrl_ready/ctrl_busy/ctrl_rdata) -- this is the real fix for the // route-delay-dominated critical path: it lets the placer/router // spread the two mux stages across two separate clock periods' // worth of physical budget instead of forcing all 20 sources to // settle into one central mux within a single cycle. // // REAL CORRECTNESS ARGUMENT (not asserted without derivation): EVERY // requester in this project (`act_tile_fetch.v`, `layer_prefetch_ // ctrl.v`, `host_mem_bridge.v`) already keeps its own `mem_active` // asserted for the FULL duration of its own outstanding transaction, // only dropping it once it has genuinely seen its own completion -- // this is the real invariant `sdram_arbiter_n.v`'s own `locked` state // already relies on for transactions that already span many real DDR3- // latency cycles today. The 2 extra real cycles of round-trip latency // this hierarchy adds (1 cycle each direction, GROUP-sourced traffic // only) are indistinguishable, from any requester's own point of view, // from "DDR3 was slightly slower this time" -- no new race is // introduced, because the LEAF's own `locked` state depends only on // its own local requesters' real `mem_active` signals, never on // response timing from the top level. // // REAL, DELIBERATE ASYMMETRY: `host_mem_bridge.v`'s own single real // requester slot BYPASSES the leaf/pipeline stage entirely, wired // directly into the top-level arbiter's own 5th slot -- it was never // the reported critical path (only one physical source, not 20), and // this keeps its own real round-trip latency completely unchanged from // today's flat design (no reason to add latency where there is no real // fan-in problem to fix). // // PRESERVES EXP-0066's real "own grant same cycle as own active" // requirement EXACTLY for all 21 real requesters: weight-fetch/PE // requesters see the LEAF's own combinational grant (unchanged // `sdram_arbiter_n.v` internals, just NUM_REQ=5 instead of 21); host // sees the TOP's own combinational grant directly. Only the underlying // ctrl_req/wdata/etc reaching mig_native_adapter.v is pipelined -- the // grant signal a requester actually polls is untouched. // // EXTERNAL PORT LIST: byte-for-byte identical shape to // `sdram_arbiter_n.v` (same NUM_REQ-wide req_active/req_grant/etc // arrays, same single ctrl_* port) -- a real drop-in replacement at // the `n16_system_ddr3_top.v` instantiation site, no change needed to // the surrounding req_active/req_grant/etc bus-slicing code there. // Real, fixed slot-index convention matched to n16_system_ddr3_top.v's // own real `GEN_GROUP` layout: slot g (g=0..N_GROUPS-1) = group g's // own weight-fetch; slot N_GROUPS+g*PES_PER_GROUP+p (p=0..PES_PER_ // GROUP-1) = group g's own PE p; slot NUM_REQ-1 = host_mem_bridge.v. // ============================================================ module sdram_arbiter_hier #( parameter N_GROUPS = 4, parameter PES_PER_GROUP = 4, parameter ADDR_WIDTH = 25, parameter BURST_LEN = 8, parameter NUM_REQ = N_GROUPS*(1+PES_PER_GROUP) + 1 // 21, this project's real N=16 topology )( input wire clk, input wire rst, input wire [NUM_REQ-1:0] req_active, output wire [NUM_REQ-1:0] req_grant, input wire [NUM_REQ-1:0] req_req, input wire [NUM_REQ-1:0] req_wr, input wire [NUM_REQ*ADDR_WIDTH-1:0] req_addr, input wire [NUM_REQ*32*BURST_LEN-1:0] req_wdata, input wire [NUM_REQ*4*BURST_LEN-1:0] req_wmask, output wire [NUM_REQ*32*BURST_LEN-1:0] req_rdata, output wire [NUM_REQ-1:0] req_ready, output wire [NUM_REQ-1:0] req_busy, output wire ctrl_req, output wire ctrl_wr, output wire [ADDR_WIDTH-1:0] ctrl_addr, output wire [32*BURST_LEN-1:0] ctrl_wdata, output wire [4*BURST_LEN-1:0] ctrl_wmask, input wire [32*BURST_LEN-1:0] ctrl_rdata, input wire ctrl_ready, input wire ctrl_busy ); localparam GROUP_SIZE = 1 + PES_PER_GROUP; // 5: weight-fetch (local idx 0) + PES_PER_GROUP PEs (local idx 1..4) // ---- per-group leaf outputs (outgoing, pre-pipeline) ---- wire [N_GROUPS-1:0] leaf_active_any; wire [N_GROUPS-1:0] leaf_ctrl_req, leaf_ctrl_wr; wire [N_GROUPS*ADDR_WIDTH-1:0] leaf_ctrl_addr; wire [N_GROUPS*32*BURST_LEN-1:0] leaf_ctrl_wdata; wire [N_GROUPS*4*BURST_LEN-1:0] leaf_ctrl_wmask; // ---- per-group leaf inputs (return path, post-pipeline) ---- reg [N_GROUPS-1:0] leaf_ctrl_ready_reg, leaf_ctrl_busy_reg; reg [N_GROUPS*32*BURST_LEN-1:0] leaf_ctrl_rdata_reg; genvar lg; generate for (lg = 0; lg < N_GROUPS; lg = lg + 1) begin : GEN_LEAF // real, fixed slot map (matches n16_system_ddr3_top.v's own // GEN_GROUP layout exactly): local idx 0 = weight-fetch // (external slot lg, same relative priority it already has // today -- lower index than its own group's PEs), local idx // 1..PES_PER_GROUP = PEs 0..PES_PER_GROUP-1 (external slots // N_GROUPS+lg*PES_PER_GROUP .. +PES_PER_GROUP-1). wire [GROUP_SIZE-1:0] g_req_active = {req_active[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP], req_active[lg]}; wire [GROUP_SIZE-1:0] g_req_req = {req_req[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP], req_req[lg]}; wire [GROUP_SIZE-1:0] g_req_wr = {req_wr[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP], req_wr[lg]}; wire [GROUP_SIZE*ADDR_WIDTH-1:0] g_req_addr = {req_addr[(N_GROUPS+lg*PES_PER_GROUP)*ADDR_WIDTH +: PES_PER_GROUP*ADDR_WIDTH], req_addr[lg*ADDR_WIDTH +: ADDR_WIDTH]}; wire [GROUP_SIZE*32*BURST_LEN-1:0] g_req_wdata = {req_wdata[(N_GROUPS+lg*PES_PER_GROUP)*32*BURST_LEN +: PES_PER_GROUP*32*BURST_LEN], req_wdata[lg*32*BURST_LEN +: 32*BURST_LEN]}; wire [GROUP_SIZE*4*BURST_LEN-1:0] g_req_wmask = {req_wmask[(N_GROUPS+lg*PES_PER_GROUP)*4*BURST_LEN +: PES_PER_GROUP*4*BURST_LEN], req_wmask[lg*4*BURST_LEN +: 4*BURST_LEN]}; wire [GROUP_SIZE-1:0] g_req_grant; wire [GROUP_SIZE*32*BURST_LEN-1:0] g_req_rdata; wire [GROUP_SIZE-1:0] g_req_ready, g_req_busy; sdram_arbiter_n #(.NUM_REQ(GROUP_SIZE), .ADDR_WIDTH(ADDR_WIDTH), .BURST_LEN(BURST_LEN)) u_leaf ( .clk(clk), .rst(rst), .req_active(g_req_active), .req_grant(g_req_grant), .req_req(g_req_req), .req_wr(g_req_wr), .req_addr(g_req_addr), .req_wdata(g_req_wdata), .req_wmask(g_req_wmask), .req_rdata(g_req_rdata), .req_ready(g_req_ready), .req_busy(g_req_busy), .ctrl_req(leaf_ctrl_req[lg]), .ctrl_wr(leaf_ctrl_wr[lg]), .ctrl_addr(leaf_ctrl_addr[lg*ADDR_WIDTH +: ADDR_WIDTH]), .ctrl_wdata(leaf_ctrl_wdata[lg*32*BURST_LEN +: 32*BURST_LEN]), .ctrl_wmask(leaf_ctrl_wmask[lg*4*BURST_LEN +: 4*BURST_LEN]), .ctrl_rdata(leaf_ctrl_rdata_reg[lg*32*BURST_LEN +: 32*BURST_LEN]), .ctrl_ready(leaf_ctrl_ready_reg[lg]), .ctrl_busy(leaf_ctrl_busy_reg[lg]) ); assign leaf_active_any[lg] = |g_req_active; // un-concatenate the leaf's own real per-slot responses back // to their real external (global) slot positions -- these are // the LEAF's own combinational grant/ready/busy/rdata, so the // real EXP-0066 "own grant same cycle as own active" // requirement is preserved EXACTLY for every weight-fetch/PE // requester (see this file's own header). assign req_grant[lg] = g_req_grant[0]; assign req_grant[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP] = g_req_grant[GROUP_SIZE-1:1]; assign req_ready[lg] = g_req_ready[0]; assign req_ready[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP] = g_req_ready[GROUP_SIZE-1:1]; assign req_busy[lg] = g_req_busy[0]; assign req_busy[N_GROUPS+lg*PES_PER_GROUP +: PES_PER_GROUP] = g_req_busy[GROUP_SIZE-1:1]; assign req_rdata[lg*32*BURST_LEN +: 32*BURST_LEN] = g_req_rdata[0 +: 32*BURST_LEN]; assign req_rdata[(N_GROUPS+lg*PES_PER_GROUP)*32*BURST_LEN +: PES_PER_GROUP*32*BURST_LEN] = g_req_rdata[32*BURST_LEN +: PES_PER_GROUP*32*BURST_LEN]; end endgenerate // ---- real pipeline register, outgoing direction (leaf -> top) ---- // REAL FIX (found via signal tracing, EXP-0094): `leaf_ctrl_req` is // a TRANSIENT one-shot pulse -- it mirrors the real underlying // requester's own one-shot ctrl_req (e.g. act_tile_fetch.v's own // S_MEMWAIT: `ctrl_req <= 1'b1` for exactly one real cycle). A bare // "register leaf_ctrl_req every cycle" pipeline (the first, broken // version of this file) loses that pulse whenever the TOP level is // still busy with a DIFFERENT group at the exact cycle it fires -- // by the time TOP gets around to this group, the transient pulse // has already reverted to 0. Real, generalizable lost-pulse bug, // same EXP-0066 class, now hit at the leaf-to-top boundary because // (unlike the flat single-level arbiter, where the winning // requester's own grant and the physical controller's own // readiness to capture it are ALWAYS the same decision) a leaf's // own LOCAL grant (which a real requester correctly waits for // before firing req, confirmed via act_tile_fetch.v) does NOT // guarantee the TOP level is free to act on it the same cycle. // FIX: latch `pending_req_r` STICKY per group, from the first real // cycle `leaf_ctrl_req` pulses until the top level has genuinely // dispatched it (see the `pending_req_r` update below, placed after // `u_top` so it can reference `top_req_grant`/`top_req_req`) -- addr/ // wr/wdata/wmask do NOT need the same treatment: the leaf stays // locked onto the SAME real requester for its entire transaction // (real requesters hold `mem_active` until truly done), so those // fields are already stable for as long as `pending_req_r` matters. reg [N_GROUPS-1:0] top_req_active_r, top_req_req_r, top_req_wr_r; reg [N_GROUPS*ADDR_WIDTH-1:0] top_req_addr_r; reg [N_GROUPS*32*BURST_LEN-1:0] top_req_wdata_r; reg [N_GROUPS*4*BURST_LEN-1:0] top_req_wmask_r; reg [N_GROUPS-1:0] pending_req_r; always @(posedge clk) begin if (rst) begin top_req_active_r <= {N_GROUPS{1'b0}}; top_req_req_r <= {N_GROUPS{1'b0}}; top_req_wr_r <= {N_GROUPS{1'b0}}; top_req_addr_r <= {(N_GROUPS*ADDR_WIDTH){1'b0}}; top_req_wdata_r <= {(N_GROUPS*32*BURST_LEN){1'b0}}; top_req_wmask_r <= {(N_GROUPS*4*BURST_LEN){1'b0}}; end else begin top_req_active_r <= leaf_active_any; top_req_req_r <= pending_req_r | leaf_ctrl_req; top_req_wr_r <= leaf_ctrl_wr; top_req_addr_r <= leaf_ctrl_addr; top_req_wdata_r <= leaf_ctrl_wdata; top_req_wmask_r <= leaf_ctrl_wmask; end end // ---- top-level arbiter: N_GROUPS (pipelined) + 1 host (direct, // unpipelined -- see this file's own header for why) ---- localparam TOP_NUM_REQ = N_GROUPS + 1; wire [TOP_NUM_REQ-1:0] top_req_active = {req_active[NUM_REQ-1], top_req_active_r}; wire [TOP_NUM_REQ-1:0] top_req_req = {req_req[NUM_REQ-1], top_req_req_r}; wire [TOP_NUM_REQ-1:0] top_req_wr = {req_wr[NUM_REQ-1], top_req_wr_r}; wire [TOP_NUM_REQ*ADDR_WIDTH-1:0] top_req_addr = {req_addr[(NUM_REQ-1)*ADDR_WIDTH +: ADDR_WIDTH], top_req_addr_r}; wire [TOP_NUM_REQ*32*BURST_LEN-1:0] top_req_wdata = {req_wdata[(NUM_REQ-1)*32*BURST_LEN +: 32*BURST_LEN], top_req_wdata_r}; wire [TOP_NUM_REQ*4*BURST_LEN-1:0] top_req_wmask = {req_wmask[(NUM_REQ-1)*4*BURST_LEN +: 4*BURST_LEN], top_req_wmask_r}; wire [TOP_NUM_REQ-1:0] top_req_grant; wire [TOP_NUM_REQ*32*BURST_LEN-1:0] top_req_rdata; wire [TOP_NUM_REQ-1:0] top_req_ready, top_req_busy; sdram_arbiter_n #(.NUM_REQ(TOP_NUM_REQ), .ADDR_WIDTH(ADDR_WIDTH), .BURST_LEN(BURST_LEN)) u_top ( .clk(clk), .rst(rst), .req_active(top_req_active), .req_grant(top_req_grant), .req_req(top_req_req), .req_wr(top_req_wr), .req_addr(top_req_addr), .req_wdata(top_req_wdata), .req_wmask(top_req_wmask), .req_rdata(top_req_rdata), .req_ready(top_req_ready), .req_busy(top_req_busy), .ctrl_req(ctrl_req), .ctrl_wr(ctrl_wr), .ctrl_addr(ctrl_addr), .ctrl_wdata(ctrl_wdata), .ctrl_wmask(ctrl_wmask), .ctrl_rdata(ctrl_rdata), .ctrl_ready(ctrl_ready), .ctrl_busy(ctrl_busy) ); // pending_req_r update (see its own declaration above for the real // rationale): set the cycle a group's own leaf_ctrl_req first // pulses; clear the cycle the top level actually dispatches it // (its own grant AND req both true for that group simultaneously -- // this can only happen several cycles after pending_req_r was set, // via the outgoing pipeline register, so there is no same-cycle // set/clear race for a single real request). wire [N_GROUPS-1:0] pending_clear = top_req_grant[N_GROUPS-1:0] & top_req_req[N_GROUPS-1:0]; always @(posedge clk) begin if (rst) pending_req_r <= {N_GROUPS{1'b0}}; else pending_req_r <= (pending_req_r | leaf_ctrl_req) & ~pending_clear; end // host's own external slot: direct, unpipelined (see header) assign req_grant[NUM_REQ-1] = top_req_grant[N_GROUPS]; assign req_ready[NUM_REQ-1] = top_req_ready[N_GROUPS]; assign req_busy[NUM_REQ-1] = top_req_busy[N_GROUPS]; assign req_rdata[(NUM_REQ-1)*32*BURST_LEN +: 32*BURST_LEN] = top_req_rdata[N_GROUPS*32*BURST_LEN +: 32*BURST_LEN]; // ---- real pipeline register, return direction (top -> leaf) ---- always @(posedge clk) begin if (rst) begin leaf_ctrl_ready_reg <= {N_GROUPS{1'b0}}; leaf_ctrl_busy_reg <= {N_GROUPS{1'b1}}; // safe default: "still busy" during reset/pipeline-fill leaf_ctrl_rdata_reg <= {(N_GROUPS*32*BURST_LEN){1'b0}}; end else begin leaf_ctrl_ready_reg <= top_req_ready[N_GROUPS-1:0]; leaf_ctrl_busy_reg <= top_req_busy[N_GROUPS-1:0]; leaf_ctrl_rdata_reg <= top_req_rdata[N_GROUPS*32*BURST_LEN-1:0]; end end endmodule