Implements the highest-leverage fix from EXP-0080's bottleneck analysis: act_tile_fetch.v now packs 2 consecutive tiles per DDR3 burst (even tile low 64 bits, odd tile high 64 bits) instead of 1 tile per burst, halving real DDR3 bytes-per-useful-byte. Timing-safe by construction: the tile-index select bit is registered at request time, long before the real DDR3 round-trip completes, never racing the arriving read data (unlike the runtime part-select pattern EXP-0079 deliberately avoided). Re-verified at all 3 levels (isolated engine 8/8, packed_slot.v 9/9 with bit-identical results to EXP-0079, full N=2 system on real DDR3 8/8) -- the real JEDEC trace now shows no half-burst padding, direct confirmation the fix works in practice, not just in theory. Also: real device data gathered on this package's I/O bank layout (only 5 banks total, 14/15/16/34/35) informing the next bandwidth step (32-bit-wide single controller recommended over a second independent channel, given the pin/logic cost comparison). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUG92aM9m68TRc4rG55BcC
171 lines
7.6 KiB
Verilog
171 lines
7.6 KiB
Verilog
`timescale 1ns/1ps
|
|
|
|
// ============================================================
|
|
// V3 -- act_tile_fetch.v: REAL activation-tile fetch engine, closing
|
|
// the gap packed_slot.v's own header has disclosed since EXP-0062
|
|
// ("a real activation fetch engine ... is a separate, later
|
|
// deliverable, NOT built here"). This is that deliverable.
|
|
//
|
|
// WHY A SEPARATE, SIMPLE ENGINE (not a prefetch/buffer pair like the
|
|
// weight path): weights are reused across M reuse-positions per
|
|
// Director-dispatched pair, so prefetching them once into an on-chip
|
|
// buffer (layer_prefetch_ctrl.v/layer_weight_buffer.v) amortizes real
|
|
// DDR3 latency across many reads. Activation data has NO such reuse
|
|
// -- each position's activation tile is read exactly once per job --
|
|
// so buffering it on-chip would only add complexity for zero benefit.
|
|
// This engine reads DIRECTLY from DDR3 per tile instead.
|
|
//
|
|
// MEMORY LAYOUT CONVENTION v2 (EXP-0081, real, disclosed, and REQUIRED
|
|
// of whoever prepares activation data in DDR3 -- documented in the
|
|
// physical realization doc too): TWO consecutive tiles (P_IN=8 INT8
|
|
// values each) share ONE full BURST_LEN=8-word (128-bit) burst -- even
|
|
// tile index in the LOW 64 bits, odd tile index in the HIGH 64 bits.
|
|
// Tile t's burst address is `base + (t>>1)*BURST_LEN`. This HALVES
|
|
// real DDR3 bytes-moved-per-useful-byte versus the original EXP-0079
|
|
// "1 tile = 1 burst" layout (real measured 1.24GB/s ceiling could only
|
|
// sustain ~25% of one core's peak DSP throughput under that layout --
|
|
// see docs/ARCHITECTURE_ANALYSIS.md S3.2 -- this doubles the real
|
|
// achievable fraction).
|
|
//
|
|
// WHY THIS IS TIMING-SAFE (the thing EXP-0079 deliberately avoided):
|
|
// the tile index's own LSB (which half of the burst to use) is known
|
|
// at REQUEST time, not at response time -- it's registered into
|
|
// `sel_lat` the SAME cycle `tcnt` is latched, many ui_clk cycles
|
|
// BEFORE the real DDR3 round-trip completes and `ctrl_rdata` becomes
|
|
// valid. The eventual data-select mux therefore selects using an
|
|
// already-long-stable registered bit, never a bit racing the read
|
|
// data itself -- this is NOT the runtime-indexed-part-select-on-the-
|
|
// critical-path pattern weight_tile_gather.v's own header (EXP-0061)
|
|
// warned about; that pattern is about a select signal arriving
|
|
// LATE/simultaneously with the data it gates. Confirmed via a real
|
|
// P&R re-check after this change (see the log), not just asserted.
|
|
//
|
|
// PROTOCOL: one request (`req` pulse + base_a/base_b/tcnt) triggers
|
|
// TWO SEQUENTIAL burst reads (lane A then lane B) over the SAME
|
|
// shared ctrl port packed_slot.v already owns -- reusing the EXACT
|
|
// port layer_prefetch_ctrl.v uses during S_PREFETCH, since that
|
|
// phase has already finished (weight data is on-chip by the time
|
|
// this engine runs) and the port is genuinely free. Follows the same
|
|
// combinational-first-grant discipline as every other one-shot-pulse
|
|
// requester in this project (EXP-0066): `mem_active` must be visible
|
|
// to the arbiter the SAME cycle it asserts, `ctrl_req` is only issued
|
|
// after `mem_grant` is observed, never blind.
|
|
// ============================================================
|
|
module act_tile_fetch #(
|
|
parameter DATA_WIDTH = 8,
|
|
parameter P_IN = 8,
|
|
parameter BURST_LEN = 8,
|
|
parameter ADDR_WIDTH = 25 // word address, matches the shared ctrl port's own convention
|
|
)(
|
|
input wire clk,
|
|
input wire rst,
|
|
|
|
input wire req, // one-shot pulse
|
|
input wire [ADDR_WIDTH-1:0] base_a,
|
|
input wire [ADDR_WIDTH-1:0] base_b,
|
|
input wire [15:0] tcnt,
|
|
output reg valid, // one-cycle pulse, data_a/data_b valid
|
|
output reg signed [DATA_WIDTH*P_IN-1:0] data_a,
|
|
output reg signed [DATA_WIDTH*P_IN-1:0] data_b,
|
|
|
|
output wire mem_active,
|
|
input wire mem_grant,
|
|
|
|
output reg ctrl_req,
|
|
output reg ctrl_wr,
|
|
output reg [ADDR_WIDTH-1:0] ctrl_addr,
|
|
output wire [16*BURST_LEN-1:0] ctrl_wdata,
|
|
output wire [2*BURST_LEN-1:0] ctrl_wmask,
|
|
input wire [16*BURST_LEN-1:0] ctrl_rdata,
|
|
input wire ctrl_ready,
|
|
input wire ctrl_busy
|
|
);
|
|
assign ctrl_wdata = {(16*BURST_LEN){1'b0}};
|
|
assign ctrl_wmask = {(2*BURST_LEN){1'b0}}; // read-only engine, mask unused
|
|
|
|
localparam S_IDLE = 3'd0,
|
|
S_MEMWAIT = 3'd1,
|
|
S_REQ_A = 3'd2,
|
|
S_GAP = 3'd3, // wait for ctrl_busy to clear before firing lane B's request
|
|
S_REQ_B = 3'd4;
|
|
|
|
reg [2:0] state;
|
|
reg [ADDR_WIDTH-1:0] base_a_lat, base_b_lat;
|
|
reg [15:0] tcnt_lat;
|
|
reg sel_lat; // registered at request time -- see header
|
|
|
|
assign mem_active = (state != S_IDLE);
|
|
|
|
// burst index = tcnt/2 (integer division -- two tiles share one burst)
|
|
wire [ADDR_WIDTH-1:0] tile_offset = {{(ADDR_WIDTH-15){1'b0}}, tcnt_lat[15:1]} * BURST_LEN[ADDR_WIDTH-1:0];
|
|
|
|
always @(posedge clk) begin
|
|
if (rst) begin
|
|
state <= S_IDLE;
|
|
ctrl_req <= 1'b0; ctrl_wr <= 1'b0; ctrl_addr <= {ADDR_WIDTH{1'b0}};
|
|
valid <= 1'b0; data_a <= {(DATA_WIDTH*P_IN){1'b0}}; data_b <= {(DATA_WIDTH*P_IN){1'b0}};
|
|
base_a_lat <= {ADDR_WIDTH{1'b0}}; base_b_lat <= {ADDR_WIDTH{1'b0}}; tcnt_lat <= 16'd0;
|
|
sel_lat <= 1'b0;
|
|
end else begin
|
|
ctrl_req <= 1'b0;
|
|
valid <= 1'b0;
|
|
|
|
case (state)
|
|
S_IDLE: begin
|
|
if (req) begin
|
|
base_a_lat <= base_a;
|
|
base_b_lat <= base_b;
|
|
tcnt_lat <= tcnt;
|
|
sel_lat <= tcnt[0];
|
|
state <= S_MEMWAIT;
|
|
end
|
|
end
|
|
|
|
S_MEMWAIT: begin
|
|
if (mem_grant) begin
|
|
ctrl_addr <= base_a_lat + tile_offset;
|
|
ctrl_wr <= 1'b0;
|
|
ctrl_req <= 1'b1;
|
|
state <= S_REQ_A;
|
|
end
|
|
end
|
|
|
|
S_REQ_A: begin
|
|
if (ctrl_ready) begin
|
|
data_a <= sel_lat ? ctrl_rdata[DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]
|
|
: ctrl_rdata[0 +: DATA_WIDTH*P_IN];
|
|
ctrl_addr <= base_b_lat + tile_offset;
|
|
ctrl_wr <= 1'b0;
|
|
state <= S_GAP;
|
|
end
|
|
end
|
|
|
|
S_GAP: begin
|
|
// the shared controller may still be finishing its
|
|
// own internal completion sequence for lane A's
|
|
// request for one more cycle after ctrl_ready
|
|
// pulsed (mig_native_adapter.v's own S_DONE state
|
|
// keeps `busy` asserted through it) -- wait for
|
|
// !ctrl_busy before firing lane B's request,
|
|
// instead of assuming back-to-back is safe.
|
|
if (!ctrl_busy) begin
|
|
ctrl_req <= 1'b1;
|
|
state <= S_REQ_B;
|
|
end
|
|
end
|
|
|
|
S_REQ_B: begin
|
|
if (ctrl_ready) begin
|
|
data_b <= sel_lat ? ctrl_rdata[DATA_WIDTH*P_IN +: DATA_WIDTH*P_IN]
|
|
: ctrl_rdata[0 +: DATA_WIDTH*P_IN];
|
|
valid <= 1'b1;
|
|
state <= S_IDLE;
|
|
end
|
|
end
|
|
|
|
default: state <= S_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
endmodule
|