FPGA-Neural/hardware/v2/nms/rtl/nms_memory_manager_stream.v

// ============================================================
// Neural Memory System (NMS) -- STEP13: per-slot memory manager,
// PIPELINED CONTINUOUS TILE STREAM variant ("_stream").
//
// Identical external interface to nms_memory_manager_pf.v (drop-in,
// same Director/Dependency-Manager side, same Neural-Processor-facing
// operand/result streams, same weight_prefetch_engine.v instance) --
// the ONLY change is INSIDE ST_RUN: the operand-delivery pipeline that
// feeds neural_processor.v.
//
// EXP-0025 traced nms_memory_manager_pf.v's own ST_RUN state with a
// zero-real-memory-latency configuration and found it costs EXACTLY 4
// cycles/tile in steady state (read_issued -> read_ready -> present
// -> consumed, a strictly sequential if/else-if chain with ZERO
// overlap between consecutive tiles), even though neither side of the
// interface requires it: the local activation/weight SRAMs
// (nms_activation_replicated.v / nms_weight_packed.v) have only a
// 1-cycle rd_en-to-data latency, and neural_processor.v's own
// operand_ready is held continuously high through the whole
// NP_WAIT_OPERANDS phase (its datapath is explicitly designed to
// accept a new tile every cycle). That 4-cycles/tile serialization
// was found to account for 93.4% of EXP-0024's real, measured
// "non-memory" cycle floor (DEC-0024) -- the dominant real bottleneck,
// NOT per-job dispatch overhead.
//
// This variant replaces ST_RUN's 4-state chain with a pipelined
// read-ahead design:
//   - `rd_ptr` (CNTW bits): the tile index whose SRAM read has been
//     (or is about to be) ISSUED -- independent of, and normally one
//     tile AHEAD of, `tile_idx` (the CONSUMPTION pointer, i.e. how
//     many tiles neural_processor.v has actually accepted).
//   - a 1-deep skid buffer (`buf_valid`/`buf_input`/`buf_weight`/
//     `buf_last`) holds one tile's fully-read SRAM data, presented to
//     NP as `operand_valid`/`input_data`/`weight_data`/`tile_last`.
//   - every cycle: if a read was issued last cycle (`rd_pending`), its
//     data is now valid (1-cycle SRAM latency) and is captured into
//     the skid buffer; independently, a NEW read is issued for
//     `rd_ptr` whenever it is legal to do so (in bounds, weight+
//     activation ready) AND the skid buffer will not overflow (it is
//     empty, or being drained -- consumed -- this very same cycle).
// Since NP's own operand_ready is continuously high through the tile-
// loading phase, the skid buffer is drained every cycle it is full,
// so a new read can be issued every cycle too: sustained ~1 cycle/
// tile, down from 4 -- a real ~4x reduction in the dominant component
// of EXP-0024's measured floor.
//
// `tile_idx` (the CONSUMPTION pointer) is still what is fed to
// weight_prefetch_engine.v's own `consumed_count` port -- its
// external contract (bound the lookahead window against how far the
// CONSUMER has progressed) is unchanged; only the local SRAM
// read-issue pointer (`rd_ptr`) is new, and it can run up to ONE tile
// ahead of `tile_idx` (the skid buffer's own depth), same as before
// conceptually (read_issued/read_ready already implied a similar
// small lookahead, just serialized rather than pipelined).
//
// nms_memory_manager_pf.v itself is UNTOUCHED -- this file exists
// alongside it (and alongside the original nms_memory_manager.v) so
// all three ("Current NMS", "NMS + weight prefetch",
// "NMS + weight prefetch + continuous tile stream") remain
// independently reproducible for A/B/C comparison.
// ============================================================
module nms_memory_manager_stream #(
    parameter DATA_WIDTH = 8,
    parameter P_IN       = 8,
    parameter ADDR_WIDTH = 23,
    parameter MAX_TILES  = 16,
    parameter PREFETCH_DISTANCE = 8,
    parameter TIW        = (MAX_TILES <= 1) ? 1 : $clog2(MAX_TILES),
    parameter CNTW        = $clog2(MAX_TILES+1)
)(
    input  wire clk,
    input  wire rst,

    input  wire                      job_start,
    input  wire [ADDR_WIDTH-1:0]     x_base,
    input  wire [ADDR_WIDTH-1:0]     w_base,
    input  wire [15:0]               n_tiles,
    input  wire [ADDR_WIDTH-1:0]     result_addr,
    output reg                       job_done,

    output wire                                operand_valid,
    input  wire                               operand_ready,
    output wire signed [DATA_WIDTH*P_IN-1:0]  input_data,
    output wire signed [DATA_WIDTH*P_IN-1:0]  weight_data,
    output wire                                tile_last,

    input  wire                       result_valid,
    output reg                        result_ready,
    input  wire signed [DATA_WIDTH-1:0] result_data,

    output wire            job_active,
    output wire [ADDR_WIDTH-1:0] job_x_base,
    output wire [15:0]     job_n_tiles,

    input  wire [ADDR_WIDTH-1:0] act_resident_tag,
    input  wire [CNTW-1:0]       act_resident_count,

    output reg              act_rd_en,
    output reg  [TIW-1:0]   act_rd_addr,
    input  wire signed [DATA_WIDTH*P_IN-1:0] act_rd_data,

    // ---- weight SRAM fill port driven by weight_prefetch_engine
    // (below), NOT by this FSM directly -- read port unchanged ----
    output wire              wgt_fill_we,
    output wire [TIW-1:0]    wgt_fill_addr,
    output wire [DATA_WIDTH*P_IN-1:0] wgt_fill_data,
    output reg              wgt_rd_en,
    output reg  [TIW-1:0]   wgt_rd_addr,
    input  wire signed [DATA_WIDTH*P_IN-1:0] wgt_rd_data,

    output wire                    mem_req,
    output wire                    mem_wr,
    output wire [ADDR_WIDTH-1:0]   mem_addr,
    output wire [15:0]             mem_wdata,
    output wire                    mem_lb_n,
    output wire                    mem_ub_n,
    input  wire [15:0]             mem_rdata,
    input  wire                    mem_ready
);

    localparam ST_IDLE        = 3'd0;
    localparam ST_RUN         = 3'd1;
    localparam ST_WAIT_RESULT = 3'd2;
    localparam ST_WRITE_RES   = 3'd3;
    localparam ST_DONE        = 3'd4;

    reg [2:0] state;
    reg job_active_reg;
    assign job_active  = job_active_reg;
    assign job_x_base  = x_base_reg;
    assign job_n_tiles = n_tiles_reg;

    reg [ADDR_WIDTH-1:0] x_base_reg, w_base_reg, result_addr_reg;
    reg [15:0]           n_tiles_reg;
    reg [CNTW-1:0]       tile_idx;   // CONSUMPTION pointer (tiles handed to NP so far)
    reg [CNTW-1:0]       rd_ptr;     // READ-ISSUE pointer (tiles whose SRAM read has been issued)

    wire [CNTW-1:0] wgt_ready_count;

    wire usable_act_count_valid = (act_resident_tag == x_base_reg);
    wire [CNTW-1:0] usable_act = usable_act_count_valid ? act_resident_count : {CNTW{1'b0}};

    // Gating for the READ-ISSUE pointer (rd_ptr), same semantics as
    // the old can_present but evaluated against rd_ptr instead of
    // tile_idx, since reads may now run ahead of consumption.
    wire can_issue_rd = ({{(16-CNTW){1'b0}}, rd_ptr} < n_tiles_reg) &&
                        (rd_ptr < wgt_ready_count) &&
                        (rd_ptr < usable_act);

    // ---- pipelined read-ahead + 1-deep skid buffer ----
    reg                          rd_pending;   // a read issued last cycle; its data is valid THIS cycle
    reg  [CNTW-1:0]              rd_pending_tile;
    reg                          rd_pending_last;
    reg                          buf_valid;
    reg  signed [DATA_WIDTH*P_IN-1:0] buf_input, buf_weight;
    reg                          buf_last;

    assign operand_valid = buf_valid;
    assign input_data    = buf_input;
    assign weight_data   = buf_weight;
    assign tile_last     = buf_last;

    // May issue a new read this cycle iff the skid buffer will not
    // overflow: it's currently empty, or it is being drained
    // (consumed) THIS cycle.
    wire buf_will_be_free = !buf_valid || (operand_valid && operand_ready);
    wire issue_rd_now     = (state == ST_RUN) && can_issue_rd && buf_will_be_free;

    always @(posedge clk) begin
        if (rst) begin
            state          <= ST_IDLE;
            job_active_reg <= 1'b0;
            job_done       <= 1'b0;
            result_ready   <= 1'b0;
            tile_idx       <= {CNTW{1'b0}};
            rd_ptr         <= {CNTW{1'b0}};
            rd_pending     <= 1'b0;
            buf_valid      <= 1'b0;
            act_rd_en      <= 1'b0;
            wgt_rd_en      <= 1'b0;
            wr_mem_req     <= 1'b0;
            wr_mem_lb_n    <= 1'b1;
            wr_mem_ub_n    <= 1'b1;
        end else begin
            job_done     <= 1'b0;
            act_rd_en    <= 1'b0;
            wgt_rd_en    <= 1'b0;
            result_ready <= 1'b0;

            case (state)
                ST_IDLE: begin
                    if (job_start) begin
                        x_base_reg      <= x_base;
                        w_base_reg      <= w_base;
                        n_tiles_reg     <= n_tiles;
                        result_addr_reg <= result_addr;
                        tile_idx        <= {CNTW{1'b0}};
                        rd_ptr          <= {CNTW{1'b0}};
                        rd_pending      <= 1'b0;
                        buf_valid       <= 1'b0;
                        job_active_reg  <= 1'b1;
                        state           <= ST_RUN;
                    end
                end

                ST_RUN: begin
                    // ---- Step 1: a read issued LAST cycle lands now ----
                    if (rd_pending) begin
                        buf_valid  <= 1'b1;
                        buf_input  <= act_rd_data;
                        buf_weight <= wgt_rd_data;
                        buf_last   <= rd_pending_last;
                    end else if (operand_valid && operand_ready) begin
                        // no new data arriving this cycle -- if the
                        // buffer is being drained and nothing refills
                        // it, it goes empty.
                        buf_valid <= 1'b0;
                    end

                    // ---- Step 2: consumption bookkeeping ----
                    if (operand_valid && operand_ready) begin
                        if ({{(16-CNTW){1'b0}}, tile_idx} + 16'd1 == n_tiles_reg) begin
                            job_active_reg <= 1'b0;
                            state          <= ST_WAIT_RESULT;
                        end else begin
                            tile_idx <= tile_idx + 1'b1;
                        end
                    end

                    // ---- Step 3: issue the NEXT read, if legal ----
                    if (issue_rd_now) begin
                        act_rd_en       <= 1'b1;
                        act_rd_addr     <= rd_ptr[TIW-1:0];
                        wgt_rd_en       <= 1'b1;
                        wgt_rd_addr     <= rd_ptr[TIW-1:0];
                        rd_pending      <= 1'b1;
                        rd_pending_tile <= rd_ptr;
                        rd_pending_last <= ({{(16-CNTW){1'b0}}, rd_ptr} == n_tiles_reg - 16'd1);
                        rd_ptr          <= rd_ptr + 1'b1;
                    end else begin
                        rd_pending <= 1'b0;
                    end
                end

                ST_WAIT_RESULT: begin
                    result_ready <= 1'b1;
                    if (result_valid && result_ready) begin
                        wr_mem_wdata <= result_addr_reg[0] ? {result_data, 8'h00} : {8'h00, result_data};
                        wr_mem_lb_n  <= result_addr_reg[0] ? 1'b1 : 1'b0;
                        wr_mem_ub_n  <= result_addr_reg[0] ? 1'b0 : 1'b1;
                        state        <= ST_WRITE_RES;
                    end
                end

                ST_WRITE_RES: begin
                    wr_mem_req  <= 1'b1;
                    wr_mem_addr <= result_addr_reg[ADDR_WIDTH-1:1];
                    state       <= ST_DONE;
                end

                ST_DONE: begin
                    wr_mem_req <= 1'b0;
                    if (mem_ready) begin
                        job_done <= 1'b1;
                        state    <= ST_IDLE;
                    end
                end

                default: state <= ST_IDLE;
            endcase
        end
    end

    // ---- REAL weight prefetch engine (STEP11): unchanged from
    // nms_memory_manager_pf.v -- consumed_count is still the
    // CONSUMPTION pointer (tile_idx), not the read-issue pointer
    // (rd_ptr): the engine's own lookahead window is bounded by how
    // far the CONSUMER has progressed, exactly as before. ----
    weight_prefetch_engine #(
        .DATA_WIDTH(DATA_WIDTH), .P_IN(P_IN), .ADDR_WIDTH(ADDR_WIDTH),
        .MAX_TILES(MAX_TILES), .PREFETCH_DISTANCE(PREFETCH_DISTANCE)
    ) u_wpf (
        .clk(clk), .rst(rst),
        .job_active(job_active_reg), .w_base(w_base_reg), .n_tiles(n_tiles_reg),
        .consumed_count(tile_idx),
        .wgt_fill_we(wgt_fill_we), .wgt_fill_addr(wgt_fill_addr), .wgt_fill_data(wgt_fill_data),
        .ready_count(wgt_ready_count),
        .mem_req(mem_req_wpf), .mem_wr(mem_wr_wpf), .mem_addr(mem_addr_wpf), .mem_wdata(mem_wdata_wpf),
        .mem_lb_n(mem_lb_n_wpf), .mem_ub_n(mem_ub_n_wpf),
        .mem_rdata(mem_rdata), .mem_ready(mem_ready)
    );
    wire mem_req_wpf, mem_wr_wpf;
    wire [ADDR_WIDTH-1:0] mem_addr_wpf;
    wire [15:0] mem_wdata_wpf;
    wire mem_lb_n_wpf, mem_ub_n_wpf;

    reg                   wr_mem_req;
    reg  [ADDR_WIDTH-1:0] wr_mem_addr;
    reg  [15:0]           wr_mem_wdata;
    reg                   wr_mem_lb_n, wr_mem_ub_n;

    wire wr_active = (state == ST_WRITE_RES) || (state == ST_DONE);
    assign mem_req   = wr_active ? wr_mem_req   : mem_req_wpf;
    assign mem_wr    = wr_active ? 1'b1         : mem_wr_wpf;
    assign mem_addr  = wr_active ? wr_mem_addr  : mem_addr_wpf;
    assign mem_wdata = wr_active ? wr_mem_wdata : mem_wdata_wpf;
    assign mem_lb_n  = wr_active ? wr_mem_lb_n  : mem_lb_n_wpf;
    assign mem_ub_n  = wr_active ? wr_mem_ub_n  : mem_ub_n_wpf;

endmodule