Feature Update: Return Code and Better Parameterization of XBar

Makefile

@@ -233,7 +233,6 @@ VLOG_DEFS = -DCACHEPOOL
 
 # Cluster configuration
 VLOG_DEFS += -DNUM_TILES=$(num_tiles)
-VLOG_DEFS += -DNumRemotePortTile=$(num_remote_ports_per_tile)
 VLOG_DEFS += -DNUM_CORES=$(num_cores)
 VLOG_DEFS += -DDATA_WIDTH=$(data_width)
 VLOG_DEFS += -DADDR_WIDTH=$(addr_width)
@@ -259,6 +258,7 @@ VLOG_DEFS += -DSPATZ_NUM_FPU=$(spatz_num_fpu)
 VLOG_DEFS += -DSPATZ_NUM_IPU=$(spatz_num_ipu)
 VLOG_DEFS += -DSPATZ_MAX_TRANS=$(spatz_max_trans)
 VLOG_DEFS += -DSNITCH_MAX_TRANS=$(snitch_max_trans)
+VLOG_DEFS += -DREMOTE_PORT_PER_CORE=$(num_remote_ports_per_tile)
 
 # AXI configuration
 VLOG_DEFS += -DAXI_USER_WIDTH=$(axi_user_width)

config/cachepool_fpu_512.mk

@@ -20,7 +20,7 @@ data_width ?= 32
 # Core addrwidth
 addr_width ?= 32
 
-num_remote_ports_per_tile ?= 1
+num_remote_ports_per_tile ?= 2
 
 
 ######################

hardware/cachepool_peripheral/cachepool_peripheral.sv

@@ -30,7 +30,7 @@ module cachepool_peripheral
   input  reg_req_t                   reg_req_i,
   output reg_rsp_t                   reg_rsp_o,
 
-  output logic                       eoc_o,
+  output logic [3:0]                 eoc_o,
   input  addr_t                      tcdm_start_address_i,
   input  addr_t                      tcdm_end_address_i,
   output addr_t                      private_start_addr_o,

hardware/cachepool_peripheral/cachepool_peripheral_reg.hjson

@@ -139,7 +139,7 @@
         hwaccess: "hro",
         resval: "0",
         fields: [{
-            bits: "0:0",
+            bits: "3:0",
             name: "EOC_EXIT",
             desc: "Indicates the end of computation and exit status."
         }]

hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv

@@ -51,7 +51,7 @@ package cachepool_peripheral_reg_pkg;
   } cachepool_peripheral_reg2hw_cluster_boot_control_reg_t;
 
   typedef struct packed {
-    logic        q;
+    logic [3:0]  q;
   } cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t;
 
   typedef struct packed {
@@ -115,15 +115,15 @@ package cachepool_peripheral_reg_pkg;
 
   // Register -> HW type
   typedef struct packed {
-    cachepool_peripheral_reg2hw_hart_select_mreg_t [1:0] hart_select; // [272:253]
-    cachepool_peripheral_reg2hw_cl_clint_set_reg_t cl_clint_set; // [252:220]
-    cachepool_peripheral_reg2hw_cl_clint_clear_reg_t cl_clint_clear; // [219:187]
-    cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [186:155]
-    cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [154:154]
-    cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [153:153]
-    cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [152:121]
-    cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [120:89]
-    cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [88:88]
+    cachepool_peripheral_reg2hw_hart_select_mreg_t [1:0] hart_select; // [275:256]
+    cachepool_peripheral_reg2hw_cl_clint_set_reg_t cl_clint_set; // [255:223]
+    cachepool_peripheral_reg2hw_cl_clint_clear_reg_t cl_clint_clear; // [222:190]
+    cachepool_peripheral_reg2hw_hw_barrier_reg_t hw_barrier; // [189:158]
+    cachepool_peripheral_reg2hw_icache_prefetch_enable_reg_t icache_prefetch_enable; // [157:157]
+    cachepool_peripheral_reg2hw_spatz_status_reg_t spatz_status; // [156:156]
+    cachepool_peripheral_reg2hw_spatz_cycle_reg_t spatz_cycle; // [155:124]
+    cachepool_peripheral_reg2hw_cluster_boot_control_reg_t cluster_boot_control; // [123:92]
+    cachepool_peripheral_reg2hw_cluster_eoc_exit_reg_t cluster_eoc_exit; // [91:88]
     cachepool_peripheral_reg2hw_cfg_l1d_spm_reg_t cfg_l1d_spm; // [87:78]
     cachepool_peripheral_reg2hw_cfg_l1d_insn_reg_t cfg_l1d_insn; // [77:76]
     cachepool_peripheral_reg2hw_cfg_l1d_tile_sel_reg_t cfg_l1d_tile_sel; // [75:44]

hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv

@@ -90,8 +90,8 @@ module cachepool_peripheral_reg_top #(
   logic [31:0] cluster_boot_control_qs;
   logic [31:0] cluster_boot_control_wd;
   logic cluster_boot_control_we;
-  logic cluster_eoc_exit_qs;
-  logic cluster_eoc_exit_wd;
+  logic [3:0] cluster_eoc_exit_qs;
+  logic [3:0] cluster_eoc_exit_wd;
   logic cluster_eoc_exit_we;
   logic [9:0] cfg_l1d_spm_qs;
   logic [9:0] cfg_l1d_spm_wd;
@@ -337,9 +337,9 @@ module cachepool_peripheral_reg_top #(
   // R[cluster_eoc_exit]: V(False)
 
   prim_subreg #(
-    .DW      (1),
+    .DW      (4),
     .SWACCESS("RW"),
-    .RESVAL  (1'h0)
+    .RESVAL  (4'h0)
   ) u_cluster_eoc_exit (
     .clk_i   (clk_i    ),
     .rst_ni  (rst_ni  ),
@@ -701,7 +701,7 @@ module cachepool_peripheral_reg_top #(
   assign cluster_boot_control_wd = reg_wdata[31:0];
 
   assign cluster_eoc_exit_we = addr_hit[9] & reg_we & !reg_error;
-  assign cluster_eoc_exit_wd = reg_wdata[0];
+  assign cluster_eoc_exit_wd = reg_wdata[3:0];
 
   assign cfg_l1d_spm_we = addr_hit[10] & reg_we & !reg_error;
   assign cfg_l1d_spm_wd = reg_wdata[9:0];
@@ -773,7 +773,7 @@ module cachepool_peripheral_reg_top #(
       end
 
       addr_hit[9]: begin
-        reg_rdata_next[0] = cluster_eoc_exit_qs;
+        reg_rdata_next[3:0] = cluster_eoc_exit_qs;
       end
 
       addr_hit[10]: begin

hardware/src/cachepool_cluster.sv

@@ -117,7 +117,7 @@ module cachepool_cluster
     /// corresponding core into debug mode. This signal is assumed to be _async_.
     input  logic          [NrCores-1:0]           debug_req_i,
     /// End of Computing indicator to notify the host/tb
-    output logic                                  eoc_o,
+    output logic          [3:0]                   eoc_o,
     /// Machine external interrupt pending. Usually those interrupts come from a
     /// platform-level interrupt controller. This signal is assumed to be _async_.
     input  logic          [NrCores-1:0]           meip_i,

hardware/src/cachepool_group.sv

@@ -219,46 +219,63 @@
 
   // Tile remote access signals
   // In/Out relative to the tile (out--leave a tile; in--enter a tile)
-  tcdm_req_t        [NumTiles-1:0][NrTCDMPortsPerCore-1:0] tile_remote_out_req;
-  tcdm_rsp_t        [NumTiles-1:0][NrTCDMPortsPerCore-1:0] tile_remote_out_rsp;
-  logic             [NumTiles-1:0][NrTCDMPortsPerCore-1:0] tile_remote_in_ready,      tile_remote_out_ready;
-  tcdm_req_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_out_req_chan;
-  logic             [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_out_req_valid, tile_remote_out_req_ready;
-  tcdm_rsp_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_out_rsp_chan;
-  logic             [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_out_rsp_valid, tile_remote_out_rsp_ready;
-
-  tcdm_req_t        [NumTiles-1:0][NrTCDMPortsPerCore-1:0] tile_remote_in_req;
-  tcdm_rsp_t        [NumTiles-1:0][NrTCDMPortsPerCore-1:0] tile_remote_in_rsp;
-  tcdm_req_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_in_req_chan;
-  logic             [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_in_req_valid,  tile_remote_in_req_ready;
-  tcdm_rsp_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_in_rsp_chan;
-  logic             [NrTCDMPortsPerCore-1:0][NumTiles-1:0] tile_remote_in_rsp_valid,  tile_remote_in_rsp_ready;
-
-  // Symmetric xbar, in/out select types are the same
-  remote_tile_sel_t [NumTiles-1:0][NrTCDMPortsPerCore-1:0] remote_out_sel_tile, remote_in_sel_tile;
-  remote_tile_sel_t [NrTCDMPortsPerCore-1:0][NumTiles-1:0] remote_out_sel_xbar, remote_in_sel_xbar;
+  // Tile-side flat layout: index = j + r*NrTCDMPortsPerCore (j=xbar idx, r=remote slot within xbar)
+  tcdm_req_t        [NumTiles-1:0][NumRemotePortTile-1:0] tile_remote_out_req;
+  tcdm_rsp_t        [NumTiles-1:0][NumRemotePortTile-1:0] tile_remote_out_rsp;
+  logic             [NumTiles-1:0][NumRemotePortTile-1:0] tile_remote_in_ready, tile_remote_out_ready;
+
+  tcdm_req_t        [NumTiles-1:0][NumRemotePortTile-1:0] tile_remote_in_req;
+  tcdm_rsp_t        [NumTiles-1:0][NumRemotePortTile-1:0] tile_remote_in_rsp;
+
+  // Xbar-side: NrTCDMPortsPerCore xbars, each with NumTiles*NumRemotePortCore ports
+  // Xbar port index = t*NumRemotePortCore + r
+  tcdm_req_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_out_req_chan;
+  logic             [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_out_req_valid, tile_remote_out_req_ready;
+  tcdm_rsp_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_out_rsp_chan;
+  logic             [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_out_rsp_valid, tile_remote_out_rsp_ready;
+
+  tcdm_req_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_in_req_chan;
+  logic             [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_in_req_valid,  tile_remote_in_req_ready;
+  tcdm_rsp_chan_t   [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_in_rsp_chan;
+  logic             [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] tile_remote_in_rsp_valid,  tile_remote_in_rsp_ready;
+
+  // Tile-side selection: narrow type, only carries tile_id
+  remote_tile_sel_t [NumTiles-1:0][NumRemotePortTile-1:0]                    remote_out_sel_tile;
+  // Xbar-side selection: wider type, encodes tile_id*NumRemotePortCore + core_id%NumRemotePortCore
+  remote_xbar_sel_t [NrTCDMPortsPerCore-1:0][NumTiles*NumRemotePortCore-1:0] remote_out_sel_xbar, remote_in_sel_xbar;
 
   for (genvar t = 0; t < NumTiles; t++) begin
-    for (genvar p = 0; p < NrTCDMPortsPerCore; p++) begin
-      assign tile_remote_out_req_chan [p][t] = tile_remote_out_req[t][p].q;
-      assign tile_remote_out_req_valid[p][t] = tile_remote_out_req[t][p].q_valid;
-      assign tile_remote_out_rsp_ready[p][t] = tile_remote_in_ready[t][p];
-
-      assign tile_remote_out_rsp[t][p].p       = tile_remote_out_rsp_chan [p][t];
-      assign tile_remote_out_rsp[t][p].p_valid = tile_remote_out_rsp_valid[p][t];
-      assign tile_remote_out_rsp[t][p].q_ready = tile_remote_out_req_ready[p][t];
-
-      assign tile_remote_in_req[t][p].q       = tile_remote_in_req_chan [p][t];
-      assign tile_remote_in_req[t][p].q_valid = tile_remote_in_req_valid[p][t];
-      assign tile_remote_out_ready[t][p]      = tile_remote_in_rsp_ready[p][t];
-
-      assign tile_remote_in_rsp_chan [p][t] = tile_remote_in_rsp[t][p].p;
-      assign tile_remote_in_rsp_valid[p][t] = tile_remote_in_rsp[t][p].p_valid;
-      assign tile_remote_in_req_ready[p][t] = tile_remote_in_rsp[t][p].q_ready;
-
-      // Selection signals
-      assign remote_out_sel_xbar[p][t] = remote_out_sel_tile[t][p];
-      assign remote_in_sel_xbar [p][t] = tile_remote_in_rsp_chan[p][t].user.tile_id;
+    for (genvar j = 0; j < NrTCDMPortsPerCore; j++) begin
+      for (genvar r = 0; r < NumRemotePortCore; r++) begin
+        // tile flat index: j + r*NrTCDMPortsPerCore
+        // xbar port index: t*NumRemotePortCore + r
+        assign tile_remote_out_req_chan [j][t*NumRemotePortCore+r] = tile_remote_out_req[t][j+r*NrTCDMPortsPerCore].q;
+        assign tile_remote_out_req_valid[j][t*NumRemotePortCore+r] = tile_remote_out_req[t][j+r*NrTCDMPortsPerCore].q_valid;
+        assign tile_remote_out_rsp_ready[j][t*NumRemotePortCore+r] = tile_remote_in_ready[t][j+r*NrTCDMPortsPerCore];
+
+        assign tile_remote_out_rsp[t][j+r*NrTCDMPortsPerCore].p       = tile_remote_out_rsp_chan [j][t*NumRemotePortCore+r];
+        assign tile_remote_out_rsp[t][j+r*NrTCDMPortsPerCore].p_valid = tile_remote_out_rsp_valid[j][t*NumRemotePortCore+r];
+        assign tile_remote_out_rsp[t][j+r*NrTCDMPortsPerCore].q_ready = tile_remote_out_req_ready[j][t*NumRemotePortCore+r];
+
+        assign tile_remote_in_req[t][j+r*NrTCDMPortsPerCore].q       = tile_remote_in_req_chan [j][t*NumRemotePortCore+r];
+        assign tile_remote_in_req[t][j+r*NrTCDMPortsPerCore].q_valid = tile_remote_in_req_valid[j][t*NumRemotePortCore+r];
+        assign tile_remote_out_ready[t][j+r*NrTCDMPortsPerCore]      = tile_remote_in_rsp_ready[j][t*NumRemotePortCore+r];
+
+        assign tile_remote_in_rsp_chan [j][t*NumRemotePortCore+r] = tile_remote_in_rsp[t][j+r*NrTCDMPortsPerCore].p;
+        assign tile_remote_in_rsp_valid[j][t*NumRemotePortCore+r] = tile_remote_in_rsp[t][j+r*NrTCDMPortsPerCore].p_valid;
+        assign tile_remote_in_req_ready[j][t*NumRemotePortCore+r] = tile_remote_in_rsp[t][j+r*NrTCDMPortsPerCore].q_ready;
+
+        // Request selection: convert narrow tile_id to wide xbar index by appending
+        // core_id % NumRemotePortCore (available in the request channel user field)
+        assign remote_out_sel_xbar[j][t*NumRemotePortCore+r] = remote_xbar_sel_t'(
+            remote_out_sel_tile[t][j+r*NrTCDMPortsPerCore] * NumRemotePortCore
+          + tile_remote_out_req_chan[j][t*NumRemotePortCore+r].user.core_id % NumRemotePortCore);
+
+        // Response selection: recover xbar port from tile_id and core_id in response user field
+        assign remote_in_sel_xbar[j][t*NumRemotePortCore+r] = remote_xbar_sel_t'(
+            tile_remote_in_rsp_chan[j][t*NumRemotePortCore+r].user.tile_id * NumRemotePortCore
+          + tile_remote_in_rsp_chan[j][t*NumRemotePortCore+r].user.core_id % NumRemotePortCore);
+      end
     end
   end
 
@@ -361,8 +378,8 @@
 
     // Decide which tile to go
     reqrsp_xbar #(
-      .NumInp           (NumTiles                     ),
-      .NumOut           (NumTiles                     ),
+      .NumInp           (NumTiles * NumRemotePortCore ),
+      .NumOut           (NumTiles * NumRemotePortCore ),
       .PipeReg          (1'b1                         ),
       .RspReg           (1'b1                         ),
       .ExtReqPrio       (1'b0                         ),
@@ -392,5 +409,4 @@
     );
   end
 
-
 endmodule

hardware/src/cachepool_pkg.sv

@@ -54,15 +54,18 @@
   // How many cores for each tile?
   localparam int unsigned NumCoresTile      = NumCores / NumTiles;
 
-  // How many remote ports for each tile? Currently needs to be 0 or 1.
-  // localparam int unsigned NumRemotePortTile = `ifdef NumRemotePortTile `NumRemotePortTile `else 0 `endif;
-  localparam int unsigned NumRemotePortTile = 1;
+  // How many remote ports for each tile per core's port?
+  localparam int unsigned NumRemotePortCore = `ifdef REMOTE_PORT_PER_CORE `REMOTE_PORT_PER_CORE `else 0 `endif;
 
   // How many cores within a tile? This is used to select the ports within a tile.
   localparam int unsigned LogNumCoresTile   = $clog2(NumCoresTile);
 
+  // 4 ports from Spatz + 1 shared port from Snitch/FPU
   localparam int unsigned NrTCDMPortsPerCore = 5;
 
+  // How many remote ports for each tile in total?
+  localparam int unsigned NumRemotePortTile = NumRemotePortCore * NrTCDMPortsPerCore;
+
   ////////////////////
   //  CLUSTER HW    //
   ////////////////////
@@ -215,6 +218,9 @@
   // Wide Data ports
   localparam int unsigned GroupWideDataPorts  = NumL1CtrlTile;
 
+  // Correct selection width for remote xbar at group level
+  localparam int unsigned RemoteXbarSelWidth = $clog2(NumTiles * NumRemotePortCore);
+
   /***** Cluster Ports *****/
   // Narrow AXI ports: 1 In from SoC, 1 Out to UART
   localparam int unsigned ClusterNarrowInAxiPorts  = 1;
@@ -352,6 +358,13 @@
     burst_req_t             burst;
   } refill_user_t;
 
+  ///////////////////
+  //  GROUP TYPES  //
+  ///////////////////
+
+  typedef logic [RemoteXbarSelWidth-1:0] remote_xbar_sel_t;
+
+
   /////////////////////
   //  CLUSTER TYPES  //
   /////////////////////