Merge branch 'valek-feat-dma_calypte_perf_cntrs' into 'devel'

RX DMA Calypte [FEATURE]: Add performance counters to measure blocking behavior

See merge request ndk/ndk-fpga!87

build/DevTree.tcl

@@ -1,5 +1,6 @@
 source [file join [file dirname [info script]] "scripts" "dts" "packed_item.tcl"]
 source [file join [file dirname [info script]] "scripts" "dts" "ndp_header.tcl"]
+source [file join [file dirname [info script]] "scripts" "dts" "dts_templates.tcl"]
 
 # ----------------------------------------------------------------------
 

build/Vivado_non_prj.inc.tcl

@@ -655,9 +655,12 @@ proc nb_sanitize_vars {synth_flags hierarchy} {
 
     set SYNTH_FLAGS(TOOL) "vivado"
 
-    global NB_PLATFORM_TAGS
-    global PLATFORM_TAGS
-    set NB_PLATFORM_TAGS "xilinx $PLATFORM_TAGS"
+    global NB_PLATFORM_TAGS env
+    if {[info exists env(PLATFORM_TAGS)]} {
+        set NB_PLATFORM_TAGS "$env(PLATFORM_TAGS)"
+    } else {
+        set NB_PLATFORM_TAGS "xilinx"
+    }
 
     if {[info commands version] != ""} {
         set SYNTH_FLAGS(TOOL_VERSION) [version -short]

build/readme.rst

@@ -385,3 +385,107 @@ The (incomplete) list of SYNTH_FLAGS array items
    - USE_XPM_LIBRARIES: includes XPM_CDC XPM_MEMORY XPM_FIFO in Vivado projects
 
 For other values and their purpose see the Vivado.inc.tcl or Quartus.inc.tcl file in the build directory.
+
+Device Tree nodes
+-----------------
+
+For the software to find internal firmware components, a *Device Tree (DT)* is
+used. This provides a tree of available parts (called nodes in the DT
+terminology) of the design that can be accessed from the host without
+restriction. A developer creates TCL procedures that generate nodes to *DT
+string (DTS)* for the components he finds fit. Since the creation of the DTS can
+be challenging, there are several TCL procedures provided that simplify the
+process. These procedures are contained within the ``dts_templates.tcl`` file
+with clarifying comments. The following examples provide an overview of their
+usage.
+
+Example 1
+~~~~~~~~~
+
+This presents a least viable code that creates a node
+``dma_calypte_rx_perf_cntrs0`` with the base address *0x8000*
+and the size *0x30*. It also contains a compatible string
+``cesnet,dma_calypte_rx_perf_cntrs``. The string property is appended to ``dts``
+variable that contains a reference to the required Device Tree string (DTS).
+
+.. code-block:: tcl
+
+    dts_create_node dts "dma_calypte_rx_perf_cntrs0" {
+        dts_appendprop_comp_node dts 0x8000 0x30 "cesnet,dma_calypte_rx_perf_cntrs"
+    }
+
+Example 2
+~~~~~~~~~
+
+A second, more complex example demonstrates addition of multiple properties to a
+node called ``dma_ctrl_calypte_$dir$id`` (string can be further adjusted through
+parameters ``dir`` and ``id``).
+
+.. code-block:: tcl
+
+    proc dts_dma_calypte_ctrl {DTS dir id base pcie} {
+        upvar 1 $DTS dts
+
+        dts_create_node dts "dma_ctrl_calypte_$dir$id" {
+            # Adding compatible string "cesnet,dma_ctrl_calypte_$dir" and the
+            # reg property with base address $base and the size 0x80.
+            dts_appendprop_comp_node dts $base 0x80 "cesnet,dma_ctrl_calypte_$dir"
+            # Integer property called "version" with the value 0x10000
+            dts_appendprop_int dts "version" 0x10000
+            # Integer prperty "pcie" with the value of $pcie
+            dts_appendprop_int dts "pcie" $pcie
+
+            # The addition of custom properties (customly named) can be done
+            # through a standard "append" macro.
+            if { $dir == "tx" } {
+                append dts "data_buff = <&dma_calypte_tx_data_buff$id>;"
+                append dts "hdr_buff = <&dma_calypte_tx_hdr_buff$id>;"
+            }
+            append dts "params = <&dma_params_$dir$pcie>;"
+        }
+    }
+
+Example 3
+~~~~~~~~~
+
+This example shows how complex node with multiple subnodes is created. The parent
+node is called ``dma_calypte_test_core0`` and contains subnodes
+``mfb_loopback0``, ``dma_calypte_debug_core0``, ``dma_calypte_latency_meter0``
+and ``dma_calypte_reset_fsm0``. Further nesting of nodes is possible as can be
+seen when adding the ``mfb_generator0`` node. Each of the called procedures
+contain a reference to the same DTS from the ``dts`` variable.
+
+.. code-block:: tcl
+
+    proc dts_calypte_test_core {DTS base_addr} {
+        # Populate reference from the calling environment
+        upvar 1 $DTS dts
+
+        set LOOPBACK_BASE_ADDR      [expr $base_addr + 0x0]
+        set TX_DBG_CORE_BASE_ADDR   [expr $base_addr + 0x10000]
+        set LATENCY_METER_BASE_ADDR [expr $base_addr + 0x20000]
+        set RESET_FSM_BASE_ADDR     [expr $base_addr + 0x30000]
+
+        dts_create_node dts "dma_calypte_test_core0" {
+
+            dts_create_node dts "mfb_loopback0" {
+                dts_appendprop_comp_node dts $LOOPBACK_BASE_ADDR 8 "cesnet,mfb_loopback"
+            }
+
+            dts_create_node dts "dma_calypte_debug_core0" {
+                dts_appendprop_comp_node dts $TX_DBG_CORE_BASE_ADDR 0x1600 "cesnet,dma_calypte_debug_core"
+
+                dts_create_node dts "mfb_generator0" {
+                    dts_appendprop_comp_node dts [expr $TX_DBG_CORE_BASE_ADDR+0x8000] 0x40 "cesnet,mfb_generator"
+                }
+            }
+
+            dts_create_node dts "dma_calypte_latency_meter0" {
+                dts_appendprop_comp_node dts $LATENCY_METER_BASE_ADDR 0x30 "cesnet,dma_calypte_latency_meter"
+            }
+
+            dts_create_node dts "dma_calypte_reset_fsm0" {
+                dts_appendprop_comp_node dts $RESET_FSM_BASE_ADDR 0x4 "cesnet,dma_calypte_reset_fsm"
+            }
+        }
+    }

build/scripts/dts/dts_templates.tcl

@@ -0,0 +1,78 @@
+# dts_templates.tcl: templates for various node types within the DeviceTree
+# Copyright (C) 2024 CESNET z. s. p. o.
+# Author(s): Vladisav Valek <[email protected]>
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Adds a string property to a Device Tree node
+# 1. DTS   - a reference to Device Tree string
+# 2. name  - name of a string property
+# 3. value - value of a string property
+proc dts_appendprop_string {DTS name value} {
+    upvar 1 $DTS dts
+    append dts "$name = \"$value\";\n"
+}
+
+# Adds integer property to a Device Tree node
+# 1. DTS   - a reference to Device Tree string
+# 2. name  - name of an integer property
+# 3. value - value of a integer property
+proc dts_appendprop_int {DTS name value} {
+    upvar 1 $DTS dts
+    append dts "$name = <$value>;\n"
+}
+
+# Adds register as a "reg" property to a Device Tree node
+# 1. DTS  - a reference to Device Tree string
+# 2. addr - base address of a register
+# 3. size - size of a register
+proc dts_appendprop_reg {DTS addr size} {
+    upvar 1 $DTS dts
+    append dts "reg = <$addr $size>;\n"
+}
+
+# Adds cells that specify processing of address and size values within reg properties of a DT node
+# 1. DTS  - a reference to Device Tree string
+# 2. addr - specifies how many cells within the reg property represent the BASE ADDRESS of a register
+# 3. size - specifies how many cells within the reg property represent the SIZE of a register
+# NOTE: A cell within a reg property is a value of type uint32.
+proc dts_add_cells {DTS {addr 1} {size 1}} {
+    upvar 1 $DTS dts
+    dts_appendprop_int dts "#address-cells" $addr
+    dts_appendprop_int dts "#size-cells" $size
+}
+
+# Adds the minimal set of properties (compatble string and a register address)
+# 1. DTS        - a reference to DTS
+# 2. base_addr  - base address in the MI address space
+# 3. size       - size of the register space in the MI address space
+# 4. compatible - compatible string
+proc dts_appendprop_comp_node {DTS base_addr size compatible} {
+    upvar 1 $DTS dts
+
+    dts_appendprop_string dts "compatible" "$compatible"
+    dts_appendprop_reg dts $base_addr $size
+}
+
+# This creates a node within a DTS
+# 1. DTS   - a reference to Device Tree string
+# 2. alias - (also called label) provides an alternative name used for cross-referencing within a
+#            Device Tree
+# 3. name  - a name of a node
+# 4. body  - a set of procedures that add properties to a node (see example within the documentation
+#            of a Build System)
+proc dts_create_labeled_node {DTS alias name body} {
+    upvar 1 $DTS dts
+
+    if {$alias ne ""} {
+        append dts "$alias: "
+    }
+    append dts "$name {\n"
+    uplevel 1 $body
+    append dts "};\n"
+}
+
+# Wrapper over dts_create_labeled_node that creates a node without a label
+proc dts_create_node {DTS name body} {
+    uplevel 1 [list dts_create_labeled_node $DTS "" $name $body]
+}

comp/dma/dma_calypte/DevTree.tcl

@@ -3,34 +3,43 @@
 # 2. id         - channel ID
 # 3. base       - base address of channel
 # 4. pcie       - index(es) of PCIe endpoint(s) which DMA controller uses.
-proc dts_dma_calypte_ctrl {dir id base pcie} {
-    set    ret ""
-    append ret "dma_ctrl_calypte" "_$dir$id {"
-    append ret "compatible = \"cesnet,dma_ctrl_calypte" "_" $dir "\";"
-    append ret "reg = <$base 0x80>;"
-    append ret "version = <0x00010000>;"
-    append ret "pcie = <$pcie>;"
-    if { $dir == "tx" } {
-        append ret "data_buff = <&dma_calypte_tx_data_buff$id>;"
-        append ret "hdr_buff = <&dma_calypte_tx_hdr_buff$id>;"
+proc dts_dma_calypte_ctrl {DTS dir id base pcie} {
+    upvar 1 $DTS dts
+
+    dts_create_node dts "dma_ctrl_calypte_$dir$id" {
+        dts_appendprop_comp_node dts $base 0x80 "cesnet,dma_ctrl_calypte_$dir"
+        dts_appendprop_int dts "version" 0x10000
+        dts_appendprop_int dts "pcie" $pcie
+        if { $dir == "tx" } {
+            append dts "data_buff = <&dma_calypte_tx_data_buff$id>;"
+            append dts "hdr_buff = <&dma_calypte_tx_hdr_buff$id>;"
+        }
+        append dts "params = <&dma_params_$dir$pcie>;"
     }
-    append ret "params = <&dma_params_$dir$pcie>;"
-    append ret "};"
-    return $ret
 }
 
 # generates Device Tree entries for data buffers in DMA Calypte
 # 1. type       - content of the buffer (header or data)
 # 2. id         - channel ID
-# 3. base       - base address for the first buffer
+# 3. base       - base address for the buffer
 # 4. size       - size of the buffer
 # 5. pcie       - index(es) of PCIe endpoint(s) which DMA controller uses.
-proc dts_dma_calypte_tx_buffer {type id base size pcie} {
-    set ret ""
-    append ret "dma_calypte_tx_${type}_buff${id}: dma_calypte_tx_${type}_buff${id} {"
-    append ret "compatible = \"cesnet,dma_calypte_tx_${type}_buff\";"
-    append ret "reg = <$base $size>;"
-    append ret "pcie = <$pcie>;"
-    append ret "};"
-    return $ret
+proc dts_dma_calypte_tx_buffer {DTS type id base size pcie} {
+    upvar 1 $DTS dts
+
+    dts_create_labeled_node dts "dma_calypte_tx_${type}_buff${id}" "dma_calypte_tx_${type}_buff${id}" {
+        dts_appendprop_comp_node dts $base $size "cesnet,dma_calypte_tx_${type}_buff"
+        dts_appendprop_int dts "pcie" $pcie
+    }
+}
+
+# Adds a node to the Device Tree for performance counters within DMA Calypte
+# 1. DTS - reference to DeviceTree string
+# 2. Base - base address of the registers in the MI address space
+proc dts_dma_perf_cntrs {DTS base} {
+    upvar 1 $DTS dts
+
+    dts_create_node dts "dma_calypte_rx_perf_cntrs0" {
+        dts_appendprop_comp_node dts $base 0x30 "cesnet,dma_calypte_rx_perf_cntrs"
+    }
 }

comp/dma/dma_calypte/comp/rx/Modules.tcl

@@ -8,13 +8,15 @@ lappend PACKAGES "$OFM_PATH/comp/base/pkg/math_pack.vhd"
 lappend PACKAGES "$OFM_PATH/comp/base/pkg/type_pack.vhd"
 lappend PACKAGES "$OFM_PATH/comp/base/pkg/pcie_meta_pack.vhd"
 
-set HDR_INSERTOR_BASE        "$ENTITY_BASE/comp/hdr_insertor"
-set HDR_MANAGER_BASE         "$ENTITY_BASE/comp/hdr_manager"
-set TRANS_BUFFER_BASE        "$ENTITY_BASE/comp/trans_buffer"
-set INPUT_BUFFER_BASE        "$ENTITY_BASE/comp/input_buffer"
-set SW_MANAGER_BASE          "$ENTITY_BASE/comp/software_manager"
-set MFB_FIFOX_BASE           "$OFM_PATH/comp/mfb_tools/storage/fifox"
-set MFB_FRAME_LNG_CHECK_BASE "$OFM_PATH/comp/mfb_tools/logic/frame_lng_check"
+set HDR_INSERTOR_BASE         "$ENTITY_BASE/comp/hdr_insertor"
+set HDR_MANAGER_BASE          "$ENTITY_BASE/comp/hdr_manager"
+set TRANS_BUFFER_BASE         "$ENTITY_BASE/comp/trans_buffer"
+set INPUT_BUFFER_BASE         "$ENTITY_BASE/comp/input_buffer"
+set SW_MANAGER_BASE           "$ENTITY_BASE/comp/software_manager"
+set MFB_FIFOX_BASE            "$OFM_PATH/comp/mfb_tools/storage/fifox"
+set MFB_FRAME_LNG_CHECK_BASE  "$OFM_PATH/comp/mfb_tools/logic/frame_lng_check"
+set DATA_LOGGER_BASE          "$OFM_PATH/comp/debug/data_logger"
+set MI_SPLITTER_PLUS_GEN_BASE "$OFM_PATH/comp/mi_tools/splitter_plus_gen"
 
 lappend COMPONENTS \
       [ list "RX_DMA_CALYPTE_HDR_INSERTOR" $HDR_INSERTOR_BASE         "FULL"] \
@@ -24,6 +26,7 @@ lappend COMPONENTS \
       [ list "RX_DMA_CALYPTE_SW_MANAGER"   $SW_MANAGER_BASE           "FULL"] \
       [ list "MFB_FIFOX"                   $MFB_FIFOX_BASE            "FULL"] \
       [ list "MFB_FRAME_LNG_CHECK"         $MFB_FRAME_LNG_CHECK_BASE  "FULL"] \
-
+      [ list "DATA_LOGGER"                 $DATA_LOGGER_BASE          "FULL"] \
+      [ list "MI_SPLITTER_PLUS_GEN"        $MI_SPLITTER_PLUS_GEN_BASE "FULL"]
 
 lappend MOD "$ENTITY_BASE/rx_dma_calypte.vhd"

comp/dma/dma_calypte/comp/rx/comp/hdr_manager/rx_dma_calypte_hdr_manager.vhd

@@ -137,7 +137,15 @@ entity RX_DMA_CALYPTE_HDR_MANAGER is
         PKT_CNTR_CHAN     : out std_logic_vector(log2(CHANNELS) -1 downto 0);
         PKT_CNTR_SENT_INC : out std_logic;
         PKT_CNTR_DISC_INC : out std_logic;
-        PKT_CNTR_PKT_SIZE : out std_logic_vector(log2(PKT_MTU+1) -1 downto 0)
+        PKT_CNTR_PKT_SIZE : out std_logic_vector(log2(PKT_MTU+1) -1 downto 0);
+
+        -- =========================================================================================
+        -- Performance counter outputs
+        -- =========================================================================================
+        DATA_ADDR_REQ_CNTR_INC    : out std_logic;
+        DMA_HDR_ADDR_REQ_CNTR_INC : out std_logic;
+        DATA_ADDR_STALL_INC       : out std_logic;
+        DMA_HDR_ADDR_STALL_INC    : out std_logic
         );
 end entity;
 
@@ -311,6 +319,12 @@ architecture FULL of RX_DMA_CALYPTE_HDR_MANAGER is
     signal ptr_fifo_rd    : std_logic;
     signal ptr_fifo_empty : std_logic;
 
+    -- =============================================================================================
+    -- Perfofmance counter logic
+    -- =============================================================================================
+    signal data_addr_next_reg    : std_logic;
+    signal dma_hdr_addr_next_reg : std_logic;
+
     -- =============================================================================================
     -- Debug signals and probes (either for verification or ILA/SignalTap)
     -- =============================================================================================
@@ -760,7 +774,7 @@ begin
             IN_VFID       => (others => '0'),
             IN_TAG        => (others => '0'),
             IN_DW_CNT     => std_logic_vector(to_unsigned(DATA_SEGMENT_SIZE/4, 11)),
-            IN_ATTRIBUTES => "000",
+            IN_ATTRIBUTES => "010",
             IN_FBE        => "1111",
             IN_LBE        => "1111",
             IN_ADDR_LEN   => pcie_addr_len_data_tran,
@@ -806,7 +820,7 @@ begin
             IN_VFID       => (others => '0'),
             IN_TAG        => (others => '0'),
             IN_DW_CNT     => std_logic_vector(to_unsigned(8/4, 11)),
-            IN_ATTRIBUTES => "000",
+            IN_ATTRIBUTES => "010",
             IN_FBE        => "1111",
             IN_LBE        => "1111",
             IN_ADDR_LEN   => pcie_addr_len_dma_hdr_tran,
@@ -1025,4 +1039,24 @@ begin
     -- signal is valid and set to 1.
     DMA_HDR_SRC_RDY <= ((not hdr_meta_fifo_empty) and (not ptr_fifo_empty) and (not pkt_size_fifo_empty) and (not discard_fifo_empty) and (not discard_fifo_do(0)))
                        or ((not discard_fifo_empty) and discard_fifo_do(0));
+
+    -- =============================================================================================
+    -- Performance counter logic
+    -- =============================================================================================
+    addr_next_reg_p: process (CLK) is
+    begin
+        if (rising_edge(CLK)) then
+            data_addr_next_reg    <= data_addr_next;
+            dma_hdr_addr_next_reg <= dma_hdr_addr_next;
+        end if;
+    end process;
+
+    -- the response for an address request comes usually one clock period delayed. If that is not a
+    -- case, the stalling occurs.
+    DATA_ADDR_STALL_INC    <= data_addr_next_reg    and (not data_pcie_addr_vld);
+    DMA_HDR_ADDR_STALL_INC <= dma_hdr_addr_next_reg and (not dma_hdr_pcie_addr_vld);
+
+    -- Counters of requests for PCIe addresses (should be equal to the overall amount of packets)
+    DATA_ADDR_REQ_CNTR_INC    <= data_pcie_addr_vld;
+    DMA_HDR_ADDR_REQ_CNTR_INC <= dma_hdr_pcie_addr_vld;
 end architecture;