Skip to content

Commit 3f85f2a

Browse files
mark-blochdledford
mark-bloch
authored and
dledford
committed
IB/mlx4: Add diagnostic hardware counters
Expose IB diagnostic hardware counters. The counters count IB events and are applicable for IB and RoCE. The counters can be divided into two groups, per device and per port. Device counters are always exposed. Port counters are exposed only if the firmware supports per port counters. rq_num_dup and sq_num_to are only exposed if we have firmware support for them, if we do, we expose them per device and per port. rq_num_udsdprd and num_cqovf are device only counters. rq - denotes responder. sq - denotes requester. |-----------------------|---------------------------------------| | Name | Description | |-----------------------|---------------------------------------| |rq_num_lle | Number of local length errors | |-----------------------|---------------------------------------| |sq_num_lle | number of local length errors | |-----------------------|---------------------------------------| |rq_num_lqpoe | Number of local QP operation errors | |-----------------------|---------------------------------------| |sq_num_lqpoe | Number of local QP operation errors | |-----------------------|---------------------------------------| |rq_num_lpe | Number of local protection errors | |-----------------------|---------------------------------------| |sq_num_lpe | Number of local protection errors | |-----------------------|---------------------------------------| |rq_num_wrfe | Number of CQEs with error | |-----------------------|---------------------------------------| |sq_num_wrfe | Number of CQEs with error | |-----------------------|---------------------------------------| |sq_num_mwbe | Number of Memory Window bind errors | |-----------------------|---------------------------------------| |sq_num_bre | Number of bad response errors | |-----------------------|---------------------------------------| |sq_num_rire | Number of Remote Invalid request | | | errors | |-----------------------|---------------------------------------| |rq_num_rire | Number of Remote Invalid request | | | errors | |-----------------------|---------------------------------------| |sq_num_rae | Number of remote access errors | |-----------------------|---------------------------------------| |rq_num_rae | Number of remote access errors | |-----------------------|---------------------------------------| |sq_num_roe | Number of remote operation errors | |-----------------------|---------------------------------------| |sq_num_tree | Number of transport retries exceeded | | | errors | |-----------------------|---------------------------------------| |sq_num_rree | Number of RNR NAK retries exceeded | | | errors | |-----------------------|---------------------------------------| |rq_num_rnr | Number of RNR NAKs sent | |-----------------------|---------------------------------------| |sq_num_rnr | Number of RNR NAKs received | |-----------------------|---------------------------------------| |rq_num_oos | Number of Out of Sequence requests | | | received | |-----------------------|---------------------------------------| |sq_num_oos | Number of Out of Sequence NAKs | | | received | |-----------------------|---------------------------------------| |rq_num_udsdprd | Number of UD packets silently | | | discarded on the Receive Queue due to | | | lack of receive descriptor | |-----------------------|---------------------------------------| |rq_num_dup | Number of duplicate requests received | |-----------------------|---------------------------------------| |sq_num_to | Number of time out received | |-----------------------|---------------------------------------| |num_cqovf | Number of CQ overflows | |-----------------------|---------------------------------------| Signed-off-by: Mark Bloch <markb@mellanox.com> Signed-off-by: Leon Romanovsky <leon@kernel.org> Signed-off-by: Doug Ledford <dledford@redhat.com>
1 parent bfaf316 commit 3f85f2a

3 files changed

Lines changed: 209 additions & 1 deletion

File tree

‎drivers/infiniband/hw/mlx4/main.c‎

Lines changed: 197 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2061,6 +2061,195 @@ static struct device_attribute *mlx4_class_attributes[] = {
20612061
&dev_attr_board_id
20622062
};
20632063

2064+
struct diag_counter {
2065+
const char *name;
2066+
u32 offset;
2067+
};
2068+
2069+
#define DIAG_COUNTER(_name, _offset) \
2070+
{ .name = #_name, .offset = _offset }
2071+
2072+
static const struct diag_counter diag_basic[] = {
2073+
DIAG_COUNTER(rq_num_lle, 0x00),
2074+
DIAG_COUNTER(sq_num_lle, 0x04),
2075+
DIAG_COUNTER(rq_num_lqpoe, 0x08),
2076+
DIAG_COUNTER(sq_num_lqpoe, 0x0C),
2077+
DIAG_COUNTER(rq_num_lpe, 0x18),
2078+
DIAG_COUNTER(sq_num_lpe, 0x1C),
2079+
DIAG_COUNTER(rq_num_wrfe, 0x20),
2080+
DIAG_COUNTER(sq_num_wrfe, 0x24),
2081+
DIAG_COUNTER(sq_num_mwbe, 0x2C),
2082+
DIAG_COUNTER(sq_num_bre, 0x34),
2083+
DIAG_COUNTER(sq_num_rire, 0x44),
2084+
DIAG_COUNTER(rq_num_rire, 0x48),
2085+
DIAG_COUNTER(sq_num_rae, 0x4C),
2086+
DIAG_COUNTER(rq_num_rae, 0x50),
2087+
DIAG_COUNTER(sq_num_roe, 0x54),
2088+
DIAG_COUNTER(sq_num_tree, 0x5C),
2089+
DIAG_COUNTER(sq_num_rree, 0x64),
2090+
DIAG_COUNTER(rq_num_rnr, 0x68),
2091+
DIAG_COUNTER(sq_num_rnr, 0x6C),
2092+
DIAG_COUNTER(rq_num_oos, 0x100),
2093+
DIAG_COUNTER(sq_num_oos, 0x104),
2094+
};
2095+
2096+
static const struct diag_counter diag_ext[] = {
2097+
DIAG_COUNTER(rq_num_dup, 0x130),
2098+
DIAG_COUNTER(sq_num_to, 0x134),
2099+
};
2100+
2101+
static const struct diag_counter diag_device_only[] = {
2102+
DIAG_COUNTER(num_cqovf, 0x1A0),
2103+
DIAG_COUNTER(rq_num_udsdprd, 0x118),
2104+
};
2105+
2106+
static struct rdma_hw_stats *mlx4_ib_alloc_hw_stats(struct ib_device *ibdev,
2107+
u8 port_num)
2108+
{
2109+
struct mlx4_ib_dev *dev = to_mdev(ibdev);
2110+
struct mlx4_ib_diag_counters *diag = dev->diag_counters;
2111+
2112+
if (!diag[!!port_num].name)
2113+
return NULL;
2114+
2115+
return rdma_alloc_hw_stats_struct(diag[!!port_num].name,
2116+
diag[!!port_num].num_counters,
2117+
RDMA_HW_STATS_DEFAULT_LIFESPAN);
2118+
}
2119+
2120+
static int mlx4_ib_get_hw_stats(struct ib_device *ibdev,
2121+
struct rdma_hw_stats *stats,
2122+
u8 port, int index)
2123+
{
2124+
struct mlx4_ib_dev *dev = to_mdev(ibdev);
2125+
struct mlx4_ib_diag_counters *diag = dev->diag_counters;
2126+
u32 hw_value[ARRAY_SIZE(diag_device_only) +
2127+
ARRAY_SIZE(diag_ext) + ARRAY_SIZE(diag_basic)] = {};
2128+
int ret;
2129+
int i;
2130+
2131+
ret = mlx4_query_diag_counters(dev->dev,
2132+
MLX4_OP_MOD_QUERY_TRANSPORT_CI_ERRORS,
2133+
diag[!!port].offset, hw_value,
2134+
diag[!!port].num_counters, port);
2135+
2136+
if (ret)
2137+
return ret;
2138+
2139+
for (i = 0; i < diag[!!port].num_counters; i++)
2140+
stats->value[i] = hw_value[i];
2141+
2142+
return diag[!!port].num_counters;
2143+
}
2144+
2145+
static int __mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev,
2146+
const char ***name,
2147+
u32 **offset,
2148+
u32 *num,
2149+
bool port)
2150+
{
2151+
u32 num_counters;
2152+
2153+
num_counters = ARRAY_SIZE(diag_basic);
2154+
2155+
if (ibdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT)
2156+
num_counters += ARRAY_SIZE(diag_ext);
2157+
2158+
if (!port)
2159+
num_counters += ARRAY_SIZE(diag_device_only);
2160+
2161+
*name = kcalloc(num_counters, sizeof(**name), GFP_KERNEL);
2162+
if (!*name)
2163+
return -ENOMEM;
2164+
2165+
*offset = kcalloc(num_counters, sizeof(**offset), GFP_KERNEL);
2166+
if (!*offset)
2167+
goto err_name;
2168+
2169+
*num = num_counters;
2170+
2171+
return 0;
2172+
2173+
err_name:
2174+
kfree(*name);
2175+
return -ENOMEM;
2176+
}
2177+
2178+
static void mlx4_ib_fill_diag_counters(struct mlx4_ib_dev *ibdev,
2179+
const char **name,
2180+
u32 *offset,
2181+
bool port)
2182+
{
2183+
int i;
2184+
int j;
2185+
2186+
for (i = 0, j = 0; i < ARRAY_SIZE(diag_basic); i++, j++) {
2187+
name[i] = diag_basic[i].name;
2188+
offset[i] = diag_basic[i].offset;
2189+
}
2190+
2191+
if (ibdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT) {
2192+
for (i = 0; i < ARRAY_SIZE(diag_ext); i++, j++) {
2193+
name[j] = diag_ext[i].name;
2194+
offset[j] = diag_ext[i].offset;
2195+
}
2196+
}
2197+
2198+
if (!port) {
2199+
for (i = 0; i < ARRAY_SIZE(diag_device_only); i++, j++) {
2200+
name[j] = diag_device_only[i].name;
2201+
offset[j] = diag_device_only[i].offset;
2202+
}
2203+
}
2204+
}
2205+
2206+
static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)
2207+
{
2208+
struct mlx4_ib_diag_counters *diag = ibdev->diag_counters;
2209+
int i;
2210+
int ret;
2211+
bool per_port = !!(ibdev->dev->caps.flags2 &
2212+
MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT);
2213+
2214+
for (i = 0; i < MLX4_DIAG_COUNTERS_TYPES; i++) {
2215+
/* i == 1 means we are building port counters */
2216+
if (i && !per_port)
2217+
continue;
2218+
2219+
ret = __mlx4_ib_alloc_diag_counters(ibdev, &diag[i].name,
2220+
&diag[i].offset,
2221+
&diag[i].num_counters, i);
2222+
if (ret)
2223+
goto err_alloc;
2224+
2225+
mlx4_ib_fill_diag_counters(ibdev, diag[i].name,
2226+
diag[i].offset, i);
2227+
}
2228+
2229+
ibdev->ib_dev.get_hw_stats = mlx4_ib_get_hw_stats;
2230+
ibdev->ib_dev.alloc_hw_stats = mlx4_ib_alloc_hw_stats;
2231+
2232+
return 0;
2233+
2234+
err_alloc:
2235+
if (i) {
2236+
kfree(diag[i - 1].name);
2237+
kfree(diag[i - 1].offset);
2238+
}
2239+
2240+
return ret;
2241+
}
2242+
2243+
static void mlx4_ib_diag_cleanup(struct mlx4_ib_dev *ibdev)
2244+
{
2245+
int i;
2246+
2247+
for (i = 0; i < MLX4_DIAG_COUNTERS_TYPES; i++) {
2248+
kfree(ibdev->diag_counters[i].offset);
2249+
kfree(ibdev->diag_counters[i].name);
2250+
}
2251+
}
2252+
20642253
#define MLX4_IB_INVALID_MAC ((u64)-1)
20652254
static void mlx4_ib_update_qps(struct mlx4_ib_dev *ibdev,
20662255
struct net_device *dev,
@@ -2552,9 +2741,12 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
25522741
for (j = 1; j <= ibdev->dev->caps.num_ports; j++)
25532742
atomic64_set(&iboe->mac[j - 1], ibdev->dev->caps.def_mac[j]);
25542743

2555-
if (ib_register_device(&ibdev->ib_dev, NULL))
2744+
if (mlx4_ib_alloc_diag_counters(ibdev))
25562745
goto err_steer_free_bitmap;
25572746

2747+
if (ib_register_device(&ibdev->ib_dev, NULL))
2748+
goto err_diag_counters;
2749+
25582750
if (mlx4_ib_mad_init(ibdev))
25592751
goto err_reg;
25602752

@@ -2620,6 +2812,9 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
26202812
err_reg:
26212813
ib_unregister_device(&ibdev->ib_dev);
26222814

2815+
err_diag_counters:
2816+
mlx4_ib_diag_cleanup(ibdev);
2817+
26232818
err_steer_free_bitmap:
26242819
kfree(ibdev->ib_uc_qpns_bitmap);
26252820

@@ -2723,6 +2918,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
27232918
mlx4_ib_close_sriov(ibdev);
27242919
mlx4_ib_mad_cleanup(ibdev);
27252920
ib_unregister_device(&ibdev->ib_dev);
2921+
mlx4_ib_diag_cleanup(ibdev);
27262922
if (ibdev->iboe.nb.notifier_call) {
27272923
if (unregister_netdevice_notifier(&ibdev->iboe.nb))
27282924
pr_warn("failure unregistering notifier\n");

‎drivers/infiniband/hw/mlx4/mlx4_ib.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -549,6 +549,14 @@ struct mlx4_ib_counters {
549549
u32 default_counter;
550550
};
551551

552+
#define MLX4_DIAG_COUNTERS_TYPES 2
553+
554+
struct mlx4_ib_diag_counters {
555+
const char **name;
556+
u32 *offset;
557+
u32 num_counters;
558+
};
559+
552560
struct mlx4_ib_dev {
553561
struct ib_device ib_dev;
554562
struct mlx4_dev *dev;
@@ -585,6 +593,7 @@ struct mlx4_ib_dev {
585593
/* protect resources needed as part of reset flow */
586594
spinlock_t reset_flow_resource_lock;
587595
struct list_head qp_list;
596+
struct mlx4_ib_diag_counters diag_counters[MLX4_DIAG_COUNTERS_TYPES];
588597
};
589598

590599
struct ib_event_work {

‎include/linux/mlx4/device.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1341,6 +1341,9 @@ enum {
13411341
VXLAN_STEER_BY_INNER_VLAN = 1 << 4,
13421342
};
13431343

1344+
enum {
1345+
MLX4_OP_MOD_QUERY_TRANSPORT_CI_ERRORS = 0x2,
1346+
};
13441347

13451348
int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, u32 qpn,
13461349
enum mlx4_net_trans_promisc_mode mode);

0 commit comments

Comments
 (0)