[ofa-general][PATCH 4/12 v2] mlx4: Pre reserved Qp regions
Yevgeny Petrilin
yevgenyp at mellanox.co.il
Wed Apr 23 07:58:32 PDT 2008
>From 2dd4f8abdedda736adca5818c98f7a67d339ba7e Mon Sep 17 00:00:00 2001
From: Yevgeny Petrilin <yevgenyp at mellanox.co.il>
Date: Mon, 21 Apr 2008 14:39:27 +0300
Subject: [PATCH] mlx4: Pre reserved Qp regions.
We reserve Qp ranges to be used by other modules in case
the ports come up as Ethernet ports.
The qps are reserved at the end of the QP table.
(This way we assure that they are aligned to their size)
We need to consider these reserved ranges in bitmap creation :
The effective max parameter.
Signed-off-by: Yevgeny Petrilin <yevgenyp at mellanox.co.il>
---
drivers/net/mlx4/alloc.c | 38 ++++++++++++++++--------
drivers/net/mlx4/fw.c | 5 +++
drivers/net/mlx4/fw.h | 2 +
drivers/net/mlx4/main.c | 65 +++++++++++++++++++++++++++++++++++++++----
drivers/net/mlx4/mlx4.h | 4 ++
drivers/net/mlx4/qp.c | 55 ++++++++++++++++++++++++++++++++++--
include/linux/mlx4/device.h | 19 ++++++++++++-
include/linux/mlx4/qp.h | 4 ++
8 files changed, 169 insertions(+), 23 deletions(-)
diff --git a/drivers/net/mlx4/alloc.c b/drivers/net/mlx4/alloc.c
index 4601506..4b6074d 100644
--- a/drivers/net/mlx4/alloc.c
+++ b/drivers/net/mlx4/alloc.c
@@ -44,15 +44,18 @@ u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
spin_lock(&bitmap->lock);
- obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
- if (obj >= bitmap->max) {
+ obj = find_next_zero_bit(bitmap->table, bitmap->effective_max,
+ bitmap->last);
+ if (obj >= bitmap->effective_max) {
bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask;
- obj = find_first_zero_bit(bitmap->table, bitmap->max);
+ obj = find_first_zero_bit(bitmap->table, bitmap->effective_max);
}
- if (obj < bitmap->max) {
+ if (obj < bitmap->effective_max) {
set_bit(obj, bitmap->table);
- bitmap->last = (obj + 1) & (bitmap->max - 1);
+ bitmap->last = (obj + 1);
+ if (bitmap->last == bitmap->effective_max)
+ bitmap->last = 0;
obj |= bitmap->top;
} else
obj = -1;
@@ -108,20 +111,20 @@ u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
spin_lock(&bitmap->lock);
obj = find_aligned_range(bitmap->table, bitmap->last,
- bitmap->max, cnt, align);
- if (obj >= bitmap->max) {
+ bitmap->effective_max, cnt, align);
+ if (obj >= bitmap->effective_max) {
bitmap->top = (bitmap->top + bitmap->max) & bitmap->mask;
obj = find_aligned_range(bitmap->table, 0,
- bitmap->max,
+ bitmap->effective_max,
cnt, align);
}
- if (obj < bitmap->max) {
+ if (obj < bitmap->effective_max) {
for (i = 0; i < cnt; i++)
set_bit(obj+i, bitmap->table);
if (obj == bitmap->last) {
bitmap->last = (obj + cnt);
- if (bitmap->last >= bitmap->max)
+ if (bitmap->last >= bitmap->effective_max)
bitmap->last = 0;
}
obj |= bitmap->top;
@@ -147,8 +150,9 @@ void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt)
spin_unlock(&bitmap->lock);
}
-int mlx4_bitmap_init(struct mlx4_bitmap *bitmap,
- u32 num, u32 mask, u32 reserved)
+int mlx4_bitmap_init_with_effective_max(struct mlx4_bitmap *bitmap,
+ u32 num, u32 mask, u32 reserved,
+ u32 effective_max)
{
int i;
@@ -160,6 +164,7 @@ int mlx4_bitmap_init(struct mlx4_bitmap *bitmap,
bitmap->top = 0;
bitmap->max = num;
bitmap->mask = mask;
+ bitmap->effective_max = effective_max;
spin_lock_init(&bitmap->lock);
bitmap->table = kzalloc(BITS_TO_LONGS(num) * sizeof (long), GFP_KERNEL);
if (!bitmap->table)
@@ -171,6 +176,13 @@ int mlx4_bitmap_init(struct mlx4_bitmap *bitmap,
return 0;
}
+int mlx4_bitmap_init(struct mlx4_bitmap *bitmap,
+ u32 num, u32 mask, u32 reserved)
+{
+ return mlx4_bitmap_init_with_effective_max(bitmap, num, mask,
+ reserved, num);
+}
+
void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
{
kfree(bitmap->table);
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index d82f275..b0ad0d1 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -325,6 +325,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
#define QUERY_PORT_MTU_OFFSET 0x01
#define QUERY_PORT_WIDTH_OFFSET 0x06
#define QUERY_PORT_MAX_GID_PKEY_OFFSET 0x07
+#define QUERY_PORT_MAX_MACVLAN_OFFSET 0x0a
#define QUERY_PORT_MAX_VL_OFFSET 0x0b
for (i = 1; i <= dev_cap->num_ports; ++i) {
@@ -342,6 +343,10 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev_cap->max_pkeys[i] = 1 << (field & 0xf);
MLX4_GET(field, outbox, QUERY_PORT_MAX_VL_OFFSET);
dev_cap->max_vl[i] = field & 0xf;
+ MLX4_GET(field, outbox, QUERY_PORT_MAX_MACVLAN_OFFSET);
+ dev_cap->log_max_macs[i] = field & 0xf;
+ dev_cap->log_max_vlans[i] = field >> 4;
+
}
}
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index 306cb9b..a2e827c 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -97,6 +97,8 @@ struct mlx4_dev_cap {
u32 reserved_lkey;
u64 max_icm_sz;
int max_gso_sz;
+ u8 log_max_macs[MLX4_MAX_PORTS + 1];
+ u8 log_max_vlans[MLX4_MAX_PORTS + 1];
};
struct mlx4_adapter {
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index a6aa49f..f309532 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -85,6 +85,22 @@ static struct mlx4_profile default_profile = {
.num_mtt = 1 << 20,
};
+static int num_mac = 1;
+module_param_named(num_mac, num_mac, int, 0444);
+MODULE_PARM_DESC(num_mac, "Maximum number of MACs per ETH port "
+ "(1-127, default 1)");
+
+static int num_vlan;
+module_param_named(num_vlan, num_vlan, int, 0444);
+MODULE_PARM_DESC(num_vlan, "Maximum number of VLANs per ETH port "
+ "(0-126, default 0)");
+
+static int use_prio;
+module_param_named(use_prio, use_prio, bool, 0444);
+MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
+ "(0/1, default 0)");
+
+
static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
int err;
@@ -134,7 +150,6 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.max_rq_sg = dev_cap->max_rq_sg;
dev->caps.max_wqes = dev_cap->max_qp_sz;
dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp;
- dev->caps.reserved_qps = dev_cap->reserved_qps;
dev->caps.max_srq_wqes = dev_cap->max_srq_sz;
dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1;
dev->caps.reserved_srqs = dev_cap->reserved_srqs;
@@ -161,6 +176,39 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.stat_rate_support = dev_cap->stat_rate_support;
dev->caps.max_gso_sz = dev_cap->max_gso_sz;
+ dev->caps.log_num_macs = ilog2(roundup_pow_of_two(num_mac + 1));
+ dev->caps.log_num_vlans = ilog2(roundup_pow_of_two(num_vlan + 2));
+ dev->caps.log_num_prios = use_prio ? 3: 0;
+
+ for (i = 1; i <= dev->caps.num_ports; ++i) {
+ if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
+ dev->caps.log_num_macs = dev_cap->log_max_macs[i];
+ mlx4_warn(dev, "Requested number of MACs is too much "
+ "for port %d, reducing to %d.\n",
+ i, 1 << dev->caps.log_num_macs);
+ }
+ if (dev->caps.log_num_vlans > dev_cap->log_max_vlans[i]) {
+ dev->caps.log_num_vlans = dev_cap->log_max_vlans[i];
+ mlx4_warn(dev, "Requested number of VLANs is too much "
+ "for port %d, reducing to %d.\n",
+ i, 1 << dev->caps.log_num_vlans);
+ }
+ }
+
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
+ (1 << dev->caps.log_num_macs)*
+ (1 << dev->caps.log_num_vlans)*
+ (1 << dev->caps.log_num_prios)*
+ dev->caps.num_ports;
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
+
+ dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] +
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
+
return 0;
}
@@ -209,7 +257,8 @@ static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
((u64) (MLX4_CMPT_TYPE_QP *
cmpt_entry_sz) << MLX4_CMPT_SHIFT),
cmpt_entry_sz, dev->caps.num_qps,
- dev->caps.reserved_qps, 0, 0);
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
+ 0, 0);
if (err)
goto err;
@@ -334,7 +383,8 @@ static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
init_hca->qpc_base,
dev_cap->qpc_entry_sz,
dev->caps.num_qps,
- dev->caps.reserved_qps, 0, 0);
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
+ 0, 0);
if (err) {
mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
goto err_unmap_dmpt;
@@ -344,7 +394,8 @@ static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
init_hca->auxc_base,
dev_cap->aux_entry_sz,
dev->caps.num_qps,
- dev->caps.reserved_qps, 0, 0);
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
+ 0, 0);
if (err) {
mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
goto err_unmap_qp;
@@ -354,7 +405,8 @@ static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
init_hca->altc_base,
dev_cap->altc_entry_sz,
dev->caps.num_qps,
- dev->caps.reserved_qps, 0, 0);
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
+ 0, 0);
if (err) {
mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
goto err_unmap_auxc;
@@ -364,7 +416,8 @@ static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
init_hca->rdmarc_base,
dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
dev->caps.num_qps,
- dev->caps.reserved_qps, 0, 0);
+ dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
+ 0, 0);
if (err) {
mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
goto err_unmap_altc;
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 89d4ccc..b74405a 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -111,6 +111,7 @@ struct mlx4_bitmap {
u32 last;
u32 top;
u32 max;
+ u32 effective_max;
u32 mask;
spinlock_t lock;
unsigned long *table;
@@ -290,6 +291,9 @@ void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj);
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align);
void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt);
int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, u32 reserved);
+int mlx4_bitmap_init_with_effective_max(struct mlx4_bitmap *bitmap,
+ u32 num, u32 mask, u32 reserved,
+ u32 effective_max);
void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap);
int mlx4_reset(struct mlx4_dev *dev);
diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c
index dff8e66..2d5be15 100644
--- a/drivers/net/mlx4/qp.c
+++ b/drivers/net/mlx4/qp.c
@@ -273,6 +273,7 @@ int mlx4_init_qp_table(struct mlx4_dev *dev)
{
struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
int err;
+ int reserved_from_top = 0;
spin_lock_init(&qp_table->lock);
INIT_RADIX_TREE(&dev->qp_table_tree, GFP_ATOMIC);
@@ -282,9 +283,43 @@ int mlx4_init_qp_table(struct mlx4_dev *dev)
* block of special QPs must be aligned to a multiple of 8, so
* round up.
*/
- dev->caps.sqp_start = ALIGN(dev->caps.reserved_qps, 8);
- err = mlx4_bitmap_init(&qp_table->bitmap, dev->caps.num_qps,
- (1 << 24) - 1, dev->caps.sqp_start + 8);
+ dev->caps.sqp_start =
+ ALIGN(dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW], 8);
+
+ {
+ int sort[MLX4_QP_REGION_COUNT];
+ int i, j, tmp;
+ int last_base = dev->caps.num_qps;
+
+ for (i = 1; i < MLX4_QP_REGION_COUNT; ++i)
+ sort[i] = i;
+
+ for (i = MLX4_QP_REGION_COUNT; i > 0; --i) {
+ for (j = 2; j < i; ++j) {
+ if (dev->caps.reserved_qps_cnt[sort[j]] >
+ dev->caps.reserved_qps_cnt[sort[j - 1]]) {
+ tmp = sort[j];
+ sort[j] = sort[j - 1];
+ sort[j - 1] = tmp;
+ }
+ }
+ }
+
+ for (i = 1; i < MLX4_QP_REGION_COUNT; ++i) {
+ last_base -= dev->caps.reserved_qps_cnt[sort[i]];
+ dev->caps.reserved_qps_base[sort[i]] = last_base;
+ reserved_from_top +=
+ dev->caps.reserved_qps_cnt[sort[i]];
+ }
+
+ }
+
+ err = mlx4_bitmap_init_with_effective_max(&qp_table->bitmap,
+ dev->caps.num_qps,
+ (1 << 23) - 1,
+ dev->caps.sqp_start + 8,
+ dev->caps.num_qps -
+ reserved_from_top);
if (err)
return err;
@@ -297,6 +332,20 @@ void mlx4_cleanup_qp_table(struct mlx4_dev *dev)
mlx4_bitmap_cleanup(&mlx4_priv(dev)->qp_table.bitmap);
}
+int mlx4_qp_get_region(struct mlx4_dev *dev,
+ enum qp_region region,
+ int *base_qpn, int *cnt)
+{
+ if ((region < 0) || (region >= MLX4_QP_REGION_COUNT))
+ return -EINVAL;
+
+ *base_qpn = dev->caps.reserved_qps_base[region];
+ *cnt = dev->caps.reserved_qps_cnt[region];
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_qp_get_region);
+
int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
struct mlx4_qp_context *context)
{
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 9c77bf3..955eeca 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -135,6 +135,18 @@ enum {
MLX4_STAT_RATE_OFFSET = 5
};
+enum qp_region {
+ MLX4_QP_REGION_FW = 0,
+ MLX4_QP_REGION_ETH_ADDR,
+ MLX4_QP_REGION_FC_ADDR,
+ MLX4_QP_REGION_FC_EXCH,
+ MLX4_QP_REGION_COUNT
+};
+
+enum {
+ MLX4_NUM_FEXCH = 64 * 1024,
+};
+
static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor)
{
return (major << 32) | (minor << 16) | subminor;
@@ -159,7 +171,6 @@ struct mlx4_caps {
int max_rq_desc_sz;
int max_qp_init_rdma;
int max_qp_dest_rdma;
- int reserved_qps;
int sqp_start;
int num_srqs;
int max_srq_wqes;
@@ -189,6 +200,12 @@ struct mlx4_caps {
u16 stat_rate_support;
u8 port_width_cap[MLX4_MAX_PORTS + 1];
int max_gso_sz;
+ int reserved_qps_cnt[MLX4_QP_REGION_COUNT];
+ int reserved_qps;
+ int reserved_qps_base[MLX4_QP_REGION_COUNT];
+ int log_num_macs;
+ int log_num_vlans;
+ int log_num_prios;
};
struct mlx4_buf_list {
diff --git a/include/linux/mlx4/qp.h b/include/linux/mlx4/qp.h
index a5e43fe..5a02980 100644
--- a/include/linux/mlx4/qp.h
+++ b/include/linux/mlx4/qp.h
@@ -303,4 +316,8 @@ static inline struct mlx4_qp *__mlx4_qp_lookup(struct mlx4_dev *dev, u32 qpn)
void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp);
+int mlx4_qp_get_region(struct mlx4_dev *dev,
+ enum qp_region region,
+ int *base_qpn, int *cnt);
+
#endif /* MLX4_QP_H */
--
1.5.4
More information about the general
mailing list