diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 84d3b917c33e..939ec3759eba 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -613,6 +613,16 @@ enum mlx5_mkey_type {
 	MLX5_MKEY_INDIRECT_DEVX,
 };
 
+struct mlx5r_cache_rb_key {
+	unsigned int access_mode;
+	unsigned int access_flags;
+	/*
+	 * keep ndescs as the last member so entries with about the same ndescs
+	 * will be close in the tree
+	 */
+	unsigned int ndescs;
+};
+
 struct mlx5_ib_mkey {
 	u32 key;
 	enum mlx5_mkey_type type;
@@ -732,10 +742,10 @@ struct mlx5_cache_ent {
 	unsigned long		stored;
 	unsigned long		reserved;
 
+	struct mlx5r_cache_rb_key rb_key;
+
 	char                    name[4];
 	u32                     order;
-	u32			access_mode;
-	unsigned int		ndescs;
 
 	u8 disabled:1;
 	u8 fill_to_high_water:1;
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index fe5567c57897..e7a3d4fa52d0 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -287,18 +287,18 @@ static int get_mkc_octo_size(unsigned int access_mode, unsigned int ndescs)
 	return ret;
 }
 
-static void set_cache_mkc(struct mlx5_ib_dev *dev, u8 access_mode,
-			  unsigned int access_flags, unsigned int ndescs,
-			  void *mkc)
+static void set_cache_mkc(struct mlx5_ib_dev *dev,
+			  struct mlx5r_cache_rb_key rb_key, void *mkc)
 {
-	set_mkc_access_pd_addr_fields(mkc, access_flags, 0, dev->umrc.pd);
+	set_mkc_access_pd_addr_fields(mkc, rb_key.access_flags, 0,
+				      dev->umrc.pd);
 	MLX5_SET(mkc, mkc, free, 1);
 	MLX5_SET(mkc, mkc, umr_en, 1);
-	MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
-	MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
+	MLX5_SET(mkc, mkc, access_mode_1_0, rb_key.access_mode & 0x3);
+	MLX5_SET(mkc, mkc, access_mode_4_2, (rb_key.access_mode >> 2) & 0x7);
 
 	MLX5_SET(mkc, mkc, translations_octword_size,
-		 get_mkc_octo_size(access_mode, ndescs));
+		 get_mkc_octo_size(rb_key.access_mode, rb_key.ndescs));
 	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 }
 
@@ -317,7 +317,7 @@ static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
 			return -ENOMEM;
 		mkc = MLX5_ADDR_OF(create_mkey_in, async_create->in,
 				   memory_key_mkey_entry);
-		set_cache_mkc(ent->dev, ent->access_mode, 0, ent->ndescs, mkc);
+		set_cache_mkc(ent->dev, ent->rb_key, mkc);
 		async_create->ent = ent;
 
 		err = push_mkey(ent, true, NULL);
@@ -343,8 +343,8 @@ static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
 }
 
 /* Synchronously create a cacheable mkey */
-static int create_cache_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
-			     unsigned int access_flags, unsigned int ndescs,
+static int create_cache_mkey(struct mlx5_ib_dev *dev,
+			     struct mlx5r_cache_rb_key rb_key,
 			     struct mlx5_ib_mkey *mkey)
 {
 	size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
@@ -356,7 +356,7 @@ static int create_cache_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
 	if (!in)
 		return -ENOMEM;
 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
-	set_cache_mkc(dev, access_mode, access_flags, ndescs, mkc);
+	set_cache_mkc(dev, rb_key, mkc);
 
 	err = mlx5_ib_create_mkey(dev, mkey, in, inlen);
 	kfree(in);
@@ -647,7 +647,7 @@ static bool mlx5_ent_get_mkey(struct mlx5_cache_ent *ent, struct mlx5_ib_mr *mr)
 	}
 
 	mr->mmkey.key = pop_stored_mkey(ent);
-	mr->mmkey.ndescs = ent->ndescs;
+	mr->mmkey.ndescs = ent->rb_key.ndescs;
 	mr->mmkey.cache_ent = ent;
 	queue_adjust_cache_locked(ent);
 	ent->in_use++;
@@ -668,29 +668,57 @@ static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,
 	return &cache->ent[order];
 }
 
-static bool mlx5_cache_get_mkey(struct mlx5_ib_dev *dev, u8 access_mode,
-				u8 access_flags, unsigned int ndescs,
+static bool mlx5_cache_get_mkey(struct mlx5_ib_dev *dev,
+				struct mlx5r_cache_rb_key rb_key,
 				struct mlx5_ib_mr *mr)
 {
 	struct mlx5_cache_ent *ent;
 
-	if (!mlx5r_umr_can_reconfig(dev, 0, access_flags))
+	if (!mlx5r_umr_can_reconfig(dev, 0, rb_key.access_flags))
 		return false;
 
-	if (access_mode == MLX5_MKC_ACCESS_MODE_KSM)
+	if (rb_key.access_mode == MLX5_MKC_ACCESS_MODE_KSM)
 		ent = &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY];
 
-	ent = mkey_cache_ent_from_order(dev, order_base_2(ndescs));
+	ent = mkey_cache_ent_from_order(dev, order_base_2(rb_key.ndescs));
 	if (!ent)
 		return false;
 
 	return mlx5_ent_get_mkey(ent, mr);
 }
 
+static int get_uchangeable_access_flags(struct mlx5_ib_dev *dev,
+					int access_flags)
+{
+	int ret = 0;
+
+	if ((access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+	    MLX5_CAP_GEN(dev->mdev, atomic) &&
+	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
+		ret |= IB_ACCESS_REMOTE_ATOMIC;
+
+	if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
+	    MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write) &&
+	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
+		ret |= IB_ACCESS_RELAXED_ORDERING;
+
+	if ((access_flags & IB_ACCESS_RELAXED_ORDERING) &&
+	    MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) &&
+	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
+		ret |= IB_ACCESS_RELAXED_ORDERING;
+
+	return ret;
+}
+
 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, u8 access_mode,
 				       unsigned int access_flags,
 				       unsigned int ndescs)
 {
+	struct mlx5r_cache_rb_key rb_key = {
+		.access_mode = access_mode,
+		.access_flags = get_uchangeable_access_flags(dev, access_flags),
+		.ndescs = ndescs
+	};
 	struct mlx5_ib_mr *mr;
 	int err;
 
@@ -698,13 +726,12 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, u8 access_mode,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	if (!mlx5_cache_get_mkey(dev, access_mode, access_flags, ndescs, mr)) {
+	if (!mlx5_cache_get_mkey(dev, rb_key, mr)) {
 		/*
 		 * Didn't find an mkey in cache.
 		 * Create an mkey with the exact needed size.
 		 */
-		err = create_cache_mkey(dev, access_mode, access_flags, ndescs,
-					&mr->mmkey);
+		err = create_cache_mkey(dev, rb_key, &mr->mmkey);
 		if (err) {
 			kfree(mr);
 			return ERR_PTR(err);
@@ -774,6 +801,8 @@ static void delay_time_func(struct timer_list *t)
 
 int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
 {
+	struct mlx5r_cache_rb_key rb_key = { .access_mode =
+						    MLX5_MKC_ACCESS_MODE_MTT };
 	struct mlx5_mkey_cache *cache = &dev->cache;
 	struct mlx5_cache_ent *ent;
 	int i;
@@ -804,8 +833,8 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
 		if (ent->order > mkey_cache_max_order(dev))
 			continue;
 
-		ent->ndescs = 1 << ent->order;
-		ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
+		rb_key.ndescs = 1 << ent->order;
+		ent->rb_key = rb_key;
 		if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
 		    !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
 		    mlx5r_umr_can_load_pas(dev, 0))
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 137143da5959..90339edddfed 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -1589,10 +1589,14 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
 
 void mlx5_odp_init_mkey_cache_entry(struct mlx5_cache_ent *ent)
 {
+	struct mlx5r_cache_rb_key rb_key = {
+		.access_mode = MLX5_MKC_ACCESS_MODE_KSM,
+		.ndescs = mlx5_imr_ksm_entries
+	};
+
 	if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
 		return;
-	ent->ndescs = mlx5_imr_ksm_entries;
-	ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
+	ent->rb_key = rb_key;
 }
 
 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
