[v1,7/7] fscrypt: add lru mechanism to choose pooled key

Commit Message

Sweet Tea Dorminy April 19, 2023, 2:42 a.m. UTC

This improves the previous mechanism to select a key to steal: it keeps
track of the LRU active key, and steals that one. While the previous
mechanism isn't an invalid strategy, it is probably inferior and results
in excessive churn in very active keys.

Signed-off-by: Sweet Tea Dorminy <[e-mail]>
---
 fs/crypto/fscrypt_private.h |  6 +++++-
 fs/crypto/keysetup.c        | 39 ++++++++++++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 4 deletions(-)

Patch

diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index cc6e4a2a942c..c23b222ca30b 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -461,7 +461,7 @@  struct fscrypt_master_key_secret {
  * fscrypt_pooled_prepared_keys (in keysetup.c).
  */
 struct fscrypt_key_pool {
-	/* Mutex protecting all the fields here */
+	/* Mutex protecting almost everything */
 	struct mutex mutex;
 	/* A list of active keys */
 	struct list_head active_keys;
@@ -471,6 +471,10 @@  struct fscrypt_key_pool {
 	size_t count;
 	/* Count of keys desired. Oft equal to count, but can be less. */
 	size_t desired;
+	/* Mutex protecting the lru list*/
+	struct mutex lru_mutex;
+	/* Same list of active keys, just ordered by usage. Head is oldest. */
+	struct list_head active_lru;
 };
 
 /*
diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
index 57c343b78abc..d498b89cd097 100644
--- a/fs/crypto/keysetup.c
+++ b/fs/crypto/keysetup.c
@@ -92,11 +92,13 @@  static DEFINE_MUTEX(fscrypt_mode_key_setup_mutex);
 struct fscrypt_pooled_prepared_key {
 	/*
 	 * Must be held when the prep key is in use or the key is being
-	 * returned.
+	 * returned. Must not be held for interactions with pool_link and
+	 * pool_lru_link, which are protected by the pool locks.
 	 */
 	struct mutex mutex;
 	struct fscrypt_prepared_key prep_key;
 	struct list_head pool_link;
+	struct list_head pool_lru_link;
 	/* NULL when it's in the pool. */
 	struct fscrypt_info *owner;
 };
@@ -201,6 +203,18 @@  get_pooled_key_owner(struct fscrypt_pooled_prepared_key *key)
 	return smp_load_acquire(&key->owner);
 }
 
+/*
+ * Must be called with the key lock held and the pool lock not held.
+ */
+static void update_lru(struct fscrypt_key_pool *pool,
+		       struct fscrypt_pooled_prepared_key *key)
+{
+	mutex_lock(&pool->lru_mutex);
+	/* Bump this key up to the most recent end of the pool's lru list. */
+	list_move_tail(&pool->active_lru, &key->pool_lru_link);
+	mutex_unlock(&pool->lru_mutex);
+}
+
 /*
  * Lock the prepared key currently in ci->ci_enc_key, if it hasn't been stolen
  * for the use of some other ci. Once this function succeeds, the prepared
@@ -215,6 +229,9 @@  static int fscrypt_lock_pooled_key(struct fscrypt_info *ci)
 	struct fscrypt_pooled_prepared_key *pooled_key =
 		container_of(ci->ci_enc_key, struct fscrypt_pooled_prepared_key,
 			     prep_key);
+	struct fscrypt_master_key *mk = ci->ci_master_key;
+	const u8 mode_num = ci->ci_mode - fscrypt_modes;
+	struct fscrypt_key_pool *pool = &mk->mk_key_pools[mode_num];
 
 	/* Peek to see if someone's definitely stolen the pooled key */
 	if (get_pooled_key_owner(pooled_key) != ci)
@@ -233,6 +250,8 @@  static int fscrypt_lock_pooled_key(struct fscrypt_info *ci)
 		goto stolen;
 	}
 
+	update_lru(pool, pooled_key);
+
 	return 0;
 
 stolen:
@@ -270,12 +289,16 @@  static void __fscrypt_free_one_free_key(struct fscrypt_key_pool *pool)
 	pool->count--;
 }
 
+/* Must be called with the pool mutex held. Releases it. */
 static void __fscrypt_return_key_to_pool(struct fscrypt_key_pool *pool,
 					 struct fscrypt_pooled_prepared_key *key)
 {
 	/* Pairs with the acquire in get_pooled_key_owner() */
 	smp_store_release(&key->owner, NULL);
 	list_move(&key->pool_link, &pool->free_keys);
+	mutex_lock(&pool->lru_mutex);
+	list_del_init(&key->pool_lru_link);
+	mutex_unlock(&pool->lru_mutex);
 	mutex_unlock(&key->mutex);
 	if (pool->count > pool->desired)
 		__fscrypt_free_one_free_key(pool);
@@ -323,6 +346,7 @@  static int fscrypt_allocate_new_pooled_key(struct fscrypt_key_pool *pool,
 	pooled_key->prep_key.type = FSCRYPT_KEY_POOLED;
 	mutex_init(&pooled_key->mutex);
 	INIT_LIST_HEAD(&pooled_key->pool_link);
+	INIT_LIST_HEAD(&pooled_key->pool_lru_link);
 	mutex_lock(&pool->mutex);
 	pool->count++;
 	pool->desired++;
@@ -334,8 +358,15 @@  static int fscrypt_allocate_new_pooled_key(struct fscrypt_key_pool *pool,
 static struct fscrypt_pooled_prepared_key *
 fscrypt_select_victim_key(struct fscrypt_key_pool *pool)
 {
-	return list_first_entry(&pool->active_keys,
-				struct fscrypt_pooled_prepared_key, pool_link);
+	struct fscrypt_pooled_prepared_key *key;
+
+	mutex_lock(&pool->lru_mutex);
+	key = list_first_entry(&pool->active_lru,
+			       struct fscrypt_pooled_prepared_key,
+			       pool_lru_link);
+	list_del_init(&key->pool_lru_link);
+	mutex_unlock(&pool->lru_mutex);
+	return key;
 }
 
 static void fscrypt_steal_an_active_key(struct fscrypt_key_pool *pool)
@@ -399,7 +430,9 @@  void fscrypt_init_key_pool(struct fscrypt_key_pool *pool, size_t modenum)
 	struct fscrypt_mode *mode = &fscrypt_modes[modenum];
 
 	mutex_init(&pool->mutex);
+	mutex_init(&pool->lru_mutex);
 	INIT_LIST_HEAD(&pool->active_keys);
+	INIT_LIST_HEAD(&pool->active_lru);
 	INIT_LIST_HEAD(&pool->free_keys);
 
 	/*