diff mbox series

[v2,1/3] vfs: support caching symlink lengths in inodes

Message ID 20241119094555.660666-2-mjguzik@gmail.com (mailing list archive)
State New
Headers show
Series symlink length caching | expand

Commit Message

Mateusz Guzik Nov. 19, 2024, 9:45 a.m. UTC 
When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
speed up when issuing readlink on /initrd.img on ext4.

Filesystems opt in by calling inode_set_cached_link() when creating an
inode.

The size is stored in what used to be a 4-byte hole. If necessary the
field can be made smaller and converted into a union with something not
used with symlinks.

Churn-wise the current readlink_copy() helper is patched to accept the
size instead of calculating it.

Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
---
 fs/namei.c                     | 34 +++++++++++++++++++---------------
 fs/proc/namespaces.c           |  2 +-
 include/linux/fs.h             | 12 ++++++++++--
 security/apparmor/apparmorfs.c |  2 +-
 4 files changed, 31 insertions(+), 19 deletions(-)

Comments

wangjianjian (C) Nov. 20, 2024, 4:15 a.m. UTC | #1 
On 2024/11/19 17:45, Mateusz Guzik wrote:
> When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
> speed up when issuing readlink on /initrd.img on ext4.
> 
> Filesystems opt in by calling inode_set_cached_link() when creating an
> inode.
> 
> The size is stored in what used to be a 4-byte hole. If necessary the
> field can be made smaller and converted into a union with something not
> used with symlinks.
> 
> Churn-wise the current readlink_copy() helper is patched to accept the
> size instead of calculating it.
> 
> Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
> ---
>   fs/namei.c                     | 34 +++++++++++++++++++---------------
>   fs/proc/namespaces.c           |  2 +-
>   include/linux/fs.h             | 12 ++++++++++--
>   security/apparmor/apparmorfs.c |  2 +-
>   4 files changed, 31 insertions(+), 19 deletions(-)
> 
> diff --git a/fs/namei.c b/fs/namei.c
> index 9d30c7aa9aa6..e56c29a22d26 100644
> --- a/fs/namei.c
> +++ b/fs/namei.c
> @@ -5272,19 +5272,16 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna
>   				getname(newname), 0);
>   }
>   
> -int readlink_copy(char __user *buffer, int buflen, const char *link)
> +int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)
>   {
> -	int len = PTR_ERR(link);
> -	if (IS_ERR(link))
> -		goto out;
> +	int copylen;
>   
> -	len = strlen(link);
> -	if (len > (unsigned) buflen)
> -		len = buflen;
> -	if (copy_to_user(buffer, link, len))
> -		len = -EFAULT;
> -out:
> -	return len;
> +	copylen = linklen;
> +	if (unlikely(copylen > (unsigned) buflen))
> +		copylen = buflen;
> +	if (copy_to_user(buffer, link, copylen))
> +		copylen = -EFAULT;
> +	return copylen;
>   }
>   
>   /**
> @@ -5304,6 +5301,9 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
>   	const char *link;
>   	int res;
>   
> +	if (inode->i_opflags & IOP_CACHED_LINK)
> +		return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);
> +
>   	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
>   		if (unlikely(inode->i_op->readlink))
>   			return inode->i_op->readlink(dentry, buffer, buflen);
> @@ -5322,7 +5322,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
>   		if (IS_ERR(link))
>   			return PTR_ERR(link);
>   	}
> -	res = readlink_copy(buffer, buflen, link);
> +	res = readlink_copy(buffer, buflen, link, strlen(link));
>   	do_delayed_call(&done);
>   	return res;
>   }
> @@ -5391,10 +5391,14 @@ EXPORT_SYMBOL(page_put_link);
>   
>   int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
>   {
> +	const char *link;
> +	int res;
> +
>   	DEFINE_DELAYED_CALL(done);
> -	int res = readlink_copy(buffer, buflen,
> -				page_get_link(dentry, d_inode(dentry),
> -					      &done));
> +	link = page_get_link(dentry, d_inode(dentry), &done);
> +	res = PTR_ERR(link);
> +	if (!IS_ERR(link))
> +		res = readlink_copy(buffer, buflen, link, strlen(link));
>   	do_delayed_call(&done);
>   	return res;
>   }
> diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
> index 8e159fc78c0a..c610224faf10 100644
> --- a/fs/proc/namespaces.c
> +++ b/fs/proc/namespaces.c
> @@ -83,7 +83,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
>   	if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
>   		res = ns_get_name(name, sizeof(name), task, ns_ops);
>   		if (res >= 0)
> -			res = readlink_copy(buffer, buflen, name);
> +			res = readlink_copy(buffer, buflen, name, strlen(name));
>   	}
>   	put_task_struct(task);
>   	return res;
> diff --git a/include/linux/fs.h b/include/linux/fs.h
> index 972147da71f9..30e332fb399d 100644
> --- a/include/linux/fs.h
> +++ b/include/linux/fs.h
> @@ -626,6 +626,7 @@ is_uncached_acl(struct posix_acl *acl)
>   #define IOP_XATTR	0x0008
>   #define IOP_DEFAULT_READLINK	0x0010
>   #define IOP_MGTIME	0x0020
> +#define IOP_CACHED_LINK	0x0040
>   
>   /*
>    * Keep mostly read-only and often accessed (especially for
> @@ -686,7 +687,7 @@ struct inode {
>   
>   	/* Misc */
>   	u32			i_state;
> -	/* 32-bit hole */
> +	int			i_linklen;	/* for symlinks */
>   	struct rw_semaphore	i_rwsem;
>   
>   	unsigned long		dirtied_when;	/* jiffies of first dirtying */
> @@ -749,6 +750,13 @@ struct inode {
>   	void			*i_private; /* fs or device private pointer */
>   } __randomize_layout;
>   
> +static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
> +{
> +	inode->i_link = link;
> +	inode->i_linklen = linklen;
Just curious, is this linklen equal to inode size? if it is, why don't 
use it?
> +	inode->i_opflags |= IOP_CACHED_LINK;
> +}
> +
>   /*
>    * Get bit address from inode->i_state to use with wait_var_event()
>    * infrastructre.
> @@ -3351,7 +3359,7 @@ extern const struct file_operations generic_ro_fops;
>   
>   #define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
>   
> -extern int readlink_copy(char __user *, int, const char *);
> +extern int readlink_copy(char __user *, int, const char *, int);
>   extern int page_readlink(struct dentry *, char __user *, int);
>   extern const char *page_get_link(struct dentry *, struct inode *,
>   				 struct delayed_call *);
> diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
> index 01b923d97a44..60959cfba672 100644
> --- a/security/apparmor/apparmorfs.c
> +++ b/security/apparmor/apparmorfs.c
> @@ -2611,7 +2611,7 @@ static int policy_readlink(struct dentry *dentry, char __user *buffer,
>   	res = snprintf(name, sizeof(name), "%s:[%lu]", AAFS_NAME,
>   		       d_inode(dentry)->i_ino);
>   	if (res > 0 && res < sizeof(name))
> -		res = readlink_copy(buffer, buflen, name);
> +		res = readlink_copy(buffer, buflen, name, strlen(name));
>   	else
>   		res = -ENOENT;
>
Matthew Wilcox Nov. 20, 2024, 5:01 a.m. UTC | #2 
On Wed, Nov 20, 2024 at 12:15:18PM +0800, wangjianjian (C) wrote:
> > +{
> > +	inode->i_link = link;
> > +	inode->i_linklen = linklen;
> Just curious, is this linklen equal to inode size? if it is, why don't use
> it?

Maybe you should read v1 of the patch series where Jan explained where
that's not true.
wangjianjian (C) Nov. 20, 2024, 5:10 a.m. UTC | #3 
On 2024/11/20 13:01, Matthew Wilcox wrote:
> On Wed, Nov 20, 2024 at 12:15:18PM +0800, wangjianjian (C) wrote:
>>> +{
>>> +	inode->i_link = link;
>>> +	inode->i_linklen = linklen;
>> Just curious, is this linklen equal to inode size? if it is, why don't use
>> it?
> 
> Maybe you should read v1 of the patch series where Jan explained where
> that's not true.
okay, I see, thanks for this.
>
diff mbox series

Patch

diff --git a/fs/namei.c b/fs/namei.c
index 9d30c7aa9aa6..e56c29a22d26 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -5272,19 +5272,16 @@  SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna
 				getname(newname), 0);
 }
 
-int readlink_copy(char __user *buffer, int buflen, const char *link)
+int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)
 {
-	int len = PTR_ERR(link);
-	if (IS_ERR(link))
-		goto out;
+	int copylen;
 
-	len = strlen(link);
-	if (len > (unsigned) buflen)
-		len = buflen;
-	if (copy_to_user(buffer, link, len))
-		len = -EFAULT;
-out:
-	return len;
+	copylen = linklen;
+	if (unlikely(copylen > (unsigned) buflen))
+		copylen = buflen;
+	if (copy_to_user(buffer, link, copylen))
+		copylen = -EFAULT;
+	return copylen;
 }
 
 /**
@@ -5304,6 +5301,9 @@  int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
 	const char *link;
 	int res;
 
+	if (inode->i_opflags & IOP_CACHED_LINK)
+		return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);
+
 	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
 		if (unlikely(inode->i_op->readlink))
 			return inode->i_op->readlink(dentry, buffer, buflen);
@@ -5322,7 +5322,7 @@  int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
 		if (IS_ERR(link))
 			return PTR_ERR(link);
 	}
-	res = readlink_copy(buffer, buflen, link);
+	res = readlink_copy(buffer, buflen, link, strlen(link));
 	do_delayed_call(&done);
 	return res;
 }
@@ -5391,10 +5391,14 @@  EXPORT_SYMBOL(page_put_link);
 
 int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
 {
+	const char *link;
+	int res;
+
 	DEFINE_DELAYED_CALL(done);
-	int res = readlink_copy(buffer, buflen,
-				page_get_link(dentry, d_inode(dentry),
-					      &done));
+	link = page_get_link(dentry, d_inode(dentry), &done);
+	res = PTR_ERR(link);
+	if (!IS_ERR(link))
+		res = readlink_copy(buffer, buflen, link, strlen(link));
 	do_delayed_call(&done);
 	return res;
 }
diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
index 8e159fc78c0a..c610224faf10 100644
--- a/fs/proc/namespaces.c
+++ b/fs/proc/namespaces.c
@@ -83,7 +83,7 @@  static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
 	if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
 		res = ns_get_name(name, sizeof(name), task, ns_ops);
 		if (res >= 0)
-			res = readlink_copy(buffer, buflen, name);
+			res = readlink_copy(buffer, buflen, name, strlen(name));
 	}
 	put_task_struct(task);
 	return res;
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 972147da71f9..30e332fb399d 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -626,6 +626,7 @@  is_uncached_acl(struct posix_acl *acl)
 #define IOP_XATTR	0x0008
 #define IOP_DEFAULT_READLINK	0x0010
 #define IOP_MGTIME	0x0020
+#define IOP_CACHED_LINK	0x0040
 
 /*
  * Keep mostly read-only and often accessed (especially for
@@ -686,7 +687,7 @@  struct inode {
 
 	/* Misc */
 	u32			i_state;
-	/* 32-bit hole */
+	int			i_linklen;	/* for symlinks */
 	struct rw_semaphore	i_rwsem;
 
 	unsigned long		dirtied_when;	/* jiffies of first dirtying */
@@ -749,6 +750,13 @@  struct inode {
 	void			*i_private; /* fs or device private pointer */
 } __randomize_layout;
 
+static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
+{
+	inode->i_link = link;
+	inode->i_linklen = linklen;
+	inode->i_opflags |= IOP_CACHED_LINK;
+}
+
 /*
  * Get bit address from inode->i_state to use with wait_var_event()
  * infrastructre.
@@ -3351,7 +3359,7 @@  extern const struct file_operations generic_ro_fops;
 
 #define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
 
-extern int readlink_copy(char __user *, int, const char *);
+extern int readlink_copy(char __user *, int, const char *, int);
 extern int page_readlink(struct dentry *, char __user *, int);
 extern const char *page_get_link(struct dentry *, struct inode *,
 				 struct delayed_call *);
diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 01b923d97a44..60959cfba672 100644
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -2611,7 +2611,7 @@  static int policy_readlink(struct dentry *dentry, char __user *buffer,
 	res = snprintf(name, sizeof(name), "%s:[%lu]", AAFS_NAME,
 		       d_inode(dentry)->i_ino);
 	if (res > 0 && res < sizeof(name))
-		res = readlink_copy(buffer, buflen, name);
+		res = readlink_copy(buffer, buflen, name, strlen(name));
 	else
 		res = -ENOENT;