@@ -1362,7 +1362,8 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
}
static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
- struct bpf_prog **progs, int prog_cnt, int stack_size)
+ struct bpf_prog **progs, int prog_cnt, int stack_size,
+ bool override_return)
{
u8 *prog = *pprog;
int cnt = 0, i;
@@ -1384,6 +1385,14 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
if (emit_call(&prog, progs[i]->bpf_func, prog))
return -EINVAL;
+
+ /* If BPF_TRAMP_F_OVERRIDE_RETURN is set, fexit trampolines can
+ * override the return value of the previous trampoline which is
+ * then passed on the stack to the next BPF program.
+ */
+ if (override_return)
+ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
+
/* arg1: mov rdi, progs[i] */
emit_mov_imm64(&prog, BPF_REG_1, (long) progs[i] >> 32,
(u32) (long) progs[i]);
@@ -1462,6 +1471,7 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end,
struct bpf_prog **fexit_progs, int fexit_cnt,
void *orig_call)
{
+ bool override_return = flags & BPF_TRAMP_F_OVERRIDE_RETURN;
int cnt = 0, nr_args = m->nr_args;
int stack_size = nr_args * 8;
u8 *prog;
@@ -1493,7 +1503,8 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end,
save_regs(m, &prog, nr_args, stack_size);
if (fentry_cnt)
- if (invoke_bpf(m, &prog, fentry_progs, fentry_cnt, stack_size))
+ if (invoke_bpf(m, &prog, fentry_progs, fentry_cnt, stack_size,
+ false))
return -EINVAL;
if (flags & BPF_TRAMP_F_CALL_ORIG) {
@@ -1503,18 +1514,20 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end,
/* call original function */
if (emit_call(&prog, orig_call, prog))
return -EINVAL;
+
/* remember return value in a stack for bpf prog to access */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
}
if (fexit_cnt)
- if (invoke_bpf(m, &prog, fexit_progs, fexit_cnt, stack_size))
+ if (invoke_bpf(m, &prog, fexit_progs, fexit_cnt, stack_size,
+ override_return))
return -EINVAL;
if (flags & BPF_TRAMP_F_RESTORE_REGS)
restore_regs(m, &prog, nr_args, stack_size);
- if (flags & BPF_TRAMP_F_CALL_ORIG)
+ if (flags & BPF_TRAMP_F_CALL_ORIG && !override_return)
/* restore original return value back into RAX */
emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
@@ -432,6 +432,10 @@ struct btf_func_model {
* programs only. Should not be used with normal calls and indirect calls.
*/
#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
+/* Override the return value of the original function. This flag only makes
+ * sense for fexit trampolines.
+ */
+#define BPF_TRAMP_F_OVERRIDE_RETURN BIT(3)
/* Different use cases for BPF trampoline:
* 1. replace nop at the function entry (kprobe equivalent)
@@ -41,6 +41,8 @@ void bpf_lsm_##NAME##_set_enabled(bool value);
})
int bpf_lsm_set_enabled(const char *name, bool value);
+int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
+ const struct bpf_prog *prog);
#else /* !CONFIG_BPF_LSM */
@@ -53,6 +55,12 @@ static inline int bpf_lsm_set_enabled(const char *name, bool value)
return -EOPNOTSUPP;
}
+static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
+ const struct bpf_prog *prog)
+{
+ return -EOPNOTSUPP;
+}
+
#endif /* CONFIG_BPF_LSM */
#endif /* _LINUX_BPF_LSM_H */
@@ -10,6 +10,7 @@
#include <linux/bpf_lsm.h>
#include <linux/jump_label.h>
#include <linux/kallsyms.h>
+#include <linux/bpf_verifier.h>
#define LSM_HOOK(RET, NAME, ...) \
DEFINE_STATIC_KEY_FALSE(bpf_lsm_key_##NAME); \
@@ -52,6 +53,32 @@ int bpf_lsm_set_enabled(const char *name, bool value)
return 0;
}
+#define BPF_LSM_SYM_PREFX "bpf_lsm_"
+
+int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
+ const struct bpf_prog *prog)
+{
+ /* Only CAP_MAC_ADMIN users are allowed to make changes to LSM hooks
+ */
+ if (!capable(CAP_MAC_ADMIN))
+ return -EPERM;
+
+ if (!prog->gpl_compatible) {
+ bpf_log(vlog,
+ "LSM programs must have a GPL compatible license\n");
+ return -EINVAL;
+ }
+
+ if (strncmp(BPF_LSM_SYM_PREFX, prog->aux->attach_func_name,
+ strlen(BPF_LSM_SYM_PREFX))) {
+ bpf_log(vlog, "attach_btf_id %u points to wrong type name %s\n",
+ prog->aux->attach_btf_id, prog->aux->attach_func_name);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
const struct bpf_prog_ops lsm_prog_ops = {
};
@@ -3710,7 +3710,8 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
nr_args--;
}
- if (prog->expected_attach_type == BPF_TRACE_FEXIT &&
+ if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
+ prog->expected_attach_type == BPF_LSM_MAC) &&
arg == nr_args) {
if (!t)
/* Default prog with 5 args. 6th arg is retval. */
@@ -25,6 +25,7 @@
#include <linux/nospec.h>
#include <linux/audit.h>
#include <uapi/linux/btf.h>
+#include <linux/bpf_lsm.h>
#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
@@ -1931,6 +1932,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
switch (prog_type) {
case BPF_PROG_TYPE_TRACING:
+ case BPF_PROG_TYPE_LSM:
case BPF_PROG_TYPE_STRUCT_OPS:
case BPF_PROG_TYPE_EXT:
break;
@@ -2169,28 +2171,53 @@ static int bpf_obj_get(const union bpf_attr *attr)
attr->file_flags);
}
-static int bpf_tracing_prog_release(struct inode *inode, struct file *filp)
+static int bpf_tramp_prog_release(struct inode *inode, struct file *filp)
{
struct bpf_prog *prog = filp->private_data;
+ /* Only CAP_MAC_ADMIN users are allowed to make changes to LSM hooks
+ */
+ if (prog->type == BPF_PROG_TYPE_LSM && !capable(CAP_MAC_ADMIN))
+ return -EPERM;
+
WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog));
bpf_prog_put(prog);
return 0;
}
-static const struct file_operations bpf_tracing_prog_fops = {
- .release = bpf_tracing_prog_release,
+static const struct file_operations bpf_tramp_prog_fops = {
+ .release = bpf_tramp_prog_release,
.read = bpf_dummy_read,
.write = bpf_dummy_write,
};
-static int bpf_tracing_prog_attach(struct bpf_prog *prog)
+static int bpf_tramp_prog_attach(struct bpf_prog *prog)
{
int tr_fd, err;
- if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
- prog->expected_attach_type != BPF_TRACE_FEXIT &&
- prog->type != BPF_PROG_TYPE_EXT) {
+ switch (prog->type) {
+ case BPF_PROG_TYPE_TRACING:
+ if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
+ prog->expected_attach_type != BPF_TRACE_FEXIT &&
+ prog->type != BPF_PROG_TYPE_EXT) {
+ err = -EINVAL;
+ goto out_put_prog;
+ }
+ break;
+ case BPF_PROG_TYPE_LSM:
+ if (prog->expected_attach_type != BPF_LSM_MAC) {
+ err = -EINVAL;
+ goto out_put_prog;
+ }
+ /* Only CAP_MAC_ADMIN users are allowed to make changes to LSM
+ * hooks.
+ */
+ if (!capable(CAP_MAC_ADMIN)) {
+ err = -EPERM;
+ goto out_put_prog;
+ }
+ break;
+ default:
err = -EINVAL;
goto out_put_prog;
}
@@ -2199,7 +2226,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog)
if (err)
goto out_put_prog;
- tr_fd = anon_inode_getfd("bpf-tracing-prog", &bpf_tracing_prog_fops,
+ tr_fd = anon_inode_getfd("bpf-tramp-prog", &bpf_tramp_prog_fops,
prog, O_CLOEXEC);
if (tr_fd < 0) {
WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog));
@@ -2258,12 +2285,14 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
prog->type != BPF_PROG_TYPE_TRACING &&
prog->type != BPF_PROG_TYPE_EXT &&
+ prog->type != BPF_PROG_TYPE_LSM &&
prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
err = -EINVAL;
goto out_put_prog;
}
if (prog->type == BPF_PROG_TYPE_TRACING ||
+ prog->type == BPF_PROG_TYPE_LSM ||
prog->type == BPF_PROG_TYPE_EXT) {
if (attr->raw_tracepoint.name) {
/* The attach point for this category of programs
@@ -2275,7 +2304,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
tp_name = prog->aux->attach_func_name;
else
- return bpf_tracing_prog_attach(prog);
+ return bpf_tramp_prog_attach(prog);
} else {
if (strncpy_from_user(buf,
u64_to_user_ptr(attr->raw_tracepoint.name),
@@ -5,6 +5,7 @@
#include <linux/filter.h>
#include <linux/ftrace.h>
#include <linux/rbtree_latch.h>
+#include <linux/bpf_lsm.h>
/* dummy _ops. The verifier will operate on target program's ops. */
const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -195,8 +196,9 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
*/
#define BPF_MAX_TRAMP_PROGS 40
-static int bpf_trampoline_update(struct bpf_trampoline *tr)
+static int bpf_trampoline_update(struct bpf_prog *prog)
{
+ struct bpf_trampoline *tr = prog->aux->trampoline;
void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2;
void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2;
struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS];
@@ -223,8 +225,11 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist)
*progs++ = aux->prog;
- if (fexit_cnt)
+ if (fexit_cnt) {
flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
+ if (prog->type == BPF_PROG_TYPE_LSM)
+ flags |= BPF_TRAMP_F_OVERRIDE_RETURN;
+ }
/* Though the second half of trampoline page is unused a task could be
* preempted in the middle of the first half of trampoline and two
@@ -261,6 +266,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t)
case BPF_TRACE_FENTRY:
return BPF_TRAMP_FENTRY;
case BPF_TRACE_FEXIT:
+ case BPF_LSM_MAC:
return BPF_TRAMP_FEXIT;
default:
return BPF_TRAMP_REPLACE;
@@ -307,11 +313,17 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog)
}
hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
tr->progs_cnt[kind]++;
- err = bpf_trampoline_update(prog->aux->trampoline);
+ err = bpf_trampoline_update(prog);
if (err) {
hlist_del(&prog->aux->tramp_hlist);
tr->progs_cnt[kind]--;
}
+
+ /* This is the first program to be attached to the LSM hook, the hook
+ * needs to be enabled.
+ */
+ if (prog->type == BPF_PROG_TYPE_LSM && tr->progs_cnt[kind] == 1)
+ err = bpf_lsm_set_enabled(prog->aux->attach_func_name, true);
out:
mutex_unlock(&tr->mutex);
return err;
@@ -336,7 +348,11 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog)
}
hlist_del(&prog->aux->tramp_hlist);
tr->progs_cnt[kind]--;
- err = bpf_trampoline_update(prog->aux->trampoline);
+ err = bpf_trampoline_update(prog);
+
+ /* There are no more LSM programs, the hook should be disabled */
+ if (prog->type == BPF_PROG_TYPE_LSM && tr->progs_cnt[kind] == 0)
+ err = bpf_lsm_set_enabled(prog->aux->attach_func_name, false);
out:
mutex_unlock(&tr->mutex);
return err;
@@ -19,6 +19,7 @@
#include <linux/sort.h>
#include <linux/perf_event.h>
#include <linux/ctype.h>
+#include <linux/bpf_lsm.h>
#include "disasm.h"
@@ -6405,8 +6406,9 @@ static int check_return_code(struct bpf_verifier_env *env)
struct tnum range = tnum_range(0, 1);
int err;
- /* The struct_ops func-ptr's return type could be "void" */
- if (env->prog->type == BPF_PROG_TYPE_STRUCT_OPS &&
+ /* LSM and struct_ops func-ptr's return type could be "void" */
+ if ((env->prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
+ env->prog->type == BPF_PROG_TYPE_LSM) &&
!prog->aux->attach_func_proto->type)
return 0;
@@ -9794,7 +9796,9 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
return check_struct_ops_btf_id(env);
- if (prog->type != BPF_PROG_TYPE_TRACING && !prog_extension)
+ if (prog->type != BPF_PROG_TYPE_TRACING &&
+ prog->type != BPF_PROG_TYPE_LSM &&
+ !prog_extension)
return 0;
if (!btf_id) {
@@ -9924,8 +9928,16 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
if (!prog_extension)
return -EINVAL;
/* fallthrough */
+ case BPF_LSM_MAC:
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ prog->aux->attach_func_name = tname;
+ if (prog->type == BPF_PROG_TYPE_LSM) {
+ ret = bpf_lsm_verify_prog(&env->log, prog);
+ if (ret < 0)
+ return ret;
+ }
+
if (!btf_type_is_func(t)) {
verbose(env, "attach_btf_id %u is not a function\n",
btf_id);
@@ -9940,7 +9952,6 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
tr = bpf_trampoline_lookup(key);
if (!tr)
return -ENOMEM;
- prog->aux->attach_func_name = tname;
/* t is either vmlinux type or another program's type */
prog->aux->attach_func_proto = t;
mutex_lock(&tr->mutex);