| Message ID | 1503015736-22315-1-git-send-email-tianyu.lan@intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
>>> On 18.08.17 at 02:22, <tianyu.lan@intel.com> wrote: > This patch is to introduct an abstract layer for arch vIOMMU implementation > to deal with requests from dom0. Arch vIOMMU code needs to provide callback > to perform create, destroy and query capabilities operation. > > Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> What is this? It looks to be a standalone patch when there was a V2 series a couple of days ago with 1/25 titled "DOMCTL: Introduce new DOMCTL commands for vIOMMU support". I'm confused. Jan
On 2017年08月18日 16:41, Jan Beulich wrote: >>>> On 18.08.17 at 02:22, <tianyu.lan@intel.com> wrote: >> This patch is to introduct an abstract layer for arch vIOMMU implementation >> to deal with requests from dom0. Arch vIOMMU code needs to provide callback >> to perform create, destroy and query capabilities operation. >> >> Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> > > What is this? It looks to be a standalone patch when there was > a V2 series a couple of days ago with 1/25 titled "DOMCTL: > Introduce new DOMCTL commands for vIOMMU support". I'm > confused. > > Jan > Hi Jan: Sorry. Missed this patch in this patchset and this should be the first one. I will send v3 with some fixs.
On Thu, Aug 17, 2017 at 08:22:16PM -0400, Lan Tianyu wrote: > diff --git a/xen/common/domain.c b/xen/common/domain.c > index b22aacc..d1f9b10 100644 > --- a/xen/common/domain.c > +++ b/xen/common/domain.c > @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, unsigned int domcr_flags, > spin_unlock(&domlist_update_lock); > } > > + if ( (err = viommu_init_domain(d)) != 0 ) > + goto fail; > + Where is the code to destroy viommu during domain destruction? I suppose you will need a viommu_destroy_domain and call it somewhere in complete_domain_destroy. > + > +#include <xen/sched.h> > +#include <xen/spinlock.h> > +#include <xen/types.h> > +#include <xen/viommu.h> > + > +bool __read_mostly opt_viommu; > +boolean_param("viommu", opt_viommu); Missing patch to xen command line option doc. > + > +static spinlock_t type_list_lock; > +static struct list_head type_list; > + > +struct viommu_type { > + u64 type; uintXX_t here and in all other places please. [...] > + > +static int viommu_create(struct domain *d, u64 type, u64 base_address, > + u64 length, u64 caps) > +{ > + struct viommu_info *info = &d->viommu; > + struct viommu *viommu; > + struct viommu_type *viommu_type = NULL; > + int rc; > + > + viommu_type = viommu_get_type(type); > + if ( !viommu_type ) > + return -EINVAL; > + > + if ( info->nr_viommu >= NR_VIOMMU_PER_DOMAIN E2BIG for this? > + || !viommu_type->ops || !viommu_type->ops->create ) > + return -EINVAL; > + > + viommu = xzalloc(struct viommu); > + if ( !viommu ) > + return -ENOMEM; > + [...] > diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h > index 6673b27..98a965a 100644 > --- a/xen/include/xen/sched.h > +++ b/xen/include/xen/sched.h > @@ -21,6 +21,7 @@ > #include <xen/perfc.h> > #include <asm/atomic.h> > #include <xen/wait.h> > +#include <xen/viommu.h> > #include <public/xen.h> > #include <public/domctl.h> > #include <public/sysctl.h> > @@ -477,6 +478,7 @@ struct domain > /* vNUMA topology accesses are protected by rwlock. */ > rwlock_t vnuma_rwlock; > struct vnuma_info *vnuma; Please add a new line here. > + struct viommu_info viommu; > > /* Common monitor options */ > struct { > diff --git a/xen/include/xen/viommu.h b/xen/include/xen/viommu.h > new file mode 100644 > index 0000000..506ea54 > --- /dev/null > +++ b/xen/include/xen/viommu.h > @@ -0,0 +1,71 @@ > +/* > + * include/xen/viommu.h > + * > + * Copyright (c) 2017, Intel Corporation > + * Author: Lan Tianyu <tianyu.lan@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License along with > + * this program; If not, see <http://www.gnu.org/licenses/>. > + * > + */ > +#ifndef __XEN_VIOMMU_H__ > +#define __XEN_VIOMMU_H__ > + > +#define NR_VIOMMU_PER_DOMAIN 1 > + > +struct viommu; > + > +struct viommu_ops { > + u64 (*query_caps)(struct domain *d); > + int (*create)(struct domain *d, struct viommu *viommu); > + int (*destroy)(struct viommu *viommu); > +}; > + > +struct viommu { > + u64 base_address; > + u64 length; > + u64 caps; > + u32 viommu_id; > + const struct viommu_ops *ops; > + void *priv; > +}; > + > +struct viommu_info { > + u32 nr_viommu; unsigned int > + struct viommu *viommu[NR_VIOMMU_PER_DOMAIN]; /* viommu array*/ > +}; > + > +#ifdef CONFIG_VIOMMU > +extern bool_t opt_viommu; bool
On Thu, Aug 17, 2017 at 08:22:16PM -0400, Lan Tianyu wrote: > This patch is to introduct an abstract layer for arch vIOMMU implementation > to deal with requests from dom0. Arch vIOMMU code needs to provide callback > to perform create, destroy and query capabilities operation. > > Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> > --- > xen/arch/x86/Kconfig | 1 + > xen/arch/x86/setup.c | 1 + > xen/common/Kconfig | 3 + > xen/common/Makefile | 1 + > xen/common/domain.c | 3 + > xen/common/viommu.c | 165 +++++++++++++++++++++++++++++++++++++++++++++++ > xen/include/xen/sched.h | 2 + > xen/include/xen/viommu.h | 71 ++++++++++++++++++++ > 8 files changed, 247 insertions(+) > create mode 100644 xen/common/viommu.c > create mode 100644 xen/include/xen/viommu.h > > diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig > index 30c2769..1f1de96 100644 > --- a/xen/arch/x86/Kconfig > +++ b/xen/arch/x86/Kconfig > @@ -23,6 +23,7 @@ config X86 > select HAS_PDX > select NUMA > select VGA > + select VIOMMU > > config ARCH_DEFCONFIG > string > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c > index db5df69..68f1631 100644 > --- a/xen/arch/x86/setup.c > +++ b/xen/arch/x86/setup.c > @@ -1513,6 +1513,7 @@ void __init noreturn __start_xen(unsigned long mbi_p) > early_msi_init(); > > iommu_setup(); /* setup iommu if available */ > + viommu_setup(); > > smp_prepare_cpus(max_cpus); > > diff --git a/xen/common/Kconfig b/xen/common/Kconfig > index dc8e876..2ad2c8d 100644 > --- a/xen/common/Kconfig > +++ b/xen/common/Kconfig > @@ -49,6 +49,9 @@ config HAS_CHECKPOLICY > string > option env="XEN_HAS_CHECKPOLICY" > > +config VIOMMU > + bool > + > config KEXEC > bool "kexec support" > default y > diff --git a/xen/common/Makefile b/xen/common/Makefile > index 26c5a64..852553d 100644 > --- a/xen/common/Makefile > +++ b/xen/common/Makefile > @@ -56,6 +56,7 @@ obj-y += time.o > obj-y += timer.o > obj-y += trace.o > obj-y += version.o > +obj-$(CONFIG_VIOMMU) += viommu.o > obj-y += virtual_region.o > obj-y += vm_event.o > obj-y += vmap.o > diff --git a/xen/common/domain.c b/xen/common/domain.c > index b22aacc..d1f9b10 100644 > --- a/xen/common/domain.c > +++ b/xen/common/domain.c > @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, unsigned int domcr_flags, > spin_unlock(&domlist_update_lock); > } > > + if ( (err = viommu_init_domain(d)) != 0 ) > + goto fail; > + > return d; > > fail: > diff --git a/xen/common/viommu.c b/xen/common/viommu.c > new file mode 100644 > index 0000000..6874d9f > --- /dev/null > +++ b/xen/common/viommu.c > @@ -0,0 +1,165 @@ > +/* > + * common/viommu.c > + * > + * Copyright (c) 2017 Intel Corporation > + * Author: Lan Tianyu <tianyu.lan@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License along with > + * this program; If not, see <http://www.gnu.org/licenses/>. > + */ > + > +#include <xen/sched.h> > +#include <xen/spinlock.h> > +#include <xen/types.h> > +#include <xen/viommu.h> > + > +bool __read_mostly opt_viommu; > +boolean_param("viommu", opt_viommu); > + > +static spinlock_t type_list_lock; static DEFINE_SPINLOCK(type_list_lock); > +static struct list_head type_list; static LIST_HEAD(type_list); > + > +struct viommu_type { > + u64 type; > + struct viommu_ops *ops; > + struct list_head node; > +}; > + > +int viommu_init_domain(struct domain *d) > +{ > + d->viommu.nr_viommu = 0; > + return 0; > +} If you don't use the viommu_info struct you can also get rid of this. The array entries will point to NULL which can be used to signal not initialized. > + > +static struct viommu_type *viommu_get_type(u64 type) > +{ > + struct viommu_type *viommu_type = NULL; > + > + spin_lock(&type_list_lock); > + list_for_each_entry( viommu_type, &type_list, node ) > + { > + if ( viommu_type->type == type ) > + { > + spin_unlock(&type_list_lock); > + return viommu_type; > + } > + } > + spin_unlock(&type_list_lock); > + > + return NULL; > +} > + > +int viommu_register_type(u64 type, struct viommu_ops * ops) > +{ > + struct viommu_type *viommu_type = NULL; > + > + if ( !viommu_enabled() ) > + return -EINVAL; ENODEV is maybe better here. > + > + if ( viommu_get_type(type) ) > + return -EEXIST; > + > + viommu_type = xzalloc(struct viommu_type); > + if ( !viommu_type ) > + return -ENOMEM; > + > + viommu_type->type = type; > + viommu_type->ops = ops; > + > + spin_lock(&type_list_lock); > + list_add_tail(&viommu_type->node, &type_list); > + spin_unlock(&type_list_lock); > + > + return 0; > +} Hm, I haven't seen the usage of this function, but from the looks of it it seems like you want to use something similar to what's used by the scheduler in order to register vIOMMU types. See the logic around REGISTER_SCHEDULER in xen/sched-if.h. > + > +static int viommu_create(struct domain *d, u64 type, u64 base_address, > + u64 length, u64 caps) > +{ > + struct viommu_info *info = &d->viommu; > + struct viommu *viommu; > + struct viommu_type *viommu_type = NULL; > + int rc; > + > + viommu_type = viommu_get_type(type); > + if ( !viommu_type ) > + return -EINVAL; > + > + if ( info->nr_viommu >= NR_VIOMMU_PER_DOMAIN > + || !viommu_type->ops || !viommu_type->ops->create ) ^ aligned with "info" above. > + return -EINVAL; > + > + viommu = xzalloc(struct viommu); > + if ( !viommu ) > + return -ENOMEM; > + > + viommu->base_address = base_address; > + viommu->length = length; > + viommu->caps = caps; > + viommu->ops = viommu_type->ops; > + viommu->viommu_id = info->nr_viommu; > + > + info->viommu[info->nr_viommu] = viommu; > + info->nr_viommu++; > + > + rc = viommu->ops->create(d, viommu); > + if ( rc < 0 ) > + { > + xfree(viommu); > + info->nr_viommu--; > + info->viommu[info->nr_viommu] = NULL; > + return rc; > + } > + > + return viommu->viommu_id; > +} > + > +static int viommu_destroy(struct domain *d, u32 viommu_id) > +{ > + struct viommu_info *info = &d->viommu; > + > + if ( viommu_id >= info->nr_viommu || !info->viommu[viommu_id] ) > + return -EINVAL; > + > + if ( info->viommu[viommu_id]->ops->destroy(info->viommu[viommu_id]) ) > + return -EFAULT; You should return the return the original return value from the "destroy" function pointer, instead of hardcoding it to EFAULT. > + > + xfree(info->viommu[viommu_id]); > + info->viommu[viommu_id] = NULL; > + return 0; > +} > + > +static u64 viommu_query_caps(struct domain *d, u64 type) > +{ > + struct viommu_type *viommu_type = viommu_get_type(type); > + > + if ( !viommu_type ) > + return -EINVAL; > + > + return viommu_type->ops->query_caps(d); > +} > + > +int __init viommu_setup(void) > +{ > + INIT_LIST_HEAD(&type_list); > + spin_lock_init(&type_list_lock); > + return 0; > +} With the suggested changes to init type_list and type_list_lock at definition time you can get rid of viommu_setup. > + > +/* > + * Local variables: > + * mode: C > + * c-file-style: "BSD" > + * c-basic-offset: 4 > + * tab-width: 4 > + * End: > + */ > diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h > index 6673b27..98a965a 100644 > --- a/xen/include/xen/sched.h > +++ b/xen/include/xen/sched.h > @@ -21,6 +21,7 @@ > #include <xen/perfc.h> > #include <asm/atomic.h> > #include <xen/wait.h> > +#include <xen/viommu.h> > #include <public/xen.h> > #include <public/domctl.h> > #include <public/sysctl.h> > @@ -477,6 +478,7 @@ struct domain > /* vNUMA topology accesses are protected by rwlock. */ > rwlock_t vnuma_rwlock; > struct vnuma_info *vnuma; > + struct viommu_info viommu; > > /* Common monitor options */ > struct { > diff --git a/xen/include/xen/viommu.h b/xen/include/xen/viommu.h > new file mode 100644 > index 0000000..506ea54 > --- /dev/null > +++ b/xen/include/xen/viommu.h > @@ -0,0 +1,71 @@ > +/* > + * include/xen/viommu.h > + * > + * Copyright (c) 2017, Intel Corporation > + * Author: Lan Tianyu <tianyu.lan@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License along with > + * this program; If not, see <http://www.gnu.org/licenses/>. > + * > + */ > +#ifndef __XEN_VIOMMU_H__ > +#define __XEN_VIOMMU_H__ > + > +#define NR_VIOMMU_PER_DOMAIN 1 > + > +struct viommu; > + > +struct viommu_ops { > + u64 (*query_caps)(struct domain *d); > + int (*create)(struct domain *d, struct viommu *viommu); > + int (*destroy)(struct viommu *viommu); > +}; > + > +struct viommu { > + u64 base_address; All those should use uint*_t instead of the u* types. > + u64 length; > + u64 caps; > + u32 viommu_id; > + const struct viommu_ops *ops; > + void *priv; > +}; > + > +struct viommu_info { > + u32 nr_viommu; > + struct viommu *viommu[NR_VIOMMU_PER_DOMAIN]; /* viommu array*/ Seems kind of pointless to have a nr_viommu field when the array is not dynamic, you could just use ARRAY_SIZE. And then in the domain struct you could directly add an array of viommu structs, getting rid of viommu_info altogether. > +}; > + > +#ifdef CONFIG_VIOMMU > +extern bool_t opt_viommu; bool > +static inline bool viommu_enabled(void) { return opt_viommu; } I think those static inline functions should also follow the coding standard. Roger.
On 2017年08月22日 23:27, Roger Pau Monné wrote: > On Thu, Aug 17, 2017 at 08:22:16PM -0400, Lan Tianyu wrote: >> This patch is to introduct an abstract layer for arch vIOMMU implementation >> to deal with requests from dom0. Arch vIOMMU code needs to provide callback >> to perform create, destroy and query capabilities operation. >> >> Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> >> --- >> xen/arch/x86/Kconfig | 1 + >> xen/arch/x86/setup.c | 1 + >> xen/common/Kconfig | 3 + >> xen/common/Makefile | 1 + >> xen/common/domain.c | 3 + >> xen/common/viommu.c | 165 +++++++++++++++++++++++++++++++++++++++++++++++ >> xen/include/xen/sched.h | 2 + >> xen/include/xen/viommu.h | 71 ++++++++++++++++++++ >> 8 files changed, 247 insertions(+) >> create mode 100644 xen/common/viommu.c >> create mode 100644 xen/include/xen/viommu.h >> >> diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig >> index 30c2769..1f1de96 100644 >> --- a/xen/arch/x86/Kconfig >> +++ b/xen/arch/x86/Kconfig >> @@ -23,6 +23,7 @@ config X86 >> select HAS_PDX >> select NUMA >> select VGA >> + select VIOMMU >> >> config ARCH_DEFCONFIG >> string >> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c >> index db5df69..68f1631 100644 >> --- a/xen/arch/x86/setup.c >> +++ b/xen/arch/x86/setup.c >> @@ -1513,6 +1513,7 @@ void __init noreturn __start_xen(unsigned long mbi_p) >> early_msi_init(); >> >> iommu_setup(); /* setup iommu if available */ >> + viommu_setup(); >> >> smp_prepare_cpus(max_cpus); >> >> diff --git a/xen/common/Kconfig b/xen/common/Kconfig >> index dc8e876..2ad2c8d 100644 >> --- a/xen/common/Kconfig >> +++ b/xen/common/Kconfig >> @@ -49,6 +49,9 @@ config HAS_CHECKPOLICY >> string >> option env="XEN_HAS_CHECKPOLICY" >> >> +config VIOMMU >> + bool >> + >> config KEXEC >> bool "kexec support" >> default y >> diff --git a/xen/common/Makefile b/xen/common/Makefile >> index 26c5a64..852553d 100644 >> --- a/xen/common/Makefile >> +++ b/xen/common/Makefile >> @@ -56,6 +56,7 @@ obj-y += time.o >> obj-y += timer.o >> obj-y += trace.o >> obj-y += version.o >> +obj-$(CONFIG_VIOMMU) += viommu.o >> obj-y += virtual_region.o >> obj-y += vm_event.o >> obj-y += vmap.o >> diff --git a/xen/common/domain.c b/xen/common/domain.c >> index b22aacc..d1f9b10 100644 >> --- a/xen/common/domain.c >> +++ b/xen/common/domain.c >> @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, unsigned int domcr_flags, >> spin_unlock(&domlist_update_lock); >> } >> >> + if ( (err = viommu_init_domain(d)) != 0 ) >> + goto fail; >> + >> return d; >> >> fail: >> diff --git a/xen/common/viommu.c b/xen/common/viommu.c >> new file mode 100644 >> index 0000000..6874d9f >> --- /dev/null >> +++ b/xen/common/viommu.c >> @@ -0,0 +1,165 @@ >> +/* >> + * common/viommu.c >> + * >> + * Copyright (c) 2017 Intel Corporation >> + * Author: Lan Tianyu <tianyu.lan@intel.com> >> + * >> + * This program is free software; you can redistribute it and/or modify it >> + * under the terms and conditions of the GNU General Public License, >> + * version 2, as published by the Free Software Foundation. >> + * >> + * This program is distributed in the hope it will be useful, but WITHOUT >> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or >> + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for >> + * more details. >> + * >> + * You should have received a copy of the GNU General Public License along with >> + * this program; If not, see <http://www.gnu.org/licenses/>. >> + */ >> + >> +#include <xen/sched.h> >> +#include <xen/spinlock.h> >> +#include <xen/types.h> >> +#include <xen/viommu.h> >> + >> +bool __read_mostly opt_viommu; >> +boolean_param("viommu", opt_viommu); >> + >> +static spinlock_t type_list_lock; > > static DEFINE_SPINLOCK(type_list_lock); > >> +static struct list_head type_list; > > static LIST_HEAD(type_list); > >> + >> +struct viommu_type { >> + u64 type; >> + struct viommu_ops *ops; >> + struct list_head node; >> +}; >> + >> +int viommu_init_domain(struct domain *d) >> +{ >> + d->viommu.nr_viommu = 0; >> + return 0; >> +} > > If you don't use the viommu_info struct you can also get rid of this. > The array entries will point to NULL which can be used to signal not > initialized. Yes, just check the memory for struct domain will be set to all zero after allocation. > >> + >> +static struct viommu_type *viommu_get_type(u64 type) >> +{ >> + struct viommu_type *viommu_type = NULL; >> + >> + spin_lock(&type_list_lock); >> + list_for_each_entry( viommu_type, &type_list, node ) >> + { >> + if ( viommu_type->type == type ) >> + { >> + spin_unlock(&type_list_lock); >> + return viommu_type; >> + } >> + } >> + spin_unlock(&type_list_lock); >> + >> + return NULL; >> +} >> + >> +int viommu_register_type(u64 type, struct viommu_ops * ops) >> +{ >> + struct viommu_type *viommu_type = NULL; >> + >> + if ( !viommu_enabled() ) >> + return -EINVAL; > > ENODEV is maybe better here. Will update. > >> + >> + if ( viommu_get_type(type) ) >> + return -EEXIST; >> + >> + viommu_type = xzalloc(struct viommu_type); >> + if ( !viommu_type ) >> + return -ENOMEM; >> + >> + viommu_type->type = type; >> + viommu_type->ops = ops; >> + >> + spin_lock(&type_list_lock); >> + list_add_tail(&viommu_type->node, &type_list); >> + spin_unlock(&type_list_lock); >> + >> + return 0; >> +} > > Hm, I haven't seen the usage of this function, but from the looks of > it it seems like you want to use something similar to what's used by > the scheduler in order to register vIOMMU types. > > See the logic around REGISTER_SCHEDULER in xen/sched-if.h. Each vIOMMU devel model needs to call viommu_register_type() to register its vIOMMU type. Tool stack will pass viommu_type and vIOMMU abstract layer call vendor's vIOMMU callback to query vIOMMU capabilities, create and destroy() vIOMMU. We just need to maintain type list. REGISTER_SCHEDULER seems to heavy which reserve a scheduler array in the Xen data section. > >> + >> +static int viommu_create(struct domain *d, u64 type, u64 base_address, >> + u64 length, u64 caps) >> +{ >> + struct viommu_info *info = &d->viommu; >> + struct viommu *viommu; >> + struct viommu_type *viommu_type = NULL; >> + int rc; >> + >> + viommu_type = viommu_get_type(type); >> + if ( !viommu_type ) >> + return -EINVAL; >> + >> + if ( info->nr_viommu >= NR_VIOMMU_PER_DOMAIN >> + || !viommu_type->ops || !viommu_type->ops->create ) > ^ aligned with "info" above. >> + return -EINVAL; >> + >> + viommu = xzalloc(struct viommu); >> + if ( !viommu ) >> + return -ENOMEM; >> + >> + viommu->base_address = base_address; >> + viommu->length = length; >> + viommu->caps = caps; >> + viommu->ops = viommu_type->ops; >> + viommu->viommu_id = info->nr_viommu; >> + >> + info->viommu[info->nr_viommu] = viommu; >> + info->nr_viommu++; >> + >> + rc = viommu->ops->create(d, viommu); >> + if ( rc < 0 ) >> + { >> + xfree(viommu); >> + info->nr_viommu--; >> + info->viommu[info->nr_viommu] = NULL; >> + return rc; >> + } >> + >> + return viommu->viommu_id; >> +} >> + >> +static int viommu_destroy(struct domain *d, u32 viommu_id) >> +{ >> + struct viommu_info *info = &d->viommu; >> + >> + if ( viommu_id >= info->nr_viommu || !info->viommu[viommu_id] ) >> + return -EINVAL; >> + >> + if ( info->viommu[viommu_id]->ops->destroy(info->viommu[viommu_id]) ) >> + return -EFAULT; > > You should return the return the original return value from the > "destroy" function pointer, instead of hardcoding it to EFAULT. OK. Will update. > >> + >> + xfree(info->viommu[viommu_id]); >> + info->viommu[viommu_id] = NULL; >> + return 0; >> +} >> + >> +static u64 viommu_query_caps(struct domain *d, u64 type) >> +{ >> + struct viommu_type *viommu_type = viommu_get_type(type); >> + >> + if ( !viommu_type ) >> + return -EINVAL; >> + >> + return viommu_type->ops->query_caps(d); >> +} >> + >> +int __init viommu_setup(void) >> +{ >> + INIT_LIST_HEAD(&type_list); >> + spin_lock_init(&type_list_lock); >> + return 0; >> +} > > With the suggested changes to init type_list and type_list_lock at > definition time you can get rid of viommu_setup. OK. Will update. > >> + >> +/* >> + * Local variables: >> + * mode: C >> + * c-file-style: "BSD" >> + * c-basic-offset: 4 >> + * tab-width: 4 >> + * End: >> + */ >> diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h >> index 6673b27..98a965a 100644 >> --- a/xen/include/xen/sched.h >> +++ b/xen/include/xen/sched.h >> @@ -21,6 +21,7 @@ >> #include <xen/perfc.h> >> #include <asm/atomic.h> >> #include <xen/wait.h> >> +#include <xen/viommu.h> >> #include <public/xen.h> >> #include <public/domctl.h> >> #include <public/sysctl.h> >> @@ -477,6 +478,7 @@ struct domain >> /* vNUMA topology accesses are protected by rwlock. */ >> rwlock_t vnuma_rwlock; >> struct vnuma_info *vnuma; >> + struct viommu_info viommu; >> >> /* Common monitor options */ >> struct { >> diff --git a/xen/include/xen/viommu.h b/xen/include/xen/viommu.h >> new file mode 100644 >> index 0000000..506ea54 >> --- /dev/null >> +++ b/xen/include/xen/viommu.h >> @@ -0,0 +1,71 @@ >> +/* >> + * include/xen/viommu.h >> + * >> + * Copyright (c) 2017, Intel Corporation >> + * Author: Lan Tianyu <tianyu.lan@intel.com> >> + * >> + * This program is free software; you can redistribute it and/or modify it >> + * under the terms and conditions of the GNU General Public License, >> + * version 2, as published by the Free Software Foundation. >> + * >> + * This program is distributed in the hope it will be useful, but WITHOUT >> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or >> + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for >> + * more details. >> + * >> + * You should have received a copy of the GNU General Public License along with >> + * this program; If not, see <http://www.gnu.org/licenses/>. >> + * >> + */ >> +#ifndef __XEN_VIOMMU_H__ >> +#define __XEN_VIOMMU_H__ >> + >> +#define NR_VIOMMU_PER_DOMAIN 1 >> + >> +struct viommu; >> + >> +struct viommu_ops { >> + u64 (*query_caps)(struct domain *d); >> + int (*create)(struct domain *d, struct viommu *viommu); >> + int (*destroy)(struct viommu *viommu); >> +}; >> + >> +struct viommu { >> + u64 base_address; > > All those should use uint*_t instead of the u* types. > >> + u64 length; >> + u64 caps; >> + u32 viommu_id; >> + const struct viommu_ops *ops; >> + void *priv; >> +}; >> + >> +struct viommu_info { >> + u32 nr_viommu; >> + struct viommu *viommu[NR_VIOMMU_PER_DOMAIN]; /* viommu array*/ > > Seems kind of pointless to have a nr_viommu field when the array is > not dynamic, you could just use ARRAY_SIZE. And then in the domain > struct you could directly add an array of viommu structs, getting rid > of viommu_info altogether. nr_viommu helps to allocate new viommu id and check whether vIOMMU id passed by tool stack is valid. Otherwise, we have to check vIOMMU array every time to get viommu number. > >> +}; >> + >> +#ifdef CONFIG_VIOMMU >> +extern bool_t opt_viommu; > > bool > >> +static inline bool viommu_enabled(void) { return opt_viommu; } > > I think those static inline functions should also follow the coding > standard. Yes, I am not sure what wrong here. Could you elaborate? Thanks.
On Wed, Aug 23, 2017 at 03:10:48PM +0800, Lan Tianyu wrote: > On 2017年08月22日 23:27, Roger Pau Monné wrote: > > On Thu, Aug 17, 2017 at 08:22:16PM -0400, Lan Tianyu wrote: > >> +int viommu_register_type(u64 type, struct viommu_ops * ops) > >> +{ > >> + struct viommu_type *viommu_type = NULL; > >> + > >> + if ( !viommu_enabled() ) > >> + return -EINVAL; > > > > ENODEV is maybe better here. > > Will update. > > > > >> + > >> + if ( viommu_get_type(type) ) > >> + return -EEXIST; > >> + > >> + viommu_type = xzalloc(struct viommu_type); > >> + if ( !viommu_type ) > >> + return -ENOMEM; > >> + > >> + viommu_type->type = type; > >> + viommu_type->ops = ops; > >> + > >> + spin_lock(&type_list_lock); > >> + list_add_tail(&viommu_type->node, &type_list); > >> + spin_unlock(&type_list_lock); > >> + > >> + return 0; > >> +} > > > > Hm, I haven't seen the usage of this function, but from the looks of > > it it seems like you want to use something similar to what's used by > > the scheduler in order to register vIOMMU types. > > > > See the logic around REGISTER_SCHEDULER in xen/sched-if.h. > > Each vIOMMU devel model needs to call viommu_register_type() to register > its vIOMMU type. Tool stack will pass viommu_type and vIOMMU abstract > layer call vendor's vIOMMU callback to query vIOMMU capabilities, create > and destroy() vIOMMU. We just need to maintain type list. > REGISTER_SCHEDULER seems to heavy which reserve a scheduler array in the > Xen data section. I will let the maintainers comment, but I find it easier to use something similar to REGISTER_SCHEDULER rather than have each IOMMU type have it's initialization function hooked up in the init procedure and calling viommu_register_type. From an abstract point of view I don't really see how this is different from a scheduler for example, the IOMMU types are fixed and compiled into Xen, and they need to be initialized/registered at boot time. > >> +static inline bool viommu_enabled(void) { return opt_viommu; } > > > > I think those static inline functions should also follow the coding > > standard. > > Yes, I am not sure what wrong here. Could you elaborate? Thanks. IMHO they should be: static inline bool viommu_enabled(void) { return opt_viommu; } Roger.
> From: Lan, Tianyu > Sent: Friday, August 18, 2017 8:22 AM > > This patch is to introduct an abstract layer for arch vIOMMU > implementation > to deal with requests from dom0. Arch vIOMMU code needs to provide > callback > to perform create, destroy and query capabilities operation. > > Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> > --- > xen/arch/x86/Kconfig | 1 + > xen/arch/x86/setup.c | 1 + > xen/common/Kconfig | 3 + > xen/common/Makefile | 1 + > xen/common/domain.c | 3 + > xen/common/viommu.c | 165 > +++++++++++++++++++++++++++++++++++++++++++++++ > xen/include/xen/sched.h | 2 + > xen/include/xen/viommu.h | 71 ++++++++++++++++++++ > 8 files changed, 247 insertions(+) > create mode 100644 xen/common/viommu.c > create mode 100644 xen/include/xen/viommu.h > > diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig > index 30c2769..1f1de96 100644 > --- a/xen/arch/x86/Kconfig > +++ b/xen/arch/x86/Kconfig > @@ -23,6 +23,7 @@ config X86 > select HAS_PDX > select NUMA > select VGA > + select VIOMMU > > config ARCH_DEFCONFIG > string > diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c > index db5df69..68f1631 100644 > --- a/xen/arch/x86/setup.c > +++ b/xen/arch/x86/setup.c > @@ -1513,6 +1513,7 @@ void __init noreturn __start_xen(unsigned long > mbi_p) > early_msi_init(); > > iommu_setup(); /* setup iommu if available */ > + viommu_setup(); start_xen is about physical bits, why placing viommu stuff here? > > smp_prepare_cpus(max_cpus); > > diff --git a/xen/common/Kconfig b/xen/common/Kconfig > index dc8e876..2ad2c8d 100644 > --- a/xen/common/Kconfig > +++ b/xen/common/Kconfig > @@ -49,6 +49,9 @@ config HAS_CHECKPOLICY > string > option env="XEN_HAS_CHECKPOLICY" > > +config VIOMMU > + bool > + > config KEXEC > bool "kexec support" > default y > diff --git a/xen/common/Makefile b/xen/common/Makefile > index 26c5a64..852553d 100644 > --- a/xen/common/Makefile > +++ b/xen/common/Makefile > @@ -56,6 +56,7 @@ obj-y += time.o > obj-y += timer.o > obj-y += trace.o > obj-y += version.o > +obj-$(CONFIG_VIOMMU) += viommu.o > obj-y += virtual_region.o > obj-y += vm_event.o > obj-y += vmap.o > diff --git a/xen/common/domain.c b/xen/common/domain.c > index b22aacc..d1f9b10 100644 > --- a/xen/common/domain.c > +++ b/xen/common/domain.c > @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, > unsigned int domcr_flags, > spin_unlock(&domlist_update_lock); > } > > + if ( (err = viommu_init_domain(d)) != 0 ) > + goto fail; > + though viommu framework is common, it's better to have arch specific code invoke above function based on their own requirement (e.g. if any dependency required to enforce) > return d; > > fail: > diff --git a/xen/common/viommu.c b/xen/common/viommu.c > new file mode 100644 > index 0000000..6874d9f > --- /dev/null > +++ b/xen/common/viommu.c > @@ -0,0 +1,165 @@ > +/* > + * common/viommu.c > + * > + * Copyright (c) 2017 Intel Corporation > + * Author: Lan Tianyu <tianyu.lan@intel.com> > + * > + * This program is free software; you can redistribute it and/or modify it > + * under the terms and conditions of the GNU General Public License, > + * version 2, as published by the Free Software Foundation. > + * > + * This program is distributed in the hope it will be useful, but WITHOUT > + * ANY WARRANTY; without even the implied warranty of > MERCHANTABILITY or > + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public > License for > + * more details. > + * > + * You should have received a copy of the GNU General Public License > along with > + * this program; If not, see <http://www.gnu.org/licenses/>. > + */ > + > +#include <xen/sched.h> > +#include <xen/spinlock.h> > +#include <xen/types.h> > +#include <xen/viommu.h> > + > +bool __read_mostly opt_viommu; > +boolean_param("viommu", opt_viommu); > + > +static spinlock_t type_list_lock; > +static struct list_head type_list; > + > +struct viommu_type { > + u64 type; please add some comment what 'type' stands for. > + struct viommu_ops *ops; > + struct list_head node; > +}; > + > +int viommu_init_domain(struct domain *d) > +{ > + d->viommu.nr_viommu = 0; > + return 0; > +} > + > +static struct viommu_type *viommu_get_type(u64 type) > +{ > + struct viommu_type *viommu_type = NULL; > + > + spin_lock(&type_list_lock); > + list_for_each_entry( viommu_type, &type_list, node ) > + { > + if ( viommu_type->type == type ) > + { > + spin_unlock(&type_list_lock); > + return viommu_type; > + } > + } > + spin_unlock(&type_list_lock); > + > + return NULL; > +} > + > +int viommu_register_type(u64 type, struct viommu_ops * ops) > +{ > + struct viommu_type *viommu_type = NULL; > + > + if ( !viommu_enabled() ) > + return -EINVAL; shouldn't above be domain specific check? Thanks Kevin
diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig index 30c2769..1f1de96 100644 --- a/xen/arch/x86/Kconfig +++ b/xen/arch/x86/Kconfig @@ -23,6 +23,7 @@ config X86 select HAS_PDX select NUMA select VGA + select VIOMMU config ARCH_DEFCONFIG string diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c index db5df69..68f1631 100644 --- a/xen/arch/x86/setup.c +++ b/xen/arch/x86/setup.c @@ -1513,6 +1513,7 @@ void __init noreturn __start_xen(unsigned long mbi_p) early_msi_init(); iommu_setup(); /* setup iommu if available */ + viommu_setup(); smp_prepare_cpus(max_cpus); diff --git a/xen/common/Kconfig b/xen/common/Kconfig index dc8e876..2ad2c8d 100644 --- a/xen/common/Kconfig +++ b/xen/common/Kconfig @@ -49,6 +49,9 @@ config HAS_CHECKPOLICY string option env="XEN_HAS_CHECKPOLICY" +config VIOMMU + bool + config KEXEC bool "kexec support" default y diff --git a/xen/common/Makefile b/xen/common/Makefile index 26c5a64..852553d 100644 --- a/xen/common/Makefile +++ b/xen/common/Makefile @@ -56,6 +56,7 @@ obj-y += time.o obj-y += timer.o obj-y += trace.o obj-y += version.o +obj-$(CONFIG_VIOMMU) += viommu.o obj-y += virtual_region.o obj-y += vm_event.o obj-y += vmap.o diff --git a/xen/common/domain.c b/xen/common/domain.c index b22aacc..d1f9b10 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, unsigned int domcr_flags, spin_unlock(&domlist_update_lock); } + if ( (err = viommu_init_domain(d)) != 0 ) + goto fail; + return d; fail: diff --git a/xen/common/viommu.c b/xen/common/viommu.c new file mode 100644 index 0000000..6874d9f --- /dev/null +++ b/xen/common/viommu.c @@ -0,0 +1,165 @@ +/* + * common/viommu.c + * + * Copyright (c) 2017 Intel Corporation + * Author: Lan Tianyu <tianyu.lan@intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; If not, see <http://www.gnu.org/licenses/>. + */ + +#include <xen/sched.h> +#include <xen/spinlock.h> +#include <xen/types.h> +#include <xen/viommu.h> + +bool __read_mostly opt_viommu; +boolean_param("viommu", opt_viommu); + +static spinlock_t type_list_lock; +static struct list_head type_list; + +struct viommu_type { + u64 type; + struct viommu_ops *ops; + struct list_head node; +}; + +int viommu_init_domain(struct domain *d) +{ + d->viommu.nr_viommu = 0; + return 0; +} + +static struct viommu_type *viommu_get_type(u64 type) +{ + struct viommu_type *viommu_type = NULL; + + spin_lock(&type_list_lock); + list_for_each_entry( viommu_type, &type_list, node ) + { + if ( viommu_type->type == type ) + { + spin_unlock(&type_list_lock); + return viommu_type; + } + } + spin_unlock(&type_list_lock); + + return NULL; +} + +int viommu_register_type(u64 type, struct viommu_ops * ops) +{ + struct viommu_type *viommu_type = NULL; + + if ( !viommu_enabled() ) + return -EINVAL; + + if ( viommu_get_type(type) ) + return -EEXIST; + + viommu_type = xzalloc(struct viommu_type); + if ( !viommu_type ) + return -ENOMEM; + + viommu_type->type = type; + viommu_type->ops = ops; + + spin_lock(&type_list_lock); + list_add_tail(&viommu_type->node, &type_list); + spin_unlock(&type_list_lock); + + return 0; +} + +static int viommu_create(struct domain *d, u64 type, u64 base_address, + u64 length, u64 caps) +{ + struct viommu_info *info = &d->viommu; + struct viommu *viommu; + struct viommu_type *viommu_type = NULL; + int rc; + + viommu_type = viommu_get_type(type); + if ( !viommu_type ) + return -EINVAL; + + if ( info->nr_viommu >= NR_VIOMMU_PER_DOMAIN + || !viommu_type->ops || !viommu_type->ops->create ) + return -EINVAL; + + viommu = xzalloc(struct viommu); + if ( !viommu ) + return -ENOMEM; + + viommu->base_address = base_address; + viommu->length = length; + viommu->caps = caps; + viommu->ops = viommu_type->ops; + viommu->viommu_id = info->nr_viommu; + + info->viommu[info->nr_viommu] = viommu; + info->nr_viommu++; + + rc = viommu->ops->create(d, viommu); + if ( rc < 0 ) + { + xfree(viommu); + info->nr_viommu--; + info->viommu[info->nr_viommu] = NULL; + return rc; + } + + return viommu->viommu_id; +} + +static int viommu_destroy(struct domain *d, u32 viommu_id) +{ + struct viommu_info *info = &d->viommu; + + if ( viommu_id >= info->nr_viommu || !info->viommu[viommu_id] ) + return -EINVAL; + + if ( info->viommu[viommu_id]->ops->destroy(info->viommu[viommu_id]) ) + return -EFAULT; + + xfree(info->viommu[viommu_id]); + info->viommu[viommu_id] = NULL; + return 0; +} + +static u64 viommu_query_caps(struct domain *d, u64 type) +{ + struct viommu_type *viommu_type = viommu_get_type(type); + + if ( !viommu_type ) + return -EINVAL; + + return viommu_type->ops->query_caps(d); +} + +int __init viommu_setup(void) +{ + INIT_LIST_HEAD(&type_list); + spin_lock_init(&type_list_lock); + return 0; +} + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * End: + */ diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h index 6673b27..98a965a 100644 --- a/xen/include/xen/sched.h +++ b/xen/include/xen/sched.h @@ -21,6 +21,7 @@ #include <xen/perfc.h> #include <asm/atomic.h> #include <xen/wait.h> +#include <xen/viommu.h> #include <public/xen.h> #include <public/domctl.h> #include <public/sysctl.h> @@ -477,6 +478,7 @@ struct domain /* vNUMA topology accesses are protected by rwlock. */ rwlock_t vnuma_rwlock; struct vnuma_info *vnuma; + struct viommu_info viommu; /* Common monitor options */ struct { diff --git a/xen/include/xen/viommu.h b/xen/include/xen/viommu.h new file mode 100644 index 0000000..506ea54 --- /dev/null +++ b/xen/include/xen/viommu.h @@ -0,0 +1,71 @@ +/* + * include/xen/viommu.h + * + * Copyright (c) 2017, Intel Corporation + * Author: Lan Tianyu <tianyu.lan@intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; If not, see <http://www.gnu.org/licenses/>. + * + */ +#ifndef __XEN_VIOMMU_H__ +#define __XEN_VIOMMU_H__ + +#define NR_VIOMMU_PER_DOMAIN 1 + +struct viommu; + +struct viommu_ops { + u64 (*query_caps)(struct domain *d); + int (*create)(struct domain *d, struct viommu *viommu); + int (*destroy)(struct viommu *viommu); +}; + +struct viommu { + u64 base_address; + u64 length; + u64 caps; + u32 viommu_id; + const struct viommu_ops *ops; + void *priv; +}; + +struct viommu_info { + u32 nr_viommu; + struct viommu *viommu[NR_VIOMMU_PER_DOMAIN]; /* viommu array*/ +}; + +#ifdef CONFIG_VIOMMU +extern bool_t opt_viommu; +static inline bool viommu_enabled(void) { return opt_viommu; } +int viommu_init_domain(struct domain *d); +int viommu_register_type(u64 type, struct viommu_ops * ops); +int viommu_setup(void); +#else +static inline int viommu_init_domain(struct domain *d) { return 0; } +static inline int viommu_register_type(u64 type, struct viommu_ops * ops) +{ return 0; } +static inline int __init viommu_setup(void) { return 0; } +static inline bool viommu_enabled(void) { return false; } +#endif + +#endif /* __XEN_VIOMMU_H__ */ + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */
This patch is to introduct an abstract layer for arch vIOMMU implementation to deal with requests from dom0. Arch vIOMMU code needs to provide callback to perform create, destroy and query capabilities operation. Signed-off-by: Lan Tianyu <tianyu.lan@intel.com> --- xen/arch/x86/Kconfig | 1 + xen/arch/x86/setup.c | 1 + xen/common/Kconfig | 3 + xen/common/Makefile | 1 + xen/common/domain.c | 3 + xen/common/viommu.c | 165 +++++++++++++++++++++++++++++++++++++++++++++++ xen/include/xen/sched.h | 2 + xen/include/xen/viommu.h | 71 ++++++++++++++++++++ 8 files changed, 247 insertions(+) create mode 100644 xen/common/viommu.c create mode 100644 xen/include/xen/viommu.h