Re: [Xen-devel] [PATCH V2 1/25] VIOMMU: Add vIOMMU helper functions to create, destroy and query capabilities

[Lan Tianyu <tianyu.lan@xxxxxxxxx>]

On 2017年08月22日 23:27, Roger Pau Monné wrote:
> On Thu, Aug 17, 2017 at 08:22:16PM -0400, Lan Tianyu wrote:
>> This patch is to introduct an abstract layer for arch vIOMMU implementation
>> to deal with requests from dom0. Arch vIOMMU code needs to provide callback
>> to perform create, destroy and query capabilities operation.
>>
>> Signed-off-by: Lan Tianyu <tianyu.lan@xxxxxxxxx>
>> ---
>>  xen/arch/x86/Kconfig     |   1 +
>>  xen/arch/x86/setup.c     |   1 +
>>  xen/common/Kconfig       |   3 +
>>  xen/common/Makefile      |   1 +
>>  xen/common/domain.c      |   3 +
>>  xen/common/viommu.c      | 165 
>> +++++++++++++++++++++++++++++++++++++++++++++++
>>  xen/include/xen/sched.h  |   2 +
>>  xen/include/xen/viommu.h |  71 ++++++++++++++++++++
>>  8 files changed, 247 insertions(+)
>>  create mode 100644 xen/common/viommu.c
>>  create mode 100644 xen/include/xen/viommu.h
>>
>> diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig
>> index 30c2769..1f1de96 100644
>> --- a/xen/arch/x86/Kconfig
>> +++ b/xen/arch/x86/Kconfig
>> @@ -23,6 +23,7 @@ config X86
>>      select HAS_PDX
>>      select NUMA
>>      select VGA
>> +    select VIOMMU
>>  
>>  config ARCH_DEFCONFIG
>>      string
>> diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
>> index db5df69..68f1631 100644
>> --- a/xen/arch/x86/setup.c
>> +++ b/xen/arch/x86/setup.c
>> @@ -1513,6 +1513,7 @@ void __init noreturn __start_xen(unsigned long mbi_p)
>>      early_msi_init();
>>  
>>      iommu_setup();    /* setup iommu if available */
>> +    viommu_setup();
>>  
>>      smp_prepare_cpus(max_cpus);
>>  
>> diff --git a/xen/common/Kconfig b/xen/common/Kconfig
>> index dc8e876..2ad2c8d 100644
>> --- a/xen/common/Kconfig
>> +++ b/xen/common/Kconfig
>> @@ -49,6 +49,9 @@ config HAS_CHECKPOLICY
>>      string
>>      option env="XEN_HAS_CHECKPOLICY"
>>  
>> +config VIOMMU
>> +    bool
>> +
>>  config KEXEC
>>      bool "kexec support"
>>      default y
>> diff --git a/xen/common/Makefile b/xen/common/Makefile
>> index 26c5a64..852553d 100644
>> --- a/xen/common/Makefile
>> +++ b/xen/common/Makefile
>> @@ -56,6 +56,7 @@ obj-y += time.o
>>  obj-y += timer.o
>>  obj-y += trace.o
>>  obj-y += version.o
>> +obj-$(CONFIG_VIOMMU) += viommu.o
>>  obj-y += virtual_region.o
>>  obj-y += vm_event.o
>>  obj-y += vmap.o
>> diff --git a/xen/common/domain.c b/xen/common/domain.c
>> index b22aacc..d1f9b10 100644
>> --- a/xen/common/domain.c
>> +++ b/xen/common/domain.c
>> @@ -396,6 +396,9 @@ struct domain *domain_create(domid_t domid, unsigned int 
>> domcr_flags,
>>          spin_unlock(&domlist_update_lock);
>>      }
>>  
>> +    if ( (err = viommu_init_domain(d)) != 0 )
>> +        goto fail;
>> +
>>      return d;
>>  
>>   fail:
>> diff --git a/xen/common/viommu.c b/xen/common/viommu.c
>> new file mode 100644
>> index 0000000..6874d9f
>> --- /dev/null
>> +++ b/xen/common/viommu.c
>> @@ -0,0 +1,165 @@
>> +/*
>> + * common/viommu.c
>> + * 
>> + * Copyright (c) 2017 Intel Corporation
>> + * Author: Lan Tianyu <tianyu.lan@xxxxxxxxx> 
>> + *
>> + * This program is free software; you can redistribute it and/or modify it
>> + * under the terms and conditions of the GNU General Public License,
>> + * version 2, as published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope it will be useful, but WITHOUT
>> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
>> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
>> + * more details.
>> + *
>> + * You should have received a copy of the GNU General Public License along 
>> with
>> + * this program; If not, see <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#include <xen/sched.h>
>> +#include <xen/spinlock.h>
>> +#include <xen/types.h>
>> +#include <xen/viommu.h>
>> +
>> +bool __read_mostly opt_viommu;
>> +boolean_param("viommu", opt_viommu);
>> +
>> +static spinlock_t type_list_lock;
> 
> static DEFINE_SPINLOCK(type_list_lock);
> 
>> +static struct list_head type_list;
> 
> static LIST_HEAD(type_list);
> 
>> +
>> +struct viommu_type {
>> +    u64 type;
>> +    struct viommu_ops *ops;
>> +    struct list_head node;
>> +};
>> +
>> +int viommu_init_domain(struct domain *d)
>> +{
>> +    d->viommu.nr_viommu = 0;
>> +    return 0;
>> +}
> 
> If you don't use the viommu_info struct you can also get rid of this.
> The array entries will point to NULL which can be used to signal not
> initialized.

Yes, just check the memory  for struct domain will be set to all zero
after allocation.

> 
>> +
>> +static struct viommu_type *viommu_get_type(u64 type)
>> +{
>> +    struct viommu_type *viommu_type = NULL;
>> +
>> +    spin_lock(&type_list_lock);
>> +    list_for_each_entry( viommu_type, &type_list, node )
>> +    {
>> +        if ( viommu_type->type == type )
>> +        {
>> +            spin_unlock(&type_list_lock);
>> +            return viommu_type;
>> +        }
>> +    }
>> +    spin_unlock(&type_list_lock);
>> +
>> +    return NULL;
>> +}
>> +
>> +int viommu_register_type(u64 type, struct viommu_ops * ops)
>> +{
>> +    struct viommu_type *viommu_type = NULL;
>> +
>> +    if ( !viommu_enabled() )
>> +        return -EINVAL;
> 
> ENODEV is maybe better here.

Will update.

> 
>> +
>> +    if ( viommu_get_type(type) )
>> +        return -EEXIST;
>> +
>> +    viommu_type = xzalloc(struct viommu_type);
>> +    if ( !viommu_type )
>> +        return -ENOMEM;
>> +
>> +    viommu_type->type = type;
>> +    viommu_type->ops = ops;
>> +
>> +    spin_lock(&type_list_lock);
>> +    list_add_tail(&viommu_type->node, &type_list);
>> +    spin_unlock(&type_list_lock);
>> +
>> +    return 0;
>> +}
> 
> Hm, I haven't seen the usage of this function, but from the looks of
> it it seems like you want to use something similar to what's used by
> the scheduler in order to register vIOMMU types.
> 
> See the logic around REGISTER_SCHEDULER in xen/sched-if.h.

Each vIOMMU devel model needs to call viommu_register_type() to register
its vIOMMU type. Tool stack will pass viommu_type and vIOMMU abstract
layer call vendor's vIOMMU callback to query vIOMMU capabilities, create
and destroy() vIOMMU. We just need to maintain type list.
REGISTER_SCHEDULER seems to heavy which reserve a scheduler array in the
Xen data section.

> 
>> +
>> +static int viommu_create(struct domain *d, u64 type, u64 base_address,
>> +    u64 length, u64 caps)
>> +{
>> +    struct viommu_info *info = &d->viommu;
>> +    struct viommu *viommu;
>> +    struct viommu_type *viommu_type = NULL;
>> +    int rc;
>> +
>> +    viommu_type = viommu_get_type(type);
>> +    if ( !viommu_type )
>> +        return -EINVAL;
>> +
>> +    if ( info->nr_viommu >= NR_VIOMMU_PER_DOMAIN
>> +        || !viommu_type->ops || !viommu_type->ops->create )
>            ^ aligned with "info" above.
>> +        return -EINVAL;
>> +
>> +    viommu = xzalloc(struct viommu);
>> +    if ( !viommu )
>> +        return -ENOMEM;
>> +
>> +    viommu->base_address = base_address;
>> +    viommu->length = length;
>> +    viommu->caps = caps;
>> +    viommu->ops = viommu_type->ops;
>> +    viommu->viommu_id = info->nr_viommu;
>> +
>> +    info->viommu[info->nr_viommu] = viommu;
>> +    info->nr_viommu++;
>> +
>> +    rc = viommu->ops->create(d, viommu);
>> +    if ( rc < 0 )
>> +    {
>> +        xfree(viommu);
>> +        info->nr_viommu--;
>> +        info->viommu[info->nr_viommu] = NULL;
>> +        return rc;
>> +    }
>> +
>> +    return viommu->viommu_id;
>> +}
>> +
>> +static int viommu_destroy(struct domain *d, u32 viommu_id)
>> +{
>> +    struct viommu_info *info = &d->viommu;
>> +
>> +    if ( viommu_id >= info->nr_viommu || !info->viommu[viommu_id] )
>> +        return -EINVAL;
>> +
>> +    if ( info->viommu[viommu_id]->ops->destroy(info->viommu[viommu_id]) )
>> +        return -EFAULT;
> 
> You should return the return the original return value from the
> "destroy" function pointer, instead of hardcoding it to EFAULT.

OK. Will update.


> 
>> +
>> +    xfree(info->viommu[viommu_id]);
>> +    info->viommu[viommu_id] = NULL;
>> +    return 0;
>> +}
>> +
>> +static u64 viommu_query_caps(struct domain *d, u64 type)
>> +{
>> +    struct viommu_type *viommu_type = viommu_get_type(type);
>> +
>> +    if ( !viommu_type )
>> +        return -EINVAL;
>> +
>> +    return viommu_type->ops->query_caps(d);
>> +}
>> +
>> +int __init viommu_setup(void)
>> +{
>> +    INIT_LIST_HEAD(&type_list);
>> +    spin_lock_init(&type_list_lock);
>> +    return 0;
>> +}
> 
> With the suggested changes to init type_list and type_list_lock at
> definition time you can get rid of viommu_setup.

OK. Will update.

> 
>> +
>> +/*
>> + * Local variables:
>> + * mode: C
>> + * c-file-style: "BSD"
>> + * c-basic-offset: 4
>> + * tab-width: 4
>> + * End:
>> + */
>> diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
>> index 6673b27..98a965a 100644
>> --- a/xen/include/xen/sched.h
>> +++ b/xen/include/xen/sched.h
>> @@ -21,6 +21,7 @@
>>  #include <xen/perfc.h>
>>  #include <asm/atomic.h>
>>  #include <xen/wait.h>
>> +#include <xen/viommu.h>
>>  #include <public/xen.h>
>>  #include <public/domctl.h>
>>  #include <public/sysctl.h>
>> @@ -477,6 +478,7 @@ struct domain
>>      /* vNUMA topology accesses are protected by rwlock. */
>>      rwlock_t vnuma_rwlock;
>>      struct vnuma_info *vnuma;
>> +    struct viommu_info viommu;
>>  
>>      /* Common monitor options */
>>      struct {
>> diff --git a/xen/include/xen/viommu.h b/xen/include/xen/viommu.h
>> new file mode 100644
>> index 0000000..506ea54
>> --- /dev/null
>> +++ b/xen/include/xen/viommu.h
>> @@ -0,0 +1,71 @@
>> +/*
>> + * include/xen/viommu.h
>> + *
>> + * Copyright (c) 2017, Intel Corporation
>> + * Author: Lan Tianyu <tianyu.lan@xxxxxxxxx> 
>> + *
>> + * This program is free software; you can redistribute it and/or modify it
>> + * under the terms and conditions of the GNU General Public License,
>> + * version 2, as published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope it will be useful, but WITHOUT
>> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
>> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
>> + * more details.
>> + *
>> + * You should have received a copy of the GNU General Public License along 
>> with
>> + * this program; If not, see <http://www.gnu.org/licenses/>.
>> + *
>> + */
>> +#ifndef __XEN_VIOMMU_H__
>> +#define __XEN_VIOMMU_H__
>> +
>> +#define NR_VIOMMU_PER_DOMAIN 1
>> +
>> +struct viommu;
>> +
>> +struct viommu_ops {
>> +    u64 (*query_caps)(struct domain *d);
>> +    int (*create)(struct domain *d, struct viommu *viommu);
>> +    int (*destroy)(struct viommu *viommu);
>> +};
>> +
>> +struct viommu {
>> +    u64 base_address;
> 
> All those should use uint*_t instead of the u* types.
> 
>> +    u64 length;
>> +    u64 caps;
>> +    u32 viommu_id;
>> +    const struct viommu_ops *ops;
>> +    void *priv;
>> +};
>> +
>> +struct viommu_info {
>> +    u32 nr_viommu;
>> +    struct viommu *viommu[NR_VIOMMU_PER_DOMAIN]; /* viommu array*/
> 
> Seems kind of pointless to have a nr_viommu field when the array is
> not dynamic, you could just use ARRAY_SIZE. And then in the domain
> struct you could directly add an array of viommu structs, getting rid
> of viommu_info altogether.

nr_viommu helps to allocate new viommu id and check whether vIOMMU id
passed by tool stack is valid. Otherwise, we have to check vIOMMU array
every time to get viommu number.

> 
>> +};
>> +
>> +#ifdef CONFIG_VIOMMU
>> +extern bool_t opt_viommu;
> 
> bool
> 
>> +static inline bool viommu_enabled(void) { return opt_viommu; }
> 
> I think those static inline functions should also follow the coding
> standard.

Yes, I am not sure what wrong here. Could you elaborate? Thanks.


-- 
Best regards
Tianyu Lan

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel