|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [Xen-devel] [PATCH v3 07/26] ARM: GICv3 ITS: introduce host LPI array
On Fri, 31 Mar 2017, Andre Przywara wrote:
> The number of LPIs on a host can be potentially huge (millions),
> although in practise will be mostly reasonable. So prematurely allocating
> an array of struct irq_desc's for each LPI is not an option.
> However Xen itself does not care about LPIs, as every LPI will be injected
> into a guest (Dom0 for now).
> Create a dense data structure (8 Bytes) for each LPI which holds just
> enough information to determine the virtual IRQ number and the VCPU into
> which the LPI needs to be injected.
> Also to not artificially limit the number of LPIs, we create a 2-level
> table for holding those structures.
> This patch introduces functions to initialize these tables and to
> create, lookup and destroy entries for a given LPI.
> By using the naturally atomic access guarantee the native uint64_t data
> type gives us, we allocate and access LPI information in a way that does
> not require a lock.
>
> Signed-off-by: Andre Przywara <andre.przywara@xxxxxxx>
See alpine.DEB.2.10.1703221552490.8001@sstabellini-ThinkPad-X260.
I'll stop here for now, I think that are enough comments already for
another version.
> ---
> xen/arch/arm/gic-v3-its.c | 89 +++++++++++++++++-
> xen/arch/arm/gic-v3-lpi.c | 196
> +++++++++++++++++++++++++++++++++++++++
> xen/include/asm-arm/gic.h | 2 +
> xen/include/asm-arm/gic_v3_its.h | 5 +
> xen/include/asm-arm/irq.h | 5 +
> 5 files changed, 295 insertions(+), 2 deletions(-)
>
> diff --git a/xen/arch/arm/gic-v3-its.c b/xen/arch/arm/gic-v3-its.c
> index 295f7dc..fa284e7 100644
> --- a/xen/arch/arm/gic-v3-its.c
> +++ b/xen/arch/arm/gic-v3-its.c
> @@ -151,6 +151,20 @@ static int its_send_cmd_sync(struct host_its *its,
> unsigned int cpu)
> return its_send_command(its, cmd);
> }
>
> +static int its_send_cmd_mapti(struct host_its *its,
> + uint32_t deviceid, uint32_t eventid,
> + uint32_t pintid, uint16_t icid)
> +{
> + uint64_t cmd[4];
> +
> + cmd[0] = GITS_CMD_MAPTI | ((uint64_t)deviceid << 32);
> + cmd[1] = eventid | ((uint64_t)pintid << 32);
> + cmd[2] = icid;
> + cmd[3] = 0x00;
> +
> + return its_send_command(its, cmd);
> +}
> +
> static int its_send_cmd_mapc(struct host_its *its, uint32_t collection_id,
> unsigned int cpu)
> {
> @@ -185,6 +199,19 @@ static int its_send_cmd_mapd(struct host_its *its,
> uint32_t deviceid,
> return its_send_command(its, cmd);
> }
>
> +static int its_send_cmd_inv(struct host_its *its,
> + uint32_t deviceid, uint32_t eventid)
> +{
> + uint64_t cmd[4];
> +
> + cmd[0] = GITS_CMD_INV | ((uint64_t)deviceid << 32);
> + cmd[1] = eventid;
> + cmd[2] = 0x00;
> + cmd[3] = 0x00;
> +
> + return its_send_command(its, cmd);
> +}
> +
> /* Set up the (1:1) collection mapping for the given host CPU. */
> int gicv3_its_setup_collection(unsigned int cpu)
> {
> @@ -469,7 +496,7 @@ int gicv3_its_init(void)
>
> static int remove_mapped_guest_device(struct its_devices *dev)
> {
> - int ret;
> + int ret, i;
>
> if ( dev->hw_its )
> {
> @@ -479,12 +506,16 @@ static int remove_mapped_guest_device(struct
> its_devices *dev)
> return ret;
> }
>
> + for ( i = 0; i < DIV_ROUND_UP(dev->eventids, LPI_BLOCK); i++ )
> + gicv3_free_host_lpi_block(dev->host_lpi_blocks[i]);
> +
> ret = gicv3_its_wait_commands(dev->hw_its);
> if ( ret )
> return ret;
>
> xfree(dev->itt_addr);
> xfree(dev->pend_irqs);
> + xfree(dev->host_lpi_blocks);
> xfree(dev);
>
> return 0;
> @@ -522,6 +553,37 @@ static int compare_its_guest_devices(struct its_devices
> *dev,
> }
>
> /*
> + * On the host ITS @its, map @nr_events consecutive LPIs.
> + * The mapping connects a device @devid and event @eventid pair to LPI @lpi,
> + * increasing both @eventid and @lpi to cover the number of requested LPIs.
> + */
> +static int gicv3_its_map_host_events(struct host_its *its,
> + uint32_t devid, uint32_t eventid,
> + uint32_t lpi, uint32_t nr_events)
> +{
> + uint32_t i;
> + int ret;
> +
> + for ( i = 0; i < nr_events; i++ )
> + {
> + /* For now we map every host LPI to host CPU 0 */
> + ret = its_send_cmd_mapti(its, devid, eventid + i, lpi + i, 0);
> + if ( ret )
> + return ret;
> +
> + ret = its_send_cmd_inv(its, devid, eventid + i);
> + if ( ret )
> + return ret;
> + }
> +
> + ret = its_send_cmd_sync(its, 0);
> + if ( ret )
> + return ret;
> +
> + return gicv3_its_wait_commands(its);
> +}
> +
> +/*
> * Map a hardware device, identified by a certain host ITS and its device ID
> * to domain d, a guest ITS (identified by its doorbell address) and device
> ID.
> * Also provide the number of events (MSIs) needed for that device.
> @@ -537,7 +599,7 @@ int gicv3_its_map_guest_device(struct domain *d,
> struct host_its *hw_its;
> struct its_devices *dev = NULL;
> struct rb_node **new = &d->arch.vgic.its_devices.rb_node, *parent = NULL;
> - int ret = -ENOENT;
> + int ret = -ENOENT, i;
>
> hw_its = gicv3_its_find_by_doorbell(host_doorbell);
> if ( !hw_its )
> @@ -595,6 +657,11 @@ int gicv3_its_map_guest_device(struct domain *d,
> if ( !dev->pend_irqs )
> goto out_unlock;
>
> + dev->host_lpi_blocks = xzalloc_array(uint32_t,
> + DIV_ROUND_UP(nr_events, LPI_BLOCK));
> + if ( !dev->host_lpi_blocks )
> + goto out_unlock;
> +
> ret = its_send_cmd_mapd(hw_its, host_devid,
> fls(ROUNDUP(nr_events, LPI_BLOCK) - 1) - 1,
> virt_to_maddr(itt_addr), true);
> @@ -613,10 +680,28 @@ int gicv3_its_map_guest_device(struct domain *d,
>
> spin_unlock(&d->arch.vgic.its_devices_lock);
>
> + /*
> + * Map all host LPIs within this device already. We can't afford to queue
> + * any host ITS commands later on during the guest's runtime.
> + */
> + for ( i = 0; i < DIV_ROUND_UP(nr_events, LPI_BLOCK); i++ )
> + {
> + ret = gicv3_allocate_host_lpi_block(d, &dev->host_lpi_blocks[i]);
> + if ( ret < 0 )
> + goto out;
> +
> + ret = gicv3_its_map_host_events(hw_its, host_devid, i * LPI_BLOCK,
> + dev->host_lpi_blocks[i], LPI_BLOCK);
> + if ( ret < 0 )
> + goto out;
> + }
> +
> return 0;
>
> out_unlock:
> spin_unlock(&d->arch.vgic.its_devices_lock);
> +
> +out:
> if ( dev )
> {
> xfree(dev->pend_irqs);
> diff --git a/xen/arch/arm/gic-v3-lpi.c b/xen/arch/arm/gic-v3-lpi.c
> index d85d63d..d642cc5 100644
> --- a/xen/arch/arm/gic-v3-lpi.c
> +++ b/xen/arch/arm/gic-v3-lpi.c
> @@ -20,25 +20,55 @@
>
> #include <xen/lib.h>
> #include <xen/mm.h>
> +#include <xen/sched.h>
> #include <xen/sizes.h>
> +#include <asm/atomic.h>
> +#include <asm/domain.h>
> #include <asm/gic.h>
> #include <asm/gic_v3_defs.h>
> #include <asm/gic_v3_its.h>
> #include <asm/io.h>
> #include <asm/page.h>
>
> +/*
> + * There could be a lot of LPIs on the host side, and they always go to
> + * a guest. So having a struct irq_desc for each of them would be wasteful
> + * and useless.
> + * Instead just store enough information to find the right VCPU to inject
> + * those LPIs into, which just requires the virtual LPI number.
> + * To avoid a global lock on this data structure, this is using a lockless
> + * approach relying on the architectural atomicty of native data types:
> + * We read or write the "data" view of this union atomically, then can
> + * access the broken-down fields in our local copy.
> + */
> +union host_lpi {
> + uint64_t data;
> + struct {
> + uint32_t virt_lpi;
> + uint16_t dom_id;
> + uint16_t vcpu_id;
> + };
> +};
> +
> #define LPI_PROPTABLE_NEEDS_FLUSHING (1U << 0)
> /* Global state */
> static struct {
> /* The global LPI property table, shared by all redistributors. */
> uint8_t *lpi_property;
> /*
> + * A two-level table to lookup LPIs firing on the host and look up the
> + * VCPU and virtual LPI number to inject into.
> + */
> + union host_lpi **host_lpis;
> + /*
> * Number of physical LPIs the host supports. This is a property of
> * the GIC hardware. We depart from the habit of naming these things
> * "physical" in Xen, as the GICv3/4 spec uses the term "physical LPI"
> * in a different context to differentiate them from "virtual LPIs".
> */
> unsigned long int nr_host_lpis;
> + /* Protects allocation and deallocation of host LPIs, but not the access
> */
> + spinlock_t host_lpis_lock;
> unsigned int flags;
> } lpi_data;
>
> @@ -51,6 +81,19 @@ struct lpi_redist_data {
> static DEFINE_PER_CPU(struct lpi_redist_data, lpi_redist);
>
> #define MAX_PHYS_LPIS (lpi_data.nr_host_lpis - LPI_OFFSET)
> +#define HOST_LPIS_PER_PAGE (PAGE_SIZE / sizeof(union host_lpi))
> +
> +static union host_lpi *gic_get_host_lpi(uint32_t plpi)
> +{
> + if ( !is_lpi(plpi) || plpi >= MAX_PHYS_LPIS + LPI_OFFSET )
> + return NULL;
> +
> + plpi -= LPI_OFFSET;
> + if ( !lpi_data.host_lpis[plpi / HOST_LPIS_PER_PAGE] )
> + return NULL;
> +
> + return &lpi_data.host_lpis[plpi / HOST_LPIS_PER_PAGE][plpi %
> HOST_LPIS_PER_PAGE];
> +}
>
> /* Stores this redistributor's physical address and ID in a per-CPU variable
> */
> void gicv3_set_redist_address(paddr_t address, unsigned int redist_id)
> @@ -212,15 +255,168 @@ int gicv3_lpi_init_rdist(void __iomem * rdist_base)
> static unsigned int max_lpi_bits = 20;
> integer_param("max_lpi_bits", max_lpi_bits);
>
> +/*
> + * Allocate the 2nd level array for host LPIs. This one holds pointers
> + * to the page with the actual "union host_lpi" entries. Our LPI limit
> + * avoids excessive memory usage.
> + */
> int gicv3_lpi_init_host_lpis(unsigned int hw_lpi_bits)
> {
> + int nr_lpi_ptrs;
> +
> + /* We rely on the data structure being atomically accessible. */
> + BUILD_BUG_ON(sizeof(union host_lpi) > sizeof(unsigned long));
> +
> lpi_data.nr_host_lpis = BIT_ULL(min(hw_lpi_bits, max_lpi_bits));
>
> + spin_lock_init(&lpi_data.host_lpis_lock);
> +
> + nr_lpi_ptrs = MAX_PHYS_LPIS / (PAGE_SIZE / sizeof(union host_lpi));
> + lpi_data.host_lpis = xzalloc_array(union host_lpi *, nr_lpi_ptrs);
> + if ( !lpi_data.host_lpis )
> + return -ENOMEM;
> +
> printk("GICv3: using at most %lu LPIs on the host.\n", MAX_PHYS_LPIS);
>
> return 0;
> }
>
> +static int find_unused_host_lpi(uint32_t start, uint32_t *index)
> +{
> + unsigned int chunk;
> + uint32_t i = *index;
> +
> + ASSERT(spin_is_locked(&lpi_data.host_lpis_lock));
> +
> + for ( chunk = start; chunk < MAX_PHYS_LPIS / HOST_LPIS_PER_PAGE; chunk++
> )
> + {
> + /* If we hit an unallocated chunk, use entry 0 in that one. */
> + if ( !lpi_data.host_lpis[chunk] )
> + {
> + *index = 0;
> + return chunk;
> + }
> +
> + /* Find an unallocated entry in this chunk. */
> + for ( ; i < HOST_LPIS_PER_PAGE; i += LPI_BLOCK )
> + {
> + if ( lpi_data.host_lpis[chunk][i].dom_id == DOMID_INVALID )
> + {
> + *index = i;
> + return chunk;
> + }
> + }
> + i = 0;
> + }
> +
> + return -1;
> +}
> +
> +/*
> + * Allocate a block of 32 LPIs on the given host ITS for device "devid",
> + * starting with "eventid". Put them into the respective ITT by issuing a
> + * MAPTI command for each of them.
> + */
> +int gicv3_allocate_host_lpi_block(struct domain *d, uint32_t *first_lpi)
> +{
> + static uint32_t next_lpi = 0;
> + uint32_t lpi, lpi_idx = next_lpi % HOST_LPIS_PER_PAGE;
> + int chunk;
> + int i;
> +
> + spin_lock(&lpi_data.host_lpis_lock);
> + chunk = find_unused_host_lpi(next_lpi / HOST_LPIS_PER_PAGE, &lpi_idx);
> +
> + if ( chunk == - 1 ) /* rescan for a hole from the beginning */
> + {
> + lpi_idx = 0;
> + chunk = find_unused_host_lpi(0, &lpi_idx);
> + if ( chunk == -1 )
> + {
> + spin_unlock(&lpi_data.host_lpis_lock);
> + return -ENOSPC;
> + }
> + }
> +
> + /* If we hit an unallocated chunk, we initialize it and use entry 0. */
> + if ( !lpi_data.host_lpis[chunk] )
> + {
> + union host_lpi *new_chunk;
> +
> + /* TODO: NUMA locality for quicker IRQ path? */
> + new_chunk = xmalloc_bytes(PAGE_SIZE);
> + if ( !new_chunk )
> + {
> + spin_unlock(&lpi_data.host_lpis_lock);
> + return -ENOMEM;
> + }
> +
> + for ( i = 0; i < HOST_LPIS_PER_PAGE; i += LPI_BLOCK )
> + new_chunk[i].dom_id = DOMID_INVALID;
> +
> + lpi_data.host_lpis[chunk] = new_chunk;
> + lpi_idx = 0;
> + }
> +
> + lpi = chunk * HOST_LPIS_PER_PAGE + lpi_idx;
> +
> + for ( i = 0; i < LPI_BLOCK; i++ )
> + {
> + union host_lpi hlpi;
> +
> + /*
> + * Mark this host LPI as belonging to the domain, but don't assign
> + * any virtual LPI or a VCPU yet.
> + */
> + hlpi.virt_lpi = INVALID_LPI;
> + hlpi.dom_id = d->domain_id;
> + hlpi.vcpu_id = ~0;
> + write_u64_atomic(&lpi_data.host_lpis[chunk][lpi_idx + i].data,
> + hlpi.data);
> +
> + /*
> + * Enable this host LPI, so we don't have to do this during the
> + * guest's runtime.
> + */
> + lpi_data.lpi_property[lpi + i] |= LPI_PROP_ENABLED;
> + }
> +
> + /*
> + * We have allocated and initialized the host LPI entries, so it's safe
> + * to drop the lock now. Access to the structures can be done
> concurrently
> + * as it involves only an atomic uint64_t access.
> + */
> + spin_unlock(&lpi_data.host_lpis_lock);
> +
> + if ( lpi_data.flags & LPI_PROPTABLE_NEEDS_FLUSHING )
> + clean_and_invalidate_dcache_va_range(&lpi_data.lpi_property[lpi],
> + LPI_BLOCK);
> +
> + next_lpi = lpi + LPI_BLOCK;
> + *first_lpi = lpi + LPI_OFFSET;
> +
> + return 0;
> +}
> +
> +void gicv3_free_host_lpi_block(uint32_t first_lpi)
> +{
> + union host_lpi *hlpi, empty_lpi = { .dom_id = DOMID_INVALID };
> + int i;
> +
> + hlpi = gic_get_host_lpi(first_lpi);
> + if ( !hlpi )
> + return; /* Nothing to free here. */
> +
> + spin_lock(&lpi_data.host_lpis_lock);
> +
> + for ( i = 0; i < LPI_BLOCK; i++ )
> + write_u64_atomic(&hlpi[i].data, empty_lpi.data);
> +
> + spin_unlock(&lpi_data.host_lpis_lock);
> +
> + return;
> +}
> +
> /*
> * Local variables:
> * mode: C
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 836a103..d04bd04 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -220,6 +220,8 @@ enum gic_version {
> GIC_V3,
> };
>
> +#define INVALID_LPI 0
> +
> extern enum gic_version gic_hw_version(void);
>
> /* Program the IRQ type into the GIC */
> diff --git a/xen/include/asm-arm/gic_v3_its.h
> b/xen/include/asm-arm/gic_v3_its.h
> index 4ade5f6..7b47596 100644
> --- a/xen/include/asm-arm/gic_v3_its.h
> +++ b/xen/include/asm-arm/gic_v3_its.h
> @@ -106,6 +106,9 @@
> #define HOST_ITS_FLUSH_CMD_QUEUE (1U << 0)
> #define HOST_ITS_USES_PTA (1U << 1)
>
> +/* We allocate LPIs on the hosts in chunks of 32 to reduce handling
> overhead. */
> +#define LPI_BLOCK 32
> +
> /* data structure for each hardware ITS */
> struct host_its {
> struct list_head entry;
> @@ -153,6 +156,8 @@ int gicv3_its_map_guest_device(struct domain *d,
> paddr_t guest_doorbell, uint32_t guest_devid,
> uint32_t nr_events, bool valid);
> void gicv3_its_unmap_all_devices(struct domain *d);
> +int gicv3_allocate_host_lpi_block(struct domain *d, uint32_t *first_lpi);
> +void gicv3_free_host_lpi_block(uint32_t first_lpi);
>
> #else
>
> diff --git a/xen/include/asm-arm/irq.h b/xen/include/asm-arm/irq.h
> index 13528c0..d16affc 100644
> --- a/xen/include/asm-arm/irq.h
> +++ b/xen/include/asm-arm/irq.h
> @@ -42,6 +42,11 @@ struct irq_desc *__irq_to_desc(int irq);
>
> void do_IRQ(struct cpu_user_regs *regs, unsigned int irq, int is_fiq);
>
> +static inline bool is_lpi(unsigned int irq)
> +{
> + return irq >= LPI_OFFSET;
> +}
> +
> #define domain_pirq_to_irq(d, pirq) (pirq)
>
> bool_t is_assignable_irq(unsigned int irq);
> --
> 2.9.0
>
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |