Re: [Xen-devel] [PATCH] xen: arm: increase priority of SGIs used as IPIs

[Stefano Stabellini <stefano.stabellini@xxxxxxxxxxxxx>]

On Tue, 28 Jan 2014, Ian Campbell wrote:
> Code such as on_selected_cpus expects/requires that an IPI can preempt a
> processor which is just handling a normal interrupt. Lacking this property can
> result in a deadlock between two CPUs trying to IPI each other from interrupt
> context.
> 
> For the time being there is only two priorities, IRQ and IPI, although it is
> also conceivable that in the future some IPIs might be higher priority than
> others. This could be used to implement a better BUG() than we have now, but I
> haven't tackled that yet.
> 
> Tested with a debug patch which sends a local IPI from a keyhandler, which is
> run in serial interrupt context.
> 
> This should also fix the issue reported by Oleksandr in "xen/arm:
> maintenance_interrupt SMP fix" without resorting to trylock.
> 
> Signed-off-by: Ian Campbell <ian.campbell@xxxxxxxxxx>
> Cc: Oleksandr Tyshchenko <oleksandr.tyshchenko@xxxxxxxxxxxxxxx>

It looks simple enough.
Oleksandr, I would appreciate if could test the patch and tell us if it
is working well for you.


> I think this is probably 4.5 material at this point.
> 
> Tested with "HACK: dump pcpu state keyhandler" which I'll post for
> completeness. It gives:
> (XEN) Xen call trace:
> (XEN)    [<0000000000212048>] dump_pcpus+0x28/0x2c (PC)
> (XEN)    [<000000000021256c>] handle_keypress+0x70/0xb0 (LR)
> (XEN)    [<000000000023ed00>] __serial_rx+0x20/0x6c
> (XEN)    [<000000000023f8ac>] serial_rx+0xb4/0xc4
> (XEN)    [<00000000002409ec>] serial_rx_interrupt+0xb0/0xd4
> (XEN)    [<00000000002404b4>] ns16550_interrupt+0x6c/0x90
> (XEN)    [<0000000000245fc0>] do_IRQ+0x144/0x1b4
> (XEN)    [<0000000000245a28>] gic_interrupt+0x60/0xf8
> (XEN)    [<000000000024be64>] do_trap_irq+0x10/0x18
> (XEN)    [<000000000024e240>] hyp_irq+0x5c/0x60
> (XEN)    [<0000000000249324>] init_done+0x10/0x18
> (XEN)    [<0000000000000080>] 0000000000000080
> ---
>  xen/arch/arm/gic.c        |   19 +++++++++++++------
>  xen/arch/arm/time.c       |    6 +++---
>  xen/include/asm-arm/gic.h |   22 ++++++++++++++++++++++
>  3 files changed, 38 insertions(+), 9 deletions(-)
> 
> diff --git a/xen/arch/arm/gic.c b/xen/arch/arm/gic.c
> index dcf9cd4..ee37019 100644
> --- a/xen/arch/arm/gic.c
> +++ b/xen/arch/arm/gic.c
> @@ -319,7 +319,8 @@ static void __init gic_dist_init(void)
>  
>      /* Default priority for global interrupts */
>      for ( i = 32; i < gic.lines; i += 4 )
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Disable all global interrupts */
>      for ( i = 32; i < gic.lines; i += 32 )
> @@ -341,8 +342,12 @@ static void __cpuinit gic_cpu_init(void)
>      GICD[GICD_ICENABLER] = 0xffff0000; /* Disable all PPI */
>      GICD[GICD_ISENABLER] = 0x0000ffff; /* Enable all SGI */
>      /* Set PPI and SGI priorities */
> -    for (i = 0; i < 32; i += 4)
> -        GICD[GICD_IPRIORITYR + i / 4] = 0xa0a0a0a0;
> +    for (i = 0; i < 16; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IPI<<24 | GIC_PRI_IPI<<16 | GIC_PRI_IPI<<8 | GIC_PRI_IPI;
> +    for (i = 16; i < 32; i += 4)
> +        GICD[GICD_IPRIORITYR + i / 4] =
> +            GIC_PRI_IRQ<<24 | GIC_PRI_IRQ<<16 | GIC_PRI_IRQ<<8 | GIC_PRI_IRQ;
>  
>      /* Local settings: interface controller */
>      GICC[GICC_PMR] = 0xff;                /* Don't mask by priority */
> @@ -538,7 +543,8 @@ void gic_disable_cpu(void)
>  void gic_route_ppis(void)
>  {
>      /* GIC maintenance */
> -    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()), 0xa0);
> +    gic_route_dt_irq(&gic.maintenance, cpumask_of(smp_processor_id()),
> +                     GIC_PRI_IRQ);
>      /* Route timer interrupt */
>      route_timer_interrupt();
>  }
> @@ -553,7 +559,8 @@ void gic_route_spis(void)
>          if ( (irq = serial_dt_irq(seridx)) == NULL )
>              continue;
>  
> -        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()), 0xa0);
> +        gic_route_dt_irq(irq, cpumask_of(smp_processor_id()),
> +                         GIC_PRI_IRQ);
>      }
>  }
>  
> @@ -777,7 +784,7 @@ int gic_route_irq_to_guest(struct domain *d, const struct 
> dt_irq *irq,
>      level = dt_irq_is_level_triggered(irq);
>  
>      gic_set_irq_properties(irq->irq, level, cpumask_of(smp_processor_id()),
> -                           0xa0);
> +                           GIC_PRI_IRQ);
>  
>      retval = __setup_irq(desc, irq->irq, action);
>      if (retval) {
> diff --git a/xen/arch/arm/time.c b/xen/arch/arm/time.c
> index 81e3e28..68b939d 100644
> --- a/xen/arch/arm/time.c
> +++ b/xen/arch/arm/time.c
> @@ -222,11 +222,11 @@ static void vtimer_interrupt(int irq, void *dev_id, 
> struct cpu_user_regs *regs)
>  void __cpuinit route_timer_interrupt(void)
>  {
>      gic_route_dt_irq(&timer_irq[TIMER_PHYS_NONSECURE_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_HYP_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>      gic_route_dt_irq(&timer_irq[TIMER_VIRT_PPI],
> -                     cpumask_of(smp_processor_id()), 0xa0);
> +                     cpumask_of(smp_processor_id()), GIC_PRI_IRQ);
>  }
>  
>  /* Set up the timer interrupt on this CPU */
> diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
> index 9c6f9bb..25b2b24 100644
> --- a/xen/include/asm-arm/gic.h
> +++ b/xen/include/asm-arm/gic.h
> @@ -129,6 +129,28 @@
>  #define GICH_LR_CPUID_SHIFT     9
>  #define GICH_VTR_NRLRGS         0x3f
>  
> +/*
> + * The minimum GICC_BPR is required to be in the range 0-3. We set
> + * GICC_BPR to 0 but we must expect that it might be 3. This means we
> + * can rely on premption between the following ranges:
> + * 0xf0..0xff
> + * 0xe0..0xdf
> + * 0xc0..0xcf
> + * 0xb0..0xbf
> + * 0xa0..0xaf
> + * 0x90..0x9f
> + * 0x80..0x8f
> + *
> + * Priorities within a range will not preempt each other.
> + *
> + * A GIC must support a mimimum of 16 priority levels.
> + */
> +#define GIC_PRI_LOWEST     0xf0
> +#define GIC_PRI_IRQ        0xa0
> +#define GIC_PRI_IPI        0x90 /* IPIs must preempt normal interrupts */
> +#define GIC_PRI_HIGHEST    0x80 /* Higher priorities belong to Secure-World 
> */
> +
> +
>  #ifndef __ASSEMBLY__
>  #include <xen/device_tree.h>
>  
> -- 
> 1.7.10.4
> 

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel