RE: [PATCH v9 6/8] common/domain: add a domain context record for shared_info...

[Paul Durrant <xadimgnik@xxxxxxxxx>]

> -----Original Message-----
> From: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
> Sent: 05 October 2020 11:40
> To: Paul Durrant <paul@xxxxxxx>; xen-devel@xxxxxxxxxxxxxxxxxxxx
> Cc: Paul Durrant <pdurrant@xxxxxxxxxx>; Ian Jackson 
> <ian.jackson@xxxxxxxxxxxxx>; Wei Liu <wl@xxxxxxx>;
> George Dunlap <george.dunlap@xxxxxxxxxx>; Jan Beulich <jbeulich@xxxxxxxx>; 
> Julien Grall
> <julien@xxxxxxx>; Stefano Stabellini <sstabellini@xxxxxxxxxx>
> Subject: Re: [PATCH v9 6/8] common/domain: add a domain context record for 
> shared_info...
> 
> On 24/09/2020 14:10, Paul Durrant wrote:
> > diff --git a/tools/misc/xen-domctx.c b/tools/misc/xen-domctx.c
> > index 243325dfce..6ead7ea89d 100644
> > --- a/tools/misc/xen-domctx.c
> > +++ b/tools/misc/xen-domctx.c
> > @@ -31,6 +31,7 @@
> >  #include <errno.h>
> >
> >  #include <xenctrl.h>
> > +#include <xen-tools/libs.h>
> >  #include <xen/xen.h>
> >  #include <xen/domctl.h>
> >  #include <xen/save.h>
> > @@ -61,6 +62,82 @@ static void dump_header(void)
> >
> >  }
> >
> > +static void print_binary(const char *prefix, const void *val, size_t size,
> > +                         const char *suffix)
> > +{
> > +    printf("%s", prefix);
> > +
> > +    while ( size-- )
> > +    {
> > +        uint8_t octet = *(const uint8_t *)val++;
> > +        unsigned int i;
> > +
> > +        for ( i = 0; i < 8; i++ )
> > +        {
> > +            printf("%u", octet & 1);
> > +            octet >>= 1;
> > +        }
> > +    }
> > +
> > +    printf("%s", suffix);
> > +}
> > +
> > +static void dump_shared_info(void)
> > +{
> > +    DOMAIN_SAVE_TYPE(SHARED_INFO) *s;
> > +    bool has_32bit_shinfo;
> > +    shared_info_any_t *info;
> > +    unsigned int i, n;
> > +
> > +    GET_PTR(s);
> > +    has_32bit_shinfo = s->flags & DOMAIN_SAVE_32BIT_SHINFO;
> > +
> > +    printf("    SHARED_INFO: has_32bit_shinfo: %s buffer_size: %u\n",
> > +           has_32bit_shinfo ? "true" : "false", s->buffer_size);
> > +
> > +    info = (shared_info_any_t *)s->buffer;
> > +
> > +#define GET_FIELD_PTR(_f)            \
> > +    (has_32bit_shinfo ?              \
> > +     (const void *)&(info->x32._f) : \
> > +     (const void *)&(info->x64._f))
> > +#define GET_FIELD_SIZE(_f) \
> > +    (has_32bit_shinfo ? sizeof(info->x32._f) : sizeof(info->x64._f))
> > +#define GET_FIELD(_f) \
> > +    (has_32bit_shinfo ? info->x32._f : info->x64._f)
> > +
> > +    n = has_32bit_shinfo ?
> > +        ARRAY_SIZE(info->x32.evtchn_pending) :
> > +        ARRAY_SIZE(info->x64.evtchn_pending);
> > +
> > +    for ( i = 0; i < n; i++ )
> > +    {
> > +        const char *prefix = !i ?
> > +            "                 evtchn_pending: " :
> > +            "                                 ";
> > +
> > +        print_binary(prefix, GET_FIELD_PTR(evtchn_pending[0]),
> > +                 GET_FIELD_SIZE(evtchn_pending[0]), "\n");
> > +    }
> > +
> > +    for ( i = 0; i < n; i++ )
> > +    {
> > +        const char *prefix = !i ?
> > +            "                    evtchn_mask: " :
> > +            "                                 ";
> > +
> > +        print_binary(prefix, GET_FIELD_PTR(evtchn_mask[0]),
> > +                 GET_FIELD_SIZE(evtchn_mask[0]), "\n");
> > +    }
> 
> What about domains using FIFO?  This is meaningless for them.
> 

Indeed, but this is essentially a debug tool so I'd rather it just dumped 
everything that might be useful.

> > +
> > +    printf("                 wc: version: %u sec: %u nsec: %u\n",
> > +           GET_FIELD(wc_version), GET_FIELD(wc_sec), GET_FIELD(wc_nsec));
> 
> wc_sec_hi is also a rather critical field in this calculation.        
> 

Ok.

> > +
> > +#undef GET_FIELD
> > +#undef GET_FIELD_SIZE
> > +#undef GET_FIELD_PTR
> > +}
> > +
> >  static void dump_end(void)
> >  {
> >      DOMAIN_SAVE_TYPE(END) *e;
> > @@ -173,6 +250,7 @@ int main(int argc, char **argv)
> >              switch (desc->typecode)
> >              {
> >              case DOMAIN_SAVE_CODE(HEADER): dump_header(); break;
> > +            case DOMAIN_SAVE_CODE(SHARED_INFO): dump_shared_info(); break;
> >              case DOMAIN_SAVE_CODE(END): dump_end(); break;
> >              default:
> >                  printf("Unknown type %u: skipping\n", desc->typecode);
> > diff --git a/xen/common/domain.c b/xen/common/domain.c
> > index 8cfa2e0b6b..6709f9c79e 100644
> > --- a/xen/common/domain.c
> > +++ b/xen/common/domain.c
> > @@ -33,6 +33,7 @@
> >  #include <xen/xenoprof.h>
> >  #include <xen/irq.h>
> >  #include <xen/argo.h>
> > +#include <xen/save.h>
> >  #include <asm/debugger.h>
> >  #include <asm/p2m.h>
> >  #include <asm/processor.h>
> > @@ -1657,6 +1658,110 @@ int continue_hypercall_on_cpu(
> >      return 0;
> >  }
> >
> > +static int save_shared_info(const struct domain *d, struct domain_context 
> > *c,
> > +                            bool dry_run)
> > +{
> > +    struct domain_shared_info_context ctxt = {
> > +#ifdef CONFIG_COMPAT
> > +        .flags = has_32bit_shinfo(d) ? DOMAIN_SAVE_32BIT_SHINFO : 0,
> > +        .buffer_size = has_32bit_shinfo(d) ?
> > +                       sizeof(struct compat_shared_info) :
> > +                       sizeof(struct shared_info),
> > +#else
> > +        .buffer_size = sizeof(struct shared_info),
> > +#endif
> > +    };
> > +    size_t hdr_size = offsetof(typeof(ctxt), buffer);
> > +    int rc;
> > +
> > +    rc = DOMAIN_SAVE_BEGIN(SHARED_INFO, c, 0);
> > +    if ( rc )
> > +        return rc;
> > +
> > +    rc = domain_save_data(c, &ctxt, hdr_size);
> > +    if ( rc )
> > +        return rc;
> > +
> > +    rc = domain_save_data(c, d->shared_info, ctxt.buffer_size);
> > +    if ( rc )
> > +        return rc;
> > +
> > +    return domain_save_end(c);
> > +}
> > +
> > +static int load_shared_info(struct domain *d, struct domain_context *c)
> > +{
> > +    struct domain_shared_info_context ctxt;
> > +    size_t hdr_size = offsetof(typeof(ctxt), buffer);
> > +    unsigned int i;
> > +    int rc;
> > +
> > +    rc = DOMAIN_LOAD_BEGIN(SHARED_INFO, c, &i);
> > +    if ( rc )
> > +        return rc;
> > +
> > +    if ( i ) /* expect only a single instance */
> > +        return -ENXIO;
> > +
> > +    rc = domain_load_data(c, &ctxt, hdr_size);
> > +    if ( rc )
> > +        return rc;
> > +
> > +    if ( ctxt.buffer_size > sizeof(shared_info_t) ||
> > +         (ctxt.flags & ~DOMAIN_SAVE_32BIT_SHINFO) )
> > +        return -EINVAL;
> > +
> > +    if ( ctxt.flags & DOMAIN_SAVE_32BIT_SHINFO )
> > +    {
> > +#ifdef CONFIG_COMPAT
> > +        has_32bit_shinfo(d) = true;
> 
> d->arch.has_32bit_shinfo
> 

If you'd prefer, ok.

> > +#else
> > +        return -EINVAL;
> > +#endif
> > +    }
> > +
> > +    if ( is_pv_domain(d) )
> > +    {
> > +        shared_info_t *shinfo = xmalloc(shared_info_t);
> > +
> > +        if ( !shinfo )
> > +            return -ENOMEM;
> > +
> > +        rc = domain_load_data(c, shinfo, sizeof(*shinfo));
> > +        if ( rc )
> > +            goto out;
> 
> There's no need for a memory allocation, or to double buffer this data.
> You can memcpy() straight out of the context record.
> 

That would mean re-working the way that domain_load_data() works. I'd really 
rather not.

> > +
> > +        memcpy(&shared_info(d, vcpu_info), &__shared_info(d, shinfo, 
> > vcpu_info),
> > +               sizeof(shared_info(d, vcpu_info)));
> > +        memcpy(&shared_info(d, arch), &__shared_info(d, shinfo, arch),
> > +               sizeof(shared_info(d, arch)));
> > +
> > +        memset(&shared_info(d, evtchn_pending), 0,
> > +               sizeof(shared_info(d, evtchn_pending)));
> > +        memset(&shared_info(d, evtchn_mask), 0xff,
> > +               sizeof(shared_info(d, evtchn_mask)));
> > +
> > +        shared_info(d, arch.pfn_to_mfn_frame_list_list) = 0;
> > +        for ( i = 0; i < XEN_LEGACY_MAX_VCPUS; i++ )
> > +            shared_info(d, vcpu_info[i].evtchn_pending_sel) = 0;
> 
> What is the plan for transparent migrate here?  While this is ok for
> regular migrate, its definitely not for transparent.
> 

Quite true, as evidenced that this is inside 'if ( is_pv_domain(d) )'. It is 
not yet clear how much of the shared info we need for transparent migrate. It 
may be nothing.

> > +
> > +        rc = domain_load_end(c, false);
> > +
> > +    out:
> > +        xfree(shinfo);
> > +    }
> > +    else
> > +        /*
> > +         * No modifications to shared_info are required for restoring 
> > non-PV
> > +         * domains.
> > +         */
> > +        rc = domain_load_end(c, true);
> > +
> > +    return rc;
> > +}
> > +
> > +DOMAIN_REGISTER_SAVE_LOAD(SHARED_INFO, save_shared_info, load_shared_info);
> > +
> >  /*
> >   * Local variables:
> >   * mode: C
> > diff --git a/xen/include/public/save.h b/xen/include/public/save.h
> > index 551dbbddb8..0e855a4b97 100644
> > --- a/xen/include/public/save.h
> > +++ b/xen/include/public/save.h
> > @@ -82,7 +82,18 @@ struct domain_save_header {
> >  };
> >  DECLARE_DOMAIN_SAVE_TYPE(HEADER, 1, struct domain_save_header);
> >
> > -#define DOMAIN_SAVE_CODE_MAX 1
> > +struct domain_shared_info_context {
> > +    uint32_t flags;
> > +
> > +#define DOMAIN_SAVE_32BIT_SHINFO 0x00000001
> > +
> > +    uint32_t buffer_size;
> 
> This struct is already wrapped with a header including a size which
> encompasses buffer.
> 
> Multiple overlapping size fields is an easy way to memory corruption,
> because it causes ambiguity as to which one is right.
> 

The record size currently includes padding. I'm re-working that in v10 and so 
this size can be dropped.

  Paul