|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v9 3/3] tools, docs: add total/local memory bandwith monitoring
Add Memory Bandwidth Monitoring(MBM) for VMs. Two types of monitoring
are supported: total and local memory bandwidth monitoring. To use it,
CMT should be enabled in hypervisor.
Signed-off-by: Chao Peng <chao.p.peng@xxxxxxxxxxxxxxx>
---
Changes in v9:
1. Refactor code in xc_psr_cmt_get_data.
2. Move bandwidth calculation(sleep) from libxl to xl.
3. Broadcast feature with LIBXL_HAVE_PSR_MBM.
4. Check event mask with libxl_psr_cmt_type_supported.
5. Coding style/Document fix.
Changes in v6:
1. Remove DISABLE_IRQ flag as hypervisor disable IRQ for MSR_IA32_TSC
implicitly.
Changes in v5:
1. Add MBM description in xen command line.
2. Use the tsc from hypervisor directly which is already ns.
3. Call resource_op with DISABLE_IRQ flag.
Changes in v4:
1. Get timestamp from hypervisor and use that for bandwidth calculation.
2. Minor document and coding style fix.
---
docs/man/xl.pod.1 | 11 +++++-
docs/misc/xen-command-line.markdown | 3 ++
tools/libxc/include/xenctrl.h | 6 +++-
tools/libxc/xc_msr_x86.h | 1 +
tools/libxc/xc_psr.c | 68 ++++++++++++++++++++++++++++++-------
tools/libxl/libxl.h | 17 ++++++++++
tools/libxl/libxl_psr.c | 44 ++++++++++++++++++++++--
tools/libxl/libxl_types.idl | 2 ++
tools/libxl/xl_cmdimpl.c | 67 ++++++++++++++++++++++++++++++++++++
tools/libxl/xl_cmdtable.c | 4 ++-
10 files changed, 204 insertions(+), 19 deletions(-)
diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index 6b89ba8..cd80ffc 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1461,6 +1461,13 @@ is domain level. To monitor a specific domain, just
attach the domain id with
the monitoring service. When the domain doesn't need to be monitored any more,
detach the domain id from the monitoring service.
+Intel Broadwell and later server platforms also offer total/local memory
+bandwidth monitoring. Xen supports per-domain monitoring for these two
+additional monitoring types. Both memory bandwidth monitoring and L3 cache
+occupancy monitoring share the same set of underlying monitoring service. Once
+a domain is attached to the monitoring service, monitoring data can be showed
+for any of these monitoring types.
+
=over 4
=item B<psr-cmt-attach> [I<domain-id>]
@@ -1475,7 +1482,9 @@ detach: Detach the platform shared resource monitoring
service from a domain.
Show monitoring data for a certain domain or all domains. Current supported
monitor types are:
- - "cache-occupancy": showing the L3 cache occupancy.
+ - "cache-occupancy": showing the L3 cache occupancy(KB).
+ - "total-mem-bandwidth": showing the total memory bandwidth(KB/s).
+ - "local-mem-bandwidth": showing the local memory bandwidth(KB/s).
=back
diff --git a/docs/misc/xen-command-line.markdown
b/docs/misc/xen-command-line.markdown
index a061aa4..0491dbb 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1097,6 +1097,9 @@ The following resources are available:
L3 cache occupancy.
* `cmt` instructs Xen to enable/disable Cache Monitoring Technology.
* `rmid_max` indicates the max value for rmid.
+* Memory Bandwidth Monitoring (Broadwell and later). Information regarding the
+ total/local memory bandwidth. Follow the same options with Cache Monitoring
+ Technology.
### reboot
> `= t[riple] | k[bd] | a[cpi] | p[ci] | n[o] [, [w]arm | [c]old]`
diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index c62f371..8f964ad 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2688,6 +2688,8 @@ int xc_resource_op(xc_interface *xch, uint32_t nr_ops,
xc_resource_op_t *ops);
#if defined(__i386__) || defined(__x86_64__)
enum xc_psr_cmt_type {
XC_PSR_CMT_L3_OCCUPANCY,
+ XC_PSR_CMT_TOTAL_MEM_BANDWIDTH,
+ XC_PSR_CMT_LOCAL_MEM_BANDWIDTH,
};
typedef enum xc_psr_cmt_type xc_psr_cmt_type;
int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid);
@@ -2697,10 +2699,12 @@ int xc_psr_cmt_get_domain_rmid(xc_interface *xch,
uint32_t domid,
int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid);
int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
uint32_t *upscaling_factor);
+int xc_psr_cmt_get_l3_event_mask(xc_interface *xch, uint32_t *event_mask);
int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu,
uint32_t *l3_cache_size);
int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu,
- uint32_t psr_cmt_type, uint64_t *monitor_data);
+ uint32_t psr_cmt_type, uint64_t *monitor_data,
+ uint64_t *tsc);
int xc_psr_cmt_enabled(xc_interface *xch);
#endif
diff --git a/tools/libxc/xc_msr_x86.h b/tools/libxc/xc_msr_x86.h
index 7c3e1a3..7f100e7 100644
--- a/tools/libxc/xc_msr_x86.h
+++ b/tools/libxc/xc_msr_x86.h
@@ -20,6 +20,7 @@
#ifndef XC_MSR_X86_H
#define XC_MSR_X86_H
+#define MSR_IA32_TSC 0x00000010
#define MSR_IA32_CMT_EVTSEL 0x00000c8d
#define MSR_IA32_CMT_CTR 0x00000c8e
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index cfae172..5482c8e 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -23,6 +23,8 @@
#define IA32_CMT_CTR_ERROR_MASK (0x3ull << 62)
#define EVTID_L3_OCCUPANCY 0x1
+#define EVTID_TOTAL_MEM_BANDWIDTH 0x2
+#define EVTID_LOCAL_MEM_BANDWIDTH 0x3
int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid)
{
@@ -112,6 +114,23 @@ int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch,
return rc;
}
+int xc_psr_cmt_get_l3_event_mask(xc_interface *xch, uint32_t *event_mask)
+{
+ int rc;
+ DECLARE_SYSCTL;
+
+ sysctl.cmd = XEN_SYSCTL_psr_cmt_op;
+ sysctl.u.psr_cmt_op.cmd =
+ XEN_SYSCTL_PSR_CMT_get_l3_event_mask;
+ sysctl.u.psr_cmt_op.flags = 0;
+
+ rc = xc_sysctl(xch, &sysctl);
+ if ( !rc )
+ *event_mask = sysctl.u.psr_cmt_op.u.data;
+
+ return rc;
+}
+
int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu,
uint32_t *l3_cache_size)
{
@@ -139,11 +158,13 @@ int xc_psr_cmt_get_l3_cache_size(xc_interface *xch,
uint32_t cpu,
}
int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu,
- xc_psr_cmt_type type, uint64_t *monitor_data)
+ xc_psr_cmt_type type, uint64_t *monitor_data,
+ uint64_t *tsc)
{
xc_resource_op_t op;
- xc_resource_entry_t entries[2];
- uint32_t evtid;
+ xc_resource_entry_t entries[3];
+ xc_resource_entry_t *tsc_entry = NULL;
+ uint32_t evtid, nr = 0;
int rc;
switch ( type )
@@ -151,33 +172,54 @@ int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid,
uint32_t cpu,
case XC_PSR_CMT_L3_OCCUPANCY:
evtid = EVTID_L3_OCCUPANCY;
break;
+ case XC_PSR_CMT_TOTAL_MEM_BANDWIDTH:
+ evtid = EVTID_TOTAL_MEM_BANDWIDTH;
+ break;
+ case XC_PSR_CMT_LOCAL_MEM_BANDWIDTH:
+ evtid = EVTID_LOCAL_MEM_BANDWIDTH;
+ break;
default:
return -1;
}
- entries[0].u.cmd = XEN_RESOURCE_OP_MSR_WRITE;
- entries[0].idx = MSR_IA32_CMT_EVTSEL;
- entries[0].val = (uint64_t)rmid << 32 | evtid;
- entries[0].rsvd = 0;
+ entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_WRITE;
+ entries[nr].idx = MSR_IA32_CMT_EVTSEL;
+ entries[nr].val = (uint64_t)rmid << 32 | evtid;
+ entries[nr].rsvd = 0;
+ nr++;
- entries[1].u.cmd = XEN_RESOURCE_OP_MSR_READ;
- entries[1].idx = MSR_IA32_CMT_CTR;
- entries[1].val = 0;
- entries[1].rsvd = 0;
+ entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_READ;
+ entries[nr].idx = MSR_IA32_CMT_CTR;
+ entries[nr].val = 0;
+ entries[nr].rsvd = 0;
+ nr++;
+
+ if ( tsc != NULL )
+ {
+ tsc_entry = &entries[nr];
+ entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_READ;
+ entries[nr].idx = MSR_IA32_TSC;
+ entries[nr].val = 0;
+ entries[nr].rsvd = 0;
+ nr++;
+ }
op.cpu = cpu;
- op.nr_entries = 2;
+ op.nr_entries = nr;
op.entries = entries;
rc = xc_resource_op(xch, 1, &op);
if ( rc < 0 )
return rc;
- if ( op.result !=2 || entries[1].val & IA32_CMT_CTR_ERROR_MASK )
+ if ( op.result != nr || entries[1].val & IA32_CMT_CTR_ERROR_MASK )
return -1;
*monitor_data = entries[1].val;
+ if ( tsc_entry != NULL )
+ *tsc = tsc_entry->val;
+
return 0;
}
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 28b6325..9b04927 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -690,6 +690,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst,
libxl_mac *src);
* If this is defined, the Cache Monitoring Technology feature is supported.
*/
#define LIBXL_HAVE_PSR_CMT 1
+
+/*
+ * LIBXL_HAVE_PSR_MBM
+ *
+ * If this is defined, the Memory Bandwidth Monitoring feature is supported.
+ */
+#define LIBXL_HAVE_PSR_MBM 1
#endif
typedef char **libxl_string_list;
@@ -1463,6 +1470,16 @@ int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx,
uint32_t *l3_cache_occupancy);
#endif
+#ifdef LIBXL_HAVE_PSR_MBM
+int libxl_psr_cmt_type_supported(libxl_ctx *ctx, libxl_psr_cmt_type type);
+int libxl_psr_cmt_get_mem_bandwidth_sample(libxl_ctx *ctx,
+ uint32_t domid,
+ uint32_t socketid,
+ bool total,
+ uint64_t *sample_r,
+ uint64_t *tsc_r);
+#endif
+
/* misc */
/* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 1e1f7ee..ed444e2 100644
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -161,11 +161,30 @@ out:
return rc;
}
+int libxl_psr_cmt_type_supported(libxl_ctx *ctx, libxl_psr_cmt_type type)
+{
+ GC_INIT(ctx);
+ uint32_t event_mask;
+ int rc;
+
+ rc = xc_psr_cmt_get_l3_event_mask(ctx->xch, &event_mask);
+ if (rc < 0) {
+ libxl__psr_cmt_log_err_msg(gc, errno);
+ rc = 0;
+ } else {
+ rc = event_mask & (1 << (type - 1));
+ }
+
+ GC_FREE;
+ return rc;
+}
+
static int libxl__psr_cmt_get_l3_monitoring_data(libxl__gc *gc,
uint32_t domid,
xc_psr_cmt_type type,
uint32_t socketid,
- uint64_t *data_r)
+ uint64_t *data_r,
+ uint64_t *tsc_r)
{
unsigned int rmid;
uint32_t upscaling_factor;
@@ -187,7 +206,7 @@ static int libxl__psr_cmt_get_l3_monitoring_data(libxl__gc
*gc,
goto out;
}
- rc = xc_psr_cmt_get_data(CTX->xch, rmid, cpu, type, &monitor_data);
+ rc = xc_psr_cmt_get_data(CTX->xch, rmid, cpu, type, &monitor_data, tsc_r);
if (rc < 0) {
LOGE(ERROR, "failed to get monitoring data");
rc = ERROR_FAIL;
@@ -218,7 +237,7 @@ int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx,
rc = libxl__psr_cmt_get_l3_monitoring_data(gc, domid,
XC_PSR_CMT_L3_OCCUPANCY,
- socketid, &data);
+ socketid, &data, NULL);
if (rc < 0)
goto out;
@@ -229,6 +248,25 @@ out:
return rc;
}
+int libxl_psr_cmt_get_mem_bandwidth_sample(libxl_ctx *ctx,
+ uint32_t domid,
+ uint32_t socketid,
+ bool total,
+ uint64_t *sample_r,
+ uint64_t *tsc_r)
+{
+ GC_INIT(ctx);
+ int rc;
+ xc_psr_cmt_type type = total ? XC_PSR_CMT_TOTAL_MEM_BANDWIDTH
+ : XC_PSR_CMT_LOCAL_MEM_BANDWIDTH;
+
+ rc = libxl__psr_cmt_get_l3_monitoring_data(gc, domid, type, socketid,
+ sample_r, tsc_r);
+
+ GC_FREE;
+ return rc;
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 1214d2e..46d160e 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -694,4 +694,6 @@ libxl_event = Struct("event",[
libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
(1, "CACHE_OCCUPANCY"),
+ (2, "TOTAL_MEM_BANDWIDTH"),
+ (3, "LOCAL_MEM_BANDWIDTH"),
])
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index a0b50e7..3378e64 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -7807,6 +7807,54 @@ out:
}
#ifdef LIBXL_HAVE_PSR_CMT
+
+#define MBM_SAMPLE_RETRY_MAX 4
+static int psr_cmt_get_mem_bandwidth(uint32_t domid,
+ uint32_t socketid,
+ bool total,
+ uint64_t *bandwidth_r)
+{
+ uint64_t sample1, sample2;
+ uint64_t tsc1, tsc2;
+ int retry_attempts = 0;
+ int rc;
+
+ while (1) {
+ rc = libxl_psr_cmt_get_mem_bandwidth_sample(ctx, domid, socketid,
+ total, &sample1, &tsc1);
+ if (rc < 0)
+ return rc;
+
+ usleep(10000);
+
+ rc = libxl_psr_cmt_get_mem_bandwidth_sample(ctx, domid, socketid,
+ total, &sample2, &tsc2);
+ if (rc < 0)
+ return rc;
+
+ if (tsc2 <= tsc1)
+ return -1;
+
+ /*
+ * Hardware guarantees at most 1 overflow can happen if the duration
+ * between two samples is less than 1 second. Note that tsc returned
+ * from hypervisor is already-scaled time(ns).
+ */
+ if (tsc2 - tsc1 < 1000000000 && sample2 >= sample1)
+ break;
+
+ if (retry_attempts < MBM_SAMPLE_RETRY_MAX) {
+ retry_attempts++;
+ } else {
+ fprintf(stderr, "event counter overflowed\n");
+ return -1;
+ }
+ }
+
+ *bandwidth_r = (sample2 - sample1) * 1000000000 / (tsc2 - tsc1) / 1024;
+ return 0;
+}
+
static void psr_cmt_print_domain_l3_info(libxl_dominfo *dominfo,
libxl_psr_cmt_type type,
uint32_t nr_sockets)
@@ -7814,6 +7862,7 @@ static void psr_cmt_print_domain_l3_info(libxl_dominfo
*dominfo,
char *domain_name;
uint32_t socketid;
uint32_t l3_cache_occupancy;
+ uint64_t mem_bandwidth;
if (!libxl_psr_cmt_domain_attached(ctx, dominfo->domid))
return;
@@ -7831,6 +7880,16 @@ static void psr_cmt_print_domain_l3_info(libxl_dominfo
*dominfo,
&l3_cache_occupancy))
printf("%13u KB", l3_cache_occupancy);
break;
+ case LIBXL_PSR_CMT_TYPE_TOTAL_MEM_BANDWIDTH:
+ if (!psr_cmt_get_mem_bandwidth(dominfo->domid, socketid, true,
+ &mem_bandwidth))
+ printf("%11"PRIu64" KB/s", mem_bandwidth);
+ break;
+ case LIBXL_PSR_CMT_TYPE_LOCAL_MEM_BANDWIDTH:
+ if (!psr_cmt_get_mem_bandwidth(dominfo->domid, socketid, false,
+ &mem_bandwidth))
+ printf("%11"PRIu64" KB/s", mem_bandwidth);
+ break;
default:
return;
}
@@ -7851,6 +7910,12 @@ static int psr_cmt_show_l3_info(libxl_psr_cmt_type type,
uint32_t domid)
return -1;
}
+ if (!libxl_psr_cmt_type_supported(ctx, type)) {
+ fprintf(stderr, "Monitor type '%s' is not supported in the system\n",
+ libxl_psr_cmt_type_to_string(type));
+ return -1;
+ }
+
libxl_physinfo_init(&info);
rc = libxl_get_physinfo(ctx, &info);
if (rc < 0) {
@@ -7968,6 +8033,8 @@ int main_psr_cmt_show(int argc, char **argv)
switch (type) {
case LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY:
+ case LIBXL_PSR_CMT_TYPE_TOTAL_MEM_BANDWIDTH:
+ case LIBXL_PSR_CMT_TYPE_LOCAL_MEM_BANDWIDTH:
ret = psr_cmt_show_l3_info(type, domid);
break;
default:
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 4b30d3d..22ab63b 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -538,7 +538,9 @@ struct cmd_spec cmd_table[] = {
"Show Cache Monitoring Technology information",
"<PSR-CMT-Type> <Domain>",
"Available monitor types:\n"
- "\"cache_occupancy\": Show L3 cache occupancy\n",
+ "\"cache_occupancy\": Show L3 cache occupancy(KB)\n"
+ "\"total_mem_bandwidth\": Show total memory bandwidth(KB/s)\n"
+ "\"local_mem_bandwidth\": Show local memory bandwidth(KB/s)\n",
},
#endif
};
--
1.9.1
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |