[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v5 12/13] tools: add tools support for Intel CAT
This is the xc/xl changes to support Intel Cache Allocation Technology(CAT). Two commands are introduced: - xl psr-cat-hwinfo Show CAT hardware information. - xl psr-cat-cbm-set [-s socket] <domain> <cbm> Set cache capacity bitmasks(CBM) for a domain. - xl psr-cat-show <domain> Show CAT domain information. Examples: [root@vmm-psr vmm]# xl psr-cat-hwinfo Cache Allocation Technology (CAT): Socket ID : 0 L3 Cache : 12288KB Maximum COS : 15 CBM length : 12 Default CBM : 0xfff [root@vmm-psr vmm]# xl psr-cat-cbm-set 0 0xff [root@vmm-psr vmm]# xl psr-cat-show Socket ID : 0 L3 Cache : 12288KB Default CBM : 0xfff ID NAME CBM 0 Domain-0 0xff Signed-off-by: Chao Peng <chao.p.peng@xxxxxxxxxxxxxxx> --- Changes in v5: * Add psr-cat-hwinfo. * Add libxl_psr_cat_info_list_free. * malloc => libxl__malloc * Other comments from Ian/Wei. Changes in v4: * Add example output in commit message. * Make libxl__count_physical_sockets private to libxl_psr.c. * Set errno in several error cases. * Change libxl_psr_cat_get_l3_info to return all sockets information. * Remove unused libxl_domain_info call. Changes in v3: * Add manpage. * libxl_psr_cat_set/get_domain_data => libxl_psr_cat_set/get_cbm. * Move libxl_count_physical_sockets into seperate patch. * Support LIBXL_PSR_TARGET_ALL for libxl_psr_cat_set_cbm. * Clean up the print codes. --- docs/man/xl.pod.1 | 35 +++++++++ tools/libxc/include/xenctrl.h | 15 ++++ tools/libxc/xc_psr.c | 76 +++++++++++++++++++ tools/libxl/libxl.h | 26 +++++++ tools/libxl/libxl_psr.c | 170 ++++++++++++++++++++++++++++++++++++++++-- tools/libxl/libxl_types.idl | 10 +++ tools/libxl/xl.h | 5 ++ tools/libxl/xl_cmdimpl.c | 169 +++++++++++++++++++++++++++++++++++++++++ tools/libxl/xl_cmdtable.c | 19 +++++ 9 files changed, 518 insertions(+), 7 deletions(-) diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index 640788f..efc6599 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1 @@ -1517,6 +1517,41 @@ monitor types are: =back +=head1 CACHE ALLOCATION TECHNOLOGY + +Intel Broadwell and later server platforms offer capabilities to configure and +make use of the Cache Allocation Technology (CAT) mechanisms, which enable more +cache resources (i.e. L3 cache) to be made available for high priority +applications. In Xen implementation, CAT is used to control cache allocation +on VM basis. To enforce cache on a specific domain, just set capacity bitmasks +(CBM) for the domain. + +=over 4 + +=item B<psr-cat-hwinfo> + +Show CAT hardware information. + +=item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm> + +Set cache capacity bitmasks(CBM) for a domain. + +B<OPTIONS> + +=over 4 + +=item B<-s SOCKET>, B<--socket=SOCKET> + +Specify the socket to process, otherwise all sockets are processed. + +=back + +=item B<psr-cat-show> [I<domain-id>] + +Show CAT settings for a certain domain or all domains. + +=back + =head1 TO BE DOCUMENTED We need better documentation for: diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h index 02d0db8..077cc1b 100644 --- a/tools/libxc/include/xenctrl.h +++ b/tools/libxc/include/xenctrl.h @@ -2696,6 +2696,12 @@ enum xc_psr_cmt_type { XC_PSR_CMT_LOCAL_MEM_COUNT, }; typedef enum xc_psr_cmt_type xc_psr_cmt_type; + +enum xc_psr_cat_type { + XC_PSR_CAT_L3_CBM = 1, +}; +typedef enum xc_psr_cat_type xc_psr_cat_type; + int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid); int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid); int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid, @@ -2710,6 +2716,15 @@ int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu, uint32_t psr_cmt_type, uint64_t *monitor_data, uint64_t *tsc); int xc_psr_cmt_enabled(xc_interface *xch); + +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid, + xc_psr_cat_type type, uint32_t target, + uint64_t data); +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid, + xc_psr_cat_type type, uint32_t target, + uint64_t *data); +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket, + uint32_t *cos_max, uint32_t *cbm_len); #endif #endif /* XENCTRL_H */ diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c index e367a80..d8b3a51 100644 --- a/tools/libxc/xc_psr.c +++ b/tools/libxc/xc_psr.c @@ -248,6 +248,82 @@ int xc_psr_cmt_enabled(xc_interface *xch) return 0; } +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid, + xc_psr_cat_type type, uint32_t target, + uint64_t data) +{ + DECLARE_DOMCTL; + uint32_t cmd; + + switch ( type ) + { + case XC_PSR_CAT_L3_CBM: + cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM; + break; + default: + errno = EINVAL; + return -1; + } + + domctl.cmd = XEN_DOMCTL_psr_cat_op; + domctl.domain = (domid_t)domid; + domctl.u.psr_cat_op.cmd = cmd; + domctl.u.psr_cat_op.target = target; + domctl.u.psr_cat_op.data = data; + + return do_domctl(xch, &domctl); +} + +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid, + xc_psr_cat_type type, uint32_t target, + uint64_t *data) +{ + int rc; + DECLARE_DOMCTL; + uint32_t cmd; + + switch ( type ) + { + case XC_PSR_CAT_L3_CBM: + cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM; + break; + default: + errno = EINVAL; + return -1; + } + + domctl.cmd = XEN_DOMCTL_psr_cat_op; + domctl.domain = (domid_t)domid; + domctl.u.psr_cat_op.cmd = cmd; + domctl.u.psr_cat_op.target = target; + + rc = do_domctl(xch, &domctl); + + if ( !rc ) + *data = domctl.u.psr_cat_op.data; + + return rc; +} + +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket, + uint32_t *cos_max, uint32_t *cbm_len) +{ + int rc; + DECLARE_SYSCTL; + + sysctl.cmd = XEN_SYSCTL_psr_cat_op; + sysctl.u.psr_cat_op.cmd = XEN_SYSCTL_PSR_CAT_get_l3_info; + sysctl.u.psr_cat_op.target = socket; + + rc = xc_sysctl(xch, &sysctl); + if ( !rc ) + { + *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max; + *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len; + } + + return rc; +} /* * Local variables: diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index 6bc75c5..e8e68f2 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -734,6 +734,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src); * If this is defined, the Memory Bandwidth Monitoring feature is supported. */ #define LIBXL_HAVE_PSR_MBM 1 + +/* + * LIBXL_HAVE_PSR_CAT + * + * If this is defined, the Cache Allocation Technology feature is supported. + */ +#define LIBXL_HAVE_PSR_CAT 1 #endif typedef char **libxl_string_list; @@ -1533,6 +1540,25 @@ int libxl_psr_cmt_get_sample(libxl_ctx *ctx, uint64_t *tsc_r); #endif +#ifdef LIBXL_HAVE_PSR_CAT + +#define LIBXL_PSR_TARGET_ALL (~0U) +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid, + libxl_psr_cbm_type type, uint32_t target, + uint64_t cbm); +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid, + libxl_psr_cbm_type type, uint32_t target, + uint64_t *cbm_r); + +/* + * On success, the function returns an array of elements in 'info', + * and the length in 'nr'. + */ +int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info, + uint32_t *nr); +void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, uint32_t nr); +#endif + /* misc */ /* Each of these sets or clears the flag according to whether the diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c index 3e1c792..9be9086 100644 --- a/tools/libxl/libxl_psr.c +++ b/tools/libxl/libxl_psr.c @@ -19,14 +19,37 @@ #define IA32_QM_CTR_ERROR_MASK (0x3ul << 62) -static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err) +static void libxl__psr_log_err_msg(libxl__gc *gc, int err) { char *msg; switch (err) { case ENOSYS: + case EOPNOTSUPP: msg = "unsupported operation"; break; + case ESRCH: + msg = "invalid domain ID"; + break; + case EBADSLT: + msg = "socket is not supported"; + break; + case EFAULT: + msg = "failed to exchange data with Xen"; + break; + default: + msg = "unknown error"; + break; + } + + LOGE(ERROR, "%s", msg); +} + +static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err) +{ + char *msg; + + switch (err) { case ENODEV: msg = "CMT is not supported in this system"; break; @@ -39,15 +62,35 @@ static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err) case EUSERS: msg = "no free RMID available"; break; - case ESRCH: - msg = "invalid domain ID"; + default: + libxl__psr_log_err_msg(gc, err); + return; + } + + LOGE(ERROR, "%s", msg); +} + +static void libxl__psr_cat_log_err_msg(libxl__gc *gc, int err) +{ + char *msg; + + switch (err) { + case ENODEV: + msg = "CAT is not supported in this system"; break; - case EFAULT: - msg = "failed to exchange data with Xen"; + case ENOENT: + msg = "CAT is not enabled on the socket"; break; - default: - msg = "unknown error"; + case EUSERS: + msg = "no free COS available"; + break; + case EEXIST: + msg = "The same CBM is already set to this domain"; break; + + default: + libxl__psr_log_err_msg(gc, err); + return; } LOGE(ERROR, "%s", msg); @@ -73,6 +116,24 @@ static int libxl__pick_socket_cpu(libxl__gc *gc, uint32_t socketid) return cpu; } +static int libxl__count_physical_sockets(libxl__gc *gc, uint32_t *sockets) +{ + int rc; + libxl_physinfo info; + + libxl_physinfo_init(&info); + + rc = libxl_get_physinfo(CTX, &info); + if (rc) + return rc; + + *sockets = info.nr_cpus / info.threads_per_core + / info.cores_per_socket; + + libxl_physinfo_dispose(&info); + return 0; +} + int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid) { GC_INIT(ctx); @@ -247,6 +308,101 @@ out: return rc; } +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid, + libxl_psr_cbm_type type, uint32_t target, + uint64_t cbm) +{ + GC_INIT(ctx); + int r, rc = 0; + uint32_t i, nr_sockets; + + if (target != LIBXL_PSR_TARGET_ALL) { + r = xc_psr_cat_set_domain_data(ctx->xch, domid, type, target, cbm); + if (r < 0) { + libxl__psr_cat_log_err_msg(gc, errno); + rc = ERROR_FAIL; + } + } else { + rc = libxl__count_physical_sockets(gc, &nr_sockets); + if (rc) { + LOGE(ERROR, "failed to get system socket count"); + goto out; + } + for (i = 0; i < nr_sockets; i++) { + r = xc_psr_cat_set_domain_data(ctx->xch, domid, type, i, cbm); + if (r < 0) { + libxl__psr_cat_log_err_msg(gc, errno); + rc = ERROR_FAIL; + goto out; + } + } + } + +out: + GC_FREE; + return rc; +} + +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid, + libxl_psr_cbm_type type, uint32_t target, + uint64_t *cbm_r) +{ + GC_INIT(ctx); + int r, rc = 0; + + r = xc_psr_cat_get_domain_data(ctx->xch, domid, type, target, cbm_r); + if (r < 0) { + libxl__psr_cat_log_err_msg(gc, errno); + rc = ERROR_FAIL; + } + + GC_FREE; + return rc; +} + +int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info, + uint32_t *nr) +{ + GC_INIT(ctx); + int rc, r; + uint32_t i, nr_sockets; + libxl_psr_cat_info *ptr; + + rc = libxl__count_physical_sockets(gc, &nr_sockets); + if (rc) { + LOGE(ERROR, "failed to get system socket count"); + goto out; + } + + ptr = libxl__malloc(NOGC, nr_sockets * sizeof(libxl_psr_cat_info)); + + for (i = 0; i < nr_sockets; i++) { + r = xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max, + &ptr[i].cbm_len); + if (r < 0) { + libxl__psr_cat_log_err_msg(gc, errno); + rc = ERROR_FAIL; + free(ptr); + goto out; + } + } + + *info = ptr; + *nr = nr_sockets; +out: + GC_FREE; + return rc; +} + +void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, uint32_t nr) +{ + int i; + + for (i = 0; i < nr; i++) + libxl_psr_cat_info_dispose(&list[i]); + free(list); +} + /* * Local variables: * mode: C diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 0866433..ee8d469 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -709,3 +709,13 @@ libxl_psr_cmt_type = Enumeration("psr_cmt_type", [ (2, "TOTAL_MEM_COUNT"), (3, "LOCAL_MEM_COUNT"), ]) + +libxl_psr_cbm_type = Enumeration("psr_cbm_type", [ + (0, "UNKNOWN"), + (1, "L3_CBM"), + ]) + +libxl_psr_cat_info = Struct("psr_cat_info", [ + ("cos_max", uint32), + ("cbm_len", uint32), + ]) diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h index 1c526dc..c87b843 100644 --- a/tools/libxl/xl.h +++ b/tools/libxl/xl.h @@ -118,6 +118,11 @@ int main_psr_cmt_attach(int argc, char **argv); int main_psr_cmt_detach(int argc, char **argv); int main_psr_cmt_show(int argc, char **argv); #endif +#ifdef LIBXL_HAVE_PSR_CAT +int main_psr_cat_hwinfo(int argc, char **argv); +int main_psr_cat_cbm_set(int argc, char **argv); +int main_psr_cat_show(int argc, char **argv); +#endif void help(const char *command); diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index 4ab5e8b..29a1c22 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -8281,6 +8281,175 @@ int main_psr_cmt_show(int argc, char **argv) } #endif +#ifdef LIBXL_HAVE_PSR_CAT +static int psr_cat_hwinfo(void) +{ + int rc; + uint32_t socket, nr_sockets, l3_cache_size; + libxl_psr_cat_info *info; + + printf("Cache Allocation Technology (CAT):\n"); + + rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets); + if (rc) { + fprintf(stderr, "Failed to get cat info\n"); + return rc; + } + + for (socket = 0; socket < nr_sockets; socket++) { + rc = libxl_psr_cmt_get_l3_cache_size(ctx, socket, &l3_cache_size); + if (rc) { + fprintf(stderr, "Failed to get l3 cache size for socket:%d\n", + socket); + goto out; + } + printf("%-16s: %u\n", "Socket ID", socket); + printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size); + printf("%-16s: %u\n", "Maximum COS", info->cos_max); + printf("%-16s: %u\n", "CBM length", info->cbm_len); + printf("%-16s: %#"PRIx64"\n", "Default CBM", + (1ul << info->cbm_len) - 1); + } + +out: + libxl_psr_cat_info_list_free(info, nr_sockets); + return rc; +} + +static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socket) +{ + char *domain_name; + uint64_t cbm; + + domain_name = libxl_domid_to_name(ctx, domid); + printf("%5d%25s", domid, domain_name); + free(domain_name); + + if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM, + socket, &cbm)) + printf("%#16"PRIx64, cbm); + + printf("\n"); +} + +static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socket) +{ + int i, nr_domains; + libxl_dominfo *list; + + if (domid != INVALID_DOMID) { + psr_cat_print_one_domain_cbm(domid, socket); + return 0; + } + + if (!(list = libxl_list_domain(ctx, &nr_domains))) { + fprintf(stderr, "Failed to get domain list for cbm display\n"); + return -1; + } + + for (i = 0; i < nr_domains; i++) + psr_cat_print_one_domain_cbm(list[i].domid, socket); + libxl_dominfo_list_free(list, nr_domains); + + return 0; +} + +static int psr_cat_print_socket(uint32_t domid, uint32_t socket, + libxl_psr_cat_info *info) +{ + int rc; + uint32_t l3_cache_size; + + rc = libxl_psr_cmt_get_l3_cache_size(ctx, socket, &l3_cache_size); + if (rc) { + fprintf(stderr, "Failed to get l3 cache size for socket:%d\n", socket); + return -1; + } + + printf("%-16s: %u\n", "Socket ID", socket); + printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size); + printf("%-16s: %#"PRIx64"\n", "Default CBM", (1ul << info->cbm_len) - 1); + printf("%5s%25s%16s\n", "ID", "NAME", "CBM"); + + return psr_cat_print_domain_cbm(domid, socket); +} + +static int psr_cat_show(uint32_t domid) +{ + uint32_t socket, nr_sockets; + int rc; + libxl_psr_cat_info *info; + + rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets); + if (rc) { + fprintf(stderr, "Failed to get cat info\n"); + return rc; + } + + for (socket = 0; socket < nr_sockets; socket++) { + rc = psr_cat_print_socket(domid, socket, info + socket); + if (rc) + goto out; + } + +out: + libxl_psr_cat_info_list_free(info, nr_sockets); + return rc; +} + +int main_psr_cat_hwinfo(int argc, char **argv) +{ + return psr_cat_hwinfo(); +} + +int main_psr_cat_cbm_set(int argc, char **argv) +{ + uint32_t domid; + uint32_t target = LIBXL_PSR_TARGET_ALL; + libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM; + uint64_t cbm; + int opt = 0; + + static struct option opts[] = { + {"socket", 0, 0, 's'}, + {0, 0, 0, 0} + }; + + SWITCH_FOREACH_OPT(opt, "s", opts, "psr-cat-cbm-set", 1) { + case 's': + target = strtol(optarg, NULL, 10); + break; + } + + domid = find_domain(argv[optind]); + cbm = strtoll(argv[optind + 1], NULL , 0); + + return libxl_psr_cat_set_cbm(ctx, domid, type, target, cbm); +} + +int main_psr_cat_show(int argc, char **argv) +{ + int opt; + uint32_t domid; + + SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cat-show", 0) { + /* No options */ + } + + if (optind >= argc) + domid = INVALID_DOMID; + else if (optind == argc - 1) + domid = find_domain(argv[optind]); + else { + help("psr-cat-show"); + return 2; + } + + return psr_cat_show(domid); +} + +#endif + /* * Local variables: * mode: C diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index dc25d1f..a3816e5 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -549,6 +549,25 @@ struct cmd_spec cmd_table[] = { "\"local_mem_bandwidth\": Show local memory bandwidth(KB/s)\n", }, #endif +#ifdef LIBXL_HAVE_PSR_CAT + { "psr-cat-hwinfo", + &main_psr_cat_hwinfo, 0, 1, + "Show hardware information for Cache Allocation Technology", + "", + }, + { "psr-cat-cbm-set", + &main_psr_cat_cbm_set, 0, 1, + "Set cache capacity bitmasks(CBM) for a domain", + "-s <socket> Specify the socket to process, otherwise all sockets are processed\n" + "<Domain> <CBM>", + }, + { "psr-cat-show", + &main_psr_cat_show, 0, 1, + "Show Cache Allocation Technology information", + "<Domain>", + }, + +#endif }; int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec); -- 1.9.1 _______________________________________________ Xen-devel mailing list Xen-devel@xxxxxxxxxxxxx http://lists.xen.org/xen-devel
|
Lists.xenproject.org is hosted with RackSpace, monitoring our |