1 /*
2  * Copyright 2009-2017 Citrix Ltd and other contributors
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU Lesser General Public License as published
6  * by the Free Software Foundation; version 2.1 only. with the special
7  * exception on linking described in file LICENSE.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU Lesser General Public License for more details.
13  */
14 
15 #include <ctype.h>
16 #include <inttypes.h>
17 #include <stdlib.h>
18 #include <unistd.h>
19 
20 #include <libxl.h>
21 #include <libxl_utils.h>
22 #include <libxlutil.h>
23 
24 #include "xl.h"
25 #include "xl_utils.h"
26 #include "xl_parse.h"
27 
psr_cmt_hwinfo(void)28 static int psr_cmt_hwinfo(void)
29 {
30     int rc;
31     int enabled;
32     uint32_t total_rmid;
33 
34     printf("Cache Monitoring Technology (CMT):\n");
35 
36     enabled = libxl_psr_cmt_enabled(ctx);
37     printf("%-16s: %s\n", "Enabled", enabled ? "1" : "0");
38     if (!enabled)
39         return 0;
40 
41     rc = libxl_psr_cmt_get_total_rmid(ctx, &total_rmid);
42     if (rc) {
43         fprintf(stderr, "Failed to get max RMID value\n");
44         return rc;
45     }
46     printf("%-16s: %u\n", "Total RMID", total_rmid);
47 
48     printf("Supported monitor types:\n");
49     if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY))
50         printf("cache-occupancy\n");
51     if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_TOTAL_MEM_COUNT))
52         printf("total-mem-bandwidth\n");
53     if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_LOCAL_MEM_COUNT))
54         printf("local-mem-bandwidth\n");
55 
56     return rc;
57 }
58 
59 #define MBM_SAMPLE_RETRY_MAX 4
psr_cmt_get_mem_bandwidth(uint32_t domid,libxl_psr_cmt_type type,uint32_t socketid,uint64_t * bandwidth_r)60 static int psr_cmt_get_mem_bandwidth(uint32_t domid,
61                                      libxl_psr_cmt_type type,
62                                      uint32_t socketid,
63                                      uint64_t *bandwidth_r)
64 {
65     uint64_t sample1, sample2;
66     uint64_t tsc1, tsc2;
67     int retry_attempts = 0;
68     int rc;
69 
70     while (1) {
71         rc = libxl_psr_cmt_get_sample(ctx, domid, type, socketid,
72                                       &sample1, &tsc1);
73         if (rc < 0)
74             return rc;
75 
76         usleep(10000);
77 
78         rc = libxl_psr_cmt_get_sample(ctx, domid, type, socketid,
79                                       &sample2, &tsc2);
80         if (rc < 0)
81             return rc;
82 
83         if (tsc2 <= tsc1)
84             return -1;
85 
86         /*
87          * Hardware guarantees at most 1 overflow can happen if the duration
88          * between two samples is less than 1 second. Note that tsc returned
89          * from hypervisor is already-scaled time(ns).
90          */
91         if (tsc2 - tsc1 < 1000000000 && sample2 >= sample1)
92             break;
93 
94         if (retry_attempts < MBM_SAMPLE_RETRY_MAX) {
95             retry_attempts++;
96         } else {
97             fprintf(stderr, "event counter overflowed\n");
98             return -1;
99         }
100     }
101 
102     *bandwidth_r = (sample2 - sample1) * 1000000000 / (tsc2 - tsc1) / 1024;
103     return 0;
104 }
105 
psr_cmt_print_domain_info(libxl_dominfo * dominfo,libxl_psr_cmt_type type,libxl_bitmap * socketmap)106 static void psr_cmt_print_domain_info(libxl_dominfo *dominfo,
107                                       libxl_psr_cmt_type type,
108                                       libxl_bitmap *socketmap)
109 {
110     char *domain_name;
111     uint32_t socketid;
112     uint64_t monitor_data;
113 
114     if (!libxl_psr_cmt_domain_attached(ctx, dominfo->domid))
115         return;
116 
117     domain_name = libxl_domid_to_name(ctx, dominfo->domid);
118     printf("%-40s %5d", domain_name, dominfo->domid);
119     free(domain_name);
120 
121     libxl_for_each_set_bit(socketid, *socketmap) {
122         switch (type) {
123         case LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY:
124             if (!libxl_psr_cmt_get_sample(ctx, dominfo->domid, type, socketid,
125                                           &monitor_data, NULL))
126                 printf("%13"PRIu64" KB", monitor_data / 1024);
127             break;
128         case LIBXL_PSR_CMT_TYPE_TOTAL_MEM_COUNT:
129         case LIBXL_PSR_CMT_TYPE_LOCAL_MEM_COUNT:
130             if (!psr_cmt_get_mem_bandwidth(dominfo->domid, type, socketid,
131                                            &monitor_data))
132                 printf("%11"PRIu64" KB/s", monitor_data);
133             break;
134         default:
135             return;
136         }
137     }
138 
139     printf("\n");
140 }
141 
psr_cmt_show(libxl_psr_cmt_type type,uint32_t domid)142 static int psr_cmt_show(libxl_psr_cmt_type type, uint32_t domid)
143 {
144     uint32_t i, socketid, total_rmid;
145     uint32_t l3_cache_size;
146     libxl_bitmap socketmap;
147     int rc, nr_domains;
148 
149     if (!libxl_psr_cmt_enabled(ctx)) {
150         fprintf(stderr, "CMT is disabled in the system\n");
151         return -1;
152     }
153 
154     if (!libxl_psr_cmt_type_supported(ctx, type)) {
155         fprintf(stderr, "Monitor type '%s' is not supported in the system\n",
156                 libxl_psr_cmt_type_to_string(type));
157         return -1;
158     }
159 
160     libxl_bitmap_init(&socketmap);
161     libxl_socket_bitmap_alloc(ctx, &socketmap, 0);
162     rc = libxl_get_online_socketmap(ctx, &socketmap);
163     if (rc < 0) {
164         fprintf(stderr, "Failed getting available sockets, rc: %d\n", rc);
165         goto out;
166     }
167 
168     rc = libxl_psr_cmt_get_total_rmid(ctx, &total_rmid);
169     if (rc < 0) {
170         fprintf(stderr, "Failed to get max RMID value\n");
171         goto out;
172     }
173 
174     printf("Total RMID: %d\n", total_rmid);
175 
176     /* Header */
177     printf("%-40s %5s", "Name", "ID");
178     libxl_for_each_set_bit(socketid, socketmap)
179         printf("%14s %d", "Socket", socketid);
180     printf("\n");
181 
182     if (type == LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY) {
183             /* Total L3 cache size */
184             printf("%-46s", "Total L3 Cache Size");
185             libxl_for_each_set_bit(socketid, socketmap) {
186                 rc = libxl_psr_cmt_get_l3_cache_size(ctx, socketid,
187                                                      &l3_cache_size);
188                 if (rc < 0) {
189                     fprintf(stderr,
190                             "Failed to get system l3 cache size for socket:%d\n",
191                             socketid);
192                     goto out;
193                 }
194                 printf("%13u KB", l3_cache_size);
195             }
196             printf("\n");
197     }
198 
199     /* Each domain */
200     if (domid != INVALID_DOMID) {
201         libxl_dominfo dominfo;
202 
203         libxl_dominfo_init(&dominfo);
204         if (libxl_domain_info(ctx, &dominfo, domid)) {
205             fprintf(stderr, "Failed to get domain info for %d\n", domid);
206             rc = -1;
207             goto out;
208         }
209         psr_cmt_print_domain_info(&dominfo, type, &socketmap);
210         libxl_dominfo_dispose(&dominfo);
211     }
212     else
213     {
214         libxl_dominfo *list;
215         if (!(list = libxl_list_domain(ctx, &nr_domains))) {
216             fprintf(stderr, "Failed to get domain info for domain list.\n");
217             rc = -1;
218             goto out;
219         }
220         for (i = 0; i < nr_domains; i++)
221             psr_cmt_print_domain_info(list + i, type, &socketmap);
222         libxl_dominfo_list_free(list, nr_domains);
223     }
224 
225 out:
226     libxl_bitmap_dispose(&socketmap);
227     return rc;
228 }
229 
main_psr_cmt_attach(int argc,char ** argv)230 int main_psr_cmt_attach(int argc, char **argv)
231 {
232     uint32_t domid;
233     int opt, ret = 0;
234 
235     SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-attach", 1) {
236         /* No options */
237     }
238 
239     domid = find_domain(argv[optind]);
240     ret = libxl_psr_cmt_attach(ctx, domid);
241 
242     return ret;
243 }
244 
main_psr_cmt_detach(int argc,char ** argv)245 int main_psr_cmt_detach(int argc, char **argv)
246 {
247     uint32_t domid;
248     int opt, ret = 0;
249 
250     SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-detach", 1) {
251         /* No options */
252     }
253 
254     domid = find_domain(argv[optind]);
255     ret = libxl_psr_cmt_detach(ctx, domid);
256 
257     return ret;
258 }
259 
main_psr_cmt_show(int argc,char ** argv)260 int main_psr_cmt_show(int argc, char **argv)
261 {
262     int opt, ret = 0;
263     uint32_t domid;
264     libxl_psr_cmt_type type;
265 
266     SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-show", 1) {
267         /* No options */
268     }
269 
270     if (!strcmp(argv[optind], "cache-occupancy"))
271         type = LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY;
272     else if (!strcmp(argv[optind], "total-mem-bandwidth"))
273         type = LIBXL_PSR_CMT_TYPE_TOTAL_MEM_COUNT;
274     else if (!strcmp(argv[optind], "local-mem-bandwidth"))
275         type = LIBXL_PSR_CMT_TYPE_LOCAL_MEM_COUNT;
276     else {
277         help("psr-cmt-show");
278         return 2;
279     }
280 
281     if (optind + 1 >= argc)
282         domid = INVALID_DOMID;
283     else if (optind + 1 == argc - 1)
284         domid = find_domain(argv[optind + 1]);
285     else {
286         help("psr-cmt-show");
287         return 2;
288     }
289 
290     ret = psr_cmt_show(type, domid);
291 
292     return ret;
293 }
294 
psr_l3_cat_hwinfo(void)295 static int psr_l3_cat_hwinfo(void)
296 {
297     int rc;
298     unsigned int i, nr;
299     uint32_t l3_cache_size;
300     libxl_psr_cat_info *info;
301 
302     rc = libxl_psr_cat_get_info(ctx, &info, &nr, 3);
303     if (rc)
304         return rc;
305 
306     printf("Cache Allocation Technology (CAT):\n");
307 
308     for (i = 0; i < nr; i++) {
309         rc = libxl_psr_cmt_get_l3_cache_size(ctx, info[i].id, &l3_cache_size);
310         if (rc) {
311             fprintf(stderr, "Failed to get l3 cache size for socket:%d\n",
312                     info[i].id);
313             goto out;
314         }
315         printf("%-16s: %u\n", "Socket ID", info[i].id);
316         printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
317         printf("%-16s: %s\n", "CDP Status",
318                info[i].cdp_enabled ? "Enabled" : "Disabled");
319         printf("%-16s: %u\n", "Maximum COS", info[i].cos_max);
320         printf("%-16s: %u\n", "CBM length", info[i].cbm_len);
321         printf("%-16s: %#llx\n", "Default CBM",
322                (1ull << info[i].cbm_len) - 1);
323     }
324 
325 out:
326     libxl_psr_cat_info_list_free(info, nr);
327     return rc;
328 }
329 
psr_cat_print_one_domain_cbm_type(uint32_t domid,uint32_t socketid,libxl_psr_cbm_type type)330 static void psr_cat_print_one_domain_cbm_type(uint32_t domid, uint32_t socketid,
331                                               libxl_psr_cbm_type type)
332 {
333     uint64_t cbm;
334 
335     if (!libxl_psr_cat_get_cbm(ctx, domid, type, socketid, &cbm))
336         printf("%#16"PRIx64, cbm);
337     else
338         printf("%16s", "error");
339 }
340 
psr_cat_print_one_domain_cbm(uint32_t domid,uint32_t socketid,bool cdp_enabled,unsigned int lvl)341 static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socketid,
342                                          bool cdp_enabled, unsigned int lvl)
343 {
344     char *domain_name;
345 
346     domain_name = libxl_domid_to_name(ctx, domid);
347     printf("%5d%25s", domid, domain_name);
348     free(domain_name);
349 
350     switch (lvl) {
351     case 3:
352         if (!cdp_enabled) {
353             psr_cat_print_one_domain_cbm_type(domid, socketid,
354                                               LIBXL_PSR_CBM_TYPE_L3_CBM);
355         } else {
356             psr_cat_print_one_domain_cbm_type(domid, socketid,
357                                               LIBXL_PSR_CBM_TYPE_L3_CBM_CODE);
358             psr_cat_print_one_domain_cbm_type(domid, socketid,
359                                               LIBXL_PSR_CBM_TYPE_L3_CBM_DATA);
360         }
361         break;
362     case 2:
363         psr_cat_print_one_domain_cbm_type(domid, socketid,
364                                           LIBXL_PSR_CBM_TYPE_L2_CBM);
365         break;
366     default:
367         printf("Input lvl %d is wrong!", lvl);
368         break;
369     }
370 
371     printf("\n");
372 }
373 
psr_cat_print_domain_cbm(uint32_t domid,uint32_t socketid,bool cdp_enabled,unsigned int lvl)374 static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socketid,
375                                     bool cdp_enabled, unsigned int lvl)
376 {
377     int i, nr_domains;
378     libxl_dominfo *list;
379 
380     if (domid != INVALID_DOMID) {
381         psr_cat_print_one_domain_cbm(domid, socketid, cdp_enabled, lvl);
382         return 0;
383     }
384 
385     if (!(list = libxl_list_domain(ctx, &nr_domains))) {
386         fprintf(stderr, "Failed to get domain list for cbm display\n");
387         return -1;
388     }
389 
390     for (i = 0; i < nr_domains; i++)
391         psr_cat_print_one_domain_cbm(list[i].domid, socketid, cdp_enabled, lvl);
392     libxl_dominfo_list_free(list, nr_domains);
393 
394     return 0;
395 }
396 
psr_cat_print_socket(uint32_t domid,libxl_psr_cat_info * info,unsigned int lvl)397 static int psr_cat_print_socket(uint32_t domid, libxl_psr_cat_info *info,
398                                 unsigned int lvl)
399 {
400     int rc;
401     uint32_t l3_cache_size;
402 
403     printf("%-16s: %u\n", "Socket ID", info->id);
404 
405     /* So far, CMT only supports L3 cache. */
406     if (lvl == 3) {
407         rc = libxl_psr_cmt_get_l3_cache_size(ctx, info->id, &l3_cache_size);
408         if (rc) {
409             fprintf(stderr, "Failed to get l3 cache size for socket:%d\n",
410                     info->id);
411             return -1;
412         }
413         printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
414     }
415 
416     printf("%-16s: %#llx\n", "Default CBM", (1ull << info->cbm_len) - 1);
417     if (info->cdp_enabled)
418         printf("%5s%25s%16s%16s\n", "ID", "NAME", "CBM (code)", "CBM (data)");
419     else
420         printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
421 
422     return psr_cat_print_domain_cbm(domid, info->id, info->cdp_enabled, lvl);
423 }
424 
psr_cat_show(uint32_t domid,unsigned int lvl)425 static int psr_cat_show(uint32_t domid, unsigned int lvl)
426 {
427     unsigned int i, nr;
428     int rc;
429     libxl_psr_cat_info *info;
430 
431     if (lvl != 2 && lvl != 3) {
432         fprintf(stderr, "Input lvl %d is wrong\n", lvl);
433         return EXIT_FAILURE;
434     }
435 
436     rc = libxl_psr_cat_get_info(ctx, &info, &nr, lvl);
437     if (rc) {
438         fprintf(stderr, "Failed to get %s cat info\n", (lvl == 3)?"L3":"L2");
439         return rc;
440     }
441 
442     for (i = 0; i < nr; i++) {
443         rc = psr_cat_print_socket(domid, info + i, lvl);
444         if (rc)
445             goto out;
446     }
447 
448 out:
449     libxl_psr_cat_info_list_free(info, nr);
450     return rc;
451 }
452 
psr_l2_cat_hwinfo(void)453 static int psr_l2_cat_hwinfo(void)
454 {
455     int rc;
456     unsigned int i, nr;
457     libxl_psr_cat_info *info;
458 
459     rc = libxl_psr_cat_get_info(ctx, &info, &nr, 2);
460     if (rc)
461         return rc;
462 
463     printf("Cache Allocation Technology (CAT): L2\n");
464 
465     for (i = 0; i < nr; i++) {
466         /* There is no CMT on L2 cache so far. */
467         printf("%-16s: %u\n", "Socket ID", info[i].id);
468         printf("%-16s: %u\n", "Maximum COS", info[i].cos_max);
469         printf("%-16s: %u\n", "CBM length", info[i].cbm_len);
470         printf("%-16s: %#llx\n", "Default CBM",
471                (1ull << info[i].cbm_len) - 1);
472     }
473 
474     libxl_psr_cat_info_list_free(info, nr);
475     return rc;
476 }
477 
main_psr_cat_cbm_set(int argc,char ** argv)478 int main_psr_cat_cbm_set(int argc, char **argv)
479 {
480     uint32_t domid;
481     libxl_psr_cbm_type type;
482     uint64_t cbm;
483     int ret, opt = 0;
484     int opt_data = 0, opt_code = 0;
485     libxl_bitmap target_map;
486     char *value;
487     libxl_string_list socket_list;
488     unsigned long start, end;
489     unsigned int i, j, len;
490     unsigned int lvl = 3;
491 
492     static struct option opts[] = {
493         {"socket", 1, 0, 's'},
494         {"data", 0, 0, 'd'},
495         {"code", 0, 0, 'c'},
496         {"level", 1, 0, 'l'},
497         COMMON_LONG_OPTS
498     };
499 
500     libxl_socket_bitmap_alloc(ctx, &target_map, 0);
501     libxl_bitmap_set_none(&target_map);
502 
503     SWITCH_FOREACH_OPT(opt, "s:l:cd", opts, "psr-cat-set", 2) {
504     case 's':
505         trim(isspace, optarg, &value);
506         split_string_into_string_list(value, ",", &socket_list);
507         len = libxl_string_list_length(&socket_list);
508         for (i = 0; i < len; i++) {
509             parse_range(socket_list[i], &start, &end);
510             for (j = start; j <= end; j++)
511                 libxl_bitmap_set(&target_map, j);
512         }
513 
514         libxl_string_list_dispose(&socket_list);
515         free(value);
516         break;
517     case 'd':
518         opt_data = 1;
519         break;
520     case 'c':
521         opt_code = 1;
522         break;
523     case 'l':
524         lvl = atoi(optarg);
525         break;
526     }
527 
528     if (lvl == 2)
529         type = LIBXL_PSR_CBM_TYPE_L2_CBM;
530     else if (lvl == 3) {
531         if (opt_data && opt_code) {
532             fprintf(stderr, "Cannot handle -c and -d at the same time\n");
533             return EXIT_FAILURE;
534         } else if (opt_data) {
535             type = LIBXL_PSR_CBM_TYPE_L3_CBM_DATA;
536         } else if (opt_code) {
537             type = LIBXL_PSR_CBM_TYPE_L3_CBM_CODE;
538         } else {
539             type = LIBXL_PSR_CBM_TYPE_L3_CBM;
540         }
541     } else {
542         type = LIBXL_PSR_CBM_TYPE_L3_CBM;
543         fprintf(stderr, "Input lvl %d is wrong\n", lvl);
544         return EXIT_FAILURE;
545     }
546 
547     if (libxl_bitmap_is_empty(&target_map))
548         libxl_bitmap_set_any(&target_map);
549 
550     if (argc != optind + 2) {
551         help("psr-cat-set");
552         return 2;
553     }
554 
555     domid = find_domain(argv[optind]);
556     cbm = strtoll(argv[optind + 1], NULL , 0);
557 
558     ret = libxl_psr_cat_set_cbm(ctx, domid, type, &target_map, cbm);
559 
560     libxl_bitmap_dispose(&target_map);
561     return ret;
562 }
563 
main_psr_cat_show(int argc,char ** argv)564 int main_psr_cat_show(int argc, char **argv)
565 {
566     int opt = 0;
567     uint32_t domid;
568     unsigned int lvl = 3;
569 
570     static struct option opts[] = {
571         {"level", 1, 0, 'l'},
572         COMMON_LONG_OPTS
573     };
574 
575     SWITCH_FOREACH_OPT(opt, "l:", opts, "psr-cat-show", 0) {
576     case 'l':
577         lvl = atoi(optarg);
578         break;
579     }
580 
581     if (optind >= argc)
582         domid = INVALID_DOMID;
583     else if (optind == argc - 1)
584         domid = find_domain(argv[optind]);
585     else {
586         help("psr-cat-show");
587         return 2;
588     }
589 
590     return psr_cat_show(domid, lvl);
591 }
592 
main_psr_hwinfo(int argc,char ** argv)593 int main_psr_hwinfo(int argc, char **argv)
594 {
595     int opt, ret = 0;
596     bool all = true, cmt = false, cat = false;
597     static struct option opts[] = {
598         {"cmt", 0, 0, 'm'},
599         {"cat", 0, 0, 'a'},
600         COMMON_LONG_OPTS
601     };
602 
603     SWITCH_FOREACH_OPT(opt, "ma", opts, "psr-hwinfo", 0) {
604     case 'm':
605         all = false; cmt = true;
606         break;
607     case 'a':
608         all = false; cat = true;
609         break;
610     }
611 
612     if (!ret && (all || cmt))
613         ret = psr_cmt_hwinfo();
614 
615     if (!ret && (all || cat))
616         ret = psr_l3_cat_hwinfo();
617 
618     /* L2 CAT is independent of CMT and L3 CAT */
619     if (all || cat)
620         ret = psr_l2_cat_hwinfo();
621 
622     return ret;
623 }
624 
625 /*
626  * Local variables:
627  * mode: C
628  * c-basic-offset: 4
629  * indent-tabs-mode: nil
630  * End:
631  */
632