From 5d3620916fa3b566fc2a76447668489ca48dae24 Mon Sep 17 00:00:00 2001
From: Bandan Das <bsd@redhat.com>
Date: Sun, 19 Mar 2017 18:16:52 +0100
Subject: [PATCH] target-i386: present virtual L3 cache info for vcpus
RH-Author: Bandan Das <bsd@redhat.com>
Message-id: <jpglgs1p1wr.fsf@linux.bootlegged.copy>
Patchwork-id: 74370
O-Subject: [RHEL-7.3.z qemu-kvm-rhev PATCH v2] target-i386: present virtual L3 cache info for vcpus
Bugzilla: 1430802
RH-Acked-by: Paolo Bonzini <pbonzini@redhat.com>
RH-Acked-by: Eduardo Habkost <ehabkost@redhat.com>
RH-Acked-by: Radim Krcmar <rkrcmar@redhat.com>
Some software algorithms are based on the hardware's cache info, for example,
for x86 linux kernel, when cpu1 want to wakeup a task on cpu2, cpu1 will trigger
a resched IPI and told cpu2 to do the wakeup if they don't share low level
cache. Oppositely, cpu1 will access cpu2's runqueue directly if they share llc.
The relevant linux-kernel code as bellow:
static void ttwu_queue(struct task_struct *p, int cpu)
{
struct rq *rq = cpu_rq(cpu);
......
if (... && !cpus_share_cache(smp_processor_id(), cpu)) {
......
ttwu_queue_remote(p, cpu); /* will trigger RES IPI */
return;
}
......
ttwu_do_activate(rq, p, 0); /* access target's rq directly */
......
}
In real hardware, the cpus on the same socket share L3 cache, so one won't
trigger a resched IPIs when wakeup a task on others. But QEMU doesn't present a
virtual L3 cache info for VM, then the linux guest will trigger lots of RES IPIs
under some workloads even if the virtual cpus belongs to the same virtual socket.
For KVM, there will be lots of vmexit due to guest send IPIs.
The workload is a SAP HANA's testsuite, we run it one round(about 40 minuates)
and observe the (Suse11sp3)Guest's amounts of RES IPIs which triggering during
the period:
No-L3 With-L3(applied this patch)
cpu0: 363890 44582
cpu1: 373405 43109
cpu2: 340783 43797
cpu3: 333854 43409
cpu4: 327170 40038
cpu5: 325491 39922
cpu6: 319129 42391
cpu7: 306480 41035
cpu8: 161139 32188
cpu9: 164649 31024
cpu10: 149823 30398
cpu11: 149823 32455
cpu12: 164830 35143
cpu13: 172269 35805
cpu14: 179979 33898
cpu15: 194505 32754
avg: 268963.6 40129.8
The VM's topology is "1*socket 8*cores 2*threads".
After present virtual L3 cache info for VM, the amounts of RES IPIs in guest
reduce 85%.
For KVM, vcpus send IPIs will cause vmexit which is expensive, so it can cause
severe performance degradation. We had tested the overall system performance if
vcpus actually run on sparate physical socket. With L3 cache, the performance
improves 7.2%~33.1%(avg:15.7%).
Signed-off-by: Longpeng(Mike) <longpeng2@huawei.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
(cherry picked from commit 14c985cffa6cb177fc01a163d8bcf227c104718c)
* RHEL7 Notes *
Included topology.h in cpu.c and set default value of enable_l3_cache to
off due to a bug in how compat props are handled.
For cases such as the SAP workload test referenced in the BZ, it's upto
libvirt to set it to "on" to take advantage of the virtual L3
cache.
Signed-off-by: Miroslav Rezanina <mrezanin@redhat.com>
Conflicts:
target-i386/cpu.c
target-i386/cpu.h
Signed-off-by: Bandan Das <bsd@redhat.com>
---
target-i386/cpu.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
target-i386/cpu.h | 6 ++++++
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 15b72df..8e593ed 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -41,6 +41,7 @@
#include "sysemu/sysemu.h"
#include "hw/qdev-properties.h"
+#include "hw/i386/topology.h"
#ifndef CONFIG_USER_ONLY
#include "exec/address-spaces.h"
#include "hw/xen/xen.h"
@@ -56,6 +57,7 @@
#define CPUID_2_L1I_32KB_8WAY_64B 0x30
#define CPUID_2_L2_2MB_8WAY_64B 0x7d
+#define CPUID_2_L3_16MB_16WAY_64B 0x4d
/* CPUID Leaf 4 constants: */
@@ -129,11 +131,18 @@
#define L2_LINES_PER_TAG 1
#define L2_SIZE_KB_AMD 512
-/* No L3 cache: */
+/* Level 3 unified cache: */
#define L3_SIZE_KB 0 /* disabled */
#define L3_ASSOCIATIVITY 0 /* disabled */
#define L3_LINES_PER_TAG 0 /* disabled */
#define L3_LINE_SIZE 0 /* disabled */
+#define L3_N_LINE_SIZE 64
+#define L3_N_ASSOCIATIVITY 16
+#define L3_N_SETS 16384
+#define L3_N_PARTITIONS 1
+#define L3_N_DESCRIPTOR CPUID_2_L3_16MB_16WAY_64B
+#define L3_N_LINES_PER_TAG 1
+#define L3_N_SIZE_KB_AMD 16384
/* TLB definitions: */
@@ -2257,6 +2266,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
{
X86CPU *cpu = x86_env_get_cpu(env);
CPUState *cs = CPU(cpu);
+ uint32_t pkg_offset;
/* test if maximum index reached */
if (index & 0x80000000) {
@@ -2310,7 +2320,11 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
}
*eax = 1; /* Number of CPUID[EAX=2] calls required */
*ebx = 0;
- *ecx = 0;
+ if (!cpu->enable_l3_cache) {
+ *ecx = 0;
+ } else {
+ *ecx = L3_N_DESCRIPTOR;
+ }
*edx = (L1D_DESCRIPTOR << 16) | \
(L1I_DESCRIPTOR << 8) | \
(L2_DESCRIPTOR);
@@ -2356,6 +2370,25 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
*ecx = L2_SETS - 1;
*edx = CPUID_4_NO_INVD_SHARING;
break;
+ case 3: /* L3 cache info */
+ if (!cpu->enable_l3_cache) {
+ *eax = 0;
+ *ebx = 0;
+ *ecx = 0;
+ *edx = 0;
+ break;
+ }
+ *eax |= CPUID_4_TYPE_UNIFIED | \
+ CPUID_4_LEVEL(3) | \
+ CPUID_4_SELF_INIT_LEVEL;
+ pkg_offset = apicid_pkg_offset(cs->nr_cores, cs->nr_threads);
+ *eax |= ((1 << pkg_offset) - 1) << 14;
+ *ebx = (L3_N_LINE_SIZE - 1) | \
+ ((L3_N_PARTITIONS - 1) << 12) | \
+ ((L3_N_ASSOCIATIVITY - 1) << 22);
+ *ecx = L3_N_SETS - 1;
+ *edx = CPUID_4_INCLUSIVE | CPUID_4_COMPLEX_IDX;
+ break;
default: /* end of info */
*eax = 0;
*ebx = 0;
@@ -2537,9 +2570,15 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
*ecx = (L2_SIZE_KB_AMD << 16) | \
(AMD_ENC_ASSOC(L2_ASSOCIATIVITY) << 12) | \
(L2_LINES_PER_TAG << 8) | (L2_LINE_SIZE);
- *edx = ((L3_SIZE_KB/512) << 18) | \
- (AMD_ENC_ASSOC(L3_ASSOCIATIVITY) << 12) | \
- (L3_LINES_PER_TAG << 8) | (L3_LINE_SIZE);
+ if (!cpu->enable_l3_cache) {
+ *edx = ((L3_SIZE_KB / 512) << 18) | \
+ (AMD_ENC_ASSOC(L3_ASSOCIATIVITY) << 12) | \
+ (L3_LINES_PER_TAG << 8) | (L3_LINE_SIZE);
+ } else {
+ *edx = ((L3_N_SIZE_KB_AMD / 512) << 18) | \
+ (AMD_ENC_ASSOC(L3_N_ASSOCIATIVITY) << 12) | \
+ (L3_N_LINES_PER_TAG << 8) | (L3_N_LINE_SIZE);
+ }
break;
case 0x80000007:
*eax = 0;
@@ -3318,6 +3357,13 @@ static Property x86_cpu_properties[] = {
DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, 0),
DEFINE_PROP_UINT32("xlevel2", X86CPU, env.cpuid_xlevel2, 0),
DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor_id),
+ /*
+ * l3-cache is set to true upstream but due to a bug, compat
+ * properties override user preferences. This bug is fixed by
+ * uptream commit 0bcba41fe379e4c6834adcf1456d9099db31a5b2 which
+ * is not present in 7.3.z.
+ */
+ DEFINE_PROP_BOOL("l3-cache", X86CPU, enable_l3_cache, false),
DEFINE_PROP_END_OF_LIST()
};
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index e13e2a1..97d7393 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -1094,6 +1094,12 @@ struct X86CPU {
*/
bool enable_pmu;
+ /* Compatibility bits for old machine types.
+ * If true present virtual l3 cache for VM, the vcpus in the same virtual
+ * socket share an virtual l3 cache.
+ */
+ bool enable_l3_cache;
+
/* in order to simplify APIC support, we leave this pointer to the
user */
struct DeviceState *apic_state;
--
1.8.3.1