common.h: Avoid using unsupported load/store instructions in arm64 VMs

Using nvme show-regs within a VM on arm64 can sometimes lead to VM
termination.

To answer why this happens: one of the deficiencies of the Arm
architecture is that there exists a range of load/store instructions
that have insufficient decode information for traps taken to the
hypervisor. KVM, for example, may raise an external abort or outright
terminate the VM depending on the configuration.

This is a known problem on the kernel side, and is fixed by using
assembly MMIO accessors w/ 'safe' load/store instructions. So do
exactly that, providing arm64-specific accessors and falling back to
plain old volatile pointer accesses for other architectures.

Reported-by: William Butler <wab@google.com>
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>

Author: oupton

common.h

@@ -17,43 +17,81 @@
 #define __packed __attribute__((__packed__))
 #endif /* __packed */
 
-static inline uint32_t mmio_read32(void *addr)
+/*
+ * VMs on arm64 can only use a subset of instructions for MMIO that provide
+ * the hypervisor with a complete instruction decode. Provide assembly MMIO
+ * accessors to prevent the compiler from using a possibly unsupported
+ * instruction.
+ *
+ * See kernel commit c726200dd106 ("KVM: arm/arm64: Allow reporting non-ISV
+ * data aborts to userspace") for more details.
+ */
+#if defined(__aarch64__)
+static inline leint32_t __raw_readl(const volatile leint32_t *addr)
+{
+	leint32_t val;
+
+	asm volatile("ldr %w0, %1" : "=r" (val) : "Qo" (*addr));
+
+	return val;
+}
+
+static inline void __raw_writel(volatile leint32_t *addr, leint32_t val)
 {
-	leint32_t *p = addr;
+	asm volatile("str %w0, %1" : : "r" (val), "Qo" (*addr));
+}
 
-	return le32_to_cpu(*p);
+static inline void __raw_writeq(volatile leint64_t *addr, leint64_t val)
+{
+	asm volatile("str %0, %1" : : "r" (val), "Qo" (*addr));
+}
+#else
+static inline leint32_t __raw_readl(volatile leint32_t *addr)
+{
+	return *addr;
+}
+
+static inline void __raw_writel(volatile leint32_t *addr, leint32_t val)
+{
+	*addr = val;
+}
+
+static inline void __raw_writeq(volatile leint64_t *addr, leint64_t val)
+{
+	*addr = val;
+}
+#endif
+
+static inline uint32_t mmio_read32(void *addr)
+{
+	return le32_to_cpu(__raw_readl(addr));
 }
 
 /* Access 64-bit registers as 2 32-bit; Some devices fail 64-bit MMIO. */
 static inline uint64_t mmio_read64(void *addr)
 {
-	const volatile uint32_t *p = addr;
 	uint32_t low, high;
 
-	low = le32_to_cpu(*p);
-	high = le32_to_cpu(*(p + 1));
+	low = le32_to_cpu(__raw_readl(addr));
+	high = le32_to_cpu(__raw_readl(addr + sizeof(leint32_t)));
 
 	return ((uint64_t)high << 32) | low;
 }
 
 static inline void mmio_write32(void *addr, uint32_t value)
 {
-	leint32_t *p = addr;
-
-	*p = cpu_to_le32(value);
+	__raw_writel(addr, cpu_to_le32(value));
 }
 
 /* Access 64-bit registers as 2 32-bit if write32 flag set; Some devices fail 64-bit MMIO. */
 static inline void mmio_write64(void *addr, uint64_t value, bool write32)
 {
-	uint64_t *p = addr;
-
 	if (write32) {
 		mmio_write32(addr, value);
 		mmio_write32((uint32_t *)addr + 1, value >> 32);
 		return;
 	}
 
-	*p = cpu_to_le64(value);
+	__raw_writeq(addr, cpu_to_le64(value));
 }
 #endif