[RFC PATCH 15/19] KVM: guest_mem: hugetlb: initialization and cleanup

[Ackerley Tng]

First stage of hugetlb support: add initialization and cleanup
routines

Signed-off-by: Ackerley Tng <ackerleytng@xxxxxxxxxx>
---
 include/uapi/linux/kvm.h | 25 ++++++++++++
 virt/kvm/guest_mem.c     | 88 +++++++++++++++++++++++++++++++++++++---
 2 files changed, 108 insertions(+), 5 deletions(-)

diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 0fa665e8862a..1df0c802c29f 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -13,6 +13,7 @@
 #include <linux/compiler.h>
 #include <linux/ioctl.h>
 #include <asm/kvm.h>
+#include <asm-generic/hugetlb_encode.h>
 
 #define KVM_API_VERSION 12
 
@@ -2280,6 +2281,30 @@ struct kvm_memory_attributes {
 #define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
 
 #define KVM_GUEST_MEMFD_HUGE_PMD		(1ULL << 0)
+#define KVM_GUEST_MEMFD_HUGETLB			(1ULL << 1)
+
+/*
+ * Huge page size encoding when KVM_GUEST_MEMFD_HUGETLB is specified, and a huge
+ * page size other than the default is desired.  See hugetlb_encode.h.  All
+ * known huge page size encodings are provided here.  It is the responsibility
+ * of the application to know which sizes are supported on the running system.
+ * See mmap(2) man page for details.
+ */
+#define KVM_GUEST_MEMFD_HUGE_SHIFT	HUGETLB_FLAG_ENCODE_SHIFT
+#define KVM_GUEST_MEMFD_HUGE_MASK	HUGETLB_FLAG_ENCODE_MASK
+
+#define KVM_GUEST_MEMFD_HUGE_64KB	HUGETLB_FLAG_ENCODE_64KB
+#define KVM_GUEST_MEMFD_HUGE_512KB	HUGETLB_FLAG_ENCODE_512KB
+#define KVM_GUEST_MEMFD_HUGE_1MB	HUGETLB_FLAG_ENCODE_1MB
+#define KVM_GUEST_MEMFD_HUGE_2MB	HUGETLB_FLAG_ENCODE_2MB
+#define KVM_GUEST_MEMFD_HUGE_8MB	HUGETLB_FLAG_ENCODE_8MB
+#define KVM_GUEST_MEMFD_HUGE_16MB	HUGETLB_FLAG_ENCODE_16MB
+#define KVM_GUEST_MEMFD_HUGE_32MB	HUGETLB_FLAG_ENCODE_32MB
+#define KVM_GUEST_MEMFD_HUGE_256MB	HUGETLB_FLAG_ENCODE_256MB
+#define KVM_GUEST_MEMFD_HUGE_512MB	HUGETLB_FLAG_ENCODE_512MB
+#define KVM_GUEST_MEMFD_HUGE_1GB	HUGETLB_FLAG_ENCODE_1GB
+#define KVM_GUEST_MEMFD_HUGE_2GB	HUGETLB_FLAG_ENCODE_2GB
+#define KVM_GUEST_MEMFD_HUGE_16GB	HUGETLB_FLAG_ENCODE_16GB
 
 struct kvm_create_guest_memfd {
 	__u64 size;
diff --git a/virt/kvm/guest_mem.c b/virt/kvm/guest_mem.c
index 13253af40be6..b533143e2878 100644
--- a/virt/kvm/guest_mem.c
+++ b/virt/kvm/guest_mem.c
@@ -19,6 +19,7 @@
 #include <linux/secretmem.h>
 #include <linux/set_memory.h>
 #include <linux/sched/signal.h>
+#include <linux/hugetlb.h>
 
 #include <uapi/linux/magic.h>
 
@@ -30,6 +31,11 @@ struct kvm_gmem {
 	struct kvm *kvm;
 	u64 flags;
 	struct xarray bindings;
+	struct {
+		struct hstate *h;
+		struct hugepage_subpool *spool;
+		struct resv_map *resv_map;
+	} hugetlb;
 };
 
 static loff_t kvm_gmem_get_size(struct file *file)
@@ -346,6 +352,46 @@ static const struct inode_operations kvm_gmem_iops = {
 	.setattr	= kvm_gmem_setattr,
 };
 
+static int kvm_gmem_hugetlb_setup(struct inode *inode, struct kvm_gmem *gmem,
+				  loff_t size, u64 flags)
+{
+	int page_size_log;
+	int hstate_idx;
+	long hpages;
+	struct resv_map *resv_map;
+	struct hugepage_subpool *spool;
+	struct hstate *h;
+
+	page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK;
+	hstate_idx = get_hstate_idx(page_size_log);
+	if (hstate_idx < 0)
+		return -ENOENT;
+
+	h = &hstates[hstate_idx];
+	/* Round up to accommodate size requests that don't align with huge pages */
+	hpages = round_up(size, huge_page_size(h)) >> huge_page_shift(h);
+	spool = hugepage_new_subpool(h, hpages, hpages);
+	if (!spool)
+		goto out;
+
+	resv_map = resv_map_alloc();
+	if (!resv_map)
+		goto out_subpool;
+
+	inode->i_blkbits = huge_page_shift(h);
+
+	gmem->hugetlb.h = h;
+	gmem->hugetlb.spool = spool;
+	gmem->hugetlb.resv_map = resv_map;
+
+	return 0;
+
+out_subpool:
+	kfree(spool);
+out:
+	return -ENOMEM;
+}
+
 static struct inode *kvm_gmem_create_inode(struct kvm *kvm, loff_t size, u64 flags,
 					   struct vfsmount *mnt)
 {
@@ -368,6 +414,12 @@ static struct inode *kvm_gmem_create_inode(struct kvm *kvm, loff_t size, u64 fla
 	if (!gmem)
 		goto err_inode;
 
+	if (flags & KVM_GUEST_MEMFD_HUGETLB) {
+		err = kvm_gmem_hugetlb_setup(inode, gmem, size, flags);
+		if (err)
+			goto err_gmem;
+	}
+
 	xa_init(&gmem->bindings);
 
 	kvm_get_kvm(kvm);
@@ -385,6 +437,8 @@ static struct inode *kvm_gmem_create_inode(struct kvm *kvm, loff_t size, u64 fla
 
 	return inode;
 
+err_gmem:
+	kfree(gmem);
 err_inode:
 	iput(inode);
 	return ERR_PTR(err);
@@ -414,6 +468,8 @@ static struct file *kvm_gmem_create_file(struct kvm *kvm, loff_t size, u64 flags
 	return file;
 }
 
+#define KVM_GUEST_MEMFD_ALL_FLAGS (KVM_GUEST_MEMFD_HUGE_PMD | KVM_GUEST_MEMFD_HUGETLB)
+
 int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *gmem)
 {
 	int fd;
@@ -424,8 +480,15 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *gmem)
 	if (size < 0 || !PAGE_ALIGNED(size))
 		return -EINVAL;
 
-	if (flags & ~KVM_GUEST_MEMFD_HUGE_PMD)
-		return -EINVAL;
+	if (!(flags & KVM_GUEST_MEMFD_HUGETLB)) {
+		if (flags & ~(unsigned int)KVM_GUEST_MEMFD_ALL_FLAGS)
+			return -EINVAL;
+	} else {
+		/* Allow huge page size encoding in flags. */
+		if (flags & ~(unsigned int)(KVM_GUEST_MEMFD_ALL_FLAGS |
+				(KVM_GUEST_MEMFD_HUGE_MASK << KVM_GUEST_MEMFD_HUGE_SHIFT)))
+			return -EINVAL;
+	}
 
 	if (flags & KVM_GUEST_MEMFD_HUGE_PMD) {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -610,7 +673,17 @@ static void kvm_gmem_evict_inode(struct inode *inode)
 	 * pointed at this file.
 	 */
 	kvm_gmem_invalidate_begin(kvm, gmem, 0, -1ul);
-	truncate_inode_pages_final(inode->i_mapping);
+	if (gmem->flags & KVM_GUEST_MEMFD_HUGETLB) {
+		truncate_inode_pages_final_prepare(inode->i_mapping);
+		remove_mapping_hugepages(
+			inode->i_mapping, gmem->hugetlb.h, gmem->hugetlb.spool,
+			gmem->hugetlb.resv_map, inode, 0, LLONG_MAX);
+
+		resv_map_release(&gmem->hugetlb.resv_map->refs);
+		hugepage_put_subpool(gmem->hugetlb.spool);
+	} else {
+		truncate_inode_pages_final(inode->i_mapping);
+	}
 	kvm_gmem_invalidate_end(kvm, gmem, 0, -1ul);
 
 	mutex_unlock(&kvm->slots_lock);
@@ -688,10 +761,15 @@ bool kvm_gmem_check_alignment(const struct kvm_userspace_memory_region2 *mem)
 {
 	size_t page_size;
 
-	if (mem->flags & KVM_GUEST_MEMFD_HUGE_PMD)
+	if (mem->flags & KVM_GUEST_MEMFD_HUGETLB) {
+		size_t page_size_log = ((mem->flags >> KVM_GUEST_MEMFD_HUGE_SHIFT)
+					& KVM_GUEST_MEMFD_HUGE_MASK);
+		page_size = 1UL << page_size_log;
+	} else if (mem->flags & KVM_GUEST_MEMFD_HUGE_PMD) {
 		page_size = HPAGE_PMD_SIZE;
-	else
+	} else {
 		page_size = PAGE_SIZE;
+	}
 
 	return (IS_ALIGNED(mem->gmem_offset, page_size) &&
 		IS_ALIGNED(mem->memory_size, page_size));
-- 
2.41.0.rc0.172.g3f132b7071-goog