#include "irq.h"
#include "mmu.h"
#include "x86.h"
#include "kvm_cache_regs.h"
#include <linux/kvm_host.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/module.h>
#include <linux/swap.h>
#include <linux/hugetlb.h>
#include <linux/compiler.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <asm/page.h>
#include <asm/cmpxchg.h>
#include <asm/io.h>
#include <asm/vmx.h>
#include <trace/events/kvm.h>
#include "mmutrace.h"
#include "paging_tmpl.h"

Data Structures
struct	pte_list_desc

struct	kvm_shadow_walk_iterator

union	split_spte

struct	rmap_iterator

struct	kvm_mmu_pages

struct	kvm_mmu_pages::mmu_page_and_offset

struct	mmu_page_path

Macros
#define	pgprintk(x...) do { } while (0)

#define	rmap_printk(x...) do { } while (0)

#define	ASSERT(x) do { } while (0)

#define	PTE_PREFETCH_NUM 8

#define	PT_FIRST_AVAIL_BITS_SHIFT 10

#define	PT64_SECOND_AVAIL_BITS_SHIFT 52

#define	PT64_LEVEL_BITS 9

#define	PT64_LEVEL_SHIFT(level) (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)

#define	PT64_INDEX(address, level) (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))

#define	PT32_LEVEL_BITS 10

#define	PT32_LEVEL_SHIFT(level) (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)

#define	PT32_LVL_OFFSET_MASK(level)

#define	PT32_INDEX(address, level) (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))

#define	PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))

#define	PT64_DIR_BASE_ADDR_MASK (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

#define	PT64_LVL_ADDR_MASK(level)

#define	PT64_LVL_OFFSET_MASK(level)

#define	PT32_BASE_ADDR_MASK PAGE_MASK

#define	PT32_DIR_BASE_ADDR_MASK (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))

#define	PT32_LVL_ADDR_MASK(level)

#define	PT64_PERM_MASK

#define	ACC_EXEC_MASK 1

#define	ACC_WRITE_MASK PT_WRITABLE_MASK

#define	ACC_USER_MASK PT_USER_MASK

#define	ACC_ALL (ACC_EXEC_MASK \| ACC_WRITE_MASK \| ACC_USER_MASK)

#define	CREATE_TRACE_POINTS

#define	SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)

#define	SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))

#define	SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)

#define	PTE_LIST_EXT 3

#define	for_each_shadow_entry(_vcpu, _addr, _walker)

#define	for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)

#define	RMAP_RECYCLE_THRESHOLD 1000

#define	KVM_PAGE_ARRAY_NR 16

#define	for_each_gfn_sp(kvm, sp, gfn, pos)

#define	for_each_gfn_indirect_valid_sp(kvm, sp, gfn, pos)

#define	for_each_sp(pvec, sp, parents, i)

#define	PTTYPE 64

#define	PTTYPE 32

Typedefs
typedef void(*	pte_list_walk_fn )(u64 *spte)

Enumerations
enum	{ AUDIT_PRE_PAGE_FAULT, AUDIT_POST_PAGE_FAULT, AUDIT_PRE_PTE_WRITE, AUDIT_POST_PTE_WRITE, AUDIT_PRE_SYNC, AUDIT_POST_SYNC }

Functions
void	kvm_mmu_set_mmio_spte_mask (u64 mmio_mask)

	EXPORT_SYMBOL_GPL (kvm_mmu_set_mmio_spte_mask)

void	kvm_mmu_set_mask_ptes (u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask)

	EXPORT_SYMBOL_GPL (kvm_mmu_set_mask_ptes)

void	kvm_mmu_write_protect_pt_masked (struct kvm kvm, struct kvm_memory_slot slot, gfn_t gfn_offset, unsigned long mask)

int	kvm_unmap_hva (struct kvm *kvm, unsigned long hva)

int	kvm_unmap_hva_range (struct kvm *kvm, unsigned long start, unsigned long end)

void	kvm_set_spte_hva (struct kvm *kvm, unsigned long hva, pte_t pte)

int	kvm_age_hva (struct kvm *kvm, unsigned long hva)

int	kvm_test_age_hva (struct kvm *kvm, unsigned long hva)

void	kvm_mmu_change_mmu_pages (struct kvm *kvm, unsigned int goal_nr_mmu_pages)

int	kvm_mmu_unprotect_page (struct kvm *kvm, gfn_t gfn)

	EXPORT_SYMBOL_GPL (kvm_mmu_unprotect_page)

u8	kvm_get_guest_memory_type (struct kvm_vcpu *vcpu, gfn_t gfn)

	EXPORT_SYMBOL_GPL (kvm_get_guest_memory_type)

void	kvm_mmu_sync_roots (struct kvm_vcpu *vcpu)

int	handle_mmio_page_fault_common (struct kvm_vcpu *vcpu, u64 addr, bool direct)

	EXPORT_SYMBOL_GPL (handle_mmio_page_fault_common)

void	kvm_mmu_flush_tlb (struct kvm_vcpu *vcpu)

int	kvm_init_shadow_mmu (struct kvm_vcpu vcpu, struct kvm_mmu context)

	EXPORT_SYMBOL_GPL (kvm_init_shadow_mmu)

int	kvm_mmu_reset_context (struct kvm_vcpu *vcpu)

	EXPORT_SYMBOL_GPL (kvm_mmu_reset_context)

int	kvm_mmu_load (struct kvm_vcpu *vcpu)

	EXPORT_SYMBOL_GPL (kvm_mmu_load)

void	kvm_mmu_unload (struct kvm_vcpu *vcpu)

	EXPORT_SYMBOL_GPL (kvm_mmu_unload)

void	kvm_mmu_pte_write (struct kvm_vcpu vcpu, gpa_t gpa, const u8 new, int bytes)

int	kvm_mmu_unprotect_page_virt (struct kvm_vcpu *vcpu, gva_t gva)

	EXPORT_SYMBOL_GPL (kvm_mmu_unprotect_page_virt)

void	__kvm_mmu_free_some_pages (struct kvm_vcpu *vcpu)

int	kvm_mmu_page_fault (struct kvm_vcpu vcpu, gva_t cr2, u32 error_code, void insn, int insn_len)

	EXPORT_SYMBOL_GPL (kvm_mmu_page_fault)

void	kvm_mmu_invlpg (struct kvm_vcpu *vcpu, gva_t gva)

	EXPORT_SYMBOL_GPL (kvm_mmu_invlpg)

void	kvm_enable_tdp (void)

	EXPORT_SYMBOL_GPL (kvm_enable_tdp)

void	kvm_disable_tdp (void)

	EXPORT_SYMBOL_GPL (kvm_disable_tdp)

int	kvm_mmu_create (struct kvm_vcpu *vcpu)

int	kvm_mmu_setup (struct kvm_vcpu *vcpu)

void	kvm_mmu_slot_remove_write_access (struct kvm *kvm, int slot)

void	kvm_mmu_zap_all (struct kvm *kvm)

int	kvm_mmu_module_init (void)

unsigned int	kvm_mmu_calculate_mmu_pages (struct kvm *kvm)

int	kvm_mmu_get_spte_hierarchy (struct kvm_vcpu *vcpu, u64 addr, u64 sptes[4])

	EXPORT_SYMBOL_GPL (kvm_mmu_get_spte_hierarchy)

void	kvm_mmu_destroy (struct kvm_vcpu *vcpu)

void	kvm_mmu_module_exit (void)

Variables
bool	tdp_enabled = false

Macro Definition Documentation

#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)

Definition at line 141 of file mmu.c.

#define ACC_EXEC_MASK 1

Definition at line 138 of file mmu.c.

#define ACC_USER_MASK PT_USER_MASK

Definition at line 140 of file mmu.c.

#define ACC_WRITE_MASK PT_WRITABLE_MASK

Definition at line 139 of file mmu.c.

#define ASSERT ( x ) do { } while (0)

Definition at line 82 of file mmu.c.

#define CREATE_TRACE_POINTS

Definition at line 145 of file mmu.c.

#define for_each_gfn_indirect_valid_sp	(	kvm,
		sp,
		gfn,
		pos
	)

Value:

hlist_for_each_entry(sp, pos,                       \
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
        if ((sp)->gfn != (gfn) || (sp)->role.direct ||      \
            (sp)->role.invalid) {} else

Definition at line 1667 of file mmu.c.

#define for_each_gfn_sp	(	kvm,
		sp,
		gfn,
		pos
	)

Value:

hlist_for_each_entry(sp, pos,                       \
   &(kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)], hash_link)   \
    if ((sp)->gfn != (gfn)) {} else

Definition at line 1662 of file mmu.c.

#define for_each_shadow_entry	(	_vcpu,
		_addr,
		_walker
	)

Value:

for (shadow_walk_init(&(_walker), _vcpu, _addr);    \
         shadow_walk_okay(&(_walker));          \
         shadow_walk_next(&(_walker)))

Definition at line 169 of file mmu.c.

#define for_each_shadow_entry_lockless	(	_vcpu,
		_addr,
		_walker,
		spte
	)

Value:

for (shadow_walk_init(&(_walker), _vcpu, _addr);        \
         shadow_walk_okay(&(_walker)) &&                \
        ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });  \
         __shadow_walk_next(&(_walker), spte))

Definition at line 174 of file mmu.c.

#define for_each_sp	(	pvec,
		sp,
		parents,
		i
	)

Value:

for (i = mmu_pages_next(&pvec, &parents, -1),   \
            sp = pvec.page[i].sp;           \
            i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});   \
            i = mmu_pages_next(&pvec, &parents, i))

Definition at line 1752 of file mmu.c.

#define KVM_PAGE_ARRAY_NR 16

Definition at line 1569 of file mmu.c.

#define pgprintk ( x... ) do { } while (0)

Definition at line 71 of file mmu.c.

#define PT32_BASE_ADDR_MASK PAGE_MASK

Definition at line 128 of file mmu.c.

#define PT32_DIR_BASE_ADDR_MASK (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))

Definition at line 129 of file mmu.c.

#define PT32_INDEX	(	address,
		level
	)	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))

Definition at line 114 of file mmu.c.

#define PT32_LEVEL_BITS 10

Definition at line 105 of file mmu.c.

#define PT32_LEVEL_SHIFT ( level ) (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)

Definition at line 107 of file mmu.c.

#define PT32_LVL_ADDR_MASK ( level )

Value:

(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \

* PT32_LEVEL_BITS))) - 1))

Definition at line 131 of file mmu.c.

#define PT32_LVL_OFFSET_MASK ( level )

Value:

(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \

* PT32_LEVEL_BITS))) - 1))

Definition at line 110 of file mmu.c.

#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))

Definition at line 118 of file mmu.c.

#define PT64_DIR_BASE_ADDR_MASK (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))

Definition at line 119 of file mmu.c.

#define PT64_INDEX	(	address,
		level
	)	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))

Definition at line 101 of file mmu.c.

#define PT64_LEVEL_BITS 9

Definition at line 96 of file mmu.c.

#define PT64_LEVEL_SHIFT ( level ) (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)

Definition at line 98 of file mmu.c.

#define PT64_LVL_ADDR_MASK ( level )

Value:

(PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \

* PT64_LEVEL_BITS))) - 1))

Definition at line 121 of file mmu.c.

#define PT64_LVL_OFFSET_MASK ( level )

Value:

(PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \

* PT64_LEVEL_BITS))) - 1))

Definition at line 124 of file mmu.c.

#define PT64_PERM_MASK

Value:

(PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \

| PT64_NX_MASK)

Definition at line 135 of file mmu.c.

#define PT64_SECOND_AVAIL_BITS_SHIFT 52

Definition at line 94 of file mmu.c.

#define PT_FIRST_AVAIL_BITS_SHIFT 10

Definition at line 93 of file mmu.c.

#define PTE_LIST_EXT 3

Definition at line 154 of file mmu.c.

#define PTE_PREFETCH_NUM 8

Definition at line 91 of file mmu.c.

#define PTTYPE 64

Definition at line 3462 of file mmu.c.

#define PTTYPE 32

Definition at line 3462 of file mmu.c.

#define rmap_printk ( x... ) do { } while (0)

Definition at line 72 of file mmu.c.

#define RMAP_RECYCLE_THRESHOLD 1000

Definition at line 1410 of file mmu.c.

#define SHADOW_PT_INDEX	(	addr,
		level
	)	PT64_INDEX(addr, level)

Definition at line 151 of file mmu.c.

#define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)

Definition at line 148 of file mmu.c.

#define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))

Definition at line 149 of file mmu.c.

Typedef Documentation

typedef void(* pte_list_walk_fn)(u64 *spte)

Definition at line 948 of file mmu.c.

Enumeration Type Documentation

anonymous enum

Enumerator:

AUDIT_PRE_PAGE_FAULT
AUDIT_POST_PAGE_FAULT
AUDIT_PRE_PTE_WRITE
AUDIT_POST_PTE_WRITE
AUDIT_PRE_SYNC
AUDIT_POST_SYNC

Definition at line 53 of file mmu.c.

Function Documentation

void __kvm_mmu_free_some_pages ( struct kvm_vcpu * vcpu )

Definition at line 4046 of file mmu.c.

EXPORT_SYMBOL_GPL ( kvm_mmu_set_mmio_spte_mask )

EXPORT_SYMBOL_GPL ( kvm_mmu_set_mask_ptes )

EXPORT_SYMBOL_GPL ( kvm_mmu_unprotect_page )

EXPORT_SYMBOL_GPL ( kvm_get_guest_memory_type )

EXPORT_SYMBOL_GPL ( handle_mmio_page_fault_common )

EXPORT_SYMBOL_GPL ( kvm_init_shadow_mmu )

EXPORT_SYMBOL_GPL ( kvm_mmu_reset_context )

EXPORT_SYMBOL_GPL ( kvm_mmu_load )

EXPORT_SYMBOL_GPL ( kvm_mmu_unload )

EXPORT_SYMBOL_GPL ( kvm_mmu_unprotect_page_virt )

EXPORT_SYMBOL_GPL ( kvm_mmu_page_fault )

EXPORT_SYMBOL_GPL ( kvm_mmu_invlpg )

EXPORT_SYMBOL_GPL ( kvm_enable_tdp )

EXPORT_SYMBOL_GPL ( kvm_disable_tdp )

EXPORT_SYMBOL_GPL ( kvm_mmu_get_spte_hierarchy )

int handle_mmio_page_fault_common	(	struct kvm_vcpu *	vcpu,
		u64	addr,
		bool	direct
	)

Definition at line 3176 of file mmu.c.

int kvm_age_hva	(	struct kvm *	kvm,
		unsigned long	hva
	)

Definition at line 1425 of file mmu.c.

void kvm_disable_tdp ( void )

Definition at line 4120 of file mmu.c.

void kvm_enable_tdp ( void )

Definition at line 4114 of file mmu.c.

u8 kvm_get_guest_memory_type	(	struct kvm_vcpu *	vcpu,
		gfn_t	gfn
	)

Definition at line 2287 of file mmu.c.

int kvm_init_shadow_mmu	(	struct kvm_vcpu *	vcpu,
		struct kvm_mmu *	context
	)

Definition at line 3683 of file mmu.c.

unsigned int kvm_mmu_calculate_mmu_pages ( struct kvm * kvm )

Definition at line 4320 of file mmu.c.

void kvm_mmu_change_mmu_pages	(	struct kvm *	kvm,
		unsigned int	goal_nr_mmu_pages
	)

Definition at line 2138 of file mmu.c.

int kvm_mmu_create ( struct kvm_vcpu * vcpu )

Definition at line 4156 of file mmu.c.

void kvm_mmu_destroy ( struct kvm_vcpu * vcpu )

Definition at line 4358 of file mmu.c.

void kvm_mmu_flush_tlb ( struct kvm_vcpu * vcpu )

Definition at line 3374 of file mmu.c.

int kvm_mmu_get_spte_hierarchy	(	struct kvm_vcpu *	vcpu,
		u64	addr,
		u64	sptes[4]
	)

Definition at line 4339 of file mmu.c.

void kvm_mmu_invlpg	(	struct kvm_vcpu *	vcpu,
		gva_t	gva
	)

Definition at line 4106 of file mmu.c.

int kvm_mmu_load ( struct kvm_vcpu * vcpu )

Definition at line 3786 of file mmu.c.

void kvm_mmu_module_exit ( void )

Definition at line 4367 of file mmu.c.

int kvm_mmu_module_init ( void )

Definition at line 4291 of file mmu.c.

int kvm_mmu_page_fault	(	struct kvm_vcpu *	vcpu,
		gva_t	cr2,
		u32	error_code,
		void *	insn,
		int	insn_len
	)

Definition at line 4070 of file mmu.c.

void kvm_mmu_pte_write	(	struct kvm_vcpu *	vcpu,
		gpa_t	gpa,
		const u8 *	new,
		int	bytes
	)

Definition at line 3961 of file mmu.c.

int kvm_mmu_reset_context ( struct kvm_vcpu * vcpu )

Definition at line 3779 of file mmu.c.

void kvm_mmu_set_mask_ptes	(	u64	user_mask,
		u64	accessed_mask,
		u64	dirty_mask,
		u64	nx_mask,
		u64	x_mask
	)

Definition at line 238 of file mmu.c.

void kvm_mmu_set_mmio_spte_mask ( u64 mmio_mask )

Definition at line 194 of file mmu.c.

int kvm_mmu_setup ( struct kvm_vcpu * vcpu )

Definition at line 4168 of file mmu.c.

void kvm_mmu_slot_remove_write_access	(	struct kvm *	kvm,
		int	slot
	)

Definition at line 4176 of file mmu.c.

void kvm_mmu_sync_roots ( struct kvm_vcpu * vcpu )

Definition at line 3110 of file mmu.c.

void kvm_mmu_unload ( struct kvm_vcpu * vcpu )

Definition at line 3806 of file mmu.c.

int kvm_mmu_unprotect_page	(	struct kvm *	kvm,
		gfn_t	gfn
	)

Definition at line 2163 of file mmu.c.

int kvm_mmu_unprotect_page_virt	(	struct kvm_vcpu *	vcpu,
		gva_t	gva
	)

Definition at line 4030 of file mmu.c.

void kvm_mmu_write_protect_pt_masked	(	struct kvm *	kvm,
		struct kvm_memory_slot *	slot,
		gfn_t	gfn_offset,
		unsigned long	mask
	)

kvm_mmu_write_protect_pt_masked - write protect selected PT level pages : kvm instance : slot to protect : start of the BITS_PER_LONG pages we care about : indicates which pages we should protect

Used when we do not need to care about huge page mappings: e.g. during dirty logging we do not have any such mappings.

Definition at line 1174 of file mmu.c.

void kvm_mmu_zap_all ( struct kvm * kvm )

Definition at line 4200 of file mmu.c.

void kvm_set_spte_hva	(	struct kvm *	kvm,
		unsigned long	hva,
		pte_t	pte
	)

Definition at line 1341 of file mmu.c.

int kvm_test_age_hva	(	struct kvm *	kvm,
		unsigned long	hva
	)

Definition at line 1430 of file mmu.c.

int kvm_unmap_hva	(	struct kvm *	kvm,
		unsigned long	hva
	)

Definition at line 1331 of file mmu.c.

int kvm_unmap_hva_range	(	struct kvm *	kvm,
		unsigned long	start,
		unsigned long	end
	)

Definition at line 1336 of file mmu.c.

Variable Documentation

bool tdp_enabled = false

Definition at line 51 of file mmu.c.

Data Structures

Macros

Typedefs

Enumerations

Functions

Variables

Macro Definition Documentation

Typedef Documentation

Enumeration Type Documentation

Function Documentation

Variable Documentation