packages/kernel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
kernel/0005-haoc-kernel.patch
liuzh c887853a77 fix haoc-kernel patch file.
2024-05-23 15:39:12 +08:00

13473 lines
401 KiB
Diff
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

From 099672494b5fbd626031f313bbdbc42d1eb3f02d Mon Sep 17 00:00:00 2001
From: liuzh <liuzhh@zgclab.edu.cn>
Date: Wed, 22 May 2024 16:36:55 +0800
Subject: [PATCH] Squashed commit of the following:
commit 909ad06b3bf629d9af4e143347c1d1ef8a3a5808
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Wed May 22 16:23:13 2024 +0800
fix mte_sync_tags() parameters after rebase to 6.6.0-27.0.0.
commit 1cbf51371b0539a45f816419b2da82cf36162b4a
Author: ljl <ljl2013@163.com>
Date: Mon Mar 25 08:01:32 2024 +0000
IEE SI: Removed redundant codes.
commit 0178bfc79ad1769a36f4165348a671d2182cff55
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Mon Mar 25 11:01:11 2024 +0800
Fix bugs on qemu when opening CONFIG_CREDP.
commit 8e714f6e8f2ace5a6fc900b4bce6b03c83c41870
Author: ljl <ljl2013@163.com>
Date: Thu Mar 21 04:44:26 2024 +0000
IEE SI: Remove PAN operations as BTLB BUG is already fixed.
commit 7b5fc74cb99e377d3bc59da81612cd6f3dd8a4d8
Author: ljl <lvjinglin2013@163.com>
Date: Wed Mar 20 18:31:47 2024 +0800
IEE SI: Migration of iee rwx gate.
commit aad2c7e89c9c4ad8ff0fb3ee53cd1b974144a283
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Mon Mar 18 15:32:43 2024 +0800
modify slub.c set_track_prepare()
commit 7452bac06ec09bf8321dfdbfb8b6a429d2cd8637
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Thu Mar 21 11:26:19 2024 +0800
Set pgd of lm Privileged.
commit 33934cfc3eed798a3a687bf86c6bd92697e68ba9
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Tue Mar 19 17:14:32 2024 +0800
Delete some redundant code and put trans_pgd into IEE.
commit 2bfe9008a72f8b8ac237bc7a5f99f9d40e84c247
Author: zhangshiyang17@mails.ucas.ac.cn <zhangshiyang17@mails.ucas.ac.cn>
Date: Mon Mar 18 11:47:50 2024 +0000
Fix bugs on physical when opening CONFIG_IEE and CONFIG_PTP.
commit dafa2df600757511ce3e8f178e05e28adabdf39b
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Mon Mar 18 10:40:42 2024 +0800
Fix bugs on qemu when opening CONFIG_IEE and CONFIG_PTP.
commit 9231a9f6b34c62090b5f202c9c64a52bfdac7a73
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Thu Mar 14 16:34:53 2024 +0800
Fix compiling bugs of CONFIG_PTP.
commit 6469df3bcce32896c2cb297d3cd7ead82c33f35d
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Thu Mar 14 11:10:00 2024 +0800
Fix bugs on qemu when opening CONFIG_IEE and CONFIG_INTERRUPTABLE.
commit 5f1773dada622a3514c9ed6aa72dd50e918f2664
Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn>
Date: Wed Mar 13 17:31:39 2024 +0800
Fix bugs on qemu when opening CONFIG_IEE.
commit 73f433a093fa84cffa5e11e86bed6f17c9b30a39
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Tue Mar 12 15:32:29 2024 +0800
fix the map of IEE_SI_TEXT.
commit 9b92deb4b2338093d9b04f4b81f162855b31c983
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Sun Mar 10 16:11:13 2024 +0800
modified to be able to compile.
can start the kernel with qemu and successfully reach `start_kernel()`.
commit e892ec4790d72e9433b48b0221e7e6dc4c361dd9
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Thu Mar 7 14:27:45 2024 +0800
fix some conflicts
commit fdec7e39345e81e867e01258487f88801b790b02
Author: liuzh <liuzhh@zgclab.edu.cn>
Date: Wed Mar 6 12:31:11 2024 +0800
migrate openeuler-commit code. (need some fix before compiling)
---
Makefile | 3 +-
arch/arm64/Kconfig | 18 +
arch/arm64/include/asm/assembler.h | 67 +
arch/arm64/include/asm/daifflags.h | 16 +
arch/arm64/include/asm/efi.h | 4 +
arch/arm64/include/asm/fixmap.h | 3 +
arch/arm64/include/asm/hw_breakpoint.h | 12 +
arch/arm64/include/asm/iee-access.h | 36 +
arch/arm64/include/asm/iee-cred.h | 150 ++
arch/arm64/include/asm/iee-def.h | 74 +
arch/arm64/include/asm/iee-si.h | 64 +
arch/arm64/include/asm/iee-slab.h | 23 +
arch/arm64/include/asm/iee-token.h | 40 +
arch/arm64/include/asm/iee.h | 10 +
arch/arm64/include/asm/kernel-pgtable.h | 21 +
arch/arm64/include/asm/koi.h | 335 +++++
arch/arm64/include/asm/memory.h | 24 +
arch/arm64/include/asm/mmu_context.h | 20 +
arch/arm64/include/asm/pgalloc.h | 4 +
arch/arm64/include/asm/pgtable-hwdef.h | 11 +
arch/arm64/include/asm/pgtable.h | 304 +++-
arch/arm64/include/asm/pointer_auth.h | 5 +
arch/arm64/include/asm/sysreg.h | 58 +
arch/arm64/include/asm/tlb.h | 9 +
arch/arm64/include/asm/tlbflush.h | 58 +-
arch/arm64/kernel/Makefile | 2 +
arch/arm64/kernel/armv8_deprecated.c | 16 +
arch/arm64/kernel/asm-offsets.c | 11 +
arch/arm64/kernel/cpu_errata.c | 12 +
arch/arm64/kernel/cpufeature.c | 79 +
arch/arm64/kernel/debug-monitors.c | 4 +
arch/arm64/kernel/entry-common.c | 4 +
arch/arm64/kernel/entry.S | 611 ++++++++
arch/arm64/kernel/fpsimd.c | 4 +
arch/arm64/kernel/head.S | 56 +
arch/arm64/kernel/hibernate.c | 14 +
arch/arm64/kernel/hw_breakpoint.c | 99 ++
arch/arm64/kernel/iee/Makefile | 1 +
arch/arm64/kernel/iee/iee-func.c | 187 +++
arch/arm64/kernel/iee/iee-gate.S | 174 +++
arch/arm64/kernel/iee/iee.c | 1360 +++++++++++++++++
arch/arm64/kernel/koi/Makefile | 1 +
arch/arm64/kernel/koi/koi.c | 1327 +++++++++++++++++
arch/arm64/kernel/mte.c | 5 +
arch/arm64/kernel/process.c | 19 +-
arch/arm64/kernel/proton-pack.c | 8 +
arch/arm64/kernel/setup.c | 33 +
arch/arm64/kernel/traps.c | 26 +
arch/arm64/kernel/vmlinux.lds.S | 61 +
arch/arm64/mm/context.c | 91 +-
arch/arm64/mm/fault.c | 9 +
arch/arm64/mm/fixmap.c | 74 +-
arch/arm64/mm/init.c | 34 +
arch/arm64/mm/mmu.c | 1780 +++++++++++++++++++----
arch/arm64/mm/pgd.c | 39 +
arch/arm64/mm/proc.S | 28 +
arch/arm64/mm/trans_pgd.c | 46 +
drivers/firmware/efi/arm-runtime.c | 4 +
drivers/firmware/efi/memmap.c | 20 +
drivers/tty/serial/earlycon.c | 4 +
drivers/usb/early/ehci-dbgp.c | 4 +
fs/coredump.c | 8 +
fs/exec.c | 20 +
fs/nfs/flexfilelayout/flexfilelayout.c | 9 +
fs/nfs/nfs4idmap.c | 9 +
fs/nfsd/auth.c | 38 +
fs/nfsd/nfs4callback.c | 12 +-
fs/nfsd/nfs4recover.c | 9 +
fs/nfsd/nfsfh.c | 9 +
fs/open.c | 26 +
fs/overlayfs/dir.c | 9 +
fs/overlayfs/super.c | 12 +
fs/smb/client/cifs_spnego.c | 9 +
fs/smb/client/cifsacl.c | 9 +
include/asm-generic/early_ioremap.h | 3 +
include/asm-generic/fixmap.h | 18 +
include/asm-generic/pgalloc.h | 54 +
include/asm-generic/vmlinux.lds.h | 24 +-
include/linux/cred.h | 45 +-
include/linux/efi.h | 9 +
include/linux/iee-func.h | 27 +
include/linux/module.h | 1 +
include/linux/sched.h | 19 +
init/main.c | 28 +-
kernel/cred.c | 182 +++
kernel/exit.c | 8 +
kernel/fork.c | 316 ++--
kernel/groups.c | 7 +
kernel/kthread.c | 13 +
kernel/smpboot.c | 9 +
kernel/sys.c | 107 ++
kernel/umh.c | 10 +
kernel/user_namespace.c | 18 +
mm/Kconfig | 12 +
mm/damon/ops-common.c | 1 +
mm/debug_vm_pgtable.c | 24 +
mm/early_ioremap.c | 57 +
mm/huge_memory.c | 30 +-
mm/init-mm.c | 17 +
mm/memory.c | 14 +
mm/slub.c | 198 ++-
mm/sparse-vmemmap.c | 21 +
mm/vmalloc.c | 2 +-
net/dns_resolver/dns_key.c | 9 +
security/commoncap.c | 169 +++
security/keys/keyctl.c | 23 +
security/keys/process_keys.c | 53 +
security/security.c | 15 +
109 files changed, 8945 insertions(+), 397 deletions(-)
create mode 100644 arch/arm64/include/asm/iee-access.h
create mode 100644 arch/arm64/include/asm/iee-cred.h
create mode 100644 arch/arm64/include/asm/iee-def.h
create mode 100644 arch/arm64/include/asm/iee-si.h
create mode 100644 arch/arm64/include/asm/iee-slab.h
create mode 100644 arch/arm64/include/asm/iee-token.h
create mode 100644 arch/arm64/include/asm/iee.h
create mode 100644 arch/arm64/include/asm/koi.h
create mode 100644 arch/arm64/kernel/iee/Makefile
create mode 100644 arch/arm64/kernel/iee/iee-func.c
create mode 100644 arch/arm64/kernel/iee/iee-gate.S
create mode 100644 arch/arm64/kernel/iee/iee.c
create mode 100644 arch/arm64/kernel/koi/Makefile
create mode 100644 arch/arm64/kernel/koi/koi.c
create mode 100644 include/linux/iee-func.h
diff --git a/Makefile b/Makefile
index 8e6d9b894b1e..20c367b5957d 100644
--- a/Makefile
+++ b/Makefile
@@ -554,7 +554,7 @@ LINUXINCLUDE := \
-I$(objtree)/include \
$(USERINCLUDE)
-KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE
+KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE -march=armv8.1-a
KBUILD_CFLAGS :=
KBUILD_CFLAGS += -std=gnu11
@@ -563,6 +563,7 @@ KBUILD_CFLAGS += -funsigned-char
KBUILD_CFLAGS += -fno-common
KBUILD_CFLAGS += -fno-PIE
KBUILD_CFLAGS += -fno-strict-aliasing
+KBUILD_CFLAGS += -march=armv8.1-a
KBUILD_CPPFLAGS := -D__KERNEL__
KBUILD_RUSTFLAGS := $(rust_common_flags) \
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 2a875546bdc7..7448afc90c0a 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1730,6 +1730,24 @@ config UNMAP_KERNEL_AT_EL0
If unsure, say Y.
+# Config for iee
+config IEE
+ depends on ARM64
+ depends on ARM64_PAN
+ depends on ARM64_VA_BITS_48
+ depends on ARM64_4K_PAGES
+ def_bool y
+
+# Config for support of interruption of iee
+config IEE_INTERRUPTABLE
+ depends on IEE
+ def_bool n
+
+# Config for credentials isolation
+config CREDP
+ depends on IEE
+ def_bool y
+
config MITIGATE_SPECTRE_BRANCH_HISTORY
bool "Mitigate Spectre style attacks against branch history" if EXPERT
default y
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 38b23786aeb4..6af10d509c2e 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -26,6 +26,41 @@
#include <asm/ptrace.h>
#include <asm/thread_info.h>
+#ifdef CONFIG_IEE
+ .macro iee_si_restore_daif, flags:req
+ msr daifclr, #0xf
+ tbnz \flags, #6, 114221f
+ tbnz \flags, #7, 114210f
+ tbnz \flags, #8, 114100f
+ msr daifset, #0b000
+ b 114514f
+114221:
+ tbnz \flags, #7, 114211f
+ tbnz \flags, #8, 114101f
+ msr daifset, #0b001
+ b 114514f
+114211:
+ tbnz \flags, #8, 114111f
+ msr daifset, #0b011
+ b 114514f
+114210:
+ tbnz \flags, #8, 114110f
+ msr daifset, #0b010
+ b 114514f
+114100:
+ msr daifset, #0b100
+ b 114514f
+114101:
+ msr daifset, #0b101
+ b 114514f
+114110:
+ msr daifset, #0b110
+ b 114514f
+114111:
+ msr daifset, #0b111
+114514:
+ .endm
+#endif
/*
* Provide a wxN alias for each wN register so what we can paste a xN
* reference after a 'w' to obtain the 32-bit version.
@@ -52,7 +87,11 @@ alternative_else_nop_endif
.macro disable_daif
disable_allint
+// #ifdef CONFIG_IEE
+// msr daifset, #0x7
+// #else
msr daifset, #0xf
+// #endif
.endm
.macro enable_daif
@@ -69,7 +108,11 @@ alternative_else_nop_endif
.endm
.macro restore_irq, flags
+// #ifdef CONFIG_IEE
+// iee_si_restore_daif \flags
+// #else
msr daif, \flags
+// #endif
.endm
.macro enable_dbg
@@ -77,20 +120,44 @@ alternative_else_nop_endif
.endm
.macro disable_step_tsk, flgs, tmp
+// #ifdef CONFIG_IEE
+// 1145:
+// tbz \flgs, #TIF_SINGLESTEP, 9990f
+// mrs \tmp, mdscr_el1
+// bic \tmp, \tmp, #DBG_MDSCR_SS
+// orr \tmp, \tmp, #DBG_MDSCR_MDE
+// msr mdscr_el1, \tmp
+// isb // Synchronise with enable_dbg
+// mrs \tmp, mdscr_el1
+// tbz \tmp, #15, 1145b
+// #else
tbz \flgs, #TIF_SINGLESTEP, 9990f
mrs \tmp, mdscr_el1
bic \tmp, \tmp, #DBG_MDSCR_SS
msr mdscr_el1, \tmp
isb // Synchronise with enable_dbg
+// #endif
9990:
.endm
/* call with daif masked */
.macro enable_step_tsk, flgs, tmp
+// #ifdef CONFIG_IEE
+// 1146:
+// tbz \flgs, #TIF_SINGLESTEP, 9990f
+// mrs \tmp, mdscr_el1
+// orr \tmp, \tmp, #DBG_MDSCR_SS
+// orr \tmp, \tmp, #DBG_MDSCR_MDE
+// msr mdscr_el1, \tmp
+// isb // Synchronise with enable_dbg
+// mrs \tmp, mdscr_el1
+// tbz \tmp, #15, 1146b
+// #else
tbz \flgs, #TIF_SINGLESTEP, 9990f
mrs \tmp, mdscr_el1
orr \tmp, \tmp, #DBG_MDSCR_SS
msr mdscr_el1, \tmp
+// #endif
9990:
.endm
diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h
index 2417cc6b1631..cb5b4c2e03b8 100644
--- a/arch/arm64/include/asm/daifflags.h
+++ b/arch/arm64/include/asm/daifflags.h
@@ -26,11 +26,19 @@ static inline void local_daif_mask(void)
(read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF |
GIC_PRIO_PSR_I_SET)));
+// #ifdef CONFIG_IEE
+// asm volatile(
+// "msr daifset, #0x7 // local_daif_mask\n"
+// :
+// :
+// : "memory");
+// #else
asm volatile(
"msr daifset, #0xf // local_daif_mask\n"
:
:
: "memory");
+// #endif
/* Don't really care for a dsb here, we don't intend to enable IRQs */
if (system_uses_irq_prio_masking())
@@ -118,7 +126,11 @@ static inline void local_daif_restore(unsigned long flags)
gic_write_pmr(pmr);
}
+// #ifdef CONFIG_IEE
+// iee_si_write_daif(flags);
+// #else
write_sysreg(flags, daif);
+// #endif
/* If we can take asynchronous errors we can take NMIs */
if (system_uses_nmi()) {
@@ -151,7 +163,11 @@ static inline void local_daif_inherit(struct pt_regs *regs)
* system_has_prio_mask_debugging() won't restore the I bit if it can
* use the pmr instead.
*/
+// #ifdef CONFIG_IEE
+// iee_si_write_daif(flags);
+// #else
write_sysreg(flags, daif);
+// #endif
/* The ALLINT field is at the same position in pstate and ALLINT */
if (system_uses_nmi()) {
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index bcd5622aa096..76c4bd6c2b20 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -58,7 +58,11 @@ void arch_efi_call_virt_teardown(void);
#define arch_efi_save_flags(state_flags) \
((void)((state_flags) = read_sysreg(daif)))
+// #ifdef CONFIG_IEE
+// #define arch_efi_restore_flags(state_flags) iee_si_write_daif(state_flags)
+// #else
#define arch_efi_restore_flags(state_flags) write_sysreg(state_flags, daif)
+// #endif
/* arch specific definitions used by the stub code */
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 58c294a96676..095a0731dce3 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -108,6 +108,9 @@ void __init fixmap_copy(pgd_t *pgdir);
#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR)
extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot);
+#ifdef CONFIG_PTP
+extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot);
+#endif
#include <asm-generic/fixmap.h>
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index 84055329cd8b..f72d89bb9a32 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -104,6 +104,18 @@ static inline void decode_ctrl_reg(u32 reg,
write_sysreg(VAL, dbg##REG##N##_el1);\
} while (0)
+#ifdef CONFIG_IEE
+#define IEE_SI_AARCH64_DBG_READ(N, REG, VAL) do{\
+ VAL = this_cpu_read(iee_si_user_##REG##N);\
+} while (0)
+
+#define IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL) do{\
+ u64 __val = (u64)(VAL); \
+ this_cpu_write(iee_si_user_##REG##N, __val);\
+ iee_rwx_gate_entry(IEE_WRITE_AFSR0);\
+} while (0)
+#endif
+
struct task_struct;
struct notifier_block;
struct perf_event_attr;
diff --git a/arch/arm64/include/asm/iee-access.h b/arch/arm64/include/asm/iee-access.h
new file mode 100644
index 000000000000..79604c21a510
--- /dev/null
+++ b/arch/arm64/include/asm/iee-access.h
@@ -0,0 +1,36 @@
+#ifndef _LINUX_IEE_ACCESS_H
+#define _LINUX_IEE_ACCESS_H
+
+#include <asm/iee-def.h>
+#include <asm/iee-slab.h>
+
+extern unsigned long long iee_rw_gate(int flag, ...);
+
+#ifdef CONFIG_IEE
+void iee_write_in_byte(void *ptr, u64 data, int length)
+{
+ iee_rw_gate(IEE_WRITE_IN_BYTE, ptr, data, length);
+}
+
+void iee_memset(void *ptr, int data, size_t n)
+{
+ iee_rw_gate(IEE_MEMSET, ptr, data, n);
+}
+
+void iee_set_track(struct track *ptr, struct track *data)
+{
+ iee_rw_gate(IEE_OP_SET_TRACK, ptr, data);
+}
+
+void iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr)
+{
+ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr);
+}
+
+void iee_write_entry_task(struct task_struct *tsk)
+{
+ iee_rw_gate(IEE_WRITE_ENTRY_TASK, tsk);
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee-cred.h b/arch/arm64/include/asm/iee-cred.h
new file mode 100644
index 000000000000..b8c3bb53f98a
--- /dev/null
+++ b/arch/arm64/include/asm/iee-cred.h
@@ -0,0 +1,150 @@
+#ifndef _LINUX_IEE_CRED_H
+#define _LINUX_IEE_CRED_H
+
+#include <linux/cred.h>
+#include <asm/iee-def.h>
+
+extern unsigned long long iee_rw_gate(int flag, ...);
+
+#ifdef CONFIG_CREDP
+static void __maybe_unused iee_copy_cred(const struct cred *old, struct cred *new)
+{
+ iee_rw_gate(IEE_OP_COPY_CRED,old,new);
+}
+
+static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_UID,cred,uid);
+}
+
+static void __maybe_unused iee_set_cred_gid(struct cred *cred, kgid_t gid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_GID,cred,gid);
+}
+
+static void __maybe_unused iee_set_cred_suid(struct cred *cred, kuid_t suid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_SUID,cred,suid);
+}
+
+static void __maybe_unused iee_set_cred_sgid(struct cred *cred, kgid_t sgid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_SGID,cred,sgid);
+}
+
+static void __maybe_unused iee_set_cred_euid(struct cred *cred, kuid_t euid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_EUID,cred,euid);
+}
+
+static void __maybe_unused iee_set_cred_egid(struct cred *cred, kgid_t egid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_EGID,cred,egid);
+}
+
+static void __maybe_unused iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_FSUID,cred,fsuid);
+}
+
+static void __maybe_unused iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_FSGID,cred,fsgid);
+}
+
+static void __maybe_unused iee_set_cred_user(struct cred *cred, struct user_struct *user)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_USER,cred,user);
+}
+
+static void __maybe_unused iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_USER_NS,cred,user_ns);
+}
+
+static void __maybe_unused iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS,cred,ucounts);
+}
+
+static void __maybe_unused iee_set_cred_group_info(struct cred *cred, struct group_info *group_info)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO,cred,group_info);
+}
+
+static void __maybe_unused iee_set_cred_securebits(struct cred *cred, unsigned securebits)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS,cred,securebits);
+}
+
+static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER,cred,cap_inheritable);
+}
+
+static void __maybe_unused iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM,cred,cap_permitted);
+}
+
+static void __maybe_unused iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT,cred,cap_effective);
+}
+
+static void __maybe_unused iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET,cred,cap_bset);
+}
+
+static void __maybe_unused iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT,cred,cap_ambient);
+}
+
+#ifdef CONFIG_KEYS
+static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING,cred,jit_keyring);
+}
+
+static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING,cred,session_keyring);
+}
+
+static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING,cred,process_keyring);
+}
+
+static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING,cred,thread_keyring);
+}
+
+static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING,cred,request_key_auth);
+}
+#endif
+
+static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE,cred,i);
+}
+
+#ifdef CONFIG_SECURITY
+static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_SECURITY,cred,security);
+}
+#endif
+
+static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_RCU,cred,rcu);
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee-def.h b/arch/arm64/include/asm/iee-def.h
new file mode 100644
index 000000000000..76e59259e4d1
--- /dev/null
+++ b/arch/arm64/include/asm/iee-def.h
@@ -0,0 +1,74 @@
+// Function Identifiers with Parameters Description
+
+#define IEE_WRITE_IN_BYTE 0 // Parameters: void *ptr, __u64 data, int length
+#define IEE_OP_SET_PTE 1 // Parameters: pte_t *ptep, pte_t pte
+#define IEE_OP_SET_PMD 2 // Parameters: pmd_t *pmdp, pmd_t pmd
+#define IEE_OP_SET_PUD 3 // Parameters: pud_t *pudp, pud_t pud
+#define IEE_OP_SET_P4D 4 // Parameters: p4d_t *p4dp, p4d_t p4d
+#define IEE_OP_SET_BM_PTE 5 // Parameters: pte_t *ptep, pte_t pte
+#define IEE_OP_SET_SWAPPER_PGD 6 // Parameters: pgd_t *pgdp, pgd_t pgd
+#define IEE_OP_SET_TRAMP_PGD 7 // Parameters: pgd_t *pgdp, pgd_t pgd
+#define IEE_OP_SET_CMPXCHG 8 // Parameters: pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval
+#define IEE_OP_SET_XCHG 9 // Parameters: pte_t *ptep, pteval_t pteval
+#define IEE_OP_COPY_CRED 10 // Parameters: struct cred *old, struct cred *new
+#define IEE_OP_SET_CRED_UID 11 // Parameters: struct cred *cred, kuid_t uid
+#define IEE_OP_SET_CRED_GID 12 // Parameters: struct cred *cred, kgid_t gid
+#define IEE_OP_SET_CRED_SUID 13 // Parameters: struct cred *cred, kuid_t suid
+#define IEE_OP_SET_CRED_SGID 14 // Parameters: struct cred *cred, kgid_t sgid
+#define IEE_OP_SET_CRED_EUID 15 // Parameters: struct cred *cred, kuid_t euid
+#define IEE_OP_SET_CRED_EGID 16 // Parameters: struct cred *cred, kgid_t egid
+#define IEE_OP_SET_CRED_FSUID 17 // Parameters: struct cred *cred, kuid_t fsuid
+#define IEE_OP_SET_CRED_FSGID 18 // Parameters: struct cred *cred, kgid_t fsgid
+#define IEE_OP_SET_CRED_USER 19 // Parameters: struct cred *cred, struct user_struct *user
+#define IEE_OP_SET_CRED_USER_NS 20 // Parameters: struct cred *cred, struct user_namespace *user_ns
+#define IEE_OP_SET_CRED_GROUP_INFO 21 // Parameters: struct cred *cred, struct group_info *group_info
+#define IEE_OP_SET_CRED_SECUREBITS 22 // Parameters: struct cred *cred, unsigned securebits
+#define IEE_OP_SET_CRED_CAP_INHER 23 // Parameters: struct cred *cred, kernel_cap_t cap_inheritable
+#define IEE_OP_SET_CRED_CAP_PERM 24 // Parameters: struct cred *cred, kernel_cap_t cap_permitted
+#define IEE_OP_SET_CRED_CAP_EFFECT 25 // Parameters: struct cred *cred, kernel_cap_t cap_effective
+#define IEE_OP_SET_CRED_CAP_BSET 26 // Parameters: struct cred *cred, kernel_cap_t cap_bset
+#define IEE_OP_SET_CRED_CAP_AMBIENT 27 // Parameters: struct cred *cred, kernel_cap_t cap_ambient
+#define IEE_OP_SET_CRED_JIT_KEYRING 28 // Parameters: struct cred *cred, unsigned char jit_keyring
+#define IEE_OP_SET_CRED_SESS_KEYRING 29 // Parameters: struct cred *cred, struct key *session_keyring
+#define IEE_OP_SET_CRED_PROC_KEYRING 30 // Parameters: struct cred *cred, struct key *process_keyring
+#define IEE_OP_SET_CRED_THREAD_KEYRING 31 // Parameters: struct cred *cred, struct key *thread_keyring
+#define IEE_OP_SET_CRED_REQ_KEYRING 32 // Parameters: struct cred *cred, struct key *request_key_auth
+#define IEE_OP_SET_CRED_NON_RCU 33 // Parameters: struct cred *cred, int non_rcu
+#define IEE_OP_SET_CRED_ATSET_USAGE 34 // Parameters: struct cred *cred, int i
+#define IEE_OP_SET_CRED_ATOP_USAGE 35 // Parameters: struct cred *cred, int flag
+#define IEE_OP_SET_CRED_SECURITY 36 // Parameters: struct cred *cred, void *security
+#define IEE_OP_SET_CRED_RCU 37 // Parameters: struct cred *cred, struct rcu_head *rcu
+#define IEE_MEMSET 38 // Parameters: void *ptr, int data, size_t n
+#define IEE_OP_SET_TRACK 39 // Parameters: struct track *ptr, struct track *data
+#define IEE_OP_SET_FREEPTR 40 // Parameters: void **pptr, void *ptr
+#define IEE_OP_SET_PTE_U 41 // Parameters: pte_t *ptep, pte_t pte
+#define IEE_OP_SET_PTE_P 42 // Parameters: pte_t *ptep, pte_t pte
+#define IEE_SET_TOKEN_MM 43 // Parameters: struct task_token *token, struct mm_struct *mm
+#define IEE_SET_TOKEN_PGD 44 // Parameters: struct task_token *token, pgd_t *pgd
+#define IEE_INIT_TOKEN 45 // Parameters: struct task_struct *tsk, void *kernel_stack, void *iee_stack
+#define IEE_FREE_TOKEN 46 // Parameters: struct task_struct *tsk
+#define IEE_READ_TOKEN_STACK 47 // Parameters: struct task_struct *tsk
+#define IEE_WRITE_ENTRY_TASK 48 // Parameters: struct task_struct *tsk
+#define IEE_OP_SET_CRED_UCOUNTS 49 // Parameters: struct cred *cred, struct ucounts *ucounts
+#ifdef CONFIG_KOI
+#define IEE_READ_KOI_STACK 50 // Parameters: struct task_struct *tsk
+#define IEE_WRITE_KOI_STACK 51 // Parameters: struct task_struct *tsk, unsigned long koi_stack
+#define IEE_READ_TOKEN_TTBR1 52 // Parameters: struct task_struct *tsk
+#define IEE_WRITE_TOKEN_TTBR1 53 // Parameters: struct task_struct *tsk, unsigned long current_ttbr1
+#define IEE_READ_KOI_KERNEL_STACK 54 // Parameters: struct task_struct *tsk
+#define IEE_WRITE_KOI_KERNEL_STACK 55 // Parameters: struct task_struct *tsk, unsigned long kernel_stack
+#define IEE_READ_KOI_STACK_BASE 56 // Parameters: struct task_struct *tsk
+#define IEE_WRITE_KOI_STACK_BASE 57 // Parameters: struct task_struct *tsk, unsigned long koi_stack_base
+#endif
+
+/* Add new IEE ops here */
+
+#define AT_ADD 1
+#define AT_INC_NOT_ZERO 2
+#define AT_SUB_AND_TEST 3
+/* Atomic ops for atomic_t */
+
+#ifdef CONFIG_KOI
+#define IEE_SWITCH_TO_KERNEL 7
+#define IEE_SWITCH_TO_KOI 8
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee-si.h b/arch/arm64/include/asm/iee-si.h
new file mode 100644
index 000000000000..e67d81db66a5
--- /dev/null
+++ b/arch/arm64/include/asm/iee-si.h
@@ -0,0 +1,64 @@
+#ifndef _LINUX_IEE_SI_H
+#define _LINUX_IEE_SI_H
+
+#include <asm/sysreg.h>
+#define __iee_si_code __section(".iee.si_text")
+#define __iee_si_data __section(".iee.si_data")
+
+/* Used for copying globals that iee rwx gate needs. */
+extern unsigned long iee_base_idmap_pg_dir;
+extern unsigned long iee_base_reserved_pg_dir;
+extern unsigned long iee_base__bp_harden_el1_vectors;
+extern bool iee_init_done;
+extern unsigned long iee_si_tcr;
+
+/* The following are __init functions used for iee si initialization. */
+extern void iee_si_prepare_data(void);
+
+extern unsigned long __iee_si_start[];
+// Handler function for sensitive inst
+u64 iee_si_handler(int flag, ...);
+/*
+ * TODO: scan a page to check whether it contains sensitive instructions
+ * return 1 when finding sensitive inst, 0 on safe page.
+ */
+extern int iee_si_scan_page(unsigned long addr);
+
+
+#define DBG_MDSCR_SS (1 << 0)
+#define DBG_MDSCR_MDE (1 << 15)
+
+#define IEE_SI_TEST 0
+#define IEE_WRITE_SCTLR 1
+#define IEE_WRITE_TTBR0 2
+#define IEE_WRITE_VBAR 3
+#define IEE_WRITE_TCR 4
+#define IEE_WRITE_MDSCR 5
+#define IEE_CONTEXT_SWITCH 6
+// #define IEE_WRITE_AFSR0 10
+/* Provide ttbr1 switch gate for KOI */
+#ifdef CONFIG_KOI
+#define IEE_SWITCH_TO_KERNEL 7
+#define IEE_SWITCH_TO_KOI 8
+#endif
+/* MASK modify-permitted bits on IEE protected sys registers */
+#define IEE_SCTLR_MASK (SCTLR_EL1_CP15BEN | SCTLR_EL1_SED | SCTLR_EL1_UCT | SCTLR_EL1_UCI |\
+ SCTLR_EL1_BT0 | SCTLR_EL1_BT1 | SCTLR_EL1_TCF0_MASK | SCTLR_ELx_DSSBS |\
+ SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB|\
+ SCTLR_EL1_SPINTMASK | SCTLR_EL1_NMI | SCTLR_EL1_TIDCP | SCTLR_EL1_MSCEn|\
+ SCTLR_ELx_ENTP2 | SCTLR_EL1_TCF_MASK)
+#define IEE_TTBR0_MASK ~0
+#define IEE_TTBR1_MASK ~0
+#define IEE_TCR_MASK (TCR_HD | TCR_T0SZ_MASK | TCR_E0PD1)
+#define IEE_MDSCR_MASK (DBG_MDSCR_SS | DBG_MDSCR_MDE)
+
+#define IEE_DBGBCR_BT 0b0000 << 20
+#define IEE_DBGBCR_SSC 0b00 << 14
+#define IEE_DBGBCR_HMC 0b1 << 13
+#define IEE_DBGBCR_BAS 0b1111 << 5
+#define IEE_DBGBCR_PMC 0b11 << 1
+#define IEE_DBGBCR_E 0b1
+#define IEE_DBGBCR IEE_DBGBCR_BT | IEE_DBGBCR_SSC | IEE_DBGBCR_HMC | IEE_DBGBCR_BAS \
+ | IEE_DBGBCR_PMC | IEE_DBGBCR_E
+
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee-slab.h b/arch/arm64/include/asm/iee-slab.h
new file mode 100644
index 000000000000..4f3c17c7da00
--- /dev/null
+++ b/arch/arm64/include/asm/iee-slab.h
@@ -0,0 +1,23 @@
+#ifndef _LINUX_IEE_SLAB_H
+#define _LINUX_IEE_SLAB_H
+/*
+ * Tracking user of a slab.
+ */
+#include <linux/stackdepot.h>
+
+#define TRACK_ADDRS_COUNT 16
+struct track {
+ unsigned long addr; /* Called from address */
+#ifdef CONFIG_STACKDEPOT
+ depot_stack_handle_t handle;
+#endif
+ int cpu; /* Was running on cpu */
+ int pid; /* Pid context */
+ unsigned long when; /* When did the operation occur */
+};
+
+enum track_item { TRACK_ALLOC, TRACK_FREE };
+
+typedef struct { unsigned long v; } freeptr_t;
+
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee-token.h b/arch/arm64/include/asm/iee-token.h
new file mode 100644
index 000000000000..152474e1a187
--- /dev/null
+++ b/arch/arm64/include/asm/iee-token.h
@@ -0,0 +1,40 @@
+#ifndef _LINUX_IEE_TOKEN_H
+#define _LINUX_IEE_TOKEN_H
+
+#include <asm/iee-def.h>
+
+extern unsigned long long iee_rw_gate(int flag, ...);
+struct task_token;
+struct task_struct;
+struct mm_struct;
+
+#ifdef CONFIG_IEE
+void iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm)
+{
+ iee_rw_gate(IEE_SET_TOKEN_MM, tsk, mm);
+}
+
+void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd)
+{
+ iee_rw_gate(IEE_SET_TOKEN_PGD, tsk, pgd);
+}
+
+void iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack)
+{
+ iee_rw_gate(IEE_INIT_TOKEN, tsk, kernel_stack, iee_stack);
+}
+
+void iee_free_token(struct task_struct *tsk)
+{
+ iee_rw_gate(IEE_FREE_TOKEN, tsk);
+}
+
+unsigned long iee_read_token_stack(struct task_struct *tsk)
+{
+ unsigned long ret;
+ ret = iee_rw_gate(IEE_READ_TOKEN_STACK, tsk);
+ return ret;
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/arch/arm64/include/asm/iee.h b/arch/arm64/include/asm/iee.h
new file mode 100644
index 000000000000..598f6d0b2626
--- /dev/null
+++ b/arch/arm64/include/asm/iee.h
@@ -0,0 +1,10 @@
+#ifndef _LINUX_IEE_H
+#define _LINUX_IEE_H
+#define __iee_code __section(".iee.text")
+#define __iee_header __section(".iee.text.header")
+
+u64 iee_dispatch(int flag, ...);
+
+#include <asm/iee-def.h>
+
+#endif
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 85d26143faa5..e7a3081ce285 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -118,4 +118,25 @@
#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY)
#endif
+#ifdef CONFIG_IEE
+
+#ifdef CONFIG_ARM64_4K_PAGES // zgcXXX: it has been deleted in 6.6.
+#define ARM64_SWAPPER_USES_SECTION_MAPS 1
+#else
+#define ARM64_SWAPPER_USES_SECTION_MAPS 0
+#endif
+
+#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) // zgcXXX: warning: 6.6 delete this macro. should delete this line later.
+
+#define SWAPPER_PTE_FLAGS_IDMAP (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED | PTE_RDONLY)
+#define SWAPPER_PMD_FLAGS_IDMAP (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S | PMD_SECT_RDONLY)
+
+#if ARM64_SWAPPER_USES_SECTION_MAPS
+#define SWAPPER_MM_MMUFLAGS_IDMAP (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS_IDMAP)
+#else
+#define SWAPPER_MM_MMUFLAGS_IDMAP (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS_IDMAP)
+#endif
+
+#endif
+
#endif /* __ASM_KERNEL_PGTABLE_H */
diff --git a/arch/arm64/include/asm/koi.h b/arch/arm64/include/asm/koi.h
new file mode 100644
index 000000000000..48d9a1378a1d
--- /dev/null
+++ b/arch/arm64/include/asm/koi.h
@@ -0,0 +1,335 @@
+#include "linux/mm.h"
+#include "asm/current.h"
+#include "asm/pgtable-hwdef.h"
+#include "asm/pgtable-types.h"
+#include "asm/pgtable.h"
+#include "linux/mm_types.h"
+#include "linux/pgtable.h"
+#include "linux/printk.h"
+#include "linux/slab.h"
+#include "linux/string.h"
+#include <linux/sched.h>
+#include "linux/hashtable.h"
+#include "linux/module.h"
+#include "linux/vmalloc.h"
+#include "stacktrace.h"
+#include "asm/mmu.h"
+#ifdef CONFIG_IEE
+#include "asm/iee-si.h"
+#include "asm/iee-def.h"
+#endif
+
+#define HASH_TABLE_BIT 10
+#define HASH_TABLE_LEN (1 << HASH_TABLE_BIT)
+#define HASH_KEY_MASK ((1 << HASH_TABLE_BIT) - 1)
+
+#define MAX_VAR_NAME 64
+#define DRIVER_ISOLATION_VAR_ARRAY_SIZE 32
+#define DRIVER_ISOLATION_MAX_VAL 256
+
+extern struct hlist_head koi_mem_htbl[1024];
+extern spinlock_t koi_mem_htbl_spin_lock;
+extern unsigned long koi_swapper_ttbr1;
+extern s64 koi_offset;
+
+#ifdef CONFIG_IEE
+extern unsigned long long iee_rw_gate(int flag, ...);
+#endif
+
+DECLARE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)],
+ koi_irq_current_ttbr1);
+
+/**
+* struct koi_mem_hash_node -
+*@mod:pointer to driver module
+*@mem_list_head:free memory list head
+*@ko_mm: mm_struct in each driver
+*@pgdp:entry to Page Global Directory :pgd
+*@node:hash linked list node
+*@addr_htbl[1 << (HASH_TABLE_BIT)]:
+*@rcu:
+*/
+struct koi_mem_hash_node {
+ struct module *mod;
+ struct list_head mem_list_head;
+ struct mm_struct *ko_mm;
+ pgd_t *pgdp;
+ unsigned long ko_ttbr1;
+ struct hlist_node node;
+ struct hlist_head addr_htbl[1 << (HASH_TABLE_BIT)];
+ struct rcu_head rcu;
+ // used to protect free mem list
+ spinlock_t spin_lock;
+ // used to protect addr hashtable
+ spinlock_t addr_htbl_spin_lock;
+};
+//describe the global shared var
+struct shared_variable_descriptor {
+ unsigned int id;
+ unsigned int type;
+ char name[MAX_VAR_NAME];
+ unsigned long offset;
+ unsigned int size;
+ unsigned int self_ptr_ids[DRIVER_ISOLATION_VAR_ARRAY_SIZE];
+};
+
+int koi_do_switch_to_kernel_pgtbl(void);
+
+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir,
+ unsigned long addr, unsigned long end);
+
+void koi_create_pagetable(struct module *mod);
+
+void koi_map_kostack(struct module *mod);
+unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr,
+ unsigned long size);
+void koi_mem_free(struct module *mod, unsigned long addr, unsigned long size,
+ bool is_const, int count, ...);
+void *koi_mem_lookup(struct module *mod, unsigned long addr);
+void koi_mem_free_callback(struct module *mod, unsigned long addr,
+ unsigned long size, void (*func)(void *));
+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size);
+void koi_mem_free_to_user(struct module *mod, unsigned long addr,
+ unsigned long size);
+
+unsigned long koi_ttbr_ctor(struct module *mod);
+extern void koi_do_switch_to_kernel_stack(void);
+extern void koi_do_switch_to_ko_stack(void);
+
+#define switch_pgtable(ttbr1) \
+ do { \
+ write_sysreg(ttbr1, ttbr1_el1); \
+ isb(); \
+ asm volatile(ALTERNATIVE("nop; nop; nop", \
+ "ic iallu; dsb nsh; isb", \
+ ARM64_WORKAROUND_CAVIUM_27456)); \
+ } while (0);
+
+#ifndef CONFIG_IEE
+#define koi_switch_to_ko() \
+ do { \
+ unsigned long flags, ko_ttbr1, cur_sp; \
+ unsigned long *ptr; \
+ struct task_token *token; \
+ asm volatile("mrs %0, daif\n" \
+ "msr daifset, #2\n" \
+ "isb\n" \
+ "mov %1, sp\n" \
+ : "=r"(flags), "=r"(cur_sp) \
+ :); \
+ if (!on_irq_stack(cur_sp, NULL)) { \
+ koi_do_switch_to_ko_stack(); \
+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \
+ token = (struct task_token *)((unsigned long)current + \
+ koi_offset); \
+ token->current_ttbr1 = ko_ttbr1 & (~TTBR_ASID_MASK); \
+ } else { \
+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \
+ __kern_my_cpu_offset()); \
+ *ptr = ko_ttbr1 & ~(TTBR_ASID_MASK); \
+ } \
+ switch_pgtable(ko_ttbr1); \
+ asm volatile("msr daif, %0\n" \
+ "isb\n" \
+ : \
+ : "r"(flags)); \
+ } while (0);
+
+#define koi_switch_to_kernel() \
+ do { \
+ unsigned long cur_sp, flags, asid; \
+ unsigned long *ptr; \
+ struct task_token *token; \
+ asm volatile("mrs %0, daif\n" \
+ "msr daifset, #2\n" \
+ "isb\n" \
+ "mov %1, sp\n" \
+ "mov %2, ttbr0_el1\n" \
+ : "=r"(flags), "=r"(cur_sp), "=r"(asid) \
+ :); \
+ asid &= ~USER_ASID_FLAG; \
+ asid &= TTBR_ASID_MASK; \
+ switch_pgtable(koi_swapper_ttbr1); \
+ if (!on_irq_stack(cur_sp, NULL)) { \
+ token = (struct task_token *)((unsigned long)current + \
+ koi_offset); \
+ token->current_ttbr1 = koi_swapper_ttbr1; \
+ koi_do_switch_to_kernel_stack(); \
+ } else { \
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \
+ __kern_my_cpu_offset()); \
+ *ptr = koi_swapper_ttbr1; \
+ } \
+ asm volatile("msr daif, %0\n" \
+ "isb\n" \
+ : \
+ : "r"(flags)); \
+ } while (0);
+#else
+#define koi_switch_to_ko() \
+ do { \
+ unsigned long cur_sp, flags, ko_ttbr1; \
+ unsigned long *ptr; \
+ asm volatile("mrs %0, daif\n" \
+ "msr daifset, #2\n" \
+ "isb\n" \
+ "mov %1, sp\n" \
+ : "=r"(flags), "=r"(cur_sp) \
+ :); \
+ if (!on_irq_stack(cur_sp, NULL)) { \
+ koi_do_switch_to_ko_stack(); \
+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \
+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \
+ ko_ttbr1 &(~TTBR_ASID_MASK)); \
+ } else { \
+ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \
+ __kern_my_cpu_offset()); \
+ *ptr = ko_ttbr1 & (~TTBR_ASID_MASK); \
+ } \
+ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, ko_ttbr1); \
+ asm volatile("msr daif, %0\n" \
+ "isb\n" \
+ : \
+ : "r"(flags)); \
+ } while (0);
+
+#define koi_switch_to_kernel() \
+ do { \
+ unsigned long flags, cur_sp; \
+ unsigned long *ptr; \
+ asm volatile("mrs %0, daif\n" \
+ "msr daifset, #2\n" \
+ "isb\n" \
+ "mov %1, sp\n" \
+ : "=r"(flags), "=r"(cur_sp) \
+ :); \
+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); \
+ if (!on_irq_stack(cur_sp, NULL)) { \
+ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \
+ koi_swapper_ttbr1); \
+ koi_do_switch_to_kernel_stack(); \
+ } else { \
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \
+ __kern_my_cpu_offset()); \
+ *ptr = koi_swapper_ttbr1; \
+ } \
+ asm volatile("msr daif, %0\n" \
+ "isb\n" \
+ : \
+ : "r"(flags)); \
+ } while (0);
+#endif
+//kzalloc function in driver space
+static __maybe_unused noinline void *
+koi_kzalloc_wrapper(struct module *mod, size_t size, gfp_t flags)
+{
+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+ void *addr;
+ struct koi_mem_hash_node *target = NULL;
+ koi_switch_to_kernel();
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk("mem node for module: %s not found\n", mod->name);
+ return NULL;
+ }
+
+ addr = kzalloc(size, flags);
+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr,
+ (unsigned long)addr + PAGE_SIZE * cnt);
+ koi_switch_to_ko();
+ return addr;
+}
+//kmalloc function in driver space
+static __maybe_unused __always_inline void *
+koi_kmalloc_wrapper(struct module *mod, size_t size, gfp_t flags)
+{
+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+ void *addr;
+ struct koi_mem_hash_node *target = NULL;
+ koi_switch_to_kernel();
+
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk("mem node for module: %s not found\n", mod->name);
+ return 0;
+ }
+
+ addr = kmalloc(cnt * PAGE_SIZE, flags);
+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr,
+ (unsigned long)addr + PAGE_SIZE * cnt);
+ koi_switch_to_ko();
+ return (void *)addr;
+}
+//vmalloc function in driver space
+static __maybe_unused void *koi_vmalloc_wrapper(struct module *mod,
+ unsigned long size)
+{
+ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+ void *addr;
+ struct koi_mem_hash_node *target = NULL;
+ koi_switch_to_kernel();
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk("mem node for module: %s not found\n", mod->name);
+ koi_switch_to_ko();
+ return 0;
+ }
+ addr = vmalloc(cnt * PAGE_SIZE);
+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr,
+ (unsigned long)addr + PAGE_SIZE * cnt);
+ koi_switch_to_ko();
+ return addr;
+}
+//kmalloc_array function in driver space
+static __maybe_unused void *koi_kmalloc_array_wrapper(struct module *mod,
+ size_t n, size_t size,
+ gfp_t flags)
+{
+ int kpage;
+ void *addr;
+ struct koi_mem_hash_node *target = NULL;
+ koi_switch_to_kernel();
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk("mem node for module: %s not found\n", mod->name);
+ koi_switch_to_ko();
+ return 0;
+ }
+ kpage = (n * size + PAGE_SIZE - 1) / PAGE_SIZE;
+ n = (kpage * PAGE_SIZE) / size;
+ addr = kmalloc_array(n, size, flags);
+ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr,
+ (unsigned long)addr + PAGE_SIZE * kpage);
+ koi_switch_to_ko();
+ return addr;
+}
\ No newline at end of file
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index fde4186cc387..6309e5514a16 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -190,6 +190,13 @@ extern u64 vabits_actual;
#endif
extern s64 memstart_addr;
+
+#if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+extern s64 memstart_addr_init;
+extern s64 iee_offset;
+#define LOGICAL_RANDOM (long long int)((long unsigned int)__va(memstart_addr_init) & (~PAGE_OFFSET))
+#endif
+
/* PHYS_OFFSET - the physical address of the start of memory. */
#define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; })
@@ -310,6 +317,23 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x);
#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET)
#define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset))
+#ifdef CONFIG_KOI
+#define KOI_OFFSET ((unsigned long)BIT(vabits_actual - 2))
+#endif
+
+#ifdef CONFIG_IEE
+#ifdef CONFIG_IEE_OFFSET
+#define IEE_OFFSET ((CONFIG_IEE_OFFSET) - LOGICAL_RANDOM)
+#else
+#define IEE_OFFSET (((unsigned long)BIT(vabits_actual - 2)) - LOGICAL_RANDOM)
+#endif
+#define __phys_to_iee(x) (__phys_to_virt(x) + IEE_OFFSET)
+#define SET_UPAGE(x) __pgprot(pgprot_val(x) | PTE_USER)
+#define SET_PPAGE(x) __pgprot(pgprot_val(x) & (~PTE_USER))
+#define SET_INVALID(x) __pgprot(pgprot_val(x) & (~PTE_VALID))
+#define SET_NG(x) __pgprot(pgprot_val(x) | PTE_NG)
+#endif
+
/*
* Convert a page to/from a physical address
*/
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index a6fb325424e7..cca5994dabfb 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -24,6 +24,9 @@
#include <asm/cputype.h>
#include <asm/sysreg.h>
#include <asm/tlbflush.h>
+#ifdef CONFIG_IEE
+#define INIT_ASID 0x2
+#endif
extern bool rodata_full;
@@ -43,7 +46,12 @@ static inline void cpu_set_reserved_ttbr0_nosync(void)
{
unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
+#ifdef CONFIG_IEE
+ ttbr |= FIELD_PREP(TTBR_ASID_MASK, 1);
+ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr);
+#else
write_sysreg(ttbr, ttbr0_el1);
+#endif
}
static inline void cpu_set_reserved_ttbr0(void)
@@ -79,7 +87,11 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz)
tcr &= ~TCR_T0SZ_MASK;
tcr |= t0sz << TCR_T0SZ_OFFSET;
+#ifdef CONFIG_IEE
+ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr);
+#else
write_sysreg(tcr, tcr_el1);
+#endif
isb();
}
@@ -144,7 +156,11 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
__cpu_set_tcr_t0sz(t0sz);
/* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */
+ #ifdef CONFIG_IEE
+ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr0);
+ #else
write_sysreg(ttbr0, ttbr0_el1);
+ #endif
isb();
}
@@ -174,6 +190,10 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
ttbr1 |= TTBR_CNP_BIT;
}
+ #ifdef CONFIG_IEE
+ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, ASID(current->active_mm));
+ #endif
+
replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
__cpu_install_idmap(idmap);
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 237224484d0f..4e3304da8421 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -63,6 +63,10 @@ static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
extern pgd_t *pgd_alloc(struct mm_struct *mm);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
+#ifdef CONFIG_KOI
+pgd_t *koi_pgd_alloc(void);
+#endif
+
static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep,
pmdval_t prot)
{
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index e4944d517c99..7f60e568c964 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -84,6 +84,13 @@
#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE)
#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1))
+#ifdef CONFIG_IEE
+#define PGD_APT_RO (_AT(pudval_t, 1) << 62)
+#endif
+#define PGD_APT (_AT(pudval_t, 1) << 61)
+#define PGD_PXN (_AT(pudval_t, 1) << 59)
+#define PGD_UXN (_AT(pudval_t, 1) << 60)
+
/*
* Hardware page table definitions.
*
@@ -285,6 +292,10 @@
#define TCR_TCMA0 (UL(1) << 57)
#define TCR_TCMA1 (UL(1) << 58)
+#ifdef CONFIG_IEE
+#define TCR_HPD1 (UL(1) << 42)
+#endif
+
/*
* TTBR.
*/
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 8d68d00de0a4..6f8d5b85bfd7 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -34,6 +34,9 @@
#include <linux/mm_types.h>
#include <linux/sched.h>
#include <linux/page_table_check.h>
+#ifdef CONFIG_PTP
+#include <asm/iee.h>
+#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
@@ -156,6 +159,30 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
#define pud_access_permitted(pud, write) \
(pte_access_permitted(pud_pte(pud), (write)))
+#ifdef CONFIG_PTP
+static inline bool in_tramp_pgdir(void *addr);
+extern unsigned long long iee_rw_gate(int flag, ...);
+
+static void iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd)
+{
+ iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd);
+}
+
+static noinline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval)
+{
+ pteval_t ret;
+ ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval);
+ return (pteval_t)ret;
+}
+
+static noinline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval)
+{
+ pteval_t ret;
+ ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval);
+ return (pteval_t)ret;
+}
+#endif
+
static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
{
pte_val(pte) &= ~pgprot_val(prot);
@@ -262,6 +289,64 @@ static inline pte_t pte_mkdevmap(pte_t pte)
static inline void __set_pte(pte_t *ptep, pte_t pte)
{
+#ifdef CONFIG_KOI
+ if (!pte_none(pte)) {
+ pte = __pte(pte_val(pte) | PTE_NG);
+ }
+#endif
+#ifdef CONFIG_PTP
+ iee_rw_gate(IEE_OP_SET_PTE, ptep, pte);
+ dsb(ishst);
+ isb();
+#else
+ WRITE_ONCE(*ptep, pte);
+
+ /*
+ * Only if the new pte is valid and kernel, otherwise TLB maintenance
+ * or update_mmu_cache() have the necessary barriers.
+ */
+ if (pte_valid_not_user(pte)) {
+ dsb(ishst);
+ isb();
+ }
+#endif
+}
+
+
+#ifdef CONFIG_PTP
+static inline void iee_set_bm_pte(pte_t *ptep, pte_t pte)
+{
+ // If it is pre init, write once.
+ // Else, write once will cause exception. So it is safe.
+ unsigned long flags;
+ unsigned long res;
+ local_irq_save(flags);
+ asm volatile("at s1e1r, %0"::"r"(__phys_to_iee(__pa_symbol(ptep))));
+ isb();
+ res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+ if(res & 0x1)
+ WRITE_ONCE(*ptep,pte);
+ else
+ iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte);
+
+ /*
+ * Only if the new pte is valid and kernel, otherwise TLB maintenance
+ * or update_mmu_cache() have the necessary barriers.
+ */
+ if (pte_valid_not_user(pte)) {
+ dsb(ishst);
+ isb();
+ }
+}
+
+static inline void iee_set_fixmap_pte_pre_init(pte_t *ptep, pte_t pte)
+{
+#ifdef CONFIG_KOI
+ if (!pte_none(pte)) {
+ pte = __pte(pte_val(pte) | PTE_NG);
+ }
+#endif
WRITE_ONCE(*ptep, pte);
/*
@@ -273,6 +358,7 @@ static inline void __set_pte(pte_t *ptep, pte_t pte)
isb();
}
}
+#endif
static inline pte_t __ptep_get(pte_t *ptep)
{
@@ -546,6 +632,95 @@ static inline void __set_pte_at(struct mm_struct *mm,
__set_pte(ptep, pte);
}
+#ifdef CONFIG_IEE
+static inline void iee_set_pte_upage(pte_t *ptep, pte_t pte)
+{
+#ifdef CONFIG_PTP
+ iee_rw_gate(IEE_OP_SET_PTE_U, ptep, pte);
+ dsb(ishst);
+ isb();
+#else
+ WRITE_ONCE(*ptep, pte);
+ if (pte_valid_not_user(pte)) {
+ dsb(ishst);
+ isb();
+ }
+#endif
+}
+
+static inline void iee_set_pte_ppage(pte_t *ptep, pte_t pte)
+{
+#ifdef CONFIG_PTP
+ iee_rw_gate(IEE_OP_SET_PTE_P, ptep, pte);
+#else
+ WRITE_ONCE(*ptep, pte);
+#endif
+ if (pte_valid_not_user(pte)) {
+ dsb(ishst);
+ isb();
+ }
+}
+#endif
+
+#ifdef CONFIG_PTP
+static inline void set_pmd(pmd_t *pmdp, pmd_t pmd);
+static inline void __set_pmd_at(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp, pmd_t pmd)
+{
+ if (pte_present(pmd_pte(pmd)) && pte_user_exec(pmd_pte(pmd)) && !pte_special(pmd_pte(pmd)))
+ __sync_icache_dcache(pmd_pte(pmd));
+
+ /*
+ * If the PTE would provide user space access to the tags associated
+ * with it then ensure that the MTE tags are synchronised. Although
+ * pte_access_permitted() returns false for exec only mappings, they
+ * don't expose tags (instruction fetches don't check tags).
+ */
+ if (system_supports_mte() && pte_access_permitted(pmd_pte(pmd), false) &&
+ !pte_special(pmd_pte(pmd)) && pte_tagged(pmd_pte(pmd)))
+ mte_sync_tags(pmd_pte(pmd), PMD_SIZE >> PAGE_SHIFT);
+
+ __check_safe_pte_update(mm, (pte_t *)pmdp, pmd_pte(pmd));
+
+ set_pmd(pmdp, pmd);
+}
+
+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+ pmd_t *pmdp, pmd_t pmd)
+{
+ page_table_check_pmd_set(mm, pmdp, pmd);
+ return __set_pmd_at(mm, addr, pmdp, pmd);
+}
+
+static inline void set_pud(pud_t *pudp, pud_t pud);
+static inline void __set_pud_at(struct mm_struct *mm, unsigned long addr,
+ pud_t *pudp, pud_t pud)
+{
+ if (pte_present(pud_pte(pud)) && pte_user_exec(pud_pte(pud)) && !pte_special(pud_pte(pud)))
+ __sync_icache_dcache(pud_pte(pud));
+
+ /*
+ * If the PTE would provide user space access to the tags associated
+ * with it then ensure that the MTE tags are synchronised. Although
+ * pte_access_permitted() returns false for exec only mappings, they
+ * don't expose tags (instruction fetches don't check tags).
+ */
+ if (system_supports_mte() && pte_access_permitted(pud_pte(pud), false) &&
+ !pte_special(pud_pte(pud)) && pte_tagged(pud_pte(pud)))
+ mte_sync_tags(pud_pte(pud), PUD_SIZE >> PAGE_SHIFT);
+
+ __check_safe_pte_update(mm, (pte_t *)pudp, pud_pte(pud));
+
+ set_pud(pudp, pud);
+}
+
+static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
+ pud_t *pudp, pud_t pud)
+{
+ page_table_check_pud_set(mm, pudp, pud);
+ return __set_pud_at(mm, addr, pudp, pud);
+}
+#else
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{
@@ -561,7 +736,7 @@ static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud),
PUD_SIZE >> PAGE_SHIFT);
}
-
+#endif
#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d))
#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys)
@@ -640,7 +815,14 @@ static inline bool in_swapper_pgdir(void *addr)
((unsigned long)swapper_pg_dir & PAGE_MASK);
}
-static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
+#ifdef CONFIG_PTP
+static inline bool in_tramp_pgdir(void *addr)
+{
+ return ((unsigned long)addr & PAGE_MASK) ==
+ ((unsigned long)tramp_pg_dir & PAGE_MASK);
+}
+
+static inline void iee_set_fixmap_pmd_pre_init(pmd_t *pmdp, pmd_t pmd)
{
#ifdef __PAGETABLE_PMD_FOLDED
if (in_swapper_pgdir(pmdp)) {
@@ -648,7 +830,6 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
return;
}
#endif /* __PAGETABLE_PMD_FOLDED */
-
WRITE_ONCE(*pmdp, pmd);
if (pmd_valid(pmd)) {
@@ -656,6 +837,32 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
isb();
}
}
+#endif
+
+static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
+{
+#ifdef __PAGETABLE_PMD_FOLDED
+ if (in_swapper_pgdir(pmdp)) {
+ set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
+ return;
+ }
+#endif /* __PAGETABLE_PMD_FOLDED */
+#ifdef CONFIG_KOI
+ pmdval_t val = pmd_val(pmd);
+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) {
+ pmd = __pmd(val | PMD_SECT_NG);
+ }
+#endif
+#ifdef CONFIG_PTP
+ iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd);
+#else
+ WRITE_ONCE(*pmdp, pmd);
+#endif
+ if (pmd_valid(pmd)) {
+ dsb(ishst);
+ isb();
+ }
+}
static inline void pmd_clear(pmd_t *pmdp)
{
@@ -675,6 +882,12 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
/* Find an entry in the third-level page table. */
#define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
+#ifdef CONFIG_PTP
+#define pte_set_fixmap_init(addr) ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr))
+#define pte_set_fixmap_offset_init(pmd, addr) pte_set_fixmap_init(pte_offset_phys(pmd, addr))
+#define pte_clear_fixmap_init() clear_fixmap_init(FIX_PTE)
+#endif
+
#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr))
#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr))
#define pte_clear_fixmap() clear_fixmap(FIX_PTE)
@@ -703,7 +916,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
#define pud_user(pud) pte_user(pud_pte(pud))
#define pud_user_exec(pud) pte_user_exec(pud_pte(pud))
-static inline void set_pud(pud_t *pudp, pud_t pud)
+
+#ifdef CONFIG_PTP
+static inline void iee_set_fixmap_pud_pre_init(pud_t *pudp, pud_t pud)
{
#ifdef __PAGETABLE_PUD_FOLDED
if (in_swapper_pgdir(pudp)) {
@@ -711,7 +926,6 @@ static inline void set_pud(pud_t *pudp, pud_t pud)
return;
}
#endif /* __PAGETABLE_PUD_FOLDED */
-
WRITE_ONCE(*pudp, pud);
if (pud_valid(pud)) {
@@ -719,6 +933,33 @@ static inline void set_pud(pud_t *pudp, pud_t pud)
isb();
}
}
+#endif
+
+static inline void set_pud(pud_t *pudp, pud_t pud)
+{
+#ifdef __PAGETABLE_PUD_FOLDED
+ if (in_swapper_pgdir(pudp)) {
+ set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
+ return;
+ }
+#endif /* __PAGETABLE_PUD_FOLDED */
+#ifdef CONFIG_KOI
+ pudval_t val = pud_val(pud);
+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) {
+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead.
+ pud = __pud(val | PMD_SECT_NG);
+ }
+#endif
+#ifdef CONFIG_PTP
+ iee_rw_gate(IEE_OP_SET_PUD, pudp, pud);
+#else
+ WRITE_ONCE(*pudp, pud);
+#endif
+ if (pud_valid(pud)) {
+ dsb(ishst);
+ isb();
+ }
+}
static inline void pud_clear(pud_t *pudp)
{
@@ -738,6 +979,12 @@ static inline pmd_t *pud_pgtable(pud_t pud)
/* Find an entry in the second-level page table. */
#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
+#ifdef CONFIG_PTP
+#define pmd_set_fixmap_init(addr) ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr))
+#define pmd_set_fixmap_offset_init(pud, addr) pmd_set_fixmap_init(pmd_offset_phys(pud, addr))
+#define pmd_clear_fixmap_init() clear_fixmap_init(FIX_PMD)
+#endif
+
#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr))
#define pmd_clear_fixmap() clear_fixmap(FIX_PMD)
@@ -769,15 +1016,26 @@ static inline pmd_t *pud_pgtable(pud_t pud)
#define p4d_none(p4d) (!p4d_val(p4d))
#define p4d_bad(p4d) (!(p4d_val(p4d) & 2))
#define p4d_present(p4d) (p4d_val(p4d))
+#define p4d_valid(p4d) pte_valid(p4d_pte(p4d))
static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
{
- if (in_swapper_pgdir(p4dp)) {
+ if (in_swapper_pgdir(p4dp))
+ {
set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
return;
}
+#ifdef CONFIG_PTP
+ if(in_tramp_pgdir(p4dp))
+ {
+ iee_set_tramp_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
+ return;
+ }
+ iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d);
+#else
WRITE_ONCE(*p4dp, p4d);
+#endif
dsb(ishst);
isb();
}
@@ -800,6 +1058,12 @@ static inline pud_t *p4d_pgtable(p4d_t p4d)
/* Find an entry in the first-level page table. */
#define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
+#ifdef CONFIG_PTP
+#define pud_set_fixmap_init(addr) ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr))
+#define pud_set_fixmap_offset_init(p4d, addr) pud_set_fixmap_init(pud_offset_phys(p4d, addr))
+#define pud_clear_fixmap_init() clear_fixmap_init(FIX_PUD)
+#endif
+
#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr))
#define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr))
#define pud_clear_fixmap() clear_fixmap(FIX_PUD)
@@ -826,6 +1090,10 @@ static inline pud_t *p4d_pgtable(p4d_t p4d)
#define pgd_ERROR(e) \
pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
+#ifdef CONFIG_PTP
+#define pgd_set_fixmap_init(addr) ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr))
+#define pgd_clear_fixmap_init() clear_fixmap_init(FIX_PGD)
+#endif
#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
@@ -912,8 +1180,13 @@ static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma,
do {
old_pte = pte;
pte = pte_mkold(pte);
+ #ifdef CONFIG_PTP
+ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep,
+ pte_val(old_pte), pte_val(pte));
+ #else
pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
pte_val(old_pte), pte_val(pte));
+ #endif
} while (pte_val(pte) != pte_val(old_pte));
return pte_young(pte);
@@ -952,8 +1225,12 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
unsigned long address, pte_t *ptep)
{
+ #ifdef CONFIG_PTP
+ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0);
+ pte_t pte = __pte(pteval);
+ #else
pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
-
+ #endif
page_table_check_pte_clear(mm, pte);
return pte;
@@ -995,7 +1272,12 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
+ #ifdef CONFIG_PTP
+ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0);
+ pmd_t pmd = __pmd(pteval);
+ #else
pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0));
+ #endif
page_table_check_pmd_clear(mm, pmd);
@@ -1012,8 +1294,12 @@ static inline void ___ptep_set_wrprotect(struct mm_struct *mm,
do {
old_pte = pte;
pte = pte_wrprotect(pte);
+ #ifdef CONFIG_PTP
+ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep,pte_val(old_pte), pte_val(pte));
+ #else
pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
pte_val(old_pte), pte_val(pte));
+ #endif
} while (pte_val(pte) != pte_val(old_pte));
}
@@ -1049,7 +1335,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
page_table_check_pmd_set(vma->vm_mm, pmdp, pmd);
+ #ifdef CONFIG_PTP
+ return __pmd((pmdval_t)iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pmdval_t)pmd_val(pmd)));
+ #else
return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
+ #endif
}
#endif
diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h
index d2e0306e65d3..8352e92d4536 100644
--- a/arch/arm64/include/asm/pointer_auth.h
+++ b/arch/arm64/include/asm/pointer_auth.h
@@ -108,8 +108,13 @@ static __always_inline void ptrauth_enable(void)
{
if (!system_supports_address_auth())
return;
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB |
+ SCTLR_ELx_ENDA | SCTLR_ELx_ENDB));
+ #else
sysreg_clear_set(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB |
SCTLR_ELx_ENDA | SCTLR_ELx_ENDB));
+ #endif
isb();
}
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 42358b8d678e..1e2d11e57fe3 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -1116,6 +1116,64 @@
write_sysreg_s(__scs_new, sysreg); \
} while (0)
+
+#ifdef CONFIG_IEE
+
+#define SYS_TCR_IEE_SI TCR_HPD1 | TCR_A1
+
+extern void iee_rwx_gate_entry(int flag, ...);
+#define IEE_SI_TEST 0
+#define IEE_WRITE_sctlr_el1 1
+#define IEE_WRITE_ttbr0_el1 2
+#define IEE_WRITE_vbar_el1 3
+#define IEE_WRITE_tcr_el1 4
+#define IEE_WRITE_mdscr_el1 5
+#define IEE_WRITE_AFSR0 10
+
+#define sysreg_clear_set_iee_si(sysreg, clear, set) do { \
+ u64 __scs_val = read_sysreg(sysreg); \
+ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \
+ if (__scs_new != __scs_val) \
+ iee_rwx_gate_entry(IEE_WRITE_##sysreg, __scs_new); \
+} while (0)
+
+#define IEE_SI_WRITE_DAIF_SEL "msr daifclr, #0xf\n\t" \
+ "tbnz %x0, #6, 114221f\n\t" \
+ "tbnz %x0, #7, 114210f\n\t" \
+ "tbnz %x0, #8, 114100f\n\t" \
+ "msr daifset, #0b000\n\t" \
+ "b 114514f\n\t" \
+"114221:\n\t" \
+ "tbnz %x0, #7, 114211f\n\t" \
+ "tbnz %x0, #8, 114101f\n\t" \
+ "msr daifset, #0b001\n\t" \
+ "b 114514f\n\t" \
+"114211:\n\t" \
+ "tbnz %x0, #8, 114111f\n\t" \
+ "msr daifset, #0b011\n\t" \
+ "b 114514f\n\t" \
+"114210:\n\t" \
+ "tbnz %x0, #8, 114110f\n\t" \
+ "msr daifset, #0b010\n\t" \
+ "b 114514f\n\t" \
+"114100:\n\t" \
+ "msr daifset, #0b100\n\t" \
+ "b 114514f\n\t" \
+"114101:\n\t" \
+ "msr daifset, #0b101\n\t" \
+ "b 114514f\n\t" \
+"114110:\n\t" \
+ "msr daifset, #0b110\n\t" \
+ "b 114514f\n\t" \
+"114111:\n\t" \
+ "msr daifset, #0b111\n\t" \
+"114514:\n\t"
+
+#define iee_si_write_daif(v) do { \
+ u64 __val = (u64)(v); \
+ asm volatile(IEE_SI_WRITE_DAIF_SEL: : "rZ" (__val));} while (0)
+#endif
+
#define read_sysreg_par() ({ \
u64 par; \
asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); \
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 2c29239d05c3..955f99317790 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -11,8 +11,17 @@
#include <linux/pagemap.h>
#include <linux/swap.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
static inline void __tlb_remove_table(void *_table)
{
+#ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(page_to_phys((struct page *)_table));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)page_address((struct page *)_table));
+#endif
free_page_and_swap_cache((struct page *)_table);
}
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 831c314d75ff..7775628528c6 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -49,6 +49,7 @@
#define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0)
+
#define __tlbi_user(op, arg) do { \
if (arm64_kernel_unmapped_at_el0()) \
__tlbi(op, (arg) | USER_ASID_FLAG); \
@@ -258,6 +259,10 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
asid = __TLBI_VADDR(0, ASID(mm));
__tlbi(aside1is, asid);
__tlbi_user(aside1is, asid);
+ #if defined(CONFIG_IEE) || defined (CONFIG_KOI)
+ if (!arm64_kernel_unmapped_at_el0())
+ __tlbi(aside1is, asid | USER_ASID_FLAG);
+ #endif
dsb(ish);
mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
}
@@ -273,6 +278,10 @@ static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
__tlbi_user(vale1is, addr);
mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
(uaddr & PAGE_MASK) + PAGE_SIZE);
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ if (!arm64_kernel_unmapped_at_el0())
+ __tlbi(vale1is, addr | USER_ASID_FLAG);
+ #endif
}
static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
@@ -366,6 +375,45 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
* 2. If there is 1 page remaining, flush it through non-range operations. Range
* operations can only span an even number of pages.
*/
+#if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+#define __flush_tlb_range_op(op, start, pages, stride, \
+ asid, tlb_level, tlbi_user) \
+do { \
+ int num = 0; \
+ int scale = 0; \
+ unsigned long addr; \
+ \
+ while (pages > 0) { \
+ if (!system_supports_tlb_range() || \
+ pages % 2 == 1) { \
+ addr = __TLBI_VADDR(start, asid); \
+ __tlbi_level(op, addr, tlb_level); \
+ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \
+ __tlbi_level(op, addr | USER_ASID_FLAG, tlb_level); \
+ if (tlbi_user) \
+ __tlbi_user_level(op, addr, tlb_level); \
+ start += stride; \
+ pages -= stride >> PAGE_SHIFT; \
+ continue; \
+ } \
+ \
+ num = __TLBI_RANGE_NUM(pages, scale); \
+ if (num >= 0) { \
+ addr = __TLBI_VADDR_RANGE(start, asid, scale, \
+ num, tlb_level); \
+ __tlbi(r##op, addr); \
+ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \
+ __tlbi(r##op, addr | USER_ASID_FLAG); \
+ if (tlbi_user) \
+ __tlbi_user(r##op, addr); \
+ start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
+ pages -= __TLBI_RANGE_PAGES(num, scale); \
+ } \
+ scale++; \
+ } \
+} while (0)
+
+#else
#define __flush_tlb_range_op(op, start, pages, stride, \
asid, tlb_level, tlbi_user) \
do { \
@@ -399,6 +447,8 @@ do { \
} \
} while (0)
+#endif //if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+
#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false)
@@ -467,7 +517,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
return;
}
- start = __TLBI_VADDR(start, 0);
+ start = __TLBI_VADDR(start, 0);
end = __TLBI_VADDR(end, 0);
dsb(ishst);
@@ -483,9 +533,9 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
*/
static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
{
- unsigned long addr = __TLBI_VADDR(kaddr, 0);
-
- dsb(ishst);
+ unsigned long addr = __TLBI_VADDR(kaddr, 0);
+
+ dsb(ishst);
__tlbi(vaae1is, addr);
dsb(ish);
isb();
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 21ef9c21a400..44eb76cc54d7 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -36,6 +36,8 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
syscall.o proton-pack.o idreg-override.o idle.o \
patching.o
+obj-y += iee/
+obj-$(CONFIG_KOI) += koi/
obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \
sys_compat.o
obj-$(CONFIG_AARCH32_EL0) += sigreturn32.o
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index fd0f291e215e..c008e46b5fc0 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -306,11 +306,19 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
static int cp15_barrier_set_hw_mode(bool enable)
{
+#ifdef CONFIG_IEE
+ if (enable)
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
+ else
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
+ return 0;
+#else
if (enable)
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
return 0;
+#endif
}
static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
@@ -341,11 +349,19 @@ static int setend_set_hw_mode(bool enable)
if (!cpu_supports_mixed_endian_el0())
return -EINVAL;
+#ifdef CONFIG_IEE
+ if (enable)
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
+ else
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
+ return 0;
+#else
if (enable)
sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
else
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
return 0;
+#endif
}
static int __a32_setend_handler(struct pt_regs *regs, u32 big_endian)
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index e997ad275afb..e105f633355a 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -97,6 +97,17 @@ int main(void)
DEFINE(FREGS_DIRECT_TRAMP, offsetof(struct ftrace_regs, direct_tramp));
#endif
DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs));
+#ifdef CONFIG_IEE
+ DEFINE(iee_from_token_offset, offsetof(struct task_token, iee_stack));
+ DEFINE(kernel_from_token_offset, offsetof(struct task_token, kernel_stack));
+ DEFINE(mm_from_task_offset, offsetof(struct task_struct, mm));
+#endif
+#ifdef CONFIG_KOI
+ DEFINE(koi_kernel_from_token_offset, offsetof(struct task_token, koi_kernel_stack));
+ DEFINE(koi_from_token_offset, offsetof(struct task_token, koi_stack));
+ DEFINE(ttbr1_from_token_offset, offsetof(struct task_token, current_ttbr1));
+ DEFINE(koi_stack_base_from_token_offset, offsetof(struct task_token, koi_stack_base));
+#endif
BLANK();
#endif
#ifdef CONFIG_AARCH32_EL0
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 46813132a09f..412006aa323c 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -80,7 +80,11 @@ hisilicon_1980005_enable(const struct arm64_cpu_capabilities *__unused)
__set_bit(ARM64_HAS_CACHE_IDC, system_cpucaps);
arm64_ftr_reg_ctrel0.sys_val |= BIT(CTR_EL0_IDC_SHIFT);
arm64_ftr_reg_ctrel0.strict_mask &= ~BIT(CTR_EL0_IDC_SHIFT);
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0);
+#else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
+#endif
}
#endif
@@ -132,7 +136,11 @@ cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap)
enable_uct_trap = true;
if (enable_uct_trap)
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0);
+#else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
+#endif
}
#ifdef CONFIG_ARM64_ERRATUM_1463225
@@ -147,7 +155,11 @@ has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry,
static void __maybe_unused
cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
{
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCI, 0);
+#else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
+#endif
}
#ifdef CONFIG_HISILICON_ERRATUM_HIP08_RU_PREFETCH
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 74e445251b51..ebc1e776b175 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -94,6 +94,10 @@
#include <asm/vectors.h>
#include <asm/virt.h>
+#ifdef CONFIG_IEE
+#include <asm/iee-si.h>
+#endif
+
/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly;
@@ -1612,7 +1616,11 @@ static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unu
* value.
*/
if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT)))
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0);
+#else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
+#endif
}
static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
@@ -1873,7 +1881,11 @@ static inline void __cpu_enable_hw_dbm(void)
{
u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
+#ifdef CONFIG_IEE
+ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr);
+#else
write_sysreg(tcr, tcr_el1);
+#endif
isb();
local_flush_tlb_all();
}
@@ -2056,7 +2068,9 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
*/
WARN_ON_ONCE(in_interrupt());
+ #ifndef CONFIG_IEE
sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
+ #endif
set_pstate_pan(1);
}
#endif /* CONFIG_ARM64_PAN */
@@ -2121,7 +2135,11 @@ static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
{
if (this_cpu_has_cap(ARM64_HAS_E0PD))
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(tcr_el1, 0, TCR_E0PD1);
+#else
sysreg_clear_set(tcr_el1, 0, TCR_E0PD1);
+#endif
}
#endif /* CONFIG_ARM64_E0PD */
@@ -2214,7 +2232,11 @@ static void nmi_enable(const struct arm64_cpu_capabilities *__unused)
* avoid leaving things masked.
*/
_allint_clear();
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI);
+ #else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI);
+ #endif
isb();
}
#endif
@@ -2229,7 +2251,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused)
* So, be strict and forbid other BRs using other registers to
* jump onto a PACIxSP instruction:
*/
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
+#else
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
+#endif
isb();
}
#endif /* CONFIG_ARM64_BTI */
@@ -2237,7 +2263,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused)
#ifdef CONFIG_ARM64_MTE
static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
{
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
+ #else
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
+ #endif
mte_cpu_setup();
@@ -2271,7 +2301,11 @@ static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, in
static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused)
{
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_TIDCP);
+ #else
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP);
+ #endif
}
static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
@@ -2281,7 +2315,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused)
{
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_MSCEn);
+ #else
sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn);
+ #endif
}
/* Internal helper functions to match cpu capability type */
@@ -3475,6 +3513,43 @@ static void __init setup_system_capabilities(void)
enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
}
+#ifdef CONFIG_IEE
+
+static void iee_si_test_end(void)
+{
+ pr_info("IEE: testing iee_exec_entry sctlr...\n");
+ iee_rwx_gate_entry(IEE_WRITE_SCTLR, read_sysreg(sctlr_el1)& ~SCTLR_ELx_M);
+ pr_info("IEE: testing iee_exec_entry ttbr0_el1...\n");
+ iee_rwx_gate_entry(IEE_WRITE_TTBR0, read_sysreg(ttbr0_el1));
+ pr_info("IEE: testing iee_exec_entry vbar...\n");
+ iee_rwx_gate_entry(IEE_WRITE_VBAR, read_sysreg(vbar_el1));
+ pr_info("IEE: testing iee_exec_entry tcr...\n");
+ iee_rwx_gate_entry(IEE_WRITE_TCR, read_sysreg(tcr_el1));
+ // pr_info("IEE: testing iee_exec_entry mdscr...\n");
+ // iee_rwx_gate_entry(IEE_WRITE_MDSCR, read_sysreg(mdscr_el1));
+ // pr_info("IEE: testing iee_exec_entry afsr0...\n");
+ // iee_rwx_gate_entry(IEE_WRITE_AFSR0);
+ #ifdef CONFIG_KOI
+ write_sysreg(read_sysreg(ttbr0_el1)+0x3000000000000, ttbr0_el1);
+ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1));
+ pr_info("IEE: testing iee_exec_entry switch to koi...\n");
+ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, phys_to_ttbr(__pa_symbol(swapper_pg_dir)));
+ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1));
+ pr_info("IEE: testing iee_exec_entry switch to kernel...\n");
+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL);
+ #endif
+}
+
+/* Finish iee rwx gate initializations. */
+static void __init iee_si_init_done(void)
+{
+ // Prepare data for iee rwx gate
+ iee_si_prepare_data();
+ // All initialization is done. Do some simple tests.
+ iee_si_test_end();
+}
+#endif
+
void __init setup_cpu_features(void)
{
u32 cwg;
@@ -3502,6 +3577,10 @@ void __init setup_cpu_features(void)
if (!cwg)
pr_warn("No Cache Writeback Granule information, assuming %d\n",
ARCH_DMA_MINALIGN);
+
+ #ifdef CONFIG_IEE
+ iee_si_init_done();
+ #endif
}
static int enable_mismatched_32bit_el0(unsigned int cpu)
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 745aefddd9a3..265417e0ad81 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -36,10 +36,14 @@ u8 debug_monitors_arch(void)
*/
static void mdscr_write(u32 mdscr)
{
+// #ifdef CONFIG_IEE
+// iee_rwx_gate_entry(IEE_WRITE_mdscr_el1, mdscr);
+// #else
unsigned long flags;
flags = local_daif_save();
write_sysreg(mdscr, mdscr_el1);
local_daif_restore(flags);
+// #endif
}
NOKPROBE_SYMBOL(mdscr_write);
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 08274e4317b2..0d259e355c90 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -156,7 +156,11 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
* mode. Before this function is called it is not safe to call regular kernel
* code, instrumentable code, or any code which may trigger an exception.
*/
+#ifdef CONFIG_IEE
+void noinstr arm64_enter_nmi(struct pt_regs *regs)
+#else
static void noinstr arm64_enter_nmi(struct pt_regs *regs)
+#endif
{
regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 7fcbee0f6c0e..dee813ee6aad 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -29,12 +29,391 @@
#include <asm/asm-uaccess.h>
#include <asm/unistd.h>
+#ifdef CONFIG_IEE
+#include <asm/iee-def.h>
+
+#define BAD_SP_EL0 0
+#define BAD_ELR_EL1 1
+#define BAD_TCR_EL1 2
+#define BAD_IEE_SI 4
+#endif
+
.macro clear_gp_regs
.irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
mov x\n, xzr
.endr
.endm
+#ifdef CONFIG_KOI
+#ifdef CONFIG_IEE
+/*
+ * This function is used to switch to ko stack in glue code
+ */
+SYM_FUNC_START(koi_do_switch_to_ko_stack)
+ sub sp, sp, #48
+ stp x29, x30, [sp]
+ str x2, [sp, #16]
+ stp x0, x1, [sp, #32]
+
+ // iee_rw_gate(IEE_WRITE_KERNEL_STACK, current, sp)
+ mov x0, #IEE_WRITE_KOI_KERNEL_STACK
+ mrs x1, sp_el0
+ add x2, sp, #48
+
+ bl iee_rw_gate
+
+ // iee_rw_gate(IEE_READ_KOI_STACK, current)
+ mov x0, #IEE_READ_KOI_STACK
+ mrs x1, sp_el0
+ bl iee_rw_gate
+
+ ldp x29, x30, [sp]
+ ldr x2, [sp, #16]
+ add x1, sp, #32
+ mov sp, x0
+ ldp x0, x1, [x1]
+
+ isb
+ ret
+SYM_FUNC_END(koi_do_switch_to_ko_stack)
+
+/*
+ * This fucntion is used to switch to kernel stack in glue code
+ */
+SYM_FUNC_START(koi_do_switch_to_kernel_stack)
+ sub sp, sp, #48
+ stp x29, x30, [sp]
+ str x2, [sp, #16]
+ stp x0, x1, [sp, #32]
+ // iee_rw_gate(IEE_WRITE_KOI_STACK, current, sp)
+ mov x0, #IEE_WRITE_KOI_STACK
+ mrs x1, sp_el0
+ add x2, sp, #48
+ bl iee_rw_gate
+
+ // iee_rw_gate(IEE_READ_KOI_KERNEL_STACK, current)
+ mov x0, #IEE_READ_KOI_KERNEL_STACK
+ mrs x1, sp_el0
+ bl iee_rw_gate
+
+ ldp x29, x30, [sp]
+ ldr x2, [sp, #16]
+ add x1, sp, #32
+ mov sp, x0
+ ldp x0, x1, [x1]
+ isb
+ ret
+SYM_FUNC_END(koi_do_switch_to_kernel_stack)
+
+/*
+ * Before switch to ko's pgtable, we must switch current stack to ko's stack.
+ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching,
+ * so we need to copy from kernel stack to ko stack
+ * the memory region to copy is [sp, stack_top)
+ * void koi_switch_to_ko_stack(void);
+ */
+SYM_FUNC_START(koi_switch_to_ko_stack)
+ mrs x17, pan
+ msr pan, 0x0
+
+ sub sp, sp, #32
+ str x17, [sp, #16]
+ stp x30, x29, [sp]
+
+ // current sp stores in x1
+ add x1, x1, #176
+ // current sp_el0 stores in x0
+ bl _iee_write_koi_kernel_stack
+
+ mrs x0, sp_el0
+ bl _iee_read_koi_stack
+
+ ldr x17, [sp, #16]
+ ldp x30, x29, [sp]
+ add sp, sp, #32
+
+ msr pan, x17
+
+ sub x0, x0, #176
+ mov x1, sp
+ mov x2, #176
+
+ // memcpy(current->driver_stack, current->kernel_stack, 176)
+ mov x16, lr
+ bl memcpy
+ mov lr, x16
+
+ mov sp, x0
+ isb
+ ret
+SYM_FUNC_END(koi_switch_to_ko_stack)
+
+SYM_FUNC_START(koi_switch_to_kernel_stack)
+ /*
+ * current sp belongs to driver stack, and the bottom 160 bytes saves registers when exception occurred,
+ * so we should add 160 to current sp, and store it in task_struct
+ * also, fetch kernel sp from task_struct, copy the bottom 160 bytes from driver stack to kernel stack
+ */
+ mrs x17, pan
+ msr pan, 0x0
+
+ sub sp, sp, #32
+ stp x30, x29, [sp]
+ str x17, [sp, #16]
+
+ mrs x0, sp_el0
+ add x1, sp, #192
+ bl _iee_write_koi_stack
+
+ mrs x0, sp_el0
+ bl _iee_read_koi_kernel_stack
+
+ ldr x17, [sp, #16]
+ ldp x30, x29, [sp]
+ add sp, sp, #32
+
+ msr pan, x17
+
+ // x0 = kernel_stack
+ sub x0, x0, #160
+ mov x1, sp
+ // x2 = 160
+ mov x2, #160
+
+ mov x16, lr
+ bl memcpy
+ mov lr, x16
+
+ mov sp, x0
+ isb
+ ret
+SYM_FUNC_END(koi_switch_to_kernel_stack)
+#else
+/*
+ * This function is used to switch to ko stack in glue code
+ */
+SYM_FUNC_START(koi_do_switch_to_ko_stack)
+ sub sp, sp, #16
+ stp x16, x17, [sp]
+ mrs x17, sp_el0
+ adrp x16, koi_offset
+ ldr x16, [x16, #:lo12:koi_offset]
+ add x17, x17, x16
+ add x16, sp, #16
+ str x16, [x17, #koi_kernel_from_token_offset]
+ ldr x16, [x17, #koi_from_token_offset]
+ mov x17, sp
+ mov sp, x16
+ ldp x16, x17, [x17]
+ isb
+ ret
+SYM_FUNC_END(koi_do_switch_to_ko_stack)
+
+/*
+ * This fucntion is used to switch to kernel stack in glue code
+ */
+SYM_FUNC_START(koi_do_switch_to_kernel_stack)
+ sub sp, sp, #16
+ stp x16, x17, [sp]
+ mrs x17, sp_el0
+ adrp x16, koi_offset
+ ldr x16, [x16, #:lo12:koi_offset]
+ add x17, x17, x16
+ add x16, sp, #16
+ str x16, [x17, #koi_from_token_offset]
+ ldr x16, [x17, #koi_kernel_from_token_offset]
+ mov x17, sp
+ mov sp, x16
+ ldp x16, x17, [x17]
+ isb
+ ret
+SYM_FUNC_END(koi_do_switch_to_kernel_stack)
+
+/*
+ * Before switch to ko's pgtable, we must switch current stack to ko's stack.
+ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching,
+ * so we need to copy from kernel stack to ko stack
+ * the memory region to copy is [sp, stack_top)
+ * void koi_switch_to_ko_stack(unsigned long stack_top);
+ */
+SYM_FUNC_START(koi_switch_to_ko_stack)
+ // current sp stores in x1
+ add x3, x1, #176
+ adrp x4, koi_offset
+ ldr x4, [x4, #:lo12:koi_offset]
+ add x4, x0, x4
+ // current sp_el0 stores in x0
+ str x3, [x4, #koi_kernel_from_token_offset]
+ ldr x0, [x4, #koi_from_token_offset]
+ sub x0, x0, #176
+ mov x2, #176
+
+ // memcpy(current->driver_stack, current->kernel_stack, 176)
+ mov x16, lr
+ bl memcpy
+ mov lr, x16
+
+ mov sp, x0
+ isb
+ ret
+SYM_FUNC_END(koi_switch_to_ko_stack)
+
+SYM_FUNC_START(koi_switch_to_kernel_stack)
+ /*
+ * current sp belongs to driver stack, and the bottom 176 bytes saves registers when exception occurred,
+ * so we should add 176 to current sp, and store it in task_struct
+ * also, fetch kernel sp from task_struct, copy the bottom 176 bytes from driver stack to kernel stack
+ */
+ mov x1, sp
+ add x3, sp, #160
+
+ mrs x16, sp_el0
+ adrp x2, koi_offset
+ ldr x2, [x2, #:lo12:koi_offset]
+ add x16, x16, x2
+ str x3, [x16, #koi_from_token_offset]
+ // sp points to kernel_stack
+ ldr x0, [x16, #koi_kernel_from_token_offset]
+
+ // x0 = kernel_stack
+ sub x0, x0, #160
+ // x2 = 160
+ mov x2, #160
+ mov x16, lr
+ // memcpy(kernel_stack, driver_stack, 160)
+ bl memcpy
+ mov lr, x16
+ mov sp, x0
+ isb
+ ret
+SYM_FUNC_END(koi_switch_to_kernel_stack)
+#endif
+
+SYM_FUNC_START(koi_switch_to_ko_pgtbl)
+ stp x0, x1, [sp, #16 * 1]
+ stp x2, x3, [sp, #16 * 2]
+ stp x4, x5, [sp, #16 * 3]
+ stp x6, x7, [sp, #16 * 4]
+ stp x8, x9, [sp, #16 * 5]
+ stp x10, x11, [sp, #16 * 6]
+ stp x12, x13, [sp, #16 * 7]
+ stp x14, x15, [sp, #16 * 8]
+ stp x16, x17, [sp, #16 * 9]
+ stp x18, x30, [sp, #16 * 10]
+
+ adrp x0, koi_swapper_ttbr1
+ ldr x0, [x0, #:lo12:koi_swapper_ttbr1]
+ cbz x0, 0f
+ bl koi_do_switch_to_ko_pgtbl
+ // if x0 == 0, don't need to switch pgtable and stack, jump to 0
+ cbz x0, 0f
+ mov x19, x0
+ // if current on task's kernel stack， switch to ko stack
+ mrs x0, sp_el0
+ mov x1, sp
+ ldr x2, [x0, TSK_STACK]
+ eor x2, x2, x1
+ and x2, x2, #~(THREAD_SIZE - 1)
+ cbnz x2, 1f
+
+ bl koi_switch_to_ko_stack
+1:
+#ifndef CONFIG_IEE
+ msr ttbr1_el1, x19
+ isb
+ nop
+ nop
+ nop
+#else
+ mov x0, #IEE_SWITCH_TO_KOI
+ mov x1, x19
+ bl iee_rwx_gate_entry
+#endif
+0:
+
+ ldp x0, x1, [sp, #16 * 1]
+ ldp x2, x3, [sp, #16 * 2]
+ ldp x4, x5, [sp, #16 * 3]
+ ldp x6, x7, [sp, #16 * 4]
+ ldp x8, x9, [sp, #16 * 5]
+ ldp x10, x11, [sp, #16 * 6]
+ ldp x12, x13, [sp, #16 * 7]
+ ldp x14, x15, [sp, #16 * 8]
+ ldp x16, x17, [sp, #16 * 9]
+ ldp x18, x30, [sp, #16 * 10]
+ ret
+SYM_FUNC_END(koi_switch_to_ko_pgtbl)
+
+.pushsection ".koi.text", "ax"
+SYM_FUNC_START(koi_switch_to_kernel_pgtbl)
+ sub sp, sp, #160
+ stp x0, x1, [sp, #16 * 0]
+ stp x2, x3, [sp, #16 * 1]
+ stp x4, x5, [sp, #16 * 2]
+ stp x6, x7, [sp, #16 * 3]
+ stp x8, x9, [sp, #16 * 4]
+ stp x10, x11, [sp, #16 * 5]
+
+ stp x12, x13, [sp, #16 * 6]
+ stp x14, x15, [sp, #16 * 7]
+ stp x16, x17, [sp, #16 * 8]
+ stp x18, x30, [sp, #16 * 9]
+ // check whether paging init finished
+ adrp x0, koi_swapper_ttbr1
+ ldr x0, [x0, #:lo12:koi_swapper_ttbr1]
+ cbz x0, 0f
+
+ bl koi_do_switch_to_kernel_pgtbl
+ /*
+ * koi_do_switch_to_kernel_pgtbl return 0 indicates
+ * that when exception occurred, the isolated ko is executing under koi pgtbl,
+ * so we need to switch stack to kernel stack after switch pgtbl back to koi_swapper_ttbr1.
+ */
+ cbz x0, 0f
+#ifndef CONFIG_IEE
+ mrs x0, sp_el0
+ adrp x1, koi_offset
+ ldr x1, [x1, #:lo12:koi_offset]
+ add x0, x0, x1
+ mov x16, sp
+ ldr x17, [x0, koi_stack_base_from_token_offset]
+ eor x17, x17, x16
+ and x17, x17, #~(THREAD_SIZE - 1)
+ cbnz x17, 0f
+#else
+ // save current pan
+ mrs x17, pan
+ // disable pan
+ msr pan, 0x0
+ mrs x0, sp_el0
+ bl _iee_read_koi_stack_base
+ // restore pan
+ msr pan, x17
+
+ mov x16, sp
+ eor x0, x0, x16
+ and x0, x0, #~(THREAD_SIZE - 1)
+ cbnz x0, 0f
+#endif
+ bl koi_switch_to_kernel_stack
+0:
+
+ ldp x0, x1, [sp, #16 * 0]
+ ldp x2, x3, [sp, #16 * 1]
+ ldp x4, x5, [sp, #16 * 2]
+ ldp x6, x7, [sp, #16 * 3]
+ ldp x8, x9, [sp, #16 * 4]
+ ldp x10, x11, [sp, #16 * 5]
+ ldp x12, x13, [sp, #16 * 6]
+ ldp x14, x15, [sp, #16 * 7]
+ ldp x16, x17, [sp, #16 * 8]
+ ldp x18, x30, [sp, #16 * 9]
+ add sp, sp, #160
+ ret
+SYM_FUNC_END(koi_switch_to_kernel_pgtbl)
+.popsection
+#endif
+
.macro kernel_ventry, el:req, ht:req, regsize:req, label:req
.align 7
.Lventry_start\@:
@@ -151,6 +530,17 @@ alternative_else_nop_endif
#endif
.endm
+#ifdef CONFIG_IEE
+// SP_EL0 check failed.
+SYM_FUNC_START_LOCAL(sp_el0_check_failed)
+ mov x0, sp
+ mov x1, #BAD_SP_EL0
+ mrs x2, esr_el1
+ bl iee_bad_mode
+ ASM_BUG()
+SYM_FUNC_END(sp_el0_check_failed)
+#endif
+
/* Clear the MTE asynchronous tag check faults */
.macro clear_mte_async_tcf thread_sctlr
#ifdef CONFIG_ARM64_MTE
@@ -224,6 +614,14 @@ alternative_cb_end
ldr_this_cpu tsk, __entry_task, x20
msr sp_el0, tsk
+#ifdef CONFIG_IEE
+ // tsk check.
+ ldr_this_cpu x19, __entry_task, x20
+ mrs x20, sp_el0
+ cmp x19, x20
+ b.ne sp_el0_check_failed
+#endif
+
/*
* Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
* when scheduling.
@@ -276,6 +674,13 @@ alternative_else_nop_endif
scs_load_current
.else
+#ifdef CONFIG_IEE
+ // tsk check.
+ ldr_this_cpu x19, __entry_task, x20
+ mrs x20, sp_el0
+ cmp x19, x20
+ b.ne sp_el0_check_failed
+#endif
add x21, sp, #PT_REGS_SIZE
get_current_task tsk
.endif /* \el == 0 */
@@ -333,9 +738,11 @@ alternative_else_nop_endif
.endm
.macro kernel_exit, el
+ #ifndef CONFIG_IEE
.if \el != 0
disable_daif
.endif
+ #endif
#ifdef CONFIG_ARM64_PSEUDO_NMI
alternative_if_not ARM64_HAS_GIC_PRIO_MASKING
@@ -411,6 +818,41 @@ alternative_else_nop_endif
msr elr_el1, x21 // set up the return data
msr spsr_el1, x22
+
+#ifdef CONFIG_IEE
+
+ .if \el == 0
+
+ #ifndef CONFIG_UNMAP_KERNEL_AT_EL0
+ // SET hpd1 = 0 start
+ mrs x0, tcr_el1
+ and x0, x0, #0xFFFFFBFFFFFFFFFF
+ and x0, x0, #0xFFFFFFFFFFBFFFFF
+ msr tcr_el1, x0
+ // SET hpd1 = 0 end
+
+ disable_daif
+
+ // Check ELR_EL1
+ mrs x0, elr_el1
+ lsr x0, x0, #48
+ tst x0, #0xffff
+ b.ne 5f
+ #endif
+
+ .endif
+
+#else
+#ifdef CONFIG_KOI
+ .if \el==0
+ mrs x0, tcr_el1
+ and x0, x0, #0xFFFFFFFFFFBFFFFF
+ msr tcr_el1,x0
+ .endif
+#endif
+
+#endif
+
ldp x0, x1, [sp, #16 * 0]
ldp x2, x3, [sp, #16 * 1]
ldp x4, x5, [sp, #16 * 2]
@@ -569,12 +1011,167 @@ SYM_CODE_START_LOCAL(__bad_stack)
SYM_CODE_END(__bad_stack)
#endif /* CONFIG_VMAP_STACK */
+/*
+ * iee exception entry
+ */
+ .macro iee_exception_entry, el
+
+ /* Check whether exception is permmited. */
+ ldr x1, =__iee_si_no_irq
+ cmp x1, x22
+ b.hi 1148f
+ ldr x1, =__iee_si_end
+ cmp x1, x22
+ b.lo 1148f
+ /* ELR check fail */
+ mov x0, sp
+ mov x1, #BAD_IEE_SI
+ mrs x2, esr_el1
+ bl iee_bad_mode
+ ASM_BUG()
+1148:
+
+ /* el0 set hpds */
+ .if \el == 0
+
+ #ifndef CONFIG_UNMAP_KERNEL_AT_EL0
+ /* SET hpd1 = 1 start */
+ mrs x0, tcr_el1
+ orr x0, x0, #0x0000040000000000
+ orr x0, x0, #0x0000000000400000
+ msr tcr_el1, x0
+ /* SET hpd1 = 1 end */
+
+ disable_daif
+
+ /* Check TCR_EL1 */
+ mrs x0, tcr_el1
+ tst x0, #0x0000040000000000
+ b.eq 5f
+ tst x0, #0x0000000000400000
+ b.ne 6f
+
+5:
+ /* TCR_EL1 check fail */
+ mov x0, sp
+ mov x1, #BAD_TCR_EL1
+ mrs x2, esr_el1
+ bl iee_bad_mode
+ ASM_BUG()
+
+6:
+ nop
+ #endif
+
+ .else
+#ifdef CONFIG_IEE_INTERRUPTABLE
+ /* el1 save elr_el1 and set pan */
+ /* Check ELR_EL1 */
+ ldr x1, =__iee_code_start
+ cmp x1, x22
+ b.hi 7f
+ ldr x1, =__iee_code_end
+ cmp x1, x22
+ b.lo 7f
+ /* Exception from iee code */
+ /* Switch to kernel stack */
+ mrs x0, sp_el0 /* x0 -> task_struct(VA) */
+ adrp x2, iee_offset
+ ldr x2, [x2, #:lo12:iee_offset]
+ add x1, x0, x2 /* x1 -> task_token(IEE) */
+ // store iee stack
+ mov x3, sp
+ str x3, [x1, #iee_from_token_offset]
+ // load kernel stack
+ ldr x3, [x1, #kernel_from_token_offset]
+ mov sp, x3
+ sub sp, sp, #PT_REGS_SIZE
+ /* Enable PAN */
+ msr pan, #0x1
+
+7:
+ /* Exception from kernel code */
+ mov x0, #0x0
+ mov x1, #0x0
+ mov x2, #0x0
+ mov x3, #0x0
+#endif
+ .endif
+ .endm
+
+/*
+ * iee exception exit
+ */
+ .macro iee_exception_exit, el
+ // Disable daif
+ disable_daif
+
+ .if \el == 1
+#ifdef CONFIG_IEE_INTERRUPTABLE
+ /* el1 pop elr_el1 and set pan */
+ /* Check ELR_EL1 */
+ ldr x1, =__iee_code_start
+ cmp x1, x22
+ b.hi 9f
+ ldr x1, =__iee_code_end
+ cmp x1, x22
+ b.lo 9f
+ /* Eret iee code */
+ /* Disable PAN */
+ msr pan, #0x0
+ /* Switch to iee stack */
+ add sp, sp, #PT_REGS_SIZE
+ mrs x0, sp_el0 /* x0 -> task_struct */
+ adrp x2, iee_offset
+ ldr x2, [x2, #:lo12:iee_offset]
+ add x1, x0, x2 /* x1 -> task_token(IEE) */
+ // store kernel stack
+ mov x3, sp
+ str x3, [x1, #kernel_from_token_offset]
+ // load iee stack
+ ldr x2, [x1, #iee_from_token_offset]
+ mov sp, x2
+ /* Load ELR_EL1 from iee stack */
+ ldr x21, [sp, #S_PC]
+ /* Check the modify of ELR_EL1 */
+ cmp x21, x22
+ b.ne 8f
+ /* ELR_EL1 not modified */
+ b 9f
+
+8:
+ // ELR_EL1 modified
+ mov x0, sp
+ mov x1, #BAD_ELR_EL1
+ mrs x2, esr_el1
+ bl iee_bad_mode
+ ASM_BUG()
+
+9:
+ // Eret kernel code
+ mov x0, #0x0
+ mov x1, #0x0
+ mov x2, #0x0
+ mov x3, #0x0
+#endif
+ .endif
+ .endm
.macro entry_handler el:req, ht:req, regsize:req, label:req
SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label)
kernel_entry \el, \regsize
+
+ #ifdef CONFIG_IEE
+ iee_exception_entry \el
+ #endif
+
mov x0, sp
bl el\el\ht\()_\regsize\()_\label\()_handler
+
+ #ifdef CONFIG_IEE
+ iee_exception_exit \el
+ #endif
+
.if \el == 0
b ret_to_user
.else
@@ -844,6 +1441,13 @@ SYM_FUNC_START(cpu_switch_to)
ldr lr, [x8]
mov sp, x9
msr sp_el0, x1
+#ifdef CONFIG_IEE
+ // tsk check.
+ ldr_this_cpu x8, __entry_task, x9
+ mrs x9, sp_el0
+ cmp x8, x9
+ b.ne sp_el0_check_failed
+#endif
ptrauth_keys_install_kernel x1, x8, x9, x10
scs_save x0
scs_load_current
@@ -1033,6 +1637,13 @@ SYM_CODE_START(__sdei_asm_handler)
mrs x28, sp_el0
ldr_this_cpu dst=x0, sym=__entry_task, tmp=x1
msr sp_el0, x0
+#ifdef CONFIG_IEE
+ // tsk check.
+ ldr_this_cpu x0, __entry_task, x1
+ mrs x1, sp_el0
+ cmp x0, x1
+ b.ne sp_el0_check_failed
+#endif
/* If we interrupted the kernel point to the previous stack/frame. */
and x0, x3, #0xc
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 5cdfcc9e3e54..c0af965bd92e 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1309,7 +1309,11 @@ void sme_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
isb();
/* Allow EL0 to access TPIDR2 */
+ #ifdef CONFIG_IEE
+ iee_rwx_gate_entry(IEE_WRITE_sctlr_el1, read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2);
+ #else
write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1);
+ #endif
isb();
}
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 6517bf2644a0..5c75e069d877 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -464,6 +464,42 @@ SYM_FUNC_END(create_kernel_mapping)
set_this_cpu_offset \tmp1
.endm
+#ifdef CONFIG_IEE
+ .macro init_cpu_task_checked tsk, tmp1, tmp2
+ msr sp_el0, \tsk
+ // tsk check.
+ adrp x29, __per_cpu_offset
+ mrs \tmp1, sp_el0
+ ldr \tmp2, [\tmp1, #TSK_TI_CPU] /* cpu number */
+1:
+ cmp \tmp2, #0
+ b.eq 2f
+ add x29, x29, #8
+ sub \tmp2, \tmp2, #1
+ b 1b
+2:
+ ldr \tmp2, [x29, #:lo12:__per_cpu_offset] /* cpu offset */
+ adr_l x29, __entry_task
+ ldr x29, [x29, \tmp2]
+ cmp x29, \tmp1
+ b.ne sp_el0_check_failed
+
+ ldr \tmp1, [\tsk, #TSK_STACK]
+ add sp, \tmp1, #THREAD_SIZE
+ sub sp, sp, #PT_REGS_SIZE
+
+ stp xzr, xzr, [sp, #S_STACKFRAME]
+ add x29, sp, #S_STACKFRAME
+
+ scs_load_current
+
+ adr_l \tmp1, __per_cpu_offset
+ ldr w\tmp2, [\tsk, #TSK_TI_CPU]
+ ldr \tmp1, [\tmp1, \tmp2, lsl #3]
+ set_this_cpu_offset \tmp1
+ .endm
+#endif
+
/*
* The following fragment of code is executed with the MMU enabled.
*
@@ -661,6 +697,18 @@ SYM_FUNC_START_LOCAL(secondary_startup)
SYM_FUNC_END(secondary_startup)
.text
+#ifdef CONFIG_IEE
+// SP_EL0 check failed.
+SYM_FUNC_START_LOCAL(sp_el0_check_failed)
+ 1:
+ nop
+ nop
+ nop
+ nop
+ b 1f
+SYM_FUNC_END(sp_el0_check_failed)
+#endif
+
SYM_FUNC_START_LOCAL(__secondary_switched)
mov x0, x20
bl set_cpu_boot_mode_flag
@@ -677,7 +725,11 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
ldr x2, [x0, #CPU_BOOT_TASK]
cbz x2, __secondary_too_slow
+#ifdef CONFIG_IEE
+ init_cpu_task_checked x2, x1, x3
+#else
init_cpu_task x2, x1, x3
+#endif
#ifdef CONFIG_ARM64_PTR_AUTH
ptrauth_keys_init_cpu x2, x3, x4, x5
@@ -746,6 +798,10 @@ SYM_FUNC_START(__enable_mmu)
cmp x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX
b.gt __no_granule_support
phys_to_ttbr x2, x2
+#ifdef CONFIG_IEE
+ mov x3, #1
+ bfi x2, x3, #48, #16 // ASID 1 is used by IEE rwx gate.
+#endif
msr ttbr0_el1, x2 // load TTBR0
load_ttbr1 x1, x1, x3
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 02870beb271e..1c14428a3ed4 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -34,6 +34,10 @@
#include <asm/trans_pgd.h>
#include <asm/virt.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
/*
* Hibernate core relies on this value being 0 on resume, and marks it
* __nosavedata assuming it will keep the resume kernel's '0' value. This
@@ -197,12 +201,22 @@ static int create_safe_exec_page(void *src_start, size_t length,
phys_addr_t trans_ttbr0;
unsigned long t0sz;
int rc;
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
if (!page)
return -ENOMEM;
memcpy(page, src_start, length);
caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length);
+
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(page_address(page)));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)page_address(page));
+ #endif
+
rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
if (rc)
return rc;
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index d39a8787edf2..b5ac4b7670bc 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -26,6 +26,10 @@
#include <asm/cputype.h>
#include <asm/system_misc.h>
+#ifdef CONFIG_IEE
+#include <asm/iee-si.h>
+#endif
+
/* Breakpoint currently in use for each BRP. */
static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
@@ -102,13 +106,68 @@ int hw_breakpoint_slots(int type)
WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
+#ifdef CONFIG_IEE
+
+#define IEE_SI_READ_WB_REG_CASE(OFF, N, REG, VAL) \
+ case (OFF + N): \
+ IEE_SI_AARCH64_DBG_READ(N, REG, VAL); \
+ break
+
+#define IEE_SI_WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
+ case (OFF + N): \
+ IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL); \
+ break
+
+#define IEE_SI_GEN_READ_REG_CASES(OFF, REG, VAL) \
+ IEE_SI_READ_WB_REG_CASE(OFF, 0, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
+
+#define IEE_SI_GEN_WRITE_REG_CASES(OFF, REG, VAL) \
+ IEE_SI_WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
+ WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
+
+#endif
+
static u64 read_wb_reg(int reg, int n)
{
u64 val = 0;
switch (reg + n) {
+// #ifdef CONFIG_IEE
+// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
+// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+// #else
GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+// #endif
GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
default:
@@ -122,8 +181,13 @@ NOKPROBE_SYMBOL(read_wb_reg);
static void write_wb_reg(int reg, int n, u64 val)
{
switch (reg + n) {
+// #ifdef CONFIG_IEE
+// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
+// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+// #else
GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
+// #endif
GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
default:
@@ -171,6 +235,10 @@ static int is_a32_compat_bp(struct perf_event *bp)
return tsk && is_a32_compat_thread(task_thread_info(tsk));
}
+#ifdef CONFIG_IEE
+int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw);
+#endif
+
/**
* hw_breakpoint_slot_setup - Find and setup a perf slot according to
* operations
@@ -191,6 +259,37 @@ static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
{
int i;
struct perf_event **slot;
+// reserve hw breakpoint 0 for iee rwx gate in kernel sapce.
+// #ifdef CONFIG_IEE
+// struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+// if (arch_check_bp_in_kernelspace(info)){
+// for (i = 1; i < max_slots; ++i) { // search from hw breakpoint 1
+// slot = &slots[i];
+// switch (ops) {
+// case HW_BREAKPOINT_INSTALL:
+// if (!*slot) {
+// *slot = bp;
+// return i;
+// }
+// break;
+// case HW_BREAKPOINT_UNINSTALL:
+// if (*slot == bp) {
+// *slot = NULL;
+// return i;
+// }
+// break;
+// case HW_BREAKPOINT_RESTORE:
+// if (*slot == bp)
+// return i;
+// break;
+// default:
+// pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
+// return -EINVAL;
+// }
+// }
+// return -ENOSPC;
+// }
+// #endif
for (i = 0; i < max_slots; ++i) {
slot = &slots[i];
diff --git a/arch/arm64/kernel/iee/Makefile b/arch/arm64/kernel/iee/Makefile
new file mode 100644
index 000000000000..123c68c5cc4e
--- /dev/null
+++ b/arch/arm64/kernel/iee/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_IEE) += iee.o iee-gate.o iee-func.o
\ No newline at end of file
diff --git a/arch/arm64/kernel/iee/iee-func.c b/arch/arm64/kernel/iee/iee-func.c
new file mode 100644
index 000000000000..7764dbd41555
--- /dev/null
+++ b/arch/arm64/kernel/iee/iee-func.c
@@ -0,0 +1,187 @@
+#include "asm/pgtable.h"
+#include <linux/memory.h>
+#include <linux/mm.h>
+#include <asm/tlb.h>
+#include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
+
+void set_iee_page_valid(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
+
+ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1))))
+ return;
+
+ pte = __pte(pte_val(pte) | 0x1);
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
+}
+
+void set_iee_page_invalid(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
+
+ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1))))
+ return;
+
+ pte = __pte(pte_val(pte) & ~0x1);
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
+}
+
+void iee_set_logical_mem_ro(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
+
+ if(addr < PAGE_OFFSET)
+ return;
+
+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM);
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
+}
+
+void iee_set_logical_mem_rw(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
+
+ if((addr < PAGE_OFFSET) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 2))))
+ return;
+
+ pte = __pte(pte_val(pte) | PTE_DBM);
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
+}
+
+void iee_set_token_page_valid(void *token, void *new)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token);
+
+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token);
+
+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token);
+
+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token);
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new)));
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE));
+ isb();
+}
+
+void iee_set_token_page_invalid(void *token)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token);
+
+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token);
+
+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token);
+
+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token);
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(((pte_val(pte) & ~((unsigned long)0x1)) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token - IEE_OFFSET)));
+ set_pte(ptep, pte);
+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE));
+ isb();
+}
+
+void iee_set_kernel_ppage(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+
+ int i;
+ for(i = 0; i < 4; i++)
+ {
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(pte_val(pte) & ~PTE_USER & ~PTE_NG);
+ iee_set_pte_ppage(ptep, pte);
+ ptep++;
+ }
+ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE);
+ isb();
+}
+
+void iee_set_kernel_upage(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+
+ int i;
+ for(i = 0; i < 4; i++)
+ {
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(pte_val(pte) | PTE_USER | PTE_NG);
+ iee_set_pte_upage(ptep, pte);
+ ptep++;
+ }
+ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE);
+ isb();
+}
\ No newline at end of file
diff --git a/arch/arm64/kernel/iee/iee-gate.S b/arch/arm64/kernel/iee/iee-gate.S
new file mode 100644
index 000000000000..6de99a018bde
--- /dev/null
+++ b/arch/arm64/kernel/iee/iee-gate.S
@@ -0,0 +1,174 @@
+#include <asm/asm-offsets.h>
+#include <linux/linkage.h>
+#include <asm/bug.h>
+#include <asm-generic/export.h>
+
+#ifdef CONFIG_IEE
+
+SYM_FUNC_START(iee_rw_gate)
+ /* save daif, close irq */
+ mrs x13, daif
+ msr daifset, #0x2
+ isb
+ /* save lr */
+ sub sp, sp, #16
+ stp x29, x30, [sp]
+ bl iee_protected_rw_gate
+ /* restore lr */
+ ldp x29, x30, [sp]
+ add sp, sp, #16
+ /* restore daif */
+ msr daif, x13
+ ret
+SYM_FUNC_END(iee_rw_gate)
+#if defined(CONFIG_CREDP) || defined(CONFIG_KOI)
+EXPORT_SYMBOL(iee_rw_gate)
+#endif
+
+ .pushsection ".iee.text.header", "ax"
+
+SYM_FUNC_START(iee_protected_rw_gate)
+ mrs x9, pan
+ /* disable PAN */
+ msr pan, #0x0
+ /* switch to iee stack */
+ mrs x9, sp_el0 /* x9 -> task_struct */
+ adrp x12, iee_offset
+ ldr x12, [x12, #:lo12:iee_offset]
+ add x11, x9, x12 /* x11 -> task_token(IEE) */
+ // store kernel stack
+ mov x10, sp
+ str x10, [x11, #kernel_from_token_offset]
+ // load iee stack
+ ldr x10, [x11, #iee_from_token_offset]
+ mov sp, x10
+#ifdef CONFIG_IEE_INTERRUPTABLE
+ isb
+ /* restore daif */
+ msr daif, x13
+ sub sp, sp, #16
+ stp x29, x30, [sp]
+#else
+ sub sp, sp, #16
+ stp x13, x30, [sp]
+#endif
+ /* call iee func */
+ bl iee_dispatch
+#ifdef CONFIG_IEE_INTERRUPTABLE
+ ldp x29, x30, [sp]
+ add sp, sp, #16
+ /* store and disable daif */
+ mrs x13, daif
+ msr daifset, #0x2
+ isb
+#else
+ ldp x13, x30, [sp]
+ add sp, sp, #16
+#endif
+ /* switch to kernel stack */
+ mrs x9, sp_el0 /* x9 -> task_struct(VA) */
+ adrp x12, iee_offset
+ ldr x12, [x12, #:lo12:iee_offset]
+ add x11, x9, x12 /* x11 -> task_token(IEE) */
+ // store iee stack
+ mov x10, sp
+ str x10, [x11, #iee_from_token_offset]
+ // load kernel stack
+ ldr x10, [x11, #kernel_from_token_offset]
+ mov sp, x10
+ /* enable PAN */
+ msr pan, #0x1
+ ret
+SYM_FUNC_END(iee_protected_rw_gate)
+
+ .popsection
+
+#include <asm/asm-bug.h>
+#define BAD_IEE 4
+#define BAD_IEE_SI 5
+
+#define SYS_TCR_EL1_HPD1 0x40000000000
+#define SYS_TCR_EL1_A1 0x400000
+
+ .pushsection ".iee.exec_entry", "ax"
+
+SYM_FUNC_START(iee_rwx_gate_entry)
+ /* Disable irq first. */
+ mrs x15, daif // use x15 to restore daif
+ msr DAIFSet, #0xf
+ isb
+
+ /* Set HPD1 = 0 to exec follwing codes in U RWX page */
+ mrs x9, tcr_el1
+ bic x9, x9, #SYS_TCR_EL1_HPD1
+ bic x9, x9, #SYS_TCR_EL1_A1
+ msr tcr_el1, x9
+ isb
+
+ b iee_rwx_gate_tramp
+SYM_FUNC_END(iee_rwx_gate_entry)
+ .popsection
+
+ .pushsection ".iee.si_text", "awx"
+
+SYM_FUNC_START(iee_rwx_gate_tramp)
+ /* Check tcr val. */
+ mrs x10, tcr_el1
+ adrp x12, iee_si_tcr // tcr val shall be const after init
+ ldr x12, [x12, #:lo12:iee_si_tcr]
+ cbz x12, 1f
+ cmp x12, x10
+ b.ne 3f
+1:
+ mov x13, sp
+ /* If iee hasn't been initialized, skip stack switch. */
+ ldr x11, =iee_init_done
+ ldr x10, [x11]
+ cbz x10, 2f
+
+ /* Switch to iee stack */
+ mrs x9, sp_el0 // x9 -> task_struct
+ adrp x12, iee_offset
+ ldr x12, [x12, #:lo12:iee_offset]
+ add x11, x9, x12 // x11 -> task_token(IEE)
+ // load iee stack
+ ldr x10, [x11, #iee_from_token_offset]
+ mov sp, x10
+
+ /* x15 stores daif and x13 stores previous sp */
+2:
+ stp x15, x13, [sp, #-32]!
+ stp x29, x30, [sp, #16]
+ bl iee_si_handler // enter actual handler
+ ldp x29, x30, [sp, #16]
+
+ b iee_rwx_gate_exit // jump to iee exit
+3:
+ mov x0, sp
+ mov x1, #BAD_IEE_SI
+ mrs x2, esr_el1
+ bl iee_bad_mode
+ ASM_BUG()
+SYM_FUNC_END(iee_rwx_gate_tramp)
+
+ .popsection
+
+ .pushsection ".iee.exec_exit", "ax"
+
+SYM_FUNC_START(iee_rwx_gate_exit)
+ ldp x15, x13, [sp], #32
+ mov sp, x13 // switch to kernel stack
+ mrs x9, tcr_el1
+ orr x9, x9, #SYS_TCR_EL1_HPD1
+ orr x9, x9, #SYS_TCR_EL1_A1
+ msr tcr_el1, x9
+/* --------Page boundary-------- */
+ isb
+ msr daif, x15
+ isb
+ ret
+SYM_FUNC_END(iee_rwx_gate_exit)
+
+ .popsection
+
+#endif
diff --git a/arch/arm64/kernel/iee/iee.c b/arch/arm64/kernel/iee/iee.c
new file mode 100644
index 000000000000..6b9f7d40df67
--- /dev/null
+++ b/arch/arm64/kernel/iee/iee.c
@@ -0,0 +1,1360 @@
+#include "linux/sched.h"
+#include <linux/stdarg.h>
+#include <asm/pgtable-types.h>
+#include <asm/iee.h>
+#include <asm/iee-si.h>
+#include <asm/sysreg.h>
+#include <linux/pgtable.h>
+#include <linux/cred.h>
+#include <asm/iee-slab.h>
+#include <asm/percpu.h>
+
+#ifdef CONFIG_IEE
+extern struct cred init_cred;
+extern s64 memstart_addr;
+
+void __iee_code _iee_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
+void __iee_code _iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd);
+void __iee_code _iee_set_pte(pte_t *ptep, pte_t pte);
+void __iee_code _iee_set_pmd(pmd_t *pmdp, pmd_t pmd);
+void __iee_code _iee_set_pud(pud_t *pudp, pud_t pud);
+void __iee_code _iee_set_p4d(p4d_t *p4dp, p4d_t p4d);
+void __iee_code _iee_set_bm_pte(pte_t *ptep, pte_t pte);
+pteval_t __iee_code _iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval);
+pteval_t __iee_code _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval);
+void __iee_code _iee_write_in_byte(void *ptr, __u64 data, int length);
+void __iee_code _iee_set_cred_uid(struct cred *cred, kuid_t uid);
+void __iee_code _iee_set_cred_gid(struct cred *cred, kgid_t gid);
+void __iee_code _iee_copy_cred(struct cred *old, struct cred *new);
+void __iee_code _iee_set_cred_suid(struct cred *cred, kuid_t suid);
+void __iee_code _iee_set_cred_sgid(struct cred *cred, kgid_t sgid);
+void __iee_code _iee_set_cred_euid(struct cred *cred, kuid_t euid);
+void __iee_code _iee_set_cred_egid(struct cred *cred, kgid_t egid);
+void __iee_code _iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid);
+void __iee_code _iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid);
+void __iee_code _iee_set_cred_user(struct cred *cred, struct user_struct *user);
+void __iee_code _iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns);
+void __iee_code _iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts);
+void __iee_code _iee_set_cred_group_info(struct cred *cred, struct group_info *group_info);
+void __iee_code _iee_set_cred_securebits(struct cred *cred, unsigned securebits);
+void __iee_code _iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable);
+void __iee_code _iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted);
+void __iee_code _iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective);
+void __iee_code _iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset);
+void __iee_code _iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient);
+void __iee_code _iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring);
+void __iee_code _iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring);
+void __iee_code _iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring);
+void __iee_code _iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring);
+void __iee_code _iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth);
+void __iee_code _iee_set_cred_non_rcu(struct cred *cred, int non_rcu);
+void __iee_code _iee_set_cred_atomic_set_usage(struct cred *cred, int i);
+bool __iee_code _iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr);
+void __iee_code _iee_set_cred_security(struct cred *cred, void *security);
+void __iee_code _iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu);
+void __iee_code _iee_memset(void *ptr, int data, size_t n);
+void __iee_code _iee_set_track(struct track *ptr, struct track *data);
+void __iee_code _iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr);
+void __iee_code _iee_set_pte_upage(pte_t *ptep, pte_t pte);
+void __iee_code _iee_set_pte_ppage(pte_t *ptep, pte_t pte);
+void __iee_code _iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm);
+void __iee_code _iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd);
+void __iee_code _iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack);
+void __iee_code _iee_free_token(struct task_struct *tsk);
+unsigned long __iee_code _iee_read_token_stack(struct task_struct *tsk);
+void __iee_code _iee_write_entry_task(struct task_struct *tsk);
+#ifdef CONFIG_KOI
+unsigned long __iee_code _iee_read_koi_stack(struct task_struct *tsk);
+void __iee_code _iee_write_koi_stack(struct task_struct *tsk, unsigned long koi_stack);
+unsigned long __iee_code _iee_read_token_ttbr1(struct task_struct *tsk);
+void __iee_code _iee_write_token_ttbr1(struct task_struct *tsk, unsigned long current_ttbr1);
+unsigned long __iee_code _iee_read_koi_kernel_stack(struct task_struct *tsk);
+void __iee_code _iee_write_koi_kernel_stack(struct task_struct *tsk, unsigned long kernel_stack);
+unsigned long __iee_code _iee_read_koi_stack_base(struct task_struct *tsk);
+void __iee_code _iee_write_koi_stack_base(struct task_struct *tsk, unsigned long koi_stack_base);
+#endif
+
+/* wrapper functions */
+void __iee_code iee_wrapper_write_in_byte(va_list args) {
+ void *ptr = va_arg(args, void *);
+ __u64 data = va_arg(args, __u64);
+ int length = va_arg(args, int);
+ _iee_write_in_byte(ptr, data, length);
+}
+
+void __iee_code iee_wrapper_set_pte(va_list args) {
+ pte_t *ptep = va_arg(args, pte_t *);
+ pte_t pte = va_arg(args, pte_t);
+ _iee_set_pte(ptep, pte);
+}
+
+void __iee_code iee_wrapper_set_pmd(va_list args) {
+ pmd_t *pmdp = va_arg(args, pmd_t *);
+ pmd_t pmd = va_arg(args, pmd_t);
+ _iee_set_pmd(pmdp, pmd);
+}
+
+void __iee_code iee_wrapper_set_p4d(va_list args) {
+ p4d_t *p4dp = va_arg(args, p4d_t *);
+ p4d_t p4d = va_arg(args, p4d_t);
+ _iee_set_p4d(p4dp, p4d);
+}
+
+void __iee_code iee_wrapper_set_pud(va_list args) {
+ pud_t *pudp = va_arg(args, pud_t *);
+ pud_t pud = va_arg(args, pud_t);
+ _iee_set_pud(pudp, pud);
+}
+
+void __iee_code iee_wrapper_set_bm_pte(va_list args) {
+ pte_t *ptep = va_arg(args, pte_t *);
+ pte_t pte = va_arg(args, pte_t);
+ _iee_set_bm_pte(ptep, pte);
+}
+
+void __iee_code iee_wrapper_set_swapper_pgd(va_list args) {
+ pgd_t *pgdp = va_arg(args, pgd_t *);
+ pgd_t pgd = va_arg(args, pgd_t);
+ _iee_set_swapper_pgd(pgdp, pgd);
+}
+
+void __iee_code iee_wrapper_set_tramp_pgd(va_list args) {
+ pgd_t *pgdp = va_arg(args, pgd_t *);
+ pgd_t pgd = va_arg(args, pgd_t);
+ _iee_set_tramp_pgd_pre_init(pgdp, pgd);
+}
+
+pteval_t __iee_code iee_wrapper_set_xchg(va_list args) {
+ pteval_t ret;
+ pte_t *ptep = va_arg(args, pte_t *);
+ pteval_t pteval = va_arg(args, pteval_t);
+ ret = _iee_set_xchg_relaxed(ptep, pteval);
+ return (u64)ret;
+}
+
+pteval_t __iee_code iee_wrapper_set_cmpxchg(va_list args) {
+ pteval_t ret;
+ pte_t *ptep = va_arg(args, pte_t *);
+ pteval_t old_pteval = va_arg(args, pteval_t);
+ pteval_t new_pteval = va_arg(args, pteval_t);
+ ret = _iee_set_cmpxchg_relaxed(ptep, old_pteval, new_pteval);
+ return (u64)ret;
+}
+
+void __iee_code iee_wrapper_set_cred_uid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kuid_t uid = va_arg(args, kuid_t);
+ _iee_set_cred_uid(cred, uid);
+}
+
+void __iee_code iee_wrapper_set_cred_gid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kgid_t gid = va_arg(args, kgid_t);
+ _iee_set_cred_gid(cred, gid);
+}
+
+void __iee_code iee_wrapper_copy_cred(va_list args) {
+ struct cred *old = va_arg(args, struct cred *);
+ struct cred *new = va_arg(args, struct cred *);
+ _iee_copy_cred(old, new);
+}
+
+void __iee_code iee_wrapper_set_cred_suid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kuid_t suid = va_arg(args, kuid_t);
+ _iee_set_cred_suid(cred, suid);
+}
+
+void __iee_code iee_wrapper_set_cred_sgid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kgid_t sgid = va_arg(args, kgid_t);
+ _iee_set_cred_sgid(cred, sgid);
+}
+
+void __iee_code iee_wrapper_set_cred_euid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kuid_t euid = va_arg(args, kuid_t);
+ _iee_set_cred_euid(cred, euid);
+}
+
+void __iee_code iee_wrapper_set_cred_egid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kgid_t egid = va_arg(args, kgid_t);
+ _iee_set_cred_egid(cred, egid);
+}
+
+void __iee_code iee_wrapper_set_cred_fsuid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kuid_t fsuid = va_arg(args, kuid_t);
+ _iee_set_cred_fsuid(cred, fsuid);
+}
+
+void __iee_code iee_wrapper_set_cred_fsgid(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kgid_t fsgid = va_arg(args, kgid_t);
+ _iee_set_cred_fsgid(cred, fsgid);
+}
+
+void __iee_code iee_wrapper_set_cred_user(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct user_struct *user = va_arg(args, struct user_struct *);
+ _iee_set_cred_user(cred, user);
+}
+
+void __iee_code iee_wrapper_set_cred_user_ns(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct user_namespace *user_ns = va_arg(args, struct user_namespace *);
+ _iee_set_cred_user_ns(cred, user_ns);
+}
+
+void __iee_code iee_wrapper_set_cred_ucounts(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct ucounts *ucounts = va_arg(args, struct ucounts *);
+ _iee_set_cred_ucounts(cred, ucounts);
+}
+
+void __iee_code iee_wrapper_set_cred_group_info(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct group_info *group_info = va_arg(args, struct group_info *);
+ _iee_set_cred_group_info(cred, group_info);
+}
+
+void __iee_code iee_wrapper_set_cred_securebits(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ unsigned securebits = va_arg(args, unsigned);
+ _iee_set_cred_securebits(cred, securebits);
+}
+
+void __iee_code iee_wrapper_set_cred_cap_inheritable(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kernel_cap_t cap_inheritable = va_arg(args, kernel_cap_t);
+ _iee_set_cred_cap_inheritable(cred, cap_inheritable);
+}
+
+void __iee_code iee_wrapper_set_cred_cap_permitted(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kernel_cap_t cap_permitted = va_arg(args, kernel_cap_t);
+ _iee_set_cred_cap_permitted(cred, cap_permitted);
+}
+
+void __iee_code iee_wrapper_set_cred_cap_effective(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kernel_cap_t cap_effective = va_arg(args, kernel_cap_t);
+ _iee_set_cred_cap_effective(cred, cap_effective);
+}
+
+void __iee_code iee_wrapper_set_cred_cap_bset(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kernel_cap_t cap_bset = va_arg(args, kernel_cap_t);
+ _iee_set_cred_cap_bset(cred, cap_bset);
+}
+
+void __iee_code iee_wrapper_set_cred_cap_ambient(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ kernel_cap_t cap_ambient = va_arg(args, kernel_cap_t);
+ _iee_set_cred_cap_ambient(cred, cap_ambient);
+}
+
+void __iee_code iee_wrapper_set_cred_jit_keyring(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ unsigned long jit_keyring = va_arg(args, unsigned long);
+ _iee_set_cred_jit_keyring(cred, (unsigned char)jit_keyring);
+}
+
+void __iee_code iee_wrapper_set_cred_session_keyring(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct key *session_keyring = va_arg(args, struct key *);
+ _iee_set_cred_session_keyring(cred, session_keyring);
+}
+
+void __iee_code iee_wrapper_set_cred_process_keyring(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct key *process_keyring = va_arg(args, struct key *);
+ _iee_set_cred_process_keyring(cred, process_keyring);
+}
+
+void __iee_code iee_wrapper_set_cred_thread_keyring(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct key *thread_keyring = va_arg(args, struct key *);
+ _iee_set_cred_thread_keyring(cred, thread_keyring);
+}
+
+void __iee_code iee_wrapper_set_cred_request_key_auth(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct key *request_key_auth = va_arg(args, struct key *);
+ _iee_set_cred_request_key_auth(cred, request_key_auth);
+}
+
+void __iee_code iee_wrapper_set_cred_non_rcu(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ int non_rcu = va_arg(args, int);
+ _iee_set_cred_non_rcu(cred, non_rcu);
+}
+
+void __iee_code iee_wrapper_set_cred_atomic_set_usage(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ int i = va_arg(args, int);
+ _iee_set_cred_atomic_set_usage(cred, i);
+}
+
+u64 __iee_code iee_wrapper_set_cred_atomic_op_usage(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ int flag = va_arg(args, int);
+ int nr = va_arg(args, int);
+ return (u64)_iee_set_cred_atomic_op_usage(cred, flag, nr);
+}
+
+void __iee_code iee_wrapper_set_cred_security(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ void *security = va_arg(args, void *);
+ _iee_set_cred_security(cred, security);
+}
+
+void __iee_code iee_wrapper_set_cred_rcu(va_list args) {
+ struct cred *cred = va_arg(args, struct cred *);
+ struct rcu_head *rcu = va_arg(args, struct rcu_head *);
+ _iee_set_cred_rcu(cred, rcu);
+}
+
+void __iee_code iee_wrapper_memset(va_list args) {
+ void *ptr = va_arg(args, void *);
+ int data = va_arg(args, int);
+ size_t n = va_arg(args, size_t);
+ _iee_memset(ptr, data, n);
+}
+
+void __iee_code iee_wrapper_set_track(va_list args) {
+ struct track *ptr = va_arg(args, struct track *);
+ struct track *data = va_arg(args, struct track *);
+ _iee_set_track(ptr, data);
+}
+
+void __iee_code iee_wrapper_set_freeptr(va_list args) {
+ freeptr_t *pptr = va_arg(args, freeptr_t *);
+ freeptr_t ptr = va_arg(args, freeptr_t);
+ _iee_set_freeptr(pptr, ptr);
+}
+
+void __iee_code iee_wrapper_set_pte_upage(va_list args) {
+ pte_t *ptep = va_arg(args, pte_t *);
+ pte_t pte = va_arg(args, pte_t);
+ _iee_set_pte_upage(ptep, pte);
+}
+
+void __iee_code iee_wrapper_set_pte_ppage(va_list args) {
+ pte_t *ptep = va_arg(args, pte_t *);
+ pte_t pte = va_arg(args, pte_t);
+ _iee_set_pte_ppage(ptep, pte);
+}
+
+void __iee_code iee_wrapper_set_token_mm(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ struct mm_struct *mm = va_arg(args, struct mm_struct *);
+ _iee_set_token_mm(tsk, mm);
+}
+
+void __iee_code iee_wrapper_set_token_pgd(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ pgd_t *pgd = va_arg(args, pgd_t *);
+ _iee_set_token_pgd(tsk, pgd);
+}
+
+void __iee_code iee_wrapper_init_token(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ void *kernel_stack = va_arg(args, void *);
+ void *iee_stack = va_arg(args, void *);
+ _iee_init_token(tsk, kernel_stack, iee_stack);
+}
+
+void __iee_code iee_wrapper_free_token(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ _iee_free_token(tsk);
+}
+
+u64 __iee_code iee_wrapper_read_token_stack(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ return (u64)_iee_read_token_stack(tsk);
+}
+
+void __iee_code iee_wrapper_write_entry_task(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ _iee_write_entry_task(tsk);
+}
+
+#ifdef CONFIG_KOI
+u64 __iee_code iee_wrapper_read_koi_stack(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ return (u64)_iee_read_koi_stack(tsk);
+}
+
+void __iee_code iee_wrapper_write_koi_stack(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ unsigned long koi_stack = va_arg(args, unsigned long);
+ _iee_write_koi_stack(tsk, koi_stack);
+}
+
+u64 __iee_code iee_wrapper_read_token_ttbr1(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ return (u64)_iee_read_token_ttbr1(tsk);
+}
+
+void __iee_code iee_wrapper_write_token_ttbr1(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ unsigned long current_ttbr1 = va_arg(args, unsigned long);
+ _iee_write_token_ttbr1(tsk, current_ttbr1);
+}
+
+u64 __iee_code iee_wrapper_read_koi_kernel_stack(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ return (u64)_iee_read_koi_kernel_stack(tsk);
+}
+
+void __iee_code iee_wrapper_write_koi_kernel_stack(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ unsigned long kernel_stack = va_arg(args, unsigned long);
+ _iee_write_koi_kernel_stack(tsk, kernel_stack);
+}
+
+u64 __iee_code iee_wrapper_read_koi_stack_base(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ return (u64)_iee_read_koi_stack_base(tsk);
+}
+
+void __iee_code iee_wrapper_write_koi_stack_base(va_list args) {
+ struct task_struct *tsk = va_arg(args, struct task_struct *);
+ unsigned long koi_stack_base = va_arg(args, unsigned long);
+ _iee_write_koi_stack_base(tsk, koi_stack_base);
+}
+#endif
+// Define the function pointer type for wrapper functions.
+// Each function pointer conforms to a standardized calling convention
+// using a variable argument list (va_list) as its parameter.
+// This allows dynamic invocation of different functions with various arguments.
+typedef void (*iee_wrapper_func)(va_list args);
+iee_wrapper_func iee_wrappers[] = {
+ iee_wrapper_write_in_byte,
+ iee_wrapper_set_pte,
+ iee_wrapper_set_pmd,
+ iee_wrapper_set_pud,
+ iee_wrapper_set_p4d,
+ iee_wrapper_set_bm_pte,
+ iee_wrapper_set_swapper_pgd,
+ iee_wrapper_set_tramp_pgd,
+ (iee_wrapper_func)iee_wrapper_set_cmpxchg,
+ (iee_wrapper_func)iee_wrapper_set_xchg,
+ iee_wrapper_copy_cred,
+ iee_wrapper_set_cred_uid,
+ iee_wrapper_set_cred_gid,
+ iee_wrapper_set_cred_suid,
+ iee_wrapper_set_cred_sgid,
+ iee_wrapper_set_cred_euid,
+ iee_wrapper_set_cred_egid,
+ iee_wrapper_set_cred_fsuid,
+ iee_wrapper_set_cred_fsgid,
+ iee_wrapper_set_cred_user,
+ iee_wrapper_set_cred_user_ns,
+ iee_wrapper_set_cred_group_info,
+ iee_wrapper_set_cred_securebits,
+ iee_wrapper_set_cred_cap_inheritable,
+ iee_wrapper_set_cred_cap_permitted,
+ iee_wrapper_set_cred_cap_effective,
+ iee_wrapper_set_cred_cap_bset,
+ iee_wrapper_set_cred_cap_ambient,
+ iee_wrapper_set_cred_jit_keyring,
+ iee_wrapper_set_cred_session_keyring,
+ iee_wrapper_set_cred_process_keyring,
+ iee_wrapper_set_cred_thread_keyring,
+ iee_wrapper_set_cred_request_key_auth,
+ iee_wrapper_set_cred_non_rcu,
+ iee_wrapper_set_cred_atomic_set_usage,
+ (iee_wrapper_func)iee_wrapper_set_cred_atomic_op_usage,
+ iee_wrapper_set_cred_security,
+ iee_wrapper_set_cred_rcu,
+ iee_wrapper_memset,
+ iee_wrapper_set_track,
+ iee_wrapper_set_freeptr,
+ iee_wrapper_set_pte_upage,
+ iee_wrapper_set_pte_ppage,
+ iee_wrapper_set_token_mm,
+ iee_wrapper_set_token_pgd,
+ iee_wrapper_init_token,
+ iee_wrapper_free_token,
+ (iee_wrapper_func)iee_wrapper_read_token_stack,
+ iee_wrapper_write_entry_task,
+ iee_wrapper_set_cred_ucounts,
+#ifdef CONFIG_KOI
+ (iee_wrapper_func)iee_wrapper_read_koi_stack,
+ iee_wrapper_write_koi_stack,
+ (iee_wrapper_func)iee_wrapper_read_token_ttbr1,
+ iee_wrapper_write_token_ttbr1,
+ (iee_wrapper_func)iee_wrapper_read_koi_kernel_stack,
+ iee_wrapper_write_koi_kernel_stack,
+ (iee_wrapper_func)iee_wrapper_read_koi_stack_base,
+ iee_wrapper_write_koi_stack_base
+#endif
+};
+
+u64 __iee_code iee_dispatch(int flag, ...){
+ va_list pArgs;
+
+ va_start(pArgs, flag);
+
+ switch(flag)
+ {
+ case IEE_OP_SET_CMPXCHG:
+ {
+ pteval_t ret = iee_wrapper_set_cmpxchg(pArgs);
+ va_end(pArgs);
+ return (u64)ret;
+ }
+ case IEE_OP_SET_XCHG:
+ {
+ pteval_t ret = iee_wrapper_set_xchg(pArgs);
+ va_end(pArgs);
+ return (u64)ret;
+ }
+ case IEE_OP_SET_CRED_ATOP_USAGE:
+ {
+ u64 ret = iee_wrapper_set_cred_atomic_op_usage(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+ case IEE_READ_TOKEN_STACK:
+ {
+ u64 ret = iee_wrapper_read_token_stack(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+#ifdef CONFIG_KOI
+ case IEE_READ_KOI_STACK:
+ {
+ u64 ret = iee_wrapper_read_koi_stack(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+ case IEE_READ_TOKEN_TTBR1:
+ {
+ u64 ret = iee_wrapper_read_token_ttbr1(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+ case IEE_READ_KOI_KERNEL_STACK:
+ {
+ u64 ret = iee_wrapper_read_koi_kernel_stack(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+ case IEE_READ_KOI_STACK_BASE:
+ {
+ u64 ret = iee_wrapper_read_koi_stack_base(pArgs);
+ va_end(pArgs);
+ return ret;
+ }
+#endif
+ default:
+ {
+ #ifndef CONFIG_KOI
+ if((flag < IEE_WRITE_IN_BYTE) | (flag > IEE_OP_SET_CRED_UCOUNTS))
+ panic("Invalid iee flag.\n");
+ #else
+ if((flag < IEE_WRITE_IN_BYTE) | (flag > IEE_WRITE_KOI_STACK_BASE))
+ panic("Invalid iee flag.\n");
+ #endif
+ iee_wrappers[flag](pArgs);
+ break;
+ }
+ }
+
+ va_end(pArgs);
+ return 0;
+}
+
+#ifdef CONFIG_KOI
+unsigned long __iee_code _iee_read_koi_stack(struct task_struct *tsk)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ return (unsigned long)token->koi_stack;
+}
+
+void __iee_code _iee_write_koi_stack(struct task_struct *tsk, unsigned long koi_stack)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->koi_stack = koi_stack;
+}
+
+unsigned long __iee_code _iee_read_token_ttbr1(struct task_struct *tsk)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ return token->current_ttbr1;
+}
+
+void __iee_code _iee_write_token_ttbr1(struct task_struct *tsk, unsigned long current_ttbr1)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->current_ttbr1 = current_ttbr1;
+}
+
+unsigned long __iee_code _iee_read_koi_kernel_stack(struct task_struct *tsk)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ return token->koi_kernel_stack;
+}
+
+void __iee_code _iee_write_koi_kernel_stack(struct task_struct *tsk, unsigned long kernel_stack)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->koi_kernel_stack = kernel_stack;
+}
+
+unsigned long __iee_code _iee_read_koi_stack_base(struct task_struct *tsk)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ return (unsigned long)token->koi_stack_base;
+}
+
+void __iee_code _iee_write_koi_stack_base(struct task_struct *tsk, unsigned long koi_stack_base)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->koi_stack_base = koi_stack_base;
+}
+#endif
+
+// Protect the __entry_task.
+__attribute__((aligned(PAGE_SIZE))) DECLARE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task);
+void __iee_code _iee_write_entry_task(struct task_struct *tsk)
+{
+ // Add check of tsk.
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+
+ unsigned long flags;
+ unsigned long res;
+ struct task_struct **entry_addr;
+ local_irq_save(flags);
+ asm volatile("at s1e1r, %0"::"r"(token));
+ isb();
+ res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+
+ // If it is logical map, that means it is not a token.
+ if(__phys_to_iee(res & PTE_ADDR_MASK) == (((unsigned long)token) & PTE_ADDR_MASK))
+ panic("Trying to forge a token.\n");
+
+ if(!token->valid)
+ panic("Trying to write a wrong task into __entry_task.\n");
+ entry_addr = (struct task_struct **)__phys_to_iee(__pa(SHIFT_PERCPU_PTR(__entry_task,__kern_my_cpu_offset())));
+ *entry_addr = tsk;
+}
+
+unsigned long __iee_code _iee_read_token_stack(struct task_struct *tsk)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ return (unsigned long)token->iee_stack;
+}
+
+void __iee_code _iee_free_token(struct task_struct *tsk)
+{
+ _iee_memset(tsk, 0, sizeof(struct task_token));
+}
+
+#ifdef CONFIG_KOI
+extern unsigned long koi_swapper_ttbr1;
+#endif
+void __iee_code _iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->kernel_stack = kernel_stack;
+ token->iee_stack = iee_stack;
+ token->valid = true;
+#ifdef CONFIG_KOI
+ token->koi_kernel_stack = NULL;
+ token->koi_stack = NULL;
+ token->koi_stack_base = NULL;
+ token->current_ttbr1 = 0;
+#endif
+}
+
+void __iee_code _iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->mm = mm;
+}
+
+void __iee_code _iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd)
+{
+ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk));
+ token->pgd = pgd;
+}
+
+void __iee_code _iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr)
+{
+ pptr = (freeptr_t *)__phys_to_iee(__pa(pptr));
+ *pptr = ptr;
+}
+
+#pragma GCC push_options
+#pragma GCC optimize("O0")
+void __iee_code _iee_memset(void *ptr, int data, size_t n)
+{
+ char *_ptr = (char *)__phys_to_iee(__pa(ptr));
+
+ while (n--)
+ *_ptr++ = data;
+}
+
+void __iee_code _iee_memcpy(void *dst, void *src, size_t n)
+{
+ char *_dst = (char *)__phys_to_iee(__pa(dst));
+ char *_src = (char *)src;
+
+ while(n--)
+ *_dst++ = *_src++;
+}
+#pragma GCC pop_options
+
+void __iee_code _iee_set_track(struct track *ptr, struct track *data)
+{
+ _iee_memcpy(ptr, data, sizeof(struct track));
+}
+
+void __iee_code _iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu)
+{
+ if(cred == &init_cred)
+ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred));
+ else
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ #ifdef CONFIG_CREDP
+ *((struct rcu_head **)(&(cred->rcu.func))) = rcu;
+ #endif
+}
+
+void __iee_code _iee_set_cred_security(struct cred *cred, void *security)
+{
+ if(cred == &init_cred)
+ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred));
+ else
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->security = security;
+}
+
+bool __iee_code _iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ switch (flag)
+ {
+ case AT_ADD: {
+ atomic_long_add(nr, &cred->usage);
+ return 0;
+ }
+ case AT_INC_NOT_ZERO: {
+ return atomic_long_inc_not_zero(&cred->usage);
+ }
+ case AT_SUB_AND_TEST: {
+ return atomic_long_sub_and_test(nr, &cred->usage);
+ }
+ }
+ return 0;
+}
+
+void __iee_code _iee_set_cred_atomic_set_usage(struct cred *cred, int i)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ atomic_long_set(&cred->usage,i);
+}
+
+void __iee_code _iee_set_cred_non_rcu(struct cred *cred, int non_rcu)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->non_rcu = non_rcu;
+}
+
+void __iee_code _iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->session_keyring = session_keyring;
+}
+
+void __iee_code _iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->process_keyring = process_keyring;
+}
+
+void __iee_code _iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->thread_keyring = thread_keyring;
+}
+
+void __iee_code _iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->request_key_auth = request_key_auth;
+}
+
+void __iee_code _iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->jit_keyring = jit_keyring;
+}
+
+void __iee_code _iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->cap_inheritable = cap_inheritable;
+}
+
+void __iee_code _iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->cap_permitted = cap_permitted;
+}
+
+void __iee_code _iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->cap_effective = cap_effective;
+}
+
+void __iee_code _iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->cap_bset = cap_bset;
+}
+
+void __iee_code _iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->cap_ambient = cap_ambient;
+}
+
+void __iee_code _iee_set_cred_securebits(struct cred *cred, unsigned securebits)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->securebits = securebits;
+}
+
+void __iee_code _iee_set_cred_group_info(struct cred *cred, struct group_info *group_info)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->group_info = group_info;
+}
+
+void __iee_code _iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->ucounts = ucounts;
+}
+
+void __iee_code _iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->user_ns = user_ns;
+}
+
+void __iee_code _iee_set_cred_user(struct cred *cred, struct user_struct *user)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->user = user;
+}
+
+void __iee_code _iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->fsgid = fsgid;
+}
+
+void __iee_code _iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->fsuid = fsuid;
+}
+
+void __iee_code _iee_set_cred_egid(struct cred *cred, kgid_t egid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->egid = egid;
+}
+
+void __iee_code _iee_set_cred_euid(struct cred *cred, kuid_t euid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->euid = euid;
+}
+
+void __iee_code _iee_set_cred_sgid(struct cred *cred, kgid_t sgid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->sgid = sgid;
+}
+
+void __iee_code _iee_set_cred_suid(struct cred *cred, kuid_t suid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->suid = suid;
+}
+
+void __iee_code _iee_copy_cred(struct cred *old, struct cred *new)
+{
+ #ifdef CONFIG_CREDP
+ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func);
+ struct cred *_new = (struct cred *)__phys_to_iee(__pa(new));
+ _iee_memcpy(new,old,sizeof(struct cred));
+ *(struct rcu_head **)(&(_new->rcu.func)) = rcu;
+ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func);
+ #endif
+}
+
+void __iee_code _iee_set_cred_gid(struct cred *cred, kgid_t gid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->gid = gid;
+}
+
+void __iee_code _iee_set_cred_uid(struct cred *cred, kuid_t uid)
+{
+ cred = (struct cred *)__phys_to_iee(__pa(cred));
+ cred->uid = uid;
+}
+
+void __iee_code _iee_write_in_byte(void *ptr, __u64 data, int length)
+{
+ ptr = (void *)__phys_to_iee(__pa(ptr));
+ switch(length) {
+ case 8: {
+ *(__u64 *)ptr = data;
+ break;
+ }
+ case 4: {
+ *(__u32 *)ptr = (__u32)data;
+ break;
+ }
+ case 2: {
+ *(__u16 *)ptr = (__u16)data;
+ break;
+ }
+ case 1: {
+ *(__u8 *)ptr = (__u8)data;
+ break;
+ }
+ }
+}
+
+pteval_t __iee_code _iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval)
+{
+ pteval_t ret = xchg_relaxed((pteval_t *)(__phys_to_iee(__pa(ptep))), pteval);
+ return ret;
+}
+
+pteval_t __iee_code _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval)
+{
+ pteval_t pteval = cmpxchg_relaxed((pteval_t *)(__phys_to_iee(__pa(ptep))), old_pteval, new_pteval);
+ return pteval;
+}
+
+/* Check if addr is allocated in IEE page */
+static inline bool check_addr_in_iee_valid(unsigned long addr)
+{
+ unsigned long flags;
+ unsigned long res;
+ local_irq_save(flags);
+ asm volatile("at s1e1r, %0"::"r"(addr));
+ isb();
+ res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+
+ // If it is not logical map, that means it is a token.
+ if(__phys_to_iee(res & PTE_ADDR_MASK) != addr)
+ return false;
+
+ return !(res & 0x1);
+}
+
+void __iee_code _iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd)
+{
+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd);
+}
+
+void __iee_code _iee_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+{
+ if(!(pgd_val(pgd) & PMD_SECT_VALID))
+ {
+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd);
+ return;
+ }
+
+ if ((pgd_val(pgd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd))))
+ panic("You can't use non-iee-pgtable\n");
+
+ if((pgdp >= pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 2))) && (pgdp < pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 1))) && !(pgd_val(pgd) & PGD_APT))
+ panic("Set IEE pgd U page.\n");
+
+ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd);
+}
+
+void __iee_code _iee_set_p4d(p4d_t *p4dp, p4d_t p4d)
+{
+ if(!(p4d_val(p4d) & PMD_SECT_VALID))
+ {
+ WRITE_ONCE(*((p4d_t *)(__phys_to_iee(__pa(p4dp)))), p4d);
+ return;
+ }
+
+ if ((p4d_val(p4d) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d))))
+ panic("You can't use non-iee-pgtable\n");
+
+ WRITE_ONCE(*((p4d_t *)(__phys_to_iee(__pa(p4dp)))), p4d);
+}
+
+void __iee_code _iee_set_pud(pud_t *pudp, pud_t pud)
+{
+ if(!(pud_val(pud) & PMD_SECT_VALID))
+ {
+ WRITE_ONCE(*((pud_t *)(__phys_to_iee(__pa(pudp)))), pud);
+ return;
+ }
+
+ if ((pud_val(pud) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud))))
+ panic("You can't use non-iee-pgtable\n");
+
+ WRITE_ONCE(*((pud_t *)(__phys_to_iee(__pa(pudp)))), pud);
+}
+
+// Return true if the modify does not break DEP.
+static inline bool check_pmd_dep(char *addr, pmd_t pmd)
+{
+ // DEP for kernel code and readonly data
+ // _text: .text start addr, __init_begin: .rodata end addr
+ if (addr >= _stext && addr < _etext)
+ {
+ if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable
+ !(PTE_RDONLY & pmd_val(pmd))) // DBM == 0 && AP[2] = 0 --> writable
+ {
+ panic("Can't make kernel's text/readonly page as writable!\n"
+ "addr = 0x%16llx, pmd_val = 0x%16llx",
+ (u64)addr, pmd_val(pmd));
+ }
+ }
+ return true;
+}
+
+// Return true if the pmd table is a part of kernel page table.
+// TODO : Optimize to get lower overhead.
+static inline bool is_kernel_pmd_table(pmd_t *pmdp, pmd_t pmd)
+{
+ int i = 0,j = 0;
+ for(i = 0; i < PAGE_SIZE/sizeof(pgd_t); i++)
+ {
+ pgd_t *pgdp = (pgd_t *)swapper_pg_dir + i;
+ if((pgd_val(*pgdp) & PMD_SECT_VALID) && (pgd_val(*pgdp) & PMD_TABLE_BIT))
+ {
+ for(j = 0; j < PAGE_SIZE/sizeof(pud_t); j++)
+ {
+ pud_t *pudp = (pud_t *)__va(__pgd_to_phys(*pgdp)) + i;
+ if((pud_val(*pudp) & PMD_SECT_VALID) && (pud_val(*pudp) & PMD_TABLE_BIT))
+ {
+ pmd_t *current_pmdp = __va(__pud_to_phys(*pudp));
+ if((unsigned long)current_pmdp == ((unsigned long)pmdp & PAGE_MASK))
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+}
+
+// Return true if it is mapped to a physical range containing IEE page.
+// TODO : Optimize to get lower overhead.
+static inline bool check_addr_range_in_iee_valid(pmd_t pmd)
+{
+ int i = 0;
+ unsigned long addr = __phys_to_iee(__pmd_to_phys(pmd));
+ for(i = 0; i < PAGE_SIZE/sizeof(pmd_t); i++)
+ {
+ if(check_addr_in_iee_valid(addr + PAGE_SIZE * i))
+ return true;
+ }
+ return false;
+}
+
+void __iee_code _iee_set_pmd(pmd_t *pmdp, pmd_t pmd)
+{
+ char * addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd));
+
+ if(!(pmd_val(pmd) & PMD_SECT_VALID))
+ {
+ WRITE_ONCE(*((pmd_t *)(__phys_to_iee(__pa(pmdp)))), pmd);
+ return;
+ }
+
+ // Check if the pte table is legally allocated.
+ if ((pmd_val(pmd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd))))
+ panic("You can't use non-iee-pgtable\n");
+
+ // Avoid mapping a huge pmd as U page.
+ if(!(pmd_val(pmd) & PMD_TABLE_BIT) && (pmd_val(pmd) & PMD_SECT_USER) && is_kernel_pmd_table(pmdp, pmd))
+ panic("Set a block descriptor in kernel space U page.\n");
+
+ // Avoid mapping a huge pmd to IEE physical page.
+ if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd))
+ panic("Mapping IEE physical page to a huge pmd.\n");
+
+ if(!check_pmd_dep(addr, pmd))
+ return;
+
+ WRITE_ONCE(*((pmd_t *)(__phys_to_iee(__pa(pmdp)))), pmd);
+}
+
+// Return true if the pte table is a part of kernel page table.
+// TODO : Optimize to get lower overhead.
+static inline bool is_kernel_pte_table(pte_t *ptep, pte_t pte)
+{
+ return false;
+}
+
+// Return true if it does not change the privilage or add new U page in kernel.
+static inline bool check_privilage_safe(pte_t *ptep, pte_t pte)
+{
+ if(!(pte_val(pte) & PTE_VALID))
+ return true;
+
+ if((pte_val(*ptep) & PTE_VALID))
+ {
+ if((pte_val(*ptep) & PTE_USER) != (pte_val(pte) & PTE_USER))
+ panic("Incorrectly change privilage.\n");
+ }
+ else
+ {
+ if((pte_val(pte) & PTE_USER) && is_kernel_pte_table(ptep, pte))
+ panic("Add new U page in kernel space.\n");
+ }
+ return true;
+}
+
+// TODO : When adding a new executable page, check it for DEP.
+static inline bool safely_adding_new_exec_page(pte_t *ptep, pte_t pte)
+{
+ return true;
+}
+
+// Return true if it is only changing prot of a pte.
+static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte)
+{
+ if(((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0)
+ return true;
+ else
+ return false;
+}
+
+// Return true if the modify does not break DEP.
+static inline bool check_pte_dep(char *addr, pte_t pte)
+{
+ // DEP for kernel code and readonly data
+ // _text: .text start addr, __init_begin: .rodata end addr
+ if (addr >= _stext && addr < _etext)
+ {
+ if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable
+ || !(PTE_RDONLY & pte_val(pte))) // DBM == 0 && AP[2] = 0 --> writable
+ {
+ panic("Can't make kernel's text/readonly page as writable!\n"
+ "addr = 0x%16llx, pte_val = 0x%16llx",
+ (u64)addr, pte_val(pte));
+ }
+ }
+ return true;
+}
+
+void __iee_code _iee_set_pte(pte_t *ptep, pte_t pte)
+{
+ char * addr = (char *)__phys_to_kimg(__pte_to_phys(pte));
+
+ if(!(pte_val(pte) & PTE_VALID))
+ {
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte);
+ return;
+ }
+
+ // Avoid modify privilage unsafely.
+ if(!check_privilage_safe(ptep, pte))
+ panic("You are modify privilage unsafely.\n");
+
+ // Avoid mapping a new executable page.
+ if(!safely_adding_new_exec_page(ptep, pte))
+ panic("You are adding a new executable page unsafely.\n");
+
+ // Avoid mapping a new VA to IEE PA.
+ if(!is_changing_pte_prot(ptep, pte) &&
+ check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte))))
+ panic("You are remmaping IEE page to other VA.\n");
+
+ // Avoid mapping a writable VA to kernel code PA.
+ if(!check_pte_dep(addr, pte))
+ return;
+
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte);
+}
+
+// Return true if it only sets U page and modify NG.
+static inline bool is_setting_upage(pte_t *ptep, pte_t pte)
+{
+ if(((pte_val(*ptep) ^ pte_val(pte)) & ~(PTE_USER | PTE_NG)) != 0)
+ panic("Incorrectly setting U page.\n");
+ if((pte_val(pte) & PTE_USER) != PTE_USER)
+ panic("Using error interface to set P page.\n");
+ return true;
+}
+
+void __iee_code _iee_set_pte_upage(pte_t *ptep, pte_t pte)
+{
+ // Check if it only change the prot.
+ if(!is_setting_upage(ptep,pte))
+ panic("Incorrectly setting U page.\n");
+
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte);
+}
+
+// Return true if it only sets P page and modify NG.
+static inline bool is_setting_ppage(pte_t *ptep, pte_t pte)
+{
+ if(((pte_val(*ptep) ^ pte_val(pte)) & ~(PTE_USER | PTE_NG)) != 0)
+ panic("Incorrectly setting P page.\n");
+ if((pte_val(pte) & PTE_USER) != 0)
+ panic("Using error interface to set U page.\n");
+ return true;
+}
+
+void __iee_code _iee_set_pte_ppage(pte_t *ptep, pte_t pte)
+{
+ // Check if it only change the prot.
+ if(!is_setting_ppage(ptep,pte))
+ panic("Incorrectly setting P page.\n");
+
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte);
+}
+
+void __iee_code _iee_set_bm_pte(pte_t *ptep, pte_t pte)
+{
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa_symbol(ptep)))), pte);
+}
+
+/* Data in iee_si_base is visible to all pgd while iee_si_data is private. */
+unsigned long iee_base_idmap_pg_dir __iee_si_data;
+unsigned long iee_base_reserved_pg_dir __iee_si_data;
+unsigned long iee_base__bp_harden_el1_vectors __iee_si_data;
+bool iee_init_done __iee_si_data;
+unsigned long iee_si_tcr __iee_si_data;
+
+static u64 __iee_si_code inline iee_si_mask(unsigned long mask, unsigned long new_val, unsigned long old_val)
+{
+ return (new_val & mask) | (old_val & ~mask);
+}
+/*
+ * handler function for requests of executing sensitive instrutions.
+ */
+u64 __iee_si_code iee_si_handler(int flag, ...)
+{
+ va_list pArgs;
+ u64 old_val, new_val;
+
+ // BUG_ON(flag > IEE_WRITE_MDSCR);
+ va_start(pArgs, flag);
+ switch (flag) {
+ case IEE_SI_TEST:
+ break;
+ case IEE_WRITE_SCTLR: {
+ old_val = read_sysreg(sctlr_el1);
+ new_val = va_arg(pArgs, u64);
+ new_val = iee_si_mask(IEE_SCTLR_MASK, new_val, old_val);
+ write_sysreg(new_val, sctlr_el1);
+ break;
+ }
+ case IEE_WRITE_TTBR0:
+ case IEE_CONTEXT_SWITCH: {
+ u64 new_asid, new_phys, old_phys, token_phys;
+ struct task_struct *tsk;
+ struct task_token *token;
+ new_val = va_arg(pArgs, u64);
+ new_phys = (new_val & PAGE_MASK) & ~TTBR_ASID_MASK;
+ new_asid = new_val >> 48;
+
+ // Check ASID first
+ if (new_phys == iee_base_reserved_pg_dir){
+ if (new_asid != 1)
+ panic("IEE SI warning: reserved_pg_dir ASID invalid: %llx:%llx", new_asid, new_val);
+ }
+ // Already reserved asid 1 for iee rwx gate.
+ else if (new_asid == 0){
+ new_val |= FIELD_PREP(TTBR_ASID_MASK, 1);
+ printk("IEE SI: Modify ASID of %llx to 1.", new_val);
+ }
+ // TO DO: operations to protect idmap_pg_dir
+ else if (new_phys == iee_base_idmap_pg_dir)
+ {
+ // printk("IEE SI: switch to idmap_pg_dir.);
+ }
+ else if (new_asid % 2 ==0)
+ panic("IEE SI warning: TTBR0 ASID invalid: %llx:%llx", new_asid, new_val);
+
+ /* Skip verification if iee hasn't been initialized. */
+ if (iee_init_done){
+ // Verify current sp_el0 with iee token info
+ asm volatile("mrs %x0, sp_el0":"=r"(tsk));
+ token = (struct task_token *)__phys_to_iee(__pa(tsk));
+
+ /*
+ * token->pgd != NULL means it is a user task, then we need to check whether current ttbr0 is correct.
+ */
+ if (token->pgd){
+ old_val = read_sysreg(ttbr0_el1);
+ // When TTBR0 is reserved_pg_dir then no checking is available.
+ if (old_val != iee_base_reserved_pg_dir){
+ old_phys = (old_val & PAGE_MASK) & ~TTBR_ASID_MASK;
+ token_phys = __pa(token->pgd);
+ if (old_phys != token_phys)
+ panic("IEE SI warning: Pgd set error. old ttbr0:%lx, token ttbr0:%lx, token pgd:%lx",
+ (unsigned long)old_phys, (unsigned long)token_phys, (unsigned long)(token->pgd));
+ }
+ }
+ }
+ // all checks are done.
+ write_sysreg(new_val, ttbr0_el1);
+
+ // SET ASID in TTBR1 when context switch
+ if (flag == IEE_CONTEXT_SWITCH){
+ new_val = (read_sysreg(ttbr1_el1) & ~TTBR_ASID_MASK) | FIELD_PREP(TTBR_ASID_MASK, new_asid-1);
+ write_sysreg(new_val, ttbr1_el1);
+ }
+ break;
+ }
+ case IEE_WRITE_VBAR: {
+ u64 el1_vector;
+ new_val = va_arg(pArgs, u64);
+ el1_vector = iee_base__bp_harden_el1_vectors;
+ if(new_val == el1_vector || new_val == el1_vector+SZ_2K ||
+ new_val == el1_vector+SZ_2K*2 || new_val == el1_vector+SZ_2K*3)
+ write_sysreg(new_val, vbar_el1);
+ break;
+ }
+ case IEE_WRITE_TCR: {
+ old_val = read_sysreg(tcr_el1);
+ new_val = va_arg(pArgs, u64);
+ new_val = iee_si_mask(IEE_TCR_MASK, new_val, old_val);
+ write_sysreg(new_val, tcr_el1);
+ break;
+ }
+ case IEE_WRITE_MDSCR: {
+ old_val = read_sysreg(mdscr_el1);
+ new_val = va_arg(pArgs, u64);
+ new_val = iee_si_mask(IEE_MDSCR_MASK, new_val, old_val);
+ write_sysreg(new_val, mdscr_el1);
+ break;
+ }
+ }
+ va_end(pArgs);
+ return 0;
+}
+/*
+ * TODO: scan a page to check whether it contains sensitive instructions
+ * return 1 when finding sensitive inst, 0 on safe page.
+ */
+int iee_si_scan_page(unsigned long addr);
+#endif
\ No newline at end of file
diff --git a/arch/arm64/kernel/koi/Makefile b/arch/arm64/kernel/koi/Makefile
new file mode 100644
index 000000000000..9be8710b714a
--- /dev/null
+++ b/arch/arm64/kernel/koi/Makefile
@@ -0,0 +1 @@
+obj-y += koi.o
\ No newline at end of file
diff --git a/arch/arm64/kernel/koi/koi.c b/arch/arm64/kernel/koi/koi.c
new file mode 100644
index 000000000000..716ba16ab358
--- /dev/null
+++ b/arch/arm64/kernel/koi/koi.c
@@ -0,0 +1,1327 @@
+#include "asm/koi.h"
+#include "linux/compiler_attributes.h"
+#include "linux/compiler_types.h"
+#include "asm/barrier.h"
+#include "asm-generic/bug.h"
+#include "asm-generic/errno-base.h"
+#include "asm-generic/memory_model.h"
+#include "asm-generic/pgtable-nop4d.h"
+#include "asm-generic/rwonce.h"
+#include "asm/pgalloc.h"
+#include "asm/memory.h"
+#include "linux/bitfield.h"
+#include "linux/compiler.h"
+#include "linux/types.h"
+#include "linux/spinlock.h"
+#include "linux/spinlock_types.h"
+#include "linux/kernel.h"
+#include "linux/rculist.h"
+#include "linux/rcupdate.h"
+#include "linux/list.h"
+#include "asm/current.h"
+#include "linux/compiler_types.h"
+#include "asm-generic/barrier.h"
+#include "asm-generic/rwonce.h"
+#include "asm-generic/pgalloc.h"
+#include "asm/cpufeature.h"
+#include "asm/kvm_hyp.h"
+#include "asm/mmu.h"
+#include "asm/mmu_context.h"
+#include "asm/page-def.h"
+#include "asm/pgalloc.h"
+#include "asm/pgtable-hwdef.h"
+#include "asm/pgtable-types.h"
+#include "asm/pgtable.h"
+#include "asm/string.h"
+#include "asm/sysreg.h"
+#include "linux/bitfield.h"
+#include "linux/compiler.h"
+#include "linux/export.h"
+#include "linux/gfp.h"
+#include "linux/huge_mm.h"
+#include "linux/kallsyms.h"
+#include "linux/kconfig.h"
+#include "linux/kern_levels.h"
+#include "linux/kernel.h"
+#include "linux/list.h"
+#include "linux/lockdep.h"
+#include "linux/mm.h"
+#include "linux/mm_types.h"
+#include "linux/pgtable.h"
+#include "linux/printk.h"
+#include "linux/rculist.h"
+#include "linux/rcupdate.h"
+#include "linux/rmap.h"
+#include "linux/sched.h"
+#include "linux/stddef.h"
+#include "linux/string.h"
+#include "linux/swap.h"
+#include "linux/swapops.h"
+#include "linux/types.h"
+#include "linux/slab.h"
+#include "linux/string.h"
+#include "linux/hashtable.h"
+
+#define __koi_code __section(".koi.text")
+#define __koi_data __section(".data..koi")
+
+extern unsigned long __koi_code_start[];
+extern unsigned long __koi_code_end[];
+extern unsigned long __koi_data_start[];
+extern unsigned long __koi_data_end[];
+#ifdef CONFIG_IEE
+extern unsigned long __iee_si_base_start[];
+extern unsigned long __iee_exec_entry_start[];
+extern unsigned long __iee_exec_entry_end[];
+#endif
+
+__koi_data unsigned long koi_swapper_ttbr1 = 0;
+EXPORT_SYMBOL(koi_swapper_ttbr1);
+#define KOI_SWAPPER_MASK 0x0000fffffffffff0
+
+__attribute__((aligned(PAGE_SIZE)))
+DEFINE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)],
+ koi_irq_current_ttbr1);
+EXPORT_SYMBOL(koi_irq_current_ttbr1);
+
+extern void koi_switch_to_ko_stack(unsigned long stack_top);
+extern void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp);
+extern void koi_check_and_switch_context(struct mm_struct *mm);
+extern int koi_add_page_mapping(unsigned long dst, unsigned long src);
+extern unsigned long _iee_read_token_ttbr1(struct task_struct *tsk);
+/**
+*struct koi_mem_list - maintain a linked list of free memory in the kernel
+*@addr: stating address of this memory
+*@size: the size of the memory
+*@list: the head of the koi_mem_list
+*@rcu: for rcu
+*/
+struct koi_mem_list {
+ unsigned long addr;
+ unsigned long size;
+ struct list_head list;
+ struct rcu_head rcu;
+};
+//mapping parameter pointer to copy
+struct koi_addr_map {
+ unsigned long buffer_addr;
+ unsigned long orig_addr;
+ int offset;
+ struct hlist_node node;
+ struct rcu_head rcu;
+};
+
+DEFINE_HASHTABLE(koi_mem_htbl, HASH_TABLE_BIT);
+EXPORT_SYMBOL(koi_mem_htbl);
+DEFINE_SPINLOCK(koi_mem_htbl_spin_lock);
+EXPORT_SYMBOL(koi_mem_htbl_spin_lock);
+
+EXPORT_SYMBOL(koi_do_switch_to_ko_stack);
+EXPORT_SYMBOL(koi_do_switch_to_kernel_stack);
+
+extern unsigned long long iee_rw_gate(int flag, ...);
+
+/**
+* koi_ttbr_ctor - return ttbr1 for the given driver module
+*/
+unsigned long koi_ttbr_ctor(struct module *mod)
+{
+ struct koi_mem_hash_node *ko;
+ struct mm_struct *ko_mm;
+ unsigned long ttbr1;
+ unsigned long asid;
+ int bkt;
+ rcu_read_lock();
+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) {
+ if (ko->mod == mod) {
+ ko_mm = ko->ko_mm;
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (!ko_mm) {
+ printk(KERN_ERR "cannot found module %s in koi_mem_htbl",
+ mod->name);
+ return 0;
+ }
+ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):);
+ asid &= TTBR_ASID_MASK;
+ ttbr1 = ko->ko_ttbr1 | asid;
+ // koi_check_and_switch_context(ko_mm);
+ // asid = ASID(ko_mm);
+ // ttbr1 = ko->ko_ttbr1 | FIELD_PREP(TTBR_ASID_MASK, asid);
+ return ttbr1;
+}
+EXPORT_SYMBOL(koi_ttbr_ctor);
+//release the hash node
+static __maybe_unused void koi_mem_hash_node_free(struct rcu_head *rcu)
+{
+ struct koi_mem_hash_node *node =
+ container_of(rcu, struct koi_mem_hash_node, rcu);
+ kfree(node);
+}
+//release free memory linked list nodes
+static void koi_mem_node_free(struct rcu_head *rcu)
+{
+ struct koi_mem_list *mem_node =
+ container_of(rcu, struct koi_mem_list, rcu);
+ kfree(mem_node);
+}
+//release the node in koi_addr_map
+static void koi_addr_map_node_free(struct rcu_head *rcu)
+{
+ struct koi_addr_map *addr_map_node =
+ container_of(rcu, struct koi_addr_map, rcu);
+ kfree(addr_map_node);
+}
+
+#ifndef CONFIG_IEE
+/*
+ * This function is used to switch to ko's pgtable.
+ */
+__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void)
+{
+ struct koi_mem_hash_node *ko;
+ // struct mm_struct *ko_mm;
+ unsigned long addr;
+ unsigned long ttbr1, asid;
+ unsigned long *ptr;
+ struct task_token *token_addr =
+ (struct task_token *)((unsigned long)current +
+ (unsigned long)koi_offset);
+ int bkt;
+ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr));
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset());
+ rcu_read_lock();
+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) {
+ if (ko->mod->init_layout.base != NULL) {
+ if (addr >= (unsigned long)ko->mod->init_layout.base &&
+ addr < (unsigned long)(ko->mod->init_layout.base +
+ ko->mod->init_layout.size)) {
+ if (token_addr->current_ttbr1 == ko->ko_ttbr1 ||
+ *ptr == ko->ko_ttbr1) {
+ // ko_mm = ko->ko_mm;
+ // koi_check_and_switch_context(ko_mm);
+ // asid = ASID(ko_mm);
+ // ttbr1 = ko->ko_ttbr1;
+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
+ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):);
+ asid &= TTBR_ASID_MASK;
+ ttbr1 = ko->ko_ttbr1 | asid;
+ rcu_read_unlock();
+ return ttbr1;
+ }
+ rcu_read_unlock();
+ return 0;
+ }
+ }
+ if (addr >= (unsigned long)ko->mod->core_layout.base &&
+ addr < (unsigned long)ko->mod->core_layout.base +
+ ko->mod->core_layout.size) {
+ if (token_addr->current_ttbr1 == ko->ko_ttbr1 ||
+ *ptr == ko->ko_ttbr1) {
+ // ko_mm = ko->ko_mm;
+ // koi_check_and_switch_context(ko_mm);
+ // asid = ASID(ko_mm);
+ // ttbr1 = ko->ko_ttbr1;
+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
+ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):);
+ asid &= TTBR_ASID_MASK;
+ ttbr1 = ko->ko_ttbr1 | asid;
+ rcu_read_unlock();
+ return ttbr1;
+ }
+ rcu_read_unlock();
+ return 0;
+ }
+ }
+ rcu_read_unlock();
+ return 0;
+}
+/**
+* koi_do_switch_to_kernel_pgtbl - switch to kernel pagetable
+*/
+__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void)
+{
+ unsigned long curr_ttbr1, asid;
+ // if (!cpu_online(smp_processor_id()))
+ // return 0;
+ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1));
+ if ((curr_ttbr1 & KOI_SWAPPER_MASK) ==
+ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) {
+ return 0;
+ }
+ if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) {
+ return 0;
+ }
+ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):);
+ asid &= ~USER_ASID_FLAG;
+ asid &= TTBR_ASID_MASK;
+ write_sysreg(koi_swapper_ttbr1 | asid, ttbr1_el1);
+ isb();
+ asm volatile(ALTERNATIVE("nop; nop; nop", "ic iallu; dsb nsh; isb",
+ ARM64_WORKAROUND_CAVIUM_27456));
+ return 1;
+}
+#else
+__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void)
+{
+ struct koi_mem_hash_node *ko;
+ struct mm_struct *ko_mm;
+ unsigned long addr, ttbr1, asid, pan_flag, current_ttbr1;
+ unsigned long *ptr;
+ int bkt;
+ asm volatile("mrs %0, pan\n"
+ "msr pan, 0x0\n"
+ : "=r"(pan_flag)
+ :);
+ current_ttbr1 = _iee_read_token_ttbr1(current);
+ asm volatile("msr pan, %0\n" : : "r"(pan_flag));
+ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset());
+ if (current_ttbr1 == 0 && *ptr == 0)
+ return 0;
+ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr));
+ rcu_read_lock();
+ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) {
+ if (ko->mod->init_layout.base != NULL) {
+ if (addr >= (unsigned long)ko->mod->init_layout.base &&
+ addr < (unsigned long)(ko->mod->init_layout.base +
+ ko->mod->init_layout.size)) {
+ rcu_read_unlock();
+ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) {
+ // ko_mm = ko->ko_mm;
+ // koi_check_and_switch_context(ko_mm);
+ // asid = ASID(ko_mm);
+ // ttbr1 = ko->ko_ttbr1;
+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK,
+ // asid);
+ return ko->ko_ttbr1;
+ }
+ return 0;
+ }
+ }
+ if (addr >= (unsigned long)ko->mod->core_layout.base &&
+ addr < (unsigned long)ko->mod->core_layout.base +
+ ko->mod->core_layout.size) {
+ rcu_read_unlock();
+ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) {
+ // ko_mm = ko->ko_mm;
+ // koi_check_and_switch_context(ko_mm);
+ // asid = ASID(ko_mm);
+ // ttbr1 = ko->ko_ttbr1;
+ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
+ return ko->ko_ttbr1;
+ }
+ return 0;
+ }
+ }
+ rcu_read_unlock();
+ return 0;
+}
+
+__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void)
+{
+ unsigned long curr_ttbr1;
+ // if (!cpu_online(smp_processor_id()))
+ // return 0;
+ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1));
+ if ((curr_ttbr1 & KOI_SWAPPER_MASK) ==
+ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) {
+ return 0;
+ }
+ if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) {
+ return 0;
+ }
+ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL);
+ return 1;
+}
+#endif
+/**
+* koi_save_ttbr - save ttbr of each driver module
+* @mod: driver module
+* @pgdp:pointer to driver module top page table,pgd
+*/
+static void koi_save_ttbr(struct module *mod, pgd_t *pgdp,
+ struct koi_mem_hash_node *node)
+{
+ phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
+ if (system_supports_cnp())
+ ttbr1 |= TTBR_CNP_BIT;
+ node->ko_ttbr1 = ttbr1;
+}
+/**
+*kio_normal_page - to obtain the pointer of the corresponding struct page structure
+*from a given page table entry(pte)
+*/
+struct page *koi_normal_page(pte_t pte)
+{
+ unsigned long pfn = pte_pfn(pte);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
+ if (likely(!pte_special(pte)))
+ goto check_pfn;
+ if (is_zero_pfn(pfn)) {
+ printk(KERN_ERR "zero pfn found! pte=0x%16lx\n", pte);
+ return NULL;
+ }
+ if (pte_devmap(pte)) {
+ printk(KERN_ERR "pte for dev found! pte=0x%16lx\n",
+ pte);
+ return NULL;
+ }
+ return NULL;
+ }
+
+check_pfn:
+ return pfn_to_page(pfn);
+}
+
+/**
+ * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
+ * is required to copy this pte.
+*/
+static inline int koi_copy_present_pte(pte_t *dst_pte, pte_t *src_pte,
+ unsigned long addr,
+ struct page **prealloc)
+{
+ pte_t pte = *src_pte;
+ struct page *page;
+
+ page = koi_normal_page(pte);
+ if (!page) {
+ printk(KERN_ERR "pte_page unavailable. Impossible.....\n");
+ return -1;
+ }
+
+ set_pte(dst_pte, pte);
+ return 0;
+}
+/**
+* copy huge pmd from kernel space to driver space.
+*/
+static int koi_copy_huge_pmd(struct mm_struct *ko_mm, pmd_t *dst_pmd,
+ pmd_t *src_pmd, unsigned long addr)
+{
+ spinlock_t *src_ptl;
+ pmd_t pmd;
+ int ret = -ENOMEM;
+
+ src_ptl = pmd_lockptr(&init_mm, src_pmd);
+ spin_lock_bh(src_ptl);
+
+ ret = -EAGAIN;
+ pmd = *src_pmd;
+
+ set_pte((pte_t *)dst_pmd, pmd_pte(pmd));
+ ret = 0;
+ spin_unlock_bh(src_ptl);
+ return ret;
+}
+
+int __koi_pte_alloc(struct mm_struct *mm, pmd_t *pmd)
+{
+ spinlock_t *ptl;
+ pgtable_t new = pte_alloc_one(mm);
+ if (!new)
+ return -ENOMEM;
+
+ /*
+ * Ensure all pte setup (eg. pte page lock and page clearing) are
+ * visible before the pte is made visible to other CPUs by being
+ * put into page tables.
+ *
+ * The other side of the story is the pointer chasing in the page
+ * table walking code (when walking the page table without locking;
+ * ie. most of the time). Fortunately, these data accesses consist
+ * of a chain of data-dependent loads, meaning most CPUs (alpha
+ * being the notable exception) will already guarantee loads are
+ * seen in-order. See the alpha page table accessors for the
+ * smp_rmb() barriers in page table walking code.
+ */
+ smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
+
+ ptl = pmd_lockptr(mm, pmd);
+ spin_lock_bh(ptl);
+ if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
+ #ifdef CONFIG_PTP
+ pte_t *pte = (pte_t *)page_address(new);
+ unsigned long iee_addr = __phys_to_iee(__pa(pte));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)pte);
+ #endif
+ mm_inc_nr_ptes(mm);
+ pmd_populate(mm, pmd, new);
+ new = NULL;
+ }
+ spin_unlock_bh(ptl);
+ if (new)
+ pte_free(mm, new);
+ return 0;
+}
+
+#define koi_pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __koi_pte_alloc(mm, pmd))
+
+#define koi_pte_offset_map_lock(mm, pmd, address, ptlp) \
+({ \
+ spinlock_t *__ptl = pte_lockptr(mm, pmd); \
+ pte_t *__pte = pte_offset_map(pmd, address); \
+ *(ptlp) = __ptl; \
+ spin_lock_bh(__ptl); \
+ __pte; \
+})
+
+#define koi_pte_alloc_map_lock(mm, pmd, address, ptlp) \
+ (koi_pte_alloc(mm, pmd) ? \
+ NULL : koi_pte_offset_map_lock(mm, pmd, address, ptlp))
+
+/**
+*koi_copy_pte_range - copy pte from kernel space to driver space
+*/
+static int koi_copy_pte_range(struct mm_struct *ko_mm, pmd_t *dst_pmd,
+ pmd_t *src_pmd, unsigned long addr,
+ unsigned long end)
+{
+ pte_t *src_pte, *dst_pte;
+ spinlock_t *src_ptl, *dst_ptl;
+ int ret = 0;
+ struct page *prealloc = NULL;
+again:
+ dst_pte = koi_pte_alloc_map_lock(ko_mm, dst_pmd, addr, &dst_ptl);
+ if (!dst_pte) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ src_pte = pte_offset_map(src_pmd, addr);
+ src_ptl = pte_lockptr(&init_mm, src_pmd);
+ spin_lock_bh(src_ptl);
+ arch_enter_lazy_mmu_mode();
+
+ do {
+ if (pte_none(*src_pte))
+ continue;
+ if (unlikely(!pte_present(*src_pte))) {
+ continue;
+ }
+ /* koi_copy_present_pte() will clear `*prealloc` if consumed */
+ ret = koi_copy_present_pte(dst_pte, src_pte, addr, &prealloc);
+ if (unlikely(ret == -EAGAIN))
+ break;
+ if (unlikely(prealloc)) {
+ put_page(prealloc);
+ prealloc = NULL;
+ }
+ } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
+ arch_leave_lazy_mmu_mode();
+ spin_unlock_bh(src_ptl);
+ spin_unlock_bh(dst_ptl);
+
+ if (ret) {
+ WARN_ON_ONCE(ret != -EAGAIN);
+ ret = 0;
+ }
+ if (addr != end)
+ goto again;
+out:
+ if (unlikely(prealloc))
+ put_page(prealloc);
+ return ret;
+}
+
+int __koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
+{
+ spinlock_t *ptl;
+ pmd_t *new = pmd_alloc_one(mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ ptl = pud_lockptr(mm, pud);
+ spin_lock_bh(ptl);
+ if (!pud_present(*pud)) {
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(new));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)new);
+ #endif
+ mm_inc_nr_pmds(mm);
+ pud_populate(mm, pud, new);
+ } else /* Another has populated it */
+ pmd_free(mm, new);
+ spin_unlock_bh(ptl);
+ return 0;
+}
+
+static inline pmd_t *koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
+{
+ return (unlikely(pud_none(*pud)) && __koi_pmd_alloc(mm, pud, address))?
+ NULL: pmd_offset(pud, address);
+}
+
+/**
+*kio_copy_pmd_range - copy pmd from kernel to driver space
+*/
+static inline int koi_copy_pmd_range(struct mm_struct *ko_mm, pud_t *dst_pud,
+ pud_t *src_pud, unsigned long addr,
+ unsigned long end)
+{
+ pmd_t *src_pmd, *dst_pmd;
+ unsigned long next;
+ int err;
+
+ dst_pmd = koi_pmd_alloc(ko_mm, dst_pud, addr);
+ if (!dst_pmd) {
+ return -ENOMEM;
+ }
+ src_pmd = pmd_offset(src_pud, addr);
+ do {
+ next = pmd_addr_end(addr, end);
+ // CONFIG_TRANSPARENT_HUGEPAGE is enabled, so we must add copy_huge_pmd
+ if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd) ||
+ (pmd_devmap(*src_pmd))) {
+ err = koi_copy_huge_pmd(ko_mm, dst_pmd, src_pmd, addr);
+ if (err == -ENOMEM)
+ return -ENOMEM;
+ if (!err)
+ continue;
+ }
+ if (pmd_none_or_clear_bad(src_pmd)) {
+ continue;
+ }
+ if (koi_copy_pte_range(ko_mm, dst_pmd, src_pmd, addr, next))
+ return -ENOMEM;
+ } while (dst_pmd++, src_pmd++, addr = next, addr != end);
+ return 0;
+}
+
+int __koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
+{
+ pud_t *new = pud_alloc_one(mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ spin_lock_bh(&mm->page_table_lock);
+ if (!p4d_present(*p4d)) {
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(new));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)new);
+ #endif
+ mm_inc_nr_puds(mm);
+ p4d_populate(mm, p4d, new);
+ } else /* Another has populated it */
+ pud_free(mm, new);
+ spin_unlock_bh(&mm->page_table_lock);
+ return 0;
+}
+
+static inline pud_t *koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d,
+ unsigned long address)
+{
+ return (unlikely(p4d_none(*p4d)) && __koi_pud_alloc(mm, p4d, address)) ?
+ NULL : pud_offset(p4d, address);
+}
+
+/**
+*koi_copy_pud_range - copy pud from kernel to driver
+*/
+static inline int koi_copy_pud_range(struct mm_struct *ko_mm, p4d_t *dst_p4d,
+ p4d_t *src_p4d, unsigned long addr,
+ unsigned long end)
+{
+ pud_t *src_pud, *dst_pud;
+ unsigned long next;
+ dst_pud = koi_pud_alloc(ko_mm, dst_p4d, addr);
+ if (!dst_pud)
+ return -ENOMEM;
+ src_pud = pud_offset(src_p4d, addr);
+ do {
+ next = pud_addr_end(addr, end);
+ if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
+ continue;
+ /* fall through */
+ }
+ if (pud_none_or_clear_bad(src_pud))
+ continue;
+ if (koi_copy_pmd_range(ko_mm, dst_pud, src_pud, addr, next))
+ return -ENOMEM;
+ } while (dst_pud++, src_pud++, addr = next, addr != end);
+ return 0;
+}
+
+/**
+* koi_copy_p4d_range - map the kernel pagetable to the driver space level by level
+* @ko_mm: the mm_struct of driver module
+* @dst_pgd: destination pgd
+* @src_pgd: source pgd
+* @addr: the start of address
+* @end: the end of address
+*/
+static inline int koi_copy_p4d_range(struct mm_struct *ko_mm, pgd_t *dst_pgd,
+ pgd_t *src_pgd, unsigned long addr,
+ unsigned long end)
+{
+ p4d_t *src_p4d, *dst_p4d;
+ unsigned long next;
+ dst_p4d = p4d_alloc(ko_mm, dst_pgd, addr);
+ if (!dst_p4d)
+ return -ENOMEM;
+ src_p4d = p4d_offset(src_pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none_or_clear_bad(src_p4d))
+ continue;
+ if (koi_copy_pud_range(ko_mm, dst_p4d, src_p4d, addr, next)) {
+ return -ENOMEM;
+ }
+ } while (dst_p4d++, src_p4d++, addr = next, addr != end);
+ return 0;
+}
+
+/**
+*int koi_copy_pagetable - map the address range from "addr" to "end" to the driver pagetable
+*@ko_mm: the mm_struct of the driver module
+*@koi_pg_dir: koi_pg_dir, related to the driver module, the entry for driver pagetable
+*@addr: the starting address of mapping zone
+*@end: the end address of mapping zone
+*/
+int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir,
+ unsigned long addr, unsigned long end)
+{
+ int ret = 0;
+ unsigned long next;
+
+ pgd_t *src_pgd, *dst_pgd;
+
+ src_pgd = pgd_offset_pgd(swapper_pg_dir, addr);
+ dst_pgd = pgd_offset_pgd(koi_pg_dir, addr);
+ do {
+ next = pgd_addr_end(addr, end);
+ if (pgd_none_or_clear_bad(src_pgd))
+ continue;
+ if (unlikely(koi_copy_p4d_range(ko_mm, dst_pgd, src_pgd, addr,
+ next))) {
+ ret = -ENOMEM;
+ break;
+ }
+ } while (dst_pgd++, src_pgd++, addr = next, addr != end);
+
+ return ret;
+}
+
+void koi_set_rdonly(unsigned long addr, pgd_t *pgdir)
+{
+ p4d_t *p4dp;
+ pud_t *pudp;
+ pmd_t *pmdp;
+ pte_t *ptep;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) {
+ return;
+ }
+
+ p4dp = p4d_offset(pgdp, addr);
+ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) {
+ return;
+ }
+
+ pudp = pud_offset(p4dp, addr);
+ if (pud_none(*pudp) || pud_bad(*pudp)) {
+ return;
+ }
+ pmdp = pmd_offset(pudp, addr);
+ if (pmd_none(*pmdp) || pmd_bad(*pmdp)) {
+ return;
+ }
+
+ ptep = pte_offset_kernel(pmdp, addr);
+ if (pte_none(*ptep)) {
+ printk(KERN_ERR "ptep 0x%llx not available\n", ptep);
+ return;
+ }
+ set_pte(ptep, __pte(pte_val(*ptep) | PTE_RDONLY));
+ printk(KERN_ERR "set_readonly successfully\n");
+ return;
+}
+
+/**
+* koi_create_pagetable - create pagetable for driver
+* @mod: driver module
+* 1.create a new koi_mem_hash_node new_node
+* 2.create page table return the pgd address, init the new_node->pgdp
+* 3.create and init the new_node->ko_mm
+* 4.map swapper_ttbr1 to the newly created pagetable
+* 5.map the interrupt vector table to the newly created pagetable
+* 6.map the init_layout of the module
+* 7.map the core_layout of the module
+* 8.map switch_to_kernel_pgtable into driver view
+* 9.map share memory
+*/
+void koi_create_pagetable(struct module *mod)
+{
+ int ret = 0, cpu;
+ unsigned long vbar, addr, ttbr1;
+ pgd_t *pgdp;
+ unsigned long *ptr;
+ struct koi_mem_list *new_mem_node;
+ struct koi_mem_hash_node *new_node =
+ kzalloc(sizeof(struct koi_mem_hash_node), GFP_KERNEL);
+ if (!new_node) {
+ printk(KERN_ERR "NULL new_node\n");
+ return;
+ };
+ if (koi_swapper_ttbr1 == 0) {
+ pgdp = lm_alias(swapper_pg_dir);
+ ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
+ if (system_supports_cnp() &&
+ !WARN_ON(pgdp != lm_alias(swapper_pg_dir)))
+ ttbr1 |= TTBR_CNP_BIT;
+#ifdef CONFIG_IEE
+ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, 1);
+#endif
+ koi_swapper_ttbr1 = ttbr1;
+ // __WRITE_ONCE(koi_swapper_ttbr1, ttbr1);
+ // koi_set_rdonly(&koi_swapper_ttbr1, swapper_pg_dir);
+ }
+ new_node->pgdp = koi_pgd_alloc();
+ new_node->ko_mm =
+ kzalloc(sizeof(struct mm_struct) +
+ sizeof(unsigned long) * BITS_TO_LONGS(NR_CPUS),
+ GFP_KERNEL);
+ init_ko_mm(new_node->ko_mm, new_node->pgdp);
+ new_node->mod = mod;
+ koi_save_ttbr(mod, new_node->pgdp, new_node);
+ printk(KERN_ERR "copying koi_data, start=0x%16llx, end=0x%16llx\n",
+ (unsigned long)__koi_data_start, (unsigned long)__koi_data_end);
+ // copy koi_swapper_ttbr1, which records page dir base for kernel view
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)__koi_data_start,
+ (unsigned long)__koi_data_end);
+ asm volatile("mrs %0, VBAR_EL1\n" : "=r"(vbar) :);
+
+ // copy interrupt vectors
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, vbar & PAGE_MASK,
+ (vbar + PAGE_SIZE) & PAGE_MASK);
+
+ // copy module init_layout, which contains init data and text in driver
+ ret = koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)mod->init_layout.base,
+ (unsigned long)mod->init_layout.base +
+ mod->init_layout.size);
+ if (ret != 0)
+ printk(KERN_ERR
+ "\033[33mError occur when copying init_layout, Eno:%d\033[0m\n",
+ ret);
+
+ // copy module core_layout, which contains non-init data and text in driver
+ ret = koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)mod->core_layout.base,
+ (unsigned long)mod->core_layout.base +
+ mod->core_layout.size);
+ if (ret != 0)
+ printk(KERN_ERR
+ "\033[33mError occur when copying core_layout, Eno: %d\033[0m\n",
+ ret);
+
+ // mapping switch_to_kernel_pgtable into driver view, which is used to switch to kernel view when entering INT
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)__koi_code_start,
+ (unsigned long)__koi_code_end);
+
+ for_each_possible_cpu (cpu) {
+ ptr = per_cpu(irq_stack_ptr, cpu);
+ printk(KERN_ERR
+ "\033[33mirq_stack_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n",
+ cpu, (unsigned long)ptr,
+ (unsigned long)ptr + IRQ_STACK_SIZE);
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)ptr,
+ (unsigned long)ptr + IRQ_STACK_SIZE);
+ }
+
+ for_each_possible_cpu (cpu) {
+ ptr = per_cpu(koi_irq_current_ttbr1, cpu);
+ printk(KERN_ERR
+ "\033[33mirq_current_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n",
+ cpu, (unsigned long)ptr, (unsigned long)ptr + PAGE_SIZE);
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)ptr,
+ (unsigned long)ptr + PAGE_SIZE);
+ }
+
+#ifdef CONFIG_IEE
+ // mapping iee_rwx_gate_entry and iee_si_base to ko's pagetable
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp,
+ (unsigned long)__iee_si_base_start,
+ (unsigned long)__iee_exec_entry_end);
+#endif
+
+ // alloc 16KB memory for new ko, and add it into hashtable
+ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL);
+ if ((void *)addr == NULL) {
+ printk(KERN_ERR "alloc buffer error\n");
+ }
+ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr,
+ addr + THREAD_SIZE);
+
+ new_mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL);
+ if (new_mem_node == NULL) {
+ printk(KERN_ERR "alloc new_mem_node error\n");
+ }
+ new_mem_node->addr = addr;
+ new_mem_node->size = THREAD_SIZE;
+
+ new_node->mem_list_head =
+ (struct list_head)LIST_HEAD_INIT(new_node->mem_list_head);
+ hash_init(new_node->addr_htbl);
+ spin_lock_init(&new_node->addr_htbl_spin_lock);
+ spin_lock_init(&new_node->spin_lock);
+
+ spin_lock(&new_node->spin_lock);
+ list_add_rcu(&new_mem_node->list, &new_node->mem_list_head);
+ spin_unlock(&new_node->spin_lock);
+
+ spin_lock(&koi_mem_htbl_spin_lock);
+ hash_add_rcu(koi_mem_htbl, &new_node->node,
+ (unsigned long)new_node->mod);
+ spin_unlock(&koi_mem_htbl_spin_lock);
+}
+/**
+* koi_mem_alloc
+*@mod: driver module
+*@orig_addr: the starting address of the parameter in kernel
+*@size: the size of the parameter
+*/
+unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr,
+ unsigned long size)
+{
+ struct koi_mem_hash_node *target = NULL;
+ struct koi_mem_list *mem_node;
+ struct koi_addr_map *new_addr_node;
+ unsigned long addr = 0, flags;
+ struct koi_mem_list *new_mem_node;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return 0;
+ }
+ spin_lock_irqsave(&target->spin_lock, flags);
+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) {
+ if (mem_node->size >= size) {
+ addr = mem_node->addr;
+ mem_node->size -= size;
+ if (mem_node->size == 0) {
+ list_del_rcu(&mem_node->list);
+ } else {
+ new_mem_node =
+ kmalloc(sizeof(struct koi_mem_list),
+ GFP_ATOMIC);
+ new_mem_node->addr = addr + size;
+ new_mem_node->size = mem_node->size;
+ list_replace_rcu(&mem_node->list,
+ &new_mem_node->list);
+ }
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ }
+ }
+ spin_unlock_irqrestore(&target->spin_lock, flags);
+ if (!addr) {
+ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL);
+ if ((void *)addr == NULL) {
+ return 0;
+ }
+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr,
+ addr + THREAD_SIZE);
+ mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL);
+ if (!mem_node) {
+ printk(KERN_ERR "NULL mem_node\n");
+ }
+ if (size > THREAD_SIZE) {
+ return 0;
+ }
+ mem_node->addr = addr + size;
+ mem_node->size = THREAD_SIZE - size;
+ spin_lock_irqsave(&target->spin_lock, flags);
+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head);
+ spin_unlock_irqrestore(&target->spin_lock, flags);
+ }
+
+ new_addr_node = kzalloc(sizeof(struct koi_addr_map), GFP_KERNEL);
+ new_addr_node->buffer_addr = addr;
+ new_addr_node->orig_addr = orig_addr;
+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags);
+ hash_add_rcu(target->addr_htbl, &new_addr_node->node,
+ new_addr_node->buffer_addr);
+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags);
+ return addr;
+}
+EXPORT_SYMBOL(koi_mem_alloc);
+// find the parameter pointer corresponding to the copy
+noinline void *koi_mem_lookup(struct module *mod, unsigned long addr)
+{
+ struct koi_mem_hash_node *target = NULL;
+ struct koi_addr_map *addr_map_node;
+ unsigned long orig_addr = addr;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return NULL;
+ }
+
+ rcu_read_lock();
+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node,
+ orig_addr) {
+ if (addr_map_node->buffer_addr == addr) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (addr_map_node) {
+ return (void *)(addr_map_node->orig_addr);
+ } else {
+ return NULL;
+ }
+}
+EXPORT_SYMBOL(koi_mem_lookup);
+/**
+* kio_mem_free - recycle a copy of the copied parameters and synchronize the parameters
+* @mod: driver module
+* @addr: the starting addr of parameter
+* @size: the size of the parameter
+* @is_const: const pointers or not
+* @count: contry the number of parameters
+*/
+noinline void koi_mem_free(struct module *mod, unsigned long addr,
+ unsigned long size, bool is_const, int count, ...)
+{
+ struct koi_mem_hash_node *target = NULL;
+ struct koi_mem_list *mem_node;
+ struct list_head *pos = NULL;
+ struct koi_addr_map *addr_map_node;
+ unsigned long orig_size = size;
+ unsigned long orig_addr = addr;
+ va_list valist;
+ int i;
+ unsigned int offset;
+ unsigned long flags;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return;
+ }
+
+ rcu_read_lock();
+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node,
+ orig_addr) {
+ if (addr_map_node->buffer_addr == orig_addr) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ va_start(valist, count);
+ for (i = 0; i < count; i++) {
+ offset = va_arg(valist, int);
+ *(unsigned long *)(addr_map_node->buffer_addr + offset) =
+ *(unsigned long *)(addr_map_node->orig_addr + offset);
+ }
+ va_end(valist);
+ memcpy((void *)addr_map_node->orig_addr,
+ (void *)addr_map_node->buffer_addr, orig_size);
+
+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags);
+ hlist_del_init_rcu(&addr_map_node->node);
+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free);
+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags);
+
+ spin_lock_irqsave(&target->spin_lock, flags);
+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) {
+ if (mem_node->addr + mem_node->size == addr) {
+ pos = mem_node->list.prev;
+ addr = mem_node->addr;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size == mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size < mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ break;
+ }
+ }
+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC);
+ mem_node->addr = addr;
+ mem_node->size = size;
+ if (pos)
+ list_add_rcu(&mem_node->list, pos);
+ else
+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head);
+ spin_unlock_irqrestore(&target->spin_lock, flags);
+}
+EXPORT_SYMBOL(koi_mem_free);
+/**
+* koi_mem_free_callback - used to recycle the copy of parameter.
+*@addr: the address of the parameter
+*@(*func)(void*): callback func, used to release the copy of the parameter pointer
+*/
+noinline void koi_mem_free_callback(struct module *mod, unsigned long addr,
+ unsigned long size, void (*func)(void *))
+{
+ struct koi_mem_hash_node *target = NULL;
+ struct koi_mem_list *mem_node;
+ struct list_head *pos = NULL;
+ struct koi_addr_map *addr_map_node;
+ unsigned long flags;
+ unsigned long orig_size = size;
+ unsigned long orig_addr = addr;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk("mem node for module: %s not found\n", mod->name);
+ return;
+ }
+
+ rcu_read_lock();
+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node,
+ orig_addr) {
+ if (addr_map_node->buffer_addr == orig_addr) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (addr_map_node != NULL) {
+ memcpy((void *)addr_map_node->orig_addr,
+ (void *)addr_map_node->buffer_addr, orig_size);
+ func((void *)addr_map_node->orig_addr);
+ } else {
+ printk("Cannot find addr_map_node in addr_htbl, maybe addr is in kernel space!!\n");
+ func((void *)orig_addr);
+ }
+
+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags);
+ if (addr_map_node != NULL) {
+ hlist_del_init_rcu(&addr_map_node->node);
+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free);
+ }
+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags);
+ spin_lock_irqsave(&target->spin_lock, flags);
+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) {
+ if (mem_node->addr + mem_node->size == addr) {
+ pos = mem_node->list.prev;
+ addr = mem_node->addr;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size == mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size < mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ break;
+ }
+ }
+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC);
+ mem_node->addr = addr;
+ mem_node->size = size;
+ if (pos)
+ list_add_rcu(&mem_node->list, pos);
+ else
+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head);
+ spin_unlock_irqrestore(&target->spin_lock, flags);
+}
+EXPORT_SYMBOL(koi_mem_free_callback);
+
+void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size)
+{
+ struct koi_mem_hash_node *target = NULL;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod)
+ break;
+ }
+ rcu_read_unlock();
+
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return;
+ }
+ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK,
+ (addr + size + PAGE_SIZE) & PAGE_MASK);
+}
+EXPORT_SYMBOL(koi_map_mem);
+/**
+* koi_mem_free_to_user - function 'copy_to_user' in driver space
+*/
+void koi_mem_free_to_user(struct module *mod, unsigned long addr,
+ unsigned long size)
+{
+ struct koi_mem_hash_node *target = NULL;
+ struct koi_mem_list *mem_node;
+ struct list_head *pos = NULL;
+ struct koi_addr_map *addr_map_node;
+ unsigned long flags;
+ unsigned long orig_size = size;
+ unsigned long orig_addr = addr;
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return;
+ }
+
+ rcu_read_lock();
+ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node,
+ orig_addr) {
+ if (addr_map_node->buffer_addr == orig_addr) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (copy_to_user((void *)addr_map_node->orig_addr,
+ (void *)addr_map_node->buffer_addr, orig_size)) {
+ return;
+ }
+
+ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags);
+ hlist_del_init_rcu(&addr_map_node->node);
+ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free);
+ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags);
+ spin_lock_irqsave(&target->spin_lock, flags);
+ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) {
+ if (mem_node->addr + mem_node->size == addr) {
+ pos = mem_node->list.prev;
+ addr = mem_node->addr;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size == mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ size += mem_node->size;
+ list_del_rcu(&mem_node->list);
+ call_rcu(&mem_node->rcu, koi_mem_node_free);
+ } else if (addr + size < mem_node->addr) {
+ if (!pos)
+ pos = mem_node->list.prev;
+ break;
+ }
+ }
+ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC);
+ mem_node->addr = addr;
+ mem_node->size = size;
+ if (pos)
+ list_add_rcu(&mem_node->list, pos);
+ else
+ list_add_tail_rcu(&mem_node->list, &target->mem_list_head);
+ spin_unlock_irqrestore(&target->spin_lock, flags);
+}
+EXPORT_SYMBOL(koi_mem_free_to_user);
+// map the driver stack to kernel
+void koi_map_kostack(struct module *mod)
+{
+ struct koi_mem_hash_node *target = NULL;
+ void *koi_stack;
+ unsigned long cur_sp;
+ asm volatile("mov %0, sp\n" : "=r"(cur_sp) :);
+ if (on_irq_stack(cur_sp, NULL)) {
+ return;
+ }
+#ifndef CONFIG_IEE
+ unsigned long res, alloc_token;
+ struct task_token *token_addr =
+ (struct task_token *)((unsigned long)current +
+ (unsigned long)koi_offset);
+ if (token_addr->koi_stack_base != NULL)
+ return;
+#else
+ koi_stack = iee_rw_gate(IEE_READ_KOI_STACK, current);
+ if (koi_stack != NULL)
+ return;
+#endif
+ koi_stack =
+ (void *)__get_free_pages(THREADINFO_GFP & ~__GFP_ACCOUNT, 3);
+ free_pages(koi_stack + 4 * PAGE_SIZE, 2);
+ printk(KERN_ERR "alloc dstack start=0x%16llx, end=0x%16llx\n",
+ koi_stack, koi_stack + THREAD_SIZE);
+#ifndef CONFIG_IEE
+ token_addr->koi_stack =
+ (struct pt_regs *)(THREAD_SIZE + (unsigned long)koi_stack) - 1;
+ token_addr->koi_stack_base = koi_stack;
+#else
+ iee_rw_gate(
+ IEE_WRITE_KOI_STACK, current,
+ (unsigned long)((struct pt_regs *)(THREAD_SIZE +
+ (unsigned long)koi_stack) -
+ 1));
+ iee_rw_gate(IEE_WRITE_KOI_STACK_BASE, current,
+ (unsigned long)koi_stack);
+#endif
+ rcu_read_lock();
+ hash_for_each_possible_rcu (koi_mem_htbl, target, node,
+ (unsigned long)mod) {
+ if (target->mod == mod) {
+ break;
+ }
+ }
+ rcu_read_unlock();
+ if (target == NULL) {
+ printk(KERN_ERR "mem node for module: %s not found\n",
+ mod->name);
+ return;
+ }
+ koi_copy_pagetable(target->ko_mm, target->pgdp,
+ (unsigned long)koi_stack,
+ (unsigned long)koi_stack + THREAD_SIZE);
+ printk(KERN_ERR "create ko stack: 0x%16llx\n",
+ (unsigned long)koi_stack);
+}
+EXPORT_SYMBOL(koi_map_kostack);
\ No newline at end of file
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index cea96ee75d22..cbddc8e464e4 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -79,8 +79,13 @@ int memcmp_pages(struct page *page1, struct page *page2)
static inline void __mte_enable_kernel(const char *mode, unsigned long tcf)
{
/* Enable MTE Sync Mode for EL1. */
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK,
+ SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf));
+#else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf));
+#endif
isb();
pr_info_once("MTE: enabled in %s mode at EL1\n", mode);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 068e5bb2661b..c98bc1a24fc7 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -467,11 +467,24 @@ static void ssbs_thread_switch(struct task_struct *next)
* This is *only* for exception entry from EL0, and is not valid until we
* __switch_to() a user task.
*/
+#ifdef CONFIG_IEE
+// Put __entry_task in a isolated page to protect it.
+__attribute__((aligned(PAGE_SIZE))) DEFINE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task);
+extern void iee_write_entry_task(struct task_struct *tsk);
+#else
DEFINE_PER_CPU(struct task_struct *, __entry_task);
+#endif
static void entry_task_switch(struct task_struct *next)
{
+ #ifdef CONFIG_IEE
+ if(next == &init_task)
+ iee_write_entry_task((struct task_struct *)__va(__pa_symbol(next)));
+ else
+ iee_write_entry_task(next);
+ #else
__this_cpu_write(__entry_task, next);
+ #endif
}
/*
@@ -506,11 +519,15 @@ static void erratum_1418040_new_exec(void)
*/
void update_sctlr_el1(u64 sctlr)
{
- /*
+ /*
* EnIA must not be cleared while in the kernel as this is necessary for
* in-kernel PAC. It will be cleared on kernel exit if needed.
*/
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
+ #else
sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
+ #endif
/* ISB required for the kernel uaccess routines when setting TCF0. */
isb();
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 58a97861bfc5..c7839247327d 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -551,7 +551,11 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
return state;
if (spectre_v4_mitigations_off()) {
+#ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_DSSBS);
+#else
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
+#endif
set_pstate_ssbs(1);
return SPECTRE_VULNERABLE;
}
@@ -975,7 +979,11 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
if (arm64_kernel_unmapped_at_el0())
return;
+#ifdef CONFIG_IEE
+ iee_rwx_gate_entry(IEE_WRITE_vbar_el1, v);
+#else
write_sysreg(v, vbar_el1);
+#endif
isb();
}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 95cb22c083c8..df73a583a733 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -33,6 +33,11 @@
#include <linux/scs.h>
#include <linux/mm.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#include <asm/iee-si.h>
+#endif
+
#include <asm/acpi.h>
#include <asm/fixmap.h>
#include <asm/cpu.h>
@@ -335,10 +340,30 @@ u64 cpu_logical_map(unsigned int cpu)
return __cpu_logical_map[cpu];
}
+#ifdef CONFIG_IEE
+/* used for secure modification of vbar*/
+extern char __bp_harden_el1_vectors[];
+/* prepare iee rwx gate for senario of ttbr1=init_pg_dir */
+static void __init iee_si_init_early(void)
+{
+ /* prepare data used for iee rwx gate. */
+ iee_base_idmap_pg_dir = phys_to_ttbr(__pa_symbol(idmap_pg_dir));
+ iee_base_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir))
+ | FIELD_PREP(TTBR_ASID_MASK, 1);
+ iee_base__bp_harden_el1_vectors = (unsigned long)__bp_harden_el1_vectors;
+ iee_si_tcr = 0;
+}
+#endif
+
void __init __no_sanitize_address setup_arch(char **cmdline_p)
{
setup_initial_init_mm(_stext, _etext, _edata, _end);
+ #ifdef CONFIG_IEE
+ init_new_context(&init_task, &init_mm);
+ atomic64_set(&init_mm.context.id, (1UL << get_cpu_asid_bits()) | INIT_ASID);
+ #endif
+
*cmdline_p = boot_command_line;
kaslr_init();
@@ -371,6 +396,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
*/
local_daif_restore(DAIF_PROCCTX_NOIRQ);
+#ifdef CONFIG_IEE
+ /*
+ * Map iee si codes to init_pg_dir to run the following
+ * cpu_uninstall_idmap() which writes ttbr0.
+ */
+ iee_si_init_early();
+#endif
+
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index b7b7afb4a8c7..168a9390d6e9 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -902,6 +902,32 @@ const char *esr_get_class_string(unsigned long esr)
return esr_class_str[ESR_ELx_EC(esr)];
}
+#ifdef CONFIG_IEE
+extern void arm64_enter_nmi(struct pt_regs *regs);
+static const char *handler[]= {
+ "SP_EL0",
+ "ELR_EL1",
+ "TCR_EL1",
+ "TTBR0 ASID"
+ "IEE_SI"
+};
+
+asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
+{
+ arm64_enter_nmi(regs);
+
+ console_verbose();
+
+ pr_crit("IEE : Bad mode in %s check detected on CPU%d, code 0x%08x -- %s\n",
+ handler[reason], smp_processor_id(), esr,
+ esr_get_class_string(esr));
+
+ __show_regs(regs);
+ local_daif_mask();
+ panic("bad mode");
+}
+#endif
+
/*
* bad_el0_sync handles unexpected, but potentially recoverable synchronous
* exceptions taken from EL0.
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 3cd7e76cc562..287eaba7a15b 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -134,6 +134,52 @@ jiffies = jiffies_64;
#define UNWIND_DATA_SECTIONS
#endif
+#ifdef CONFIG_IEE
+#define IEE_TEXT \
+ . = ALIGN(PAGE_SIZE); \
+ __iee_code_start = .; \
+ *(.iee.text.header) \
+ *(.iee.text) \
+ . = ALIGN(PAGE_SIZE); \
+ __iee_code_end = .;
+#else
+#define IEE_TEXT
+#endif
+
+#ifdef CONFIG_IEE
+#define IEE_SI_TEXT \
+ . = ALIGN(PAGE_SIZE); \
+ __iee_si_data_start = .; \
+ *(.iee.si_data) \
+ . = ALIGN(PAGE_SIZE); \
+ __iee_exec_entry_start = .; \
+ __iee_si_no_irq = . + (16); \
+ *(.iee.exec_entry) \
+ . = ALIGN(PAGE_SIZE); \
+ __iee_si_start = .; \
+ *(.iee.si_text) \
+ . = ALIGN(PAGE_SIZE); \
+ . += PAGE_SIZE - (24); \
+ __iee_si_end = . + (24); \
+ __iee_exec_exit = .; \
+ *(.iee.exec_exit) \
+ . = ALIGN(PAGE_SIZE);
+
+#else
+#define IEE_SI_TEXT
+#endif
+
+#ifdef CONFIG_KOI
+#define KOI_TEXT \
+ . = ALIGN(PAGE_SIZE); \
+ __koi_code_start = .; \
+ *(.koi.text) \
+ . = ALIGN(PAGE_SIZE); \
+ __koi_code_end = .;
+#else
+#define KOI_TEXT
+#endif
+
/*
* The size of the PE/COFF section that covers the kernel image, which
* runs from _stext to _edata, must be a round multiple of the PE/COFF
@@ -176,10 +222,13 @@ SECTIONS
SOFTIRQENTRY_TEXT
ENTRY_TEXT
TEXT_TEXT
+ IEE_TEXT
SCHED_TEXT
LOCK_TEXT
KPROBES_TEXT
HYPERVISOR_TEXT
+ IEE_SI_TEXT
+ KOI_TEXT
*(.gnu.warning)
}
@@ -318,6 +367,18 @@ SECTIONS
. += INIT_DIR_SIZE;
init_pg_end = .;
+ #ifdef CONFIG_IEE
+ . = ALIGN(PAGE_SIZE*8);
+ init_iee_stack_begin = .;
+ . += PAGE_SIZE*4;
+ init_iee_stack_end = .;
+
+ . = ALIGN(PAGE_SIZE);
+ init_iee_si_stack_begin = .;
+ . += PAGE_SIZE*4;
+ init_iee_si_stack_end = .;
+ #endif
+
. = ALIGN(SEGMENT_ALIGN);
__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
_end = .;
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 188197590fc9..97e1f86046cf 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -17,6 +17,10 @@
#include <asm/smp.h>
#include <asm/tlbflush.h>
+#ifdef CONFIG_IEE
+#include <asm/iee-si.h>
+#endif
+
static u32 asid_bits;
static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
@@ -39,7 +43,11 @@ static unsigned long *pinned_asid_map;
#define asid2ctxid(asid, genid) ((asid) | (genid))
/* Get the ASIDBits supported by the current CPU */
+#ifdef CONFIG_IEE
+u32 get_cpu_asid_bits(void)
+#else
static u32 get_cpu_asid_bits(void)
+#endif
{
u32 asid;
int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1),
@@ -212,6 +220,38 @@ static u64 new_context(struct mm_struct *mm)
return asid2ctxid(asid, generation);
}
+#ifdef CONFIG_KOI
+/*
+ * This function is used to check and allocate ASID for ko's pgd
+ * The mm MUST point to the isolated kos' mm_struct, other behaviours are undefined.
+ */
+void koi_check_and_switch_context(struct mm_struct *mm) {
+ u64 asid = atomic64_read(&mm->context.id);
+ u64 old_active_asid;
+ unsigned long flags;
+ unsigned int cpu;
+
+ old_active_asid = atomic64_read(this_cpu_ptr(&active_asids));
+ if (old_active_asid && asid_gen_match(asid) && atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_asids), old_active_asid, asid)) {
+ return;
+ }
+
+ raw_spin_lock_irqsave(&cpu_asid_lock, flags);
+ asid = atomic64_read(&mm->context.id);
+ if (!asid_gen_match(asid)) {
+ asid = new_context(mm);
+ atomic64_set(&mm->context.id, asid);
+ }
+
+ cpu = smp_processor_id();
+ if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
+ local_flush_tlb_all();
+
+ atomic64_set(this_cpu_ptr(&active_asids), asid);
+ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
+}
+#endif
+
void check_and_switch_context(struct mm_struct *mm)
{
unsigned long flags;
@@ -348,7 +388,9 @@ asmlinkage void post_ttbr_update_workaround(void)
void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm)
{
+ #ifndef CONFIG_IEE
unsigned long ttbr1 = read_sysreg(ttbr1_el1);
+ #endif
unsigned long asid = ASID(mm);
unsigned long ttbr0 = phys_to_ttbr(pgd_phys);
@@ -360,14 +402,28 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm)
if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN))
ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid);
- /* Set ASID in TTBR1 since TCR.A1 is set */
+ #ifdef CONFIG_IEE
+ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1);
+ iee_rwx_gate_entry(IEE_CONTEXT_SWITCH, ttbr0);
+ // TODO : if defined CONFIG_IEE and defined CONFIG_KOI
+ #else
+ /* Set ASID in TTBR0 since TCR.A1 is set 0*/
+
+ #ifdef CONFIG_KOI
+ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1);
+ ttbr1 &= ~TTBR_ASID_MASK;
+ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
+ #else
ttbr1 &= ~TTBR_ASID_MASK;
ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid);
-
+
+ #endif
cpu_set_reserved_ttbr0_nosync();
write_sysreg(ttbr1, ttbr1_el1);
write_sysreg(ttbr0, ttbr0_el1);
isb();
+ #endif
+
post_ttbr_update_workaround();
}
@@ -375,11 +431,21 @@ static int asids_update_limit(void)
{
unsigned long num_available_asids = NUM_USER_ASIDS;
- if (arm64_kernel_unmapped_at_el0()) {
- num_available_asids /= 2;
- if (pinned_asid_map)
- set_kpti_asid_bits(pinned_asid_map);
- }
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ num_available_asids /= 2;
+ if (pinned_asid_map) {
+ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);
+ memset(pinned_asid_map, 0xaa, len);
+ __set_bit(INIT_ASID, pinned_asid_map);
+ }
+ #else
+ if (arm64_kernel_unmapped_at_el0()) {
+ num_available_asids /= 2;
+ if (pinned_asid_map)
+ set_kpti_asid_bits(pinned_asid_map);
+ }
+ #endif
+
/*
* Expect allocation after rollover to fail if we don't have at least
* one more ASID than CPUs. ASID #0 is reserved for init_mm.
@@ -400,6 +466,10 @@ arch_initcall(asids_update_limit);
static int asids_init(void)
{
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ unsigned int len;
+ #endif
+
asid_bits = get_cpu_asid_bits();
atomic64_set(&asid_generation, ASID_FIRST_VERSION);
asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
@@ -410,6 +480,11 @@ static int asids_init(void)
pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
nr_pinned_asids = 0;
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);
+ memset(asid_map, 0xaa, len);
+ __set_bit(INIT_ASID, asid_map);
+ #else
/*
* We cannot call set_reserved_asid_bits() here because CPU
* caps are not finalized yet, so it is safer to assume KPTI
@@ -417,6 +492,8 @@ static int asids_init(void)
*/
if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))
set_kpti_asid_bits(asid_map);
+ #endif
+
return 0;
}
early_initcall(asids_init);
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 4ea07caba71c..de1d57c2b30f 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -261,7 +261,11 @@ int __ptep_set_access_flags(struct vm_area_struct *vma,
pteval ^= PTE_RDONLY;
pteval |= pte_val(entry);
pteval ^= PTE_RDONLY;
+ #ifdef CONFIG_PTP
+ pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval);
+ #else
pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
+ #endif
} while (pteval != old_pteval);
/* Invalidate a stale read-only entry */
@@ -376,8 +380,13 @@ static void do_tag_recovery(unsigned long addr, unsigned long esr,
* It will be done lazily on the other CPUs when they will hit a
* tag fault.
*/
+ #ifdef CONFIG_IEE
+ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK,
+ SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE));
+ #else
sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE));
+ #endif
isb();
}
diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index bfc02568805a..580ecb596d2d 100644
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -32,6 +32,22 @@ static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss;
static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+#ifdef CONFIG_IEE
+void *bm_pte_addr = (void *)bm_pte;
+void *bm_pmd_addr = (void *)bm_pmd;
+void *bm_pud_addr = (void *)bm_pud;
+#endif
+
+#ifdef CONFIG_PTP
+extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot);
+extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot);
+extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep,
+ pmdval_t prot);
+
+extern void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d);
+#define set_pgd_init(pgdptr, pgdval) iee_set_p4d_pre_init((p4d_t *)(pgdptr), (p4d_t) { pgdval })
+#endif
+
static inline pte_t *fixmap_pte(unsigned long addr)
{
return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)];
@@ -44,7 +60,11 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
if (pmd_none(pmd)) {
ptep = bm_pte[BM_PTE_TABLE_IDX(addr)];
+ #ifdef CONFIG_PTP
+ __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE);
+ #else
__pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE);
+ #endif
}
}
@@ -55,8 +75,13 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr,
pud_t pud = READ_ONCE(*pudp);
pmd_t *pmdp;
- if (pud_none(pud))
+ if (pud_none(pud)) {
+ #ifdef CONFIG_PTP
+ __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
+ #else
__pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
+ #endif
+ }
pmdp = pmd_offset_kimg(pudp, addr);
do {
@@ -82,8 +107,13 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
}
- if (p4d_none(p4d))
+ if (p4d_none(p4d)) {
+ #ifdef CONFIG_PTP
+ __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
+ #else
__p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
+ #endif
+ }
pudp = pud_offset_kimg(p4dp, addr);
early_fixmap_init_pmd(pudp, addr, end);
@@ -106,6 +136,27 @@ void __init early_fixmap_init(void)
early_fixmap_init_pud(p4dp, addr, end);
}
+#ifdef CONFIG_PTP
+extern void iee_set_pte_pre_init(pte_t *ptep, pte_t pte);
+void __iee_set_fixmap_pre_init(enum fixed_addresses idx,
+ phys_addr_t phys, pgprot_t flags)
+{
+ unsigned long addr = __fix_to_virt(idx);
+ pte_t *ptep;
+
+ BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
+
+ ptep = fixmap_pte(addr);
+
+ if (pgprot_val(flags)) {
+ iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
+ } else {
+ iee_set_pte_pre_init(ptep, __pte(0));
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ }
+}
+#endif
+
/*
* Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
* ever need to use IPIs for TLB broadcasting, then we're in trouble here.
@@ -121,9 +172,17 @@ void __set_fixmap(enum fixed_addresses idx,
ptep = fixmap_pte(addr);
if (pgprot_val(flags)) {
+ #ifdef CONFIG_PTP
+ iee_set_bm_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
+ #else
__set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
+ #endif
} else {
+ #ifdef CONFIG_PTP
+ iee_set_bm_pte(ptep, __pte(0));
+ #else
__pte_clear(&init_mm, addr, ptep);
+ #endif
flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
}
}
@@ -179,8 +238,13 @@ void __init fixmap_copy(pgd_t *pgdir)
* live in the carveout for the swapper_pg_dir. We can simply
* re-use the existing dir for the fixmap.
*/
+ #ifdef CONFIG_PTP
+ set_pgd_init(pgd_offset_pgd(pgdir, FIXADDR_TOT_START),
+ READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START)));
+ #else
set_pgd(pgd_offset_pgd(pgdir, FIXADDR_TOT_START),
READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START)));
+ #endif
} else if (CONFIG_PGTABLE_LEVELS > 3) {
pgd_t *bm_pgdp;
p4d_t *bm_p4dp;
@@ -194,9 +258,15 @@ void __init fixmap_copy(pgd_t *pgdir)
BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
bm_pgdp = pgd_offset_pgd(pgdir, FIXADDR_TOT_START);
bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_TOT_START);
+ #ifdef CONFIG_PTP
+ bm_pudp = pud_set_fixmap_offset_init(bm_p4dp, FIXADDR_TOT_START);
+ __iee_pud_populate_pre_init(bm_pudp, __pa(lm_alias(bm_pmd)), PMD_TYPE_TABLE);
+ pud_clear_fixmap_init();
+ #else
bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_TOT_START);
pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd));
pud_clear_fixmap();
+ #endif
} else {
BUG();
}
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 8c8d7653ba84..4190f5c10d68 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -55,8 +55,19 @@
* that cannot be mistaken for a real physical address.
*/
s64 memstart_addr __ro_after_init = -1;
+#if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+s64 memstart_addr_init __ro_after_init = -1;
+#endif
+#ifdef CONFIG_KOI
+s64 koi_offset __ro_after_init = -1;
+EXPORT_SYMBOL(koi_offset);
+#endif
+#ifdef CONFIG_IEE
+s64 iee_offset __ro_after_init = -1;
+#endif
EXPORT_SYMBOL(memstart_addr);
+
/*
* If the corresponding config options are enabled, we create both ZONE_DMA
* and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory
@@ -421,7 +432,11 @@ early_param("memmap", parse_memmap_opt);
void __init arm64_memblock_init(void)
{
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ s64 linear_region_size = BIT(vabits_actual - 2);
+ #else
s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
+ #endif
/*
* Corner case: 52-bit VA capable systems running KVM in nVHE mode may
@@ -438,13 +453,24 @@ void __init arm64_memblock_init(void)
}
/* Remove memory above our supported physical address size */
+ #ifdef CONFIG_IEE
+ // If config iee, phys size can not be above 0x400000000000
+ if(__pa_symbol(_end) > BIT_ULL(vabits_actual - 2))
+ panic("Image on too high phys mem.\n");
+ else
+ memblock_remove(BIT_ULL(vabits_actual - 2), ULLONG_MAX);
+ #else
memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX);
+ #endif
/*
* Select a suitable value for the base of physical memory.
*/
memstart_addr = round_down(memblock_start_of_DRAM(),
ARM64_MEMSTART_ALIGN);
+ #if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+ memstart_addr_init = memstart_addr;
+ #endif
if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size)
pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n");
@@ -531,6 +557,14 @@ void __init arm64_memblock_init(void)
((range * memstart_offset_seed) >> 16);
}
}
+
+ #ifdef CONFIG_KOI
+ koi_offset = memstart_addr - memstart_addr_init + KOI_OFFSET;
+ #endif
+ #ifdef CONFIG_IEE
+ iee_offset = memstart_addr - memstart_addr_init + ((unsigned long)BIT(vabits_actual - 2));
+ #endif
+ //printk(KERN_ERR "koi_offset: 0x%16llx\n", koi_offset);
/*
* Register the kernel text, kernel data, initrd, and initial
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 4142a75a414e..094f3798441d 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -6,6 +6,7 @@
* Copyright (C) 2012 ARM Ltd.
*/
+#include "asm/pgtable.h"
#include <linux/cache.h>
#include <linux/export.h>
#include <linux/kernel.h>
@@ -40,6 +41,11 @@
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>
#include <asm/kfence.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#include <asm/iee.h>
+#include <asm/iee-si.h>
+#endif
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
@@ -76,8 +82,282 @@ EXPORT_SYMBOL(empty_zero_page);
static DEFINE_SPINLOCK(swapper_pgdir_lock);
static DEFINE_MUTEX(fixmap_lock);
+#ifdef CONFIG_IEE
+extern struct cred init_cred;
+
+extern unsigned long init_iee_stack_begin[];
+extern unsigned long init_iee_stack_end[];
+extern unsigned long __iee_si_data_start[];
+extern unsigned long __iee_exec_entry_start[];
+extern unsigned long __iee_si_start[];
+extern unsigned long __iee_si_end[];
+
+extern void *bm_pte_addr;
+extern void *bm_pmd_addr;
+extern void *bm_pud_addr;
+
+#ifdef CONFIG_PTP
+
+/* Funcs to set pgtable before iee initialized. */
+static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd)
+{
+ pgd_t *fixmap_pgdp;
+
+ spin_lock(&swapper_pgdir_lock);
+ fixmap_pgdp = pgd_set_fixmap_init(__pa_symbol(pgdp));
+ WRITE_ONCE(*fixmap_pgdp, pgd);
+ /*
+ * We need dsb(ishst) here to ensure the page-table-walker sees
+ * our new entry before set_p?d() returns. The fixmap's
+ * flush_tlb_kernel_range() via clear_fixmap() does this for us.
+ */
+ pgd_clear_fixmap_init();
+ spin_unlock(&swapper_pgdir_lock);
+}
+
+void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d)
+{
+ if (in_swapper_pgdir(p4dp)) {
+ iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
+ return;
+ }
+
+ WRITE_ONCE(*p4dp, p4d);
+ dsb(ishst);
+ isb();
+}
+
+static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud)
+{
+#ifdef __PAGETABLE_PUD_FOLDED
+ if (in_swapper_pgdir(pudp)) {
+ iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud)));
+ return;
+ }
+#endif /* __PAGETABLE_PUD_FOLDED */
+#ifdef CONFIG_KOI
+ pudval_t val = pud_val(pud);
+ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) {
+ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead.
+ pud = __pud(val | PMD_SECT_NG);
+ }
+#endif
+ WRITE_ONCE(*pudp, pud);
+
+ if (pud_valid(pud)) {
+ dsb(ishst);
+ isb();
+ }
+}
+
+static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd)
+{
+#ifdef __PAGETABLE_PMD_FOLDED
+ if (in_swapper_pgdir(pmdp)) {
+ iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
+ return;
+ }
+#endif /* __PAGETABLE_PMD_FOLDED */
+#ifdef CONFIG_KOI
+ pmdval_t val = pmd_val(pmd);
+ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) {
+ pmd = __pmd(val | PMD_SECT_NG);
+ }
+#endif
+ WRITE_ONCE(*pmdp, pmd);
+
+ if (pmd_valid(pmd)) {
+ dsb(ishst);
+ isb();
+ }
+}
+
+
+void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
+{
+ iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));
+}
+
+void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
+{
+ iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot));
+}
+
+void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep,
+ pmdval_t prot)
+{
+ iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot));
+}
+
+/* Funcs to set fixmap before iee initialized. */
+bool pgattr_change_is_safe(u64 old, u64 new);
+static int iee_pud_set_huge_fixmap(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
+{
+ pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot));
+
+ /* Only allow permission changes for now */
+ if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
+ pud_val(new_pud)))
+ return 0;
+
+ VM_BUG_ON(phys & ~PUD_MASK);
+ iee_set_fixmap_pud_pre_init(pudp, new_pud);
+ return 1;
+}
+
+static int iee_pmd_set_huge_fixmap(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
+{
+ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot));
+
+ /* Only allow permission changes for now */
+ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
+ pmd_val(new_pmd)))
+ return 0;
+
+ VM_BUG_ON(phys & ~PMD_MASK);
+ iee_set_fixmap_pmd_pre_init(pmdp, new_pmd);
+ return 1;
+}
+
+static inline void __iee_pmd_populate_fixmap(pmd_t *pmdp, phys_addr_t ptep,
+ pmdval_t prot)
+{
+ iee_set_fixmap_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot));
+}
+
+static inline void __iee_pud_populate_fixmap(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
+{
+ iee_set_fixmap_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot));
+}
+#endif /* END CONFIG_PTP*/
+
+void iee_set_pte_pre_init(pte_t *ptep, pte_t pte)
+{
+#ifdef CONFIG_KOI
+ if (!pte_none(pte)) {
+ pte = __pte(pte_val(pte) | PTE_NG);
+ }
+#endif
+ WRITE_ONCE(*ptep, pte);
+
+ /*
+ * Only if the new pte is valid and kernel, otherwise TLB maintenance
+ * or update_mmu_cache() have the necessary barriers.
+ */
+ if (pte_valid_not_user(pte)) {
+ dsb(ishst);
+ isb();
+ }
+}
+
+static void __init iee_set_token_page_valid_pre_init(void *token, void *new)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token);
+
+ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token);
+
+ pud_t *pudp = pud_offset(p4dp, (unsigned long)token);
+
+ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token);
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new)));
+ iee_set_pte_pre_init(ptep, pte);
+ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE));
+ isb();
+}
+#endif /* END CONFIG_IEE*/
+
+#if defined(CONFIG_KOI) && !defined(CONFIG_IEE)
+int koi_add_page_mapping(unsigned long dst, unsigned long src)
+{
+ pgd_t *src_pgdp, *dst_pgdp;
+ p4d_t *src_p4dp, *dst_p4dp;
+ pud_t *src_pudp, *dst_pudp;
+ pmd_t *src_pmdp, *dst_pmdp;
+ pte_t *src_ptep, *dst_ptep;
+
+ src_pgdp = pgd_offset_pgd(swapper_pg_dir, src);
+ dst_pgdp = pgd_offset_pgd(swapper_pg_dir, dst);
+
+ src_p4dp = p4d_offset(src_pgdp, src);
+ dst_p4dp = p4d_alloc(&init_mm, dst_pgdp, dst);
+ if (!dst_p4dp) {
+ return -ENOMEM;
+ }
+ src_pudp = pud_offset(src_p4dp, src);
+ dst_pudp = pud_alloc(&init_mm, dst_p4dp, dst);
+ if (!dst_pudp) {
+ return -ENOMEM;
+ }
+ if (pud_val(*src_pudp) & PMD_TABLE_BIT) {
+ src_pmdp = pmd_offset(src_pudp, src);
+ dst_pmdp = pmd_alloc(&init_mm, dst_pudp, dst);
+ if (!dst_pmdp) {
+ return -ENOMEM;
+ }
+ if (pmd_val(*src_pmdp) & PMD_TABLE_BIT) {
+ src_ptep = pte_offset_kernel(src_pmdp, src);
+ dst_ptep = pte_alloc_map(&init_mm, dst_pmdp, dst);
+ set_pte(dst_ptep, *src_ptep);
+ } else {
+ set_pte((pte_t *)dst_pmdp, pmd_pte(*src_pmdp));
+ }
+ } else {
+ set_pte((pte_t *)dst_pudp, pud_pte(*src_pudp));
+ }
+
+
+ flush_tlb_kernel_range(dst, dst+PAGE_SIZE);
+ isb();
+ return 0;
+}
+
+void koi_remove_page_mapping(unsigned long addr) {
+ pgd_t *src_pgdp;
+ p4d_t *src_p4dp;
+ pud_t *src_pudp;
+ pmd_t *src_pmdp;
+ pte_t *src_ptep;
+
+ src_pgdp = pgd_offset_pgd(swapper_pg_dir, addr);
+ if (pgd_none(*src_pgdp) || pgd_bad(*src_pgdp))
+ return;
+ src_p4dp = p4d_offset(src_pgdp, addr);
+ if (p4d_none(*src_p4dp) || p4d_bad(*src_p4dp))
+ return;
+ src_pudp = pud_offset(src_p4dp, addr);
+ if (pud_none(*src_pudp))
+ return;
+ if (pud_val(*src_pudp) & PMD_TABLE_BIT) {
+ src_pmdp = pmd_offset(src_pudp, addr);
+ if (pmd_none(*src_pmdp))
+ return;
+ if (pmd_val(*src_pmdp) & PMD_TABLE_BIT) {
+ src_ptep = pte_offset_kernel(src_pmdp, addr);
+ if(!pte_none(*src_ptep))
+ pte_clear(&init_mm, addr, src_ptep);
+ } else {
+ pmd_clear(src_pmdp);
+ }
+ } else {
+ pud_clear(src_pudp);
+ }
+
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
+}
+#endif
+
void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
{
+ #ifdef CONFIG_PTP
+ spin_lock(&swapper_pgdir_lock);
+ iee_rw_gate(IEE_OP_SET_SWAPPER_PGD, pgdp, pgd);
+ spin_unlock(&swapper_pgdir_lock);
+ #else
pgd_t *fixmap_pgdp;
spin_lock(&swapper_pgdir_lock);
@@ -90,6 +370,7 @@ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
*/
pgd_clear_fixmap();
spin_unlock(&swapper_pgdir_lock);
+ #endif
}
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
@@ -118,7 +399,11 @@ static phys_addr_t __init early_pgtable_alloc(int shift)
* slot will be free, so we can (ab)use the FIX_PTE slot to initialise
* any level of table.
*/
+ #ifdef CONFIG_PTP
+ ptr = pte_set_fixmap_init(phys);
+ #else
ptr = pte_set_fixmap(phys);
+ #endif
memset(ptr, 0, PAGE_SIZE);
@@ -126,11 +411,16 @@ static phys_addr_t __init early_pgtable_alloc(int shift)
* Implicit barriers also ensure the zeroed page is visible to the page
* table walker
*/
+ #ifdef CONFIG_PTP
+ pte_clear_fixmap_init();
+ #else
pte_clear_fixmap();
+ #endif
return phys;
}
+
bool pgattr_change_is_safe(u64 old, u64 new)
{
/*
@@ -178,7 +468,11 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
do {
pte_t old_pte = __ptep_get(ptep);
+ #ifdef CONFIG_PTP
+ iee_set_fixmap_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot));
+ #else
__set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
+ #endif
/*
* After the PTE entry has been populated once, we
@@ -211,7 +505,11 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
pmdval |= PMD_TABLE_PXN;
BUG_ON(!pgtable_alloc);
pte_phys = pgtable_alloc(PAGE_SHIFT);
+ #ifdef CONFIG_PTP
+ __iee_pmd_populate_fixmap(pmdp, pte_phys, pmdval);
+ #else
__pmd_populate(pmdp, pte_phys, pmdval);
+ #endif
pmd = READ_ONCE(*pmdp);
}
BUG_ON(pmd_bad(pmd));
@@ -248,7 +546,11 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
/* try section mapping first */
if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
+ #ifdef CONFIG_PTP
+ iee_pmd_set_huge_fixmap(pmdp, phys, prot);
+ #else
pmd_set_huge(pmdp, phys, prot);
+ #endif
/*
* After the PMD entry has been populated once, we
@@ -289,7 +591,11 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
pudval |= PUD_TABLE_PXN;
BUG_ON(!pgtable_alloc);
pmd_phys = pgtable_alloc(PMD_SHIFT);
- __pud_populate(pudp, pmd_phys, pudval);
+ #ifdef CONFIG_PTP
+ __iee_pud_populate_fixmap(pudp, pmd_phys, PUD_TYPE_TABLE);
+ #else
+ __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
+ #endif
pud = READ_ONCE(*pudp);
}
BUG_ON(pud_bad(pud));
@@ -345,7 +651,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
if (pud_sect_supported() &&
((addr | next | phys) & ~PUD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
+ #ifdef CONFIG_PTP
+ iee_pud_set_huge_fixmap(pudp, phys, prot);
+ #else
pud_set_huge(pudp, phys, prot);
+ #endif
/*
* After the PUD entry has been populated once, we
@@ -374,6 +684,10 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
{
unsigned long addr, end, next;
pgd_t *pgdp = pgd_offset_pgd(pgdir, virt);
+ #ifdef CONFIG_IEE
+ p4d_t *p4dp;
+ p4d_t p4d;
+ #endif
/*
* If the virtual and physical address don't have the same offset
@@ -390,10 +704,14 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
next = pgd_addr_end(addr, end);
alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
flags);
+ #ifdef CONFIG_IEE
+ p4dp = p4d_offset(pgdp, addr);
+ p4d = READ_ONCE(*p4dp);
+ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE));
+ #endif
phys += next - addr;
} while (pgdp++, addr = next, addr != end);
}
-
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
@@ -413,162 +731,844 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
phys_addr_t (*pgtable_alloc)(int), int flags);
#endif
-static phys_addr_t __pgd_pgtable_alloc(int shift)
+#ifdef CONFIG_PTP
+static int __init iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
{
- void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
- BUG_ON(!ptr);
+ pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot));
- /* Ensure the zeroed page is visible to the page table walker */
- dsb(ishst);
- return __pa(ptr);
+ /* Only allow permission changes for now */
+ if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
+ pud_val(new_pud)))
+ return 0;
+
+ VM_BUG_ON(phys & ~PUD_MASK);
+ iee_set_pud_pre_init(pudp, new_pud);
+ return 1;
}
-static phys_addr_t pgd_pgtable_alloc(int shift)
+static int __init iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
{
- phys_addr_t pa = __pgd_pgtable_alloc(shift);
- struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa));
-
- /*
- * Call proper page table ctor in case later we need to
- * call core mm functions like apply_to_page_range() on
- * this pre-allocated page table.
- *
- * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
- * folded, and if so pagetable_pte_ctor() becomes nop.
- */
- if (shift == PAGE_SHIFT)
- BUG_ON(!pagetable_pte_ctor(ptdesc));
- else if (shift == PMD_SHIFT)
- BUG_ON(!pagetable_pmd_ctor(ptdesc));
+ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot));
- return pa;
-}
+ /* Only allow permission changes for now */
+ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
+ pmd_val(new_pmd)))
+ return 0;
-/*
- * This function can only be used to modify existing table entries,
- * without allocating new levels of table. Note that this permits the
- * creation of new section or page entries.
- */
-void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
- phys_addr_t size, pgprot_t prot)
-{
- if (virt < PAGE_OFFSET) {
- pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
- &phys, virt);
- return;
- }
- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
- NO_CONT_MAPPINGS);
+ VM_BUG_ON(phys & ~PMD_MASK);
+ iee_set_pmd_pre_init(pmdp, new_pmd);
+ return 1;
}
-void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
- unsigned long virt, phys_addr_t size,
- pgprot_t prot, bool page_mappings_only)
+static __init void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end,
+ phys_addr_t phys, pgprot_t prot)
{
- int flags = 0;
-
- BUG_ON(mm == &init_mm);
+ pte_t *ptep;
- if (page_mappings_only)
- flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+ ptep = pte_set_fixmap_offset_init(pmdp, addr);
+ do {
+ pte_t old_pte = READ_ONCE(*ptep);
- __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
- pgd_pgtable_alloc, flags);
-}
+ iee_set_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot));
-static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
- phys_addr_t size, pgprot_t prot)
-{
- if (virt < PAGE_OFFSET) {
- pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
- &phys, virt);
- return;
- }
+ /*
+ * After the PTE entry has been populated once, we
+ * only allow updates to the permission attributes.
+ */
+ BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
+ READ_ONCE(pte_val(*ptep))));
- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
- NO_CONT_MAPPINGS);
+ phys += PAGE_SIZE;
+ } while (ptep++, addr += PAGE_SIZE, addr != end);
- /* flush the TLBs after updating live kernel mappings */
- flush_tlb_kernel_range(virt, virt + size);
+ pte_clear_fixmap_init();
}
-static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
- phys_addr_t end, pgprot_t prot, int flags)
+static __init void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr,
+ unsigned long end, phys_addr_t phys,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
{
- __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
- prot, early_pgtable_alloc, flags);
-}
+ unsigned long next;
+ pmd_t pmd = READ_ONCE(*pmdp);
-void __init mark_linear_text_alias_ro(void)
-{
- /*
- * Remove the write permissions from the linear alias of .text/.rodata
- */
- update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext),
- (unsigned long)__init_begin - (unsigned long)_stext,
- PAGE_KERNEL_RO);
-}
+ BUG_ON(pmd_sect(pmd));
+ if (pmd_none(pmd)) {
+ pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
+ phys_addr_t pte_phys;
-#ifdef CONFIG_KFENCE
+ if (flags & NO_EXEC_MAPPINGS)
+ pmdval |= PMD_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
+ pte_phys = pgtable_alloc(PAGE_SHIFT);
+ __iee_pmd_populate_pre_init(pmdp, pte_phys, pmdval);
+ pmd = READ_ONCE(*pmdp);
+ }
+ BUG_ON(pmd_bad(pmd));
-bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL;
+ do {
+ pgprot_t __prot = prot;
-/* early_param() will be parsed before map_mem() below. */
-static int __init parse_kfence_early_init(char *arg)
-{
- int val;
+ next = pte_cont_addr_end(addr, end);
- if (get_option(&arg, &val))
- kfence_early_init = !!val;
+ /* use a contiguous mapping if the range is suitably aligned */
+ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
+ (flags & NO_CONT_MAPPINGS) == 0)
+ __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
-#if IS_ENABLED(CONFIG_KFENCE_MUST_EARLY_INIT)
- kfence_must_early_init = (val == -1) ? true : false;
-#endif
+ iee_init_pte_pre_init(pmdp, addr, next, phys, __prot);
- return 0;
+ phys += next - addr;
+ } while (addr = next, addr != end);
}
-early_param("kfence.sample_interval", parse_kfence_early_init);
-static phys_addr_t __init arm64_kfence_alloc_pool(void)
+static __init void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end,
+ phys_addr_t phys, pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int), int flags)
{
- phys_addr_t kfence_pool;
+ unsigned long next;
+ pmd_t *pmdp;
- if (!kfence_early_init)
- return 0;
+ pmdp = pmd_set_fixmap_offset_init(pudp, addr);
+ do {
+ pmd_t old_pmd = READ_ONCE(*pmdp);
- kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
- if (!kfence_pool) {
- pr_err("failed to allocate kfence pool\n");
- kfence_early_init = false;
- return 0;
- }
+ next = pmd_addr_end(addr, end);
- /* Temporarily mark as NOMAP. */
- memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
+ /* try section mapping first */
+ if (((addr | next | phys) & ~PMD_MASK) == 0 &&
+ (flags & NO_BLOCK_MAPPINGS) == 0) {
+ iee_pmd_set_huge_pre_init(pmdp, phys, prot);
+
+ /*
+ * After the PMD entry has been populated once, we
+ * only allow updates to the permission attributes.
+ */
+ BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
+ READ_ONCE(pmd_val(*pmdp))));
+ } else {
+ iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot,
+ pgtable_alloc, flags);
+
+ BUG_ON(pmd_val(old_pmd) != 0 &&
+ pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
+ }
+ phys += next - addr;
+ } while (pmdp++, addr = next, addr != end);
+
+ pmd_clear_fixmap_init();
+}
+
+static __init void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr,
+ unsigned long end, phys_addr_t phys,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int), int flags)
+{
+ unsigned long next;
+ pud_t pud = READ_ONCE(*pudp);
+
+ /*
+ * Check for initial section mappings in the pgd/pud.
+ */
+ BUG_ON(pud_sect(pud));
+ if (pud_none(pud)) {
+ pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
+ phys_addr_t pmd_phys;
+
+ if (flags & NO_EXEC_MAPPINGS)
+ pudval |= PUD_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
+ pmd_phys = pgtable_alloc(PMD_SHIFT);
+ __iee_pud_populate_pre_init(pudp, pmd_phys, pudval);
+ pud = READ_ONCE(*pudp);
+ }
+ BUG_ON(pud_bad(pud));
+
+ do {
+ pgprot_t __prot = prot;
+
+ next = pmd_cont_addr_end(addr, end);
+
+ /* use a contiguous mapping if the range is suitably aligned */
+ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
+ (flags & NO_CONT_MAPPINGS) == 0)
+ __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
+
+ iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags);
+
+ phys += next - addr;
+ } while (addr = next, addr != end);
+}
+
+static __init void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end,
+ phys_addr_t phys, pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ unsigned long next;
+ pud_t *pudp;
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
+
+ if (p4d_none(p4d)) {
+ p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
+ phys_addr_t pud_phys;
+
+ if (flags & NO_EXEC_MAPPINGS)
+ p4dval |= P4D_TABLE_PXN;
+ BUG_ON(!pgtable_alloc);
+ pud_phys = pgtable_alloc(PUD_SHIFT);
+ __iee_p4d_populate_pre_init(p4dp, pud_phys, p4dval);
+ p4d = READ_ONCE(*p4dp);
+ }
+ BUG_ON(p4d_bad(p4d));
+
+ pudp = pud_set_fixmap_offset_init(p4dp, addr);
+ do {
+ pud_t old_pud = READ_ONCE(*pudp);
+
+ next = pud_addr_end(addr, end);
+
+ /*
+ * For 4K granule only, attempt to put down a 1GB block
+ */
+ if (pud_sect_supported() &&
+ ((addr | next | phys) & ~PUD_MASK) == 0 &&
+ (flags & NO_BLOCK_MAPPINGS) == 0) {
+ iee_pud_set_huge_pre_init(pudp, phys, prot);
+
+ /*
+ * After the PUD entry has been populated once, we
+ * only allow updates to the permission attributes.
+ */
+ BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
+ READ_ONCE(pud_val(*pudp))));
+ } else {
+ iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot,
+ pgtable_alloc, flags);
+
+ BUG_ON(pud_val(old_pud) != 0 &&
+ pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
+ }
+ phys += next - addr;
+ } while (pudp++, addr = next, addr != end);
+
+ pud_clear_fixmap_init();
+}
+
+static __init void __iee_create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ unsigned long addr, end, next;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt);
+ p4d_t *p4dp;
+ p4d_t p4d;
+
+ /*
+ * If the virtual and physical address don't have the same offset
+ * within a page, we cannot map the region as the caller expects.
+ */
+ if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
+ return;
+
+ phys &= PAGE_MASK;
+ addr = virt & PAGE_MASK;
+ end = PAGE_ALIGN(virt + size);
+
+ do {
+ next = pgd_addr_end(addr, end);
+ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc,
+ flags);
+ p4dp = p4d_offset(pgdp, addr);
+ p4d = READ_ONCE(*p4dp);
+ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE));
+ phys += next - addr;
+ } while (pgdp++, addr = next, addr != end);
+}
+
+static __init void __iee_create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ mutex_lock(&fixmap_lock);
+ __iee_create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot,
+ pgtable_alloc, flags);
+ mutex_unlock(&fixmap_lock);
+}
+#endif
+
+static phys_addr_t __pgd_pgtable_alloc(int shift)
+{
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
+ void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
+ BUG_ON(!ptr);
+
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(ptr));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)ptr);
+ #endif
+
+ /* Ensure the zeroed page is visible to the page table walker */
+ dsb(ishst);
+ return __pa(ptr);
+}
+
+static phys_addr_t pgd_pgtable_alloc(int shift)
+{
+ phys_addr_t pa = __pgd_pgtable_alloc(shift);
+ struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa));
+
+ /*
+ * Call proper page table ctor in case later we need to
+ * call core mm functions like apply_to_page_range() on
+ * this pre-allocated page table.
+ *
+ * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
+ * folded, and if so pagetable_pte_ctor() becomes nop.
+ */
+ if (shift == PAGE_SHIFT)
+ BUG_ON(!pagetable_pte_ctor(ptdesc));
+ else if (shift == PMD_SHIFT)
+ BUG_ON(!pagetable_pmd_ctor(ptdesc));
+
+ return pa;
+}
+
+/*
+ * This function can only be used to modify existing table entries,
+ * without allocating new levels of table. Note that this permits the
+ * creation of new section or page entries.
+ */
+void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
+ phys_addr_t size, pgprot_t prot)
+{
+ if (virt < PAGE_OFFSET) {
+ pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
+ &phys, virt);
+ return;
+ }
+
+ #ifdef CONFIG_PTP
+ __iee_create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL,
+ NO_CONT_MAPPINGS);
+ #else
+ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
+ NO_CONT_MAPPINGS);
+ #endif
+}
+
+void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot, bool page_mappings_only)
+{
+ int flags = 0;
+
+ BUG_ON(mm == &init_mm);
+
+ if (page_mappings_only)
+ flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+
+ __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
+ pgd_pgtable_alloc, flags);
+}
+
+static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
+ phys_addr_t size, pgprot_t prot)
+{
+ if (virt < PAGE_OFFSET) {
+ pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
+ &phys, virt);
+ return;
+ }
+
+ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
+ NO_CONT_MAPPINGS);
+
+ /* flush the TLBs after updating live kernel mappings */
+ flush_tlb_kernel_range(virt, virt + size);
+}
+
+static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
+ phys_addr_t end, pgprot_t prot, int flags)
+{
+ #ifdef CONFIG_PTP
+ __iee_create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), end - start,
+ prot, early_pgtable_alloc, flags);
+ #else
+ __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
+ prot, early_pgtable_alloc, flags);
+ #endif
+}
+
+void __init mark_linear_text_alias_ro(void)
+{
+ /*
+ * Remove the write permissions from the linear alias of .text/.rodata
+ */
+ update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext),
+ (unsigned long)__init_begin - (unsigned long)_stext,
+ PAGE_KERNEL_RO);
+}
+
+#ifdef CONFIG_KFENCE
+
+bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL;
+
+/* early_param() will be parsed before map_mem() below. */
+static int __init parse_kfence_early_init(char *arg)
+{
+ int val;
+
+ if (get_option(&arg, &val))
+ kfence_early_init = !!val;
+
+#if IS_ENABLED(CONFIG_KFENCE_MUST_EARLY_INIT)
+ kfence_must_early_init = (val == -1) ? true : false;
+#endif
+
+ return 0;
+}
+early_param("kfence.sample_interval", parse_kfence_early_init);
+
+static phys_addr_t __init arm64_kfence_alloc_pool(void)
+{
+ phys_addr_t kfence_pool;
+
+ if (!kfence_early_init)
+ return 0;
+
+ kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
+ if (!kfence_pool) {
+ pr_err("failed to allocate kfence pool\n");
+ kfence_early_init = false;
+ return 0;
+ }
+
+ /* Temporarily mark as NOMAP. */
+ memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
+
+ return kfence_pool;
+}
+
+static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp)
+{
+ if (!kfence_pool)
+ return;
+
+ /* KFENCE pool needs page-level mapping. */
+ __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
+ pgprot_tagged(PAGE_KERNEL),
+ NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
+ memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
+ __kfence_pool = phys_to_virt(kfence_pool);
+}
+#else /* CONFIG_KFENCE */
+
+static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; }
+static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { }
+
+#endif /* CONFIG_KFENCE */
+
+static void __init map_mem(pgd_t *pgdp)
+{
+ static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
+ phys_addr_t kernel_start = __pa_symbol(_stext);
+ phys_addr_t kernel_end = __pa_symbol(__init_begin);
+ phys_addr_t start, end;
+ phys_addr_t early_kfence_pool;
+ int flags = NO_EXEC_MAPPINGS;
+ u64 i;
+
+ /*
+ * Setting hierarchical PXNTable attributes on table entries covering
+ * the linear region is only possible if it is guaranteed that no table
+ * entries at any level are being shared between the linear region and
+ * the vmalloc region. Check whether this is true for the PGD level, in
+ * which case it is guaranteed to be true for all other levels as well.
+ */
+ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
+
+ early_kfence_pool = arm64_kfence_alloc_pool();
+
+ if (can_set_direct_map())
+ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+
+ #ifdef CONFIG_IEE
+ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+ #endif
+ /*
+ * Take care not to create a writable alias for the
+ * read-only text and rodata sections of the kernel image.
+ * So temporarily mark them as NOMAP to skip mappings in
+ * the following for-loop
+ */
+ memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
+
+ /* map all the memory banks */
+ for_each_mem_range(i, &start, &end) {
+ if (start >= end)
+ break;
+ /*
+ * The linear map must allow allocation tags reading/writing
+ * if MTE is present. Otherwise, it has the same attributes as
+ * PAGE_KERNEL.
+ */
+ __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL),
+ flags);
+ }
- return kfence_pool;
+ /*
+ * Map the linear alias of the [_stext, __init_begin) interval
+ * as non-executable now, and remove the write permission in
+ * mark_linear_text_alias_ro() below (which will be called after
+ * alternative patching has completed). This makes the contents
+ * of the region accessible to subsystems such as hibernate,
+ * but protects it from inadvertent modification or execution.
+ * Note that contiguous mappings cannot be remapped in this way,
+ * so we should avoid them here.
+ */
+ #ifdef CONFIG_IEE
+ __map_memblock(pgdp, kernel_start, kernel_end,
+ PAGE_KERNEL, flags);
+ #else
+ __map_memblock(pgdp, kernel_start, kernel_end,
+ PAGE_KERNEL, NO_CONT_MAPPINGS);
+ #endif
+ memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
+ arm64_kfence_map_pool(early_kfence_pool, pgdp);
+}
+
+void mark_rodata_ro(void)
+{
+ unsigned long section_size;
+
+ /*
+ * mark .rodata as read only. Use __init_begin rather than __end_rodata
+ * to cover NOTES and EXCEPTION_TABLE.
+ */
+ section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
+ update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
+ section_size, PAGE_KERNEL_RO);
+
+ debug_checkwx();
+}
+
+static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
+ pgprot_t prot, struct vm_struct *vma,
+ int flags, unsigned long vm_flags)
+{
+ phys_addr_t pa_start = __pa_symbol(va_start);
+ unsigned long size = va_end - va_start;
+
+ BUG_ON(!PAGE_ALIGNED(pa_start));
+ BUG_ON(!PAGE_ALIGNED(size));
+
+ #ifdef CONFIG_PTP
+ __iee_create_pgd_mapping_pre_init(pgdp, pa_start, (unsigned long)va_start, size, prot,
+ early_pgtable_alloc, flags);
+ #else
+ __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
+ early_pgtable_alloc, flags);
+ #endif
+
+ if (!(vm_flags & VM_NO_GUARD))
+ size += PAGE_SIZE;
+
+ vma->addr = va_start;
+ vma->phys_addr = pa_start;
+ vma->size = size;
+ vma->flags = VM_MAP | vm_flags;
+ vma->caller = __builtin_return_address(0);
+
+ vm_area_add_early(vma);
+}
+
+static pgprot_t kernel_exec_prot(void)
+{
+ return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+}
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static int __init map_entry_trampoline(void)
+{
+ int i;
+
+ pgprot_t prot = kernel_exec_prot();
+ phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+
+ /* The trampoline is always mapped and can therefore be global */
+ pgprot_val(prot) &= ~PTE_NG;
+
+ /* Map only the text into the trampoline page table */
+ memset(tramp_pg_dir, 0, PGD_SIZE);
+ #ifdef CONFIG_PTP
+ iee_set_logical_mem_ro((unsigned long)tramp_pg_dir);
+ #endif
+ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
+ entry_tramp_text_size(), prot,
+ __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
+
+ /* Map both the text and data into the kernel page table */
+ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
+ pa_start + i * PAGE_SIZE, prot);
+
+ if (IS_ENABLED(CONFIG_RELOCATABLE))
+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
+ pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
+
+ return 0;
+}
+core_initcall(map_entry_trampoline);
+#endif
+
+/*
+ * Open coded check for BTI, only for use to determine configuration
+ * for early mappings for before the cpufeature code has run.
+ */
+static bool arm64_early_this_cpu_has_bti(void)
+{
+ u64 pfr1;
+
+ if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
+ return false;
+
+ pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1);
+ return cpuid_feature_extract_unsigned_field(pfr1,
+ ID_AA64PFR1_EL1_BT_SHIFT);
+}
+
+#ifdef CONFIG_IEE
+/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */
+static void __init iee_si_set_pmd_APtable(unsigned long addr, pgd_t *pgdir)
+{
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
+
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ #ifdef CONFIG_PTP
+ pud_t *pudp = pud_set_fixmap_offset_init(p4dp, addr);
+
+ pmd_t *pmdp = pmd_set_fixmap_offset_init(pudp, addr);
+
+ pmd_t pmd = READ_ONCE(*pmdp);
+
+ __iee_pmd_populate_pre_init(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE);
+
+ pud_clear_fixmap_init();
+ pmd_clear_fixmap_init();
+ #else
+ pud_t *pudp = pud_set_fixmap_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_set_fixmap_offset(pudp, addr);
+
+ pmd_t pmd = READ_ONCE(*pmdp);
+
+ __pmd_populate(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE);
+
+ pud_clear_fixmap();
+ pmd_clear_fixmap();
+ #endif
+}
+/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */
+static void __init mark_iee_si_pmd_APtable(pgd_t *pgdir)
+{
+ unsigned long addr = (unsigned long)__iee_si_start;
+ iee_si_set_pmd_APtable(addr, pgdir);
+ // iee rwx gate exit may be mapped by another pmd.
+ iee_si_set_pmd_APtable(addr + PAGE_SIZE, pgdir);
+}
+#endif
+
+/*
+ * Create fine-grained mappings for the kernel.
+ */
+static void __init map_kernel(pgd_t *pgdp)
+{
+ static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
+ vmlinux_initdata, vmlinux_data;
+
+ #ifdef CONFIG_IEE
+ static struct vm_struct vmlinux_iee_code, vmlinux_iee_data, vmlinux_iee_gate, vmlinux_text_end;
+ #endif
+
+ /*
+ * External debuggers may need to write directly to the text
+ * mapping to install SW breakpoints. Allow this (only) when
+ * explicitly requested with rodata=off.
+ */
+ pgprot_t text_prot = kernel_exec_prot();
+
+ /*
+ * If we have a CPU that supports BTI and a kernel built for
+ * BTI then mark the kernel executable text as guarded pages
+ * now so we don't have to rewrite the page tables later.
+ */
+ if (arm64_early_this_cpu_has_bti())
+ text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP);
+
+ /*
+ * Only rodata will be remapped with different permissions later on,
+ * all other segments are allowed to use contiguous mappings.
+ */
+ #ifdef CONFIG_IEE
+ map_kernel_segment(pgdp, _stext, __iee_si_data_start, text_prot, &vmlinux_text,
+ 0, VM_NO_GUARD);
+ /* Set iee si data RW. */
+ map_kernel_segment(pgdp, __iee_si_data_start, __iee_exec_entry_start, SET_NG(PAGE_KERNEL),
+ &vmlinux_iee_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD);
+ /* Set iee entry codes NG. */
+ map_kernel_segment(pgdp, __iee_exec_entry_start, __iee_si_start, SET_NG(text_prot), &vmlinux_iee_gate,
+ NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD);
+ /* Map __iee_si_start - __iee_si_end as U RWX pages and set PMD APTABLE = (1,1). */
+ map_kernel_segment(pgdp, __iee_si_start, __iee_si_end, SET_NG((PAGE_KERNEL_EXEC)),
+ &vmlinux_iee_code, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD);
+ mark_iee_si_pmd_APtable(pgdp);
+
+ map_kernel_segment(pgdp, __iee_si_end, _etext, text_prot, &vmlinux_text_end, 0,
+ VM_NO_GUARD);
+
+ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
+ &vmlinux_rodata, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD);
+ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
+ &vmlinux_inittext, 0, VM_NO_GUARD);
+ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
+ &vmlinux_initdata, 0, VM_NO_GUARD);
+ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, 0);
+ #else
+ map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0,
+ VM_NO_GUARD);
+ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
+ &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
+ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
+ &vmlinux_inittext, 0, VM_NO_GUARD);
+ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
+ &vmlinux_initdata, 0, VM_NO_GUARD);
+ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
+ #endif
+
+
+ fixmap_copy(pgdp);
+ kasan_copy_shadow(pgdp);
+}
+
+static void __init create_idmap(void)
+{
+ u64 start = __pa_symbol(__idmap_text_start);
+ u64 size = __pa_symbol(__idmap_text_end) - start;
+ pgd_t *pgd = idmap_pg_dir;
+ u64 pgd_phys;
+
+ /* check if we need an additional level of translation */
+ if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) {
+ pgd_phys = early_pgtable_alloc(PAGE_SHIFT);
+ set_pgd(&idmap_pg_dir[start >> VA_BITS],
+ __pgd(pgd_phys | P4D_TYPE_TABLE));
+ pgd = __va(pgd_phys);
+ }
+ #ifdef CONFIG_PTP
+ __iee_create_pgd_mapping_pre_init(pgd, start, start, size, PAGE_KERNEL_ROX,
+ early_pgtable_alloc, 0);
+ #else
+ __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX,
+ early_pgtable_alloc, 0);
+ #endif
+
+ if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
+ extern u32 __idmap_kpti_flag;
+ u64 pa = __pa_symbol(&__idmap_kpti_flag);
+
+ /*
+ * The KPTI G-to-nG conversion code needs a read-write mapping
+ * of its synchronization flag in the ID map.
+ */
+ #ifdef CONFIG_PTP
+ __iee_create_pgd_mapping_pre_init(pgd, pa, pa, sizeof(u32), PAGE_KERNEL,
+ early_pgtable_alloc, 0);
+ #else
+ __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL,
+ early_pgtable_alloc, 0);
+ #endif
+ }
+}
+
+#ifdef CONFIG_IEE
+static void __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ unsigned long addr, end, next;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt);
+ p4d_t *p4dp;
+ p4d_t p4d;
+
+ /*
+ * If the virtual and physical address don't have the same offset
+ * within a page, we cannot map the region as the caller expects.
+ */
+ if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
+ return;
+
+ phys &= PAGE_MASK;
+ addr = virt & PAGE_MASK;
+ end = PAGE_ALIGN(virt + size);
+
+ do {
+ next = pgd_addr_end(addr, end);
+ #ifdef CONFIG_PTP
+ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc,
+ flags);
+ #else
+ alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
+ flags);
+ #endif
+ p4dp = p4d_offset(pgdp, addr);
+ p4d = READ_ONCE(*p4dp);
+ #ifdef CONFIG_PTP
+ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE));
+ #else
+ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE));
+ #endif
+ phys += next - addr;
+ } while (pgdp++, addr = next, addr != end);
+}
+
+static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys,
+ unsigned long virt, phys_addr_t size,
+ pgprot_t prot,
+ phys_addr_t (*pgtable_alloc)(int),
+ int flags)
+{
+ mutex_lock(&fixmap_lock);
+ __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot,
+ pgtable_alloc, flags);
+ mutex_unlock(&fixmap_lock);
}
-static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp)
+static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start,
+ phys_addr_t end, pgprot_t prot, int flags)
{
- if (!kfence_pool)
- return;
-
- /* KFENCE pool needs page-level mapping. */
- __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
- pgprot_tagged(PAGE_KERNEL),
- NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
- memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
- __kfence_pool = phys_to_virt(kfence_pool);
+ #ifdef CONFIG_PTP
+ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start,
+ prot, early_pgtable_alloc, flags);
+ #else
+ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start,
+ prot, early_pgtable_alloc, flags);
+ #endif
}
-#else /* CONFIG_KFENCE */
-static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; }
-static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { }
-
-#endif /* CONFIG_KFENCE */
-
-static void __init map_mem(pgd_t *pgdp)
+static void __init map_iee(pgd_t *pgdp)
{
static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
phys_addr_t kernel_start = __pa_symbol(_stext);
@@ -578,6 +1578,8 @@ static void __init map_mem(pgd_t *pgdp)
int flags = NO_EXEC_MAPPINGS;
u64 i;
+ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
+
/*
* Setting hierarchical PXNTable attributes on table entries covering
* the linear region is only possible if it is guaranteed that no table
@@ -589,9 +1591,6 @@ static void __init map_mem(pgd_t *pgdp)
early_kfence_pool = arm64_kfence_alloc_pool();
- if (can_set_direct_map())
- flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
-
/*
* Take care not to create a writable alias for the
* read-only text and rodata sections of the kernel image.
@@ -609,12 +1608,11 @@ static void __init map_mem(pgd_t *pgdp)
* if MTE is present. Otherwise, it has the same attributes as
* PAGE_KERNEL.
*/
- __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL),
- flags);
+ __map_memblock_for_iee(pgdp, start, end, SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags);
}
/*
- * Map the linear alias of the [_stext, __init_begin) interval
+ * Map the linear alias of the [_text, __init_begin) interval
* as non-executable now, and remove the write permission in
* mark_linear_text_alias_ro() below (which will be called after
* alternative patching has completed). This makes the contents
@@ -623,178 +1621,384 @@ static void __init map_mem(pgd_t *pgdp)
* Note that contiguous mappings cannot be remapped in this way,
* so we should avoid them here.
*/
- __map_memblock(pgdp, kernel_start, kernel_end,
- PAGE_KERNEL, NO_CONT_MAPPINGS);
+ __map_memblock_for_iee(pgdp, kernel_start, kernel_end,
+ SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags);
memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
arm64_kfence_map_pool(early_kfence_pool, pgdp);
}
-void mark_rodata_ro(void)
+/*
+ * Change page access permission, whereas not handling huge pages.
+ * Only used on IEE init functions.
+ */
+static void __init iee_si_set_page_attr(unsigned long addr, pteval_t attr)
{
- unsigned long section_size;
+ unsigned long flag;
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
- /*
- * mark .rodata as read only. Use __init_begin rather than __end_rodata
- * to cover NOTES and EXCEPTION_TABLE.
- */
- section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
- update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
- section_size, PAGE_KERNEL_RO);
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
- debug_checkwx();
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
+
+ if(attr & PTE_RDONLY)
+ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM);
+ pte = __pte(pte_val(pte) | attr);
+ #ifdef CONFIG_PTP
+ // Write pgtable in IEE directly.
+ flag = local_daif_save();
+ asm volatile ("msr pan, #0");
+ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte);
+ asm volatile ("msr pan, #1");
+ local_daif_restore(flag);
+ #else
+ WRITE_ONCE(*ptep, pte);
+ #endif
+}
+
+/* Prepare data used for iee rwx gates. These data are setted only once. */
+void __init iee_si_prepare_data(void)
+{
+ unsigned long va;
+ // Record current TCR val after system init.
+ iee_si_tcr = read_sysreg(tcr_el1) & ~(SYS_TCR_IEE_SI);
+ // Mark iee data as RO and move it to iee after setting up.
+ va = (unsigned long)__iee_si_data_start;
+ iee_si_set_page_attr(va, PTE_RDONLY);
+ iee_si_set_page_attr(lm_alias(va)+iee_offset, 0x1 | PTE_RDONLY);
+ // Set iee sensitive inst code page U RWX here to hide it from kernel.
+ va = (unsigned long)__iee_si_start;
+ iee_si_set_page_attr(va, PTE_USER);
+ va = (unsigned long)__iee_si_start + PAGE_SIZE;
+ iee_si_set_page_attr(va, PTE_USER);
+ flush_tlb_all();
}
-static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
- pgprot_t prot, struct vm_struct *vma,
- int flags, unsigned long vm_flags)
+#endif
+
+#ifdef CONFIG_PTP
+// Attention : Using set_xxx without adding offset.
+static void __init set_iee_valid_pre_init(unsigned long addr)
{
- phys_addr_t pa_start = __pa_symbol(va_start);
- unsigned long size = va_end - va_start;
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
- BUG_ON(!PAGE_ALIGNED(pa_start));
- BUG_ON(!PAGE_ALIGNED(size));
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
- __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
- early_pgtable_alloc, flags);
+ pud_t *pudp = pud_offset(p4dp, addr);
- if (!(vm_flags & VM_NO_GUARD))
- size += PAGE_SIZE;
+ pmd_t *pmdp = pmd_offset(pudp, addr);
- vma->addr = va_start;
- vma->phys_addr = pa_start;
- vma->size = size;
- vma->flags = VM_MAP | vm_flags;
- vma->caller = __builtin_return_address(0);
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+ pte_t pte = READ_ONCE(*ptep);
- vm_area_add_early(vma);
+ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1))))
+ return;
+
+ pte = __pte(pte_val(pte) | 0x1);
+ iee_set_pte_pre_init(ptep, pte);
+ flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
+ isb();
}
-static pgprot_t kernel_exec_prot(void)
+static void __init move_pte_table_into_iee(pmd_t *pmdp, unsigned long addr, unsigned long end)
{
- return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+ pmd_t pmd = READ_ONCE(*pmdp);
+ unsigned long iee_addr = __phys_to_iee(__pmd_to_phys(pmd));
+ set_iee_valid_pre_init(iee_addr);
}
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-static int __init map_entry_trampoline(void)
+static void __init move_pmd_table_into_iee(pud_t *pudp, unsigned long addr, unsigned long end)
{
- int i;
+ unsigned long next;
+ pud_t pud = READ_ONCE(*pudp);
+ pmd_t *pmdp;
+ pmd_t pmd;
+ unsigned long iee_addr = __phys_to_iee(__pud_to_phys(pud));
+ set_iee_valid_pre_init(iee_addr);
- pgprot_t prot = kernel_exec_prot();
- phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+ pmdp = pmd_offset(pudp, addr);
+ do {
+ next = pmd_addr_end(addr, end);
+ pmd = READ_ONCE(*pmdp);
+ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0)
+ {
+ continue;
+ }
+ else
+ {
+ move_pte_table_into_iee(pmdp, addr, next);
+ }
+ } while (pmdp++, addr = next, addr != end);
+}
- /* The trampoline is always mapped and can therefore be global */
- pgprot_val(prot) &= ~PTE_NG;
+static void __init move_pud_table_into_iee(pgd_t *pgdp, unsigned long addr, unsigned long end)
+{
+ unsigned long next;
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
+ pud_t *pudp;
+ pud_t pud;
+ unsigned long iee_addr = __phys_to_iee(__p4d_to_phys(p4d));
+ set_iee_valid_pre_init(iee_addr);
- /* Map only the text into the trampoline page table */
- memset(tramp_pg_dir, 0, PGD_SIZE);
- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
- entry_tramp_text_size(), prot,
- __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
+ pudp = pud_offset(p4dp, addr);
+ do {
+ next = pud_addr_end(addr, end);
+ pud = READ_ONCE(*pudp);
+ if ((pud_val(pud) & PUD_TABLE_BIT) == 0)
+ {
+ continue;
+ }
+ else
+ {
+ move_pmd_table_into_iee(pudp, addr, next);
+ }
+ } while (pudp++, addr = next, addr != end);
+}
- /* Map both the text and data into the kernel page table */
- for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
- pa_start + i * PAGE_SIZE, prot);
+static void __init init_iee_for_one_region(pgd_t *pgdir, unsigned long va_start, unsigned long va_end)
+{
+ unsigned long addr, end, next;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start);
- if (IS_ENABLED(CONFIG_RELOCATABLE))
- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
- pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
+ addr = va_start & PAGE_MASK;
+ end = PAGE_ALIGN(va_end);
- return 0;
+ do {
+ next = pgd_addr_end(addr, end);
+ move_pud_table_into_iee(pgdp, addr, next);
+ } while (pgdp++, addr = next, addr != end);
}
-core_initcall(map_entry_trampoline);
-#endif
-/*
- * Open coded check for BTI, only for use to determine configuration
- * for early mappings for before the cpufeature code has run.
- */
-static bool arm64_early_this_cpu_has_bti(void)
+static void __init init_iee(void)
{
- u64 pfr1;
+ unsigned long iee_addr;
+ phys_addr_t start, end;
+ u64 i;
+ pgd_t *pgdp;
+
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ // handling 1-level tramp page table tramp_pg_dir
+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(tramp_pg_dir));
+ set_iee_valid_pre_init(iee_addr);
+ #endif
+ // handling 1-level page table swapper_pg_dir
+ pgdp = swapper_pg_dir;
+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(swapper_pg_dir));
+ set_iee_valid_pre_init(iee_addr);
+ // handling 2/3/4-level page table for kernel
+ init_iee_for_one_region(pgdp, (unsigned long)_text, (unsigned long)_etext);
+ init_iee_for_one_region(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin);
+ init_iee_for_one_region(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end);
+ init_iee_for_one_region(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end);
+ init_iee_for_one_region(pgdp, (unsigned long)_data, (unsigned long)_end);
+ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx
+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pte_addr));
+ set_iee_valid_pre_init(iee_addr);
+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pmd_addr));
+ set_iee_valid_pre_init(iee_addr);
+ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pud_addr));
+ set_iee_valid_pre_init(iee_addr);
+ // handling 2/3/4-level page table for logical mem and iee
+ for_each_mem_range(i, &start, &end) {
+ if (start >= end)
+ break;
+ /*
+ * The linear map must allow allocation tags reading/writing
+ * if MTE is present. Otherwise, it has the same attributes as
+ * PAGE_KERNEL.
+ */
+ init_iee_for_one_region(pgdp, (unsigned long)__va(start), (unsigned long)__va(end));
+ init_iee_for_one_region(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end));
+ }
+}
- if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
- return false;
+static void iee_set_kernel_upage_pre_init(unsigned long addr)
+{
+ pgd_t *pgdir = swapper_pg_dir;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr);
- pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1);
- return cpuid_feature_extract_unsigned_field(pfr1,
- ID_AA64PFR1_EL1_BT_SHIFT);
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
+
+ pud_t *pudp = pud_offset(p4dp, addr);
+
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+
+ int i;
+ for(i = 0; i < 4; i++)
+ {
+ pte_t pte = READ_ONCE(*ptep);
+ pte = __pte(pte_val(pte) | PTE_USER | PTE_NG);
+ iee_set_pte_pre_init(ptep, pte);
+ ptep++;
+ }
+ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE);
+ isb();
}
-/*
- * Create fine-grained mappings for the kernel.
- */
-static void __init map_kernel(pgd_t *pgdp)
+static void __init iee_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end)
{
- static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
- vmlinux_initdata, vmlinux_data;
+ pmd_t pmd = READ_ONCE(*pmdp);
+ unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd));
+ iee_set_logical_mem_ro(logical_addr);
+}
- /*
- * External debuggers may need to write directly to the text
- * mapping to install SW breakpoints. Allow this (only) when
- * explicitly requested with rodata=off.
- */
- pgprot_t text_prot = kernel_exec_prot();
+static void __init iee_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end)
+{
+ unsigned long next;
+ pud_t pud = READ_ONCE(*pudp);
+ pmd_t *pmdp;
+ pmd_t pmd;
+ unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud));
+ iee_set_logical_mem_ro(logical_addr);
- /*
- * If we have a CPU that supports BTI and a kernel built for
- * BTI then mark the kernel executable text as guarded pages
- * now so we don't have to rewrite the page tables later.
- */
- if (arm64_early_this_cpu_has_bti())
- text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP);
+ pmdp = pmd_offset(pudp, addr);
+ do {
+ next = pmd_addr_end(addr, end);
+ pmd = READ_ONCE(*pmdp);
+ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0)
+ {
+ continue;
+ }
+ else
+ {
+ iee_set_pte_table_ro(pmdp, addr, next);
+ }
+ } while (pmdp++, addr = next, addr != end);
+}
- /*
- * Only rodata will be remapped with different permissions later on,
- * all other segments are allowed to use contiguous mappings.
- */
- map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0,
- VM_NO_GUARD);
- map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
- &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
- map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
- &vmlinux_inittext, 0, VM_NO_GUARD);
- map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
- &vmlinux_initdata, 0, VM_NO_GUARD);
- map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
+static void __init iee_set_pud_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end)
+{
+ unsigned long next;
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
+ pud_t *pudp;
+ pud_t pud;
+ unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d));
+ iee_set_logical_mem_ro(logical_addr);
- fixmap_copy(pgdp);
- kasan_copy_shadow(pgdp);
+ pudp = pud_offset(p4dp, addr);
+ do {
+ next = pud_addr_end(addr, end);
+ pud = READ_ONCE(*pudp);
+ if ((pud_val(pud) & PUD_TABLE_BIT) == 0)
+ {
+ continue;
+ }
+ else
+ {
+ iee_set_pmd_table_ro(pudp, addr, next);
+ }
+ } while (pudp++, addr = next, addr != end);
}
-static void __init create_idmap(void)
+static void __init iee_mark_pgtable_for_one_region_ro(pgd_t *pgdir, unsigned long va_start, unsigned long va_end)
{
- u64 start = __pa_symbol(__idmap_text_start);
- u64 size = __pa_symbol(__idmap_text_end) - start;
- pgd_t *pgd = idmap_pg_dir;
- u64 pgd_phys;
+ unsigned long addr, end, next;
+ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start);
- /* check if we need an additional level of translation */
- if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) {
- pgd_phys = early_pgtable_alloc(PAGE_SHIFT);
- set_pgd(&idmap_pg_dir[start >> VA_BITS],
- __pgd(pgd_phys | P4D_TYPE_TABLE));
- pgd = __va(pgd_phys);
- }
- __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX,
- early_pgtable_alloc, 0);
+ addr = va_start & PAGE_MASK;
+ end = PAGE_ALIGN(va_end);
- if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
- extern u32 __idmap_kpti_flag;
- u64 pa = __pa_symbol(&__idmap_kpti_flag);
+ do {
+ next = pgd_addr_end(addr, end);
+ iee_set_pud_table_ro(pgdp, addr, next);
+ } while (pgdp++, addr = next, addr != end);
+}
+static void __init iee_mark_all_lm_pgtable_ro(void)
+{
+ unsigned long logical_addr;
+ phys_addr_t start, end;
+ u64 i;
+ pgd_t *pgdp;
+
+ // handling static allocated page table
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ // handling 1-level tramp page table tramp_pg_dir
+ logical_addr = (unsigned long)__va(__pa_symbol(tramp_pg_dir));
+ iee_set_logical_mem_ro(logical_addr);
+ #endif
+ // handling 1-level page table swapper_pg_dir
+ pgdp = swapper_pg_dir;
+ iee_set_logical_mem_ro((unsigned long)swapper_pg_dir);
+ logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir));
+ iee_set_logical_mem_ro(logical_addr);
+
+ // handling 2/3/4-level page table for kernel
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_text, (unsigned long)_etext);
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin);
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end);
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end);
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_data, (unsigned long)_end);
+
+ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx
+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pte_addr));
+ iee_set_logical_mem_ro(logical_addr);
+
+ iee_set_logical_mem_ro((unsigned long)bm_pmd_addr);
+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pmd_addr));
+ iee_set_logical_mem_ro(logical_addr);
+
+ iee_set_logical_mem_ro((unsigned long)bm_pud_addr);
+ logical_addr = (unsigned long)__va(__pa_symbol(bm_pud_addr));
+ iee_set_logical_mem_ro(logical_addr);
+
+ // handling 2/3/4-level page table for logical mem and iee
+ for_each_mem_range(i, &start, &end) {
+ if (start >= end)
+ break;
/*
- * The KPTI G-to-nG conversion code needs a read-write mapping
- * of its synchronization flag in the ID map.
+ * The linear map must allow allocation tags reading/writing
+ * if MTE is present. Otherwise, it has the same attributes as
+ * PAGE_KERNEL.
*/
- __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL,
- early_pgtable_alloc, 0);
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__va(start), (unsigned long)__va(end));
+ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end));
}
}
+#endif
+
+#ifdef CONFIG_KOI
+extern s64 koi_offset;
+#endif
void __init paging_init(void)
{
+ #ifdef CONFIG_IEE
+ unsigned long SP_EL0;
+ void *new;
+ void *init_token;
+ struct task_token *token;
+ unsigned long tcr;
+
+ // Check if cpu has PAN and HPDS.
+ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1),
+ ID_AA64MMFR1_EL1_PAN_SHIFT))
+ panic("Architecture doesn't support PAN, please disable CONFIG_IEE.\n");
+
+ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1),
+ ID_AA64MMFR1_EL1_HPDS_SHIFT))
+ panic("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n");
+ #endif
+
+ // Avoid using iee code to modify pgtable before iee initialized.
+ #ifdef CONFIG_PTP
+ pgd_t *pgdp = pgd_set_fixmap_init(__pa_symbol(swapper_pg_dir));
+ #else
pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir));
+ #endif
+
+
extern pgd_t init_idmap_pg_dir[];
idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0));
@@ -802,7 +2006,17 @@ void __init paging_init(void)
map_kernel(pgdp);
map_mem(pgdp);
+ // Map the whole physical mem into IEE, but set invalid.
+ #ifdef CONFIG_IEE
+ map_iee(pgdp);
+ #endif
+
+ // Avoid using iee code to modify pgtable before iee initialized.
+ #ifdef CONFIG_PTP
+ pgd_clear_fixmap_init();
+ #else
pgd_clear_fixmap();
+ #endif
cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir);
init_mm.pgd = swapper_pg_dir;
@@ -813,6 +2027,80 @@ void __init paging_init(void)
memblock_allow_resize();
create_idmap();
+
+ #ifdef CONFIG_IEE
+ // test iee_exec_entry
+ iee_rwx_gate_entry(IEE_SI_TEST);
+ // Initialize init iee stack.
+ #ifdef CONFIG_PTP
+ iee_set_kernel_upage_pre_init((unsigned long)init_iee_stack_begin);
+ iee_set_kernel_upage_pre_init((unsigned long)__va(__pa_symbol(init_iee_stack_begin)));
+ #else
+ iee_set_kernel_upage((unsigned long)init_iee_stack_begin);
+ iee_set_kernel_upage((unsigned long)__va(__pa_symbol(init_iee_stack_begin)));
+ #endif
+ #endif
+
+ // Init token for init_task.
+ #ifdef CONFIG_IEE
+ // Change SP_EL0 from Image VA to Logical VA.
+ SP_EL0 = (unsigned long)__va(__pa_symbol(&init_task));
+ write_sysreg(SP_EL0, sp_el0);
+ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask);
+ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev)));
+ init_task.children.next = (__va(__pa_symbol(init_task.children.next)));
+ // Set init_task into __entry_task before per_cpu init.
+ *(struct task_struct **)__entry_task = __va(__pa_symbol(&init_task));
+ // Alloc a page for init_token.
+ new = __va(early_pgtable_alloc(0));
+ init_token = (void *)__phys_to_iee(__pa_symbol(&init_task));
+ #ifdef CONFIG_PTP
+ iee_set_token_page_valid_pre_init(init_token, new);
+ #else
+ iee_set_token_page_valid(init_token, new);
+ #endif
+ // Use lm to write token before IEE initialized.
+ token = (struct task_token *)((unsigned long)new + (((unsigned long)&init_task) & ~PAGE_MASK));
+ token->mm = &init_mm;
+ token->pgd = NULL;
+ token->iee_stack = (void *)init_iee_stack_end;
+ token->valid = true;
+ #endif
+
+ #ifdef CONFIG_PTP
+ // Map the existing pgtable into IEE, set valid.
+ init_iee();
+ #endif
+
+ #ifdef CONFIG_IEE
+ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SPAN);
+ #endif
+
+ #ifdef CONFIG_PTP
+ // IEE ready.
+ // Pgtable writing before uses logical memory and after uses IEE memory.
+
+ // Set the logical va of existing pgtable readonly.
+ iee_mark_all_lm_pgtable_ro();
+ #endif
+
+ // Set the init token readonly.
+ #ifdef CONFIG_IEE
+ set_iee_page_valid(__phys_to_iee(__pa(new)));
+ iee_set_logical_mem_ro((unsigned long)new);
+
+ // Set HPD1 as 1.
+ tcr = read_sysreg(tcr_el1);
+ tcr |= ((unsigned long)0x1 << 42);
+ write_sysreg(tcr, tcr_el1);
+ isb();
+
+ // Flush tlb to enable IEE.
+ flush_tlb_all();
+
+ // mark that iee is prepared.
+ iee_init_done = true;
+ #endif
}
#ifdef CONFIG_MEMORY_HOTPLUG
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 4a64089e5771..894bda11c389 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -15,14 +15,44 @@
#include <asm/page.h>
#include <asm/tlbflush.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
static struct kmem_cache *pgd_cache __ro_after_init;
+#ifdef CONFIG_KOI
+pgd_t *koi_pgd_alloc(void)
+{
+ pgd_t *pgd;
+#ifdef CONFIG_PTP
+ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL);
+ unsigned long iee_addr = __phys_to_iee(__pa(pgd));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)pgd);
+#else
+ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL);
+#endif
+ return pgd;
+}
+#endif
+
pgd_t *pgd_alloc(struct mm_struct *mm)
{
gfp_t gfp = GFP_PGTABLE_USER;
if (PGD_SIZE == PAGE_SIZE)
+#ifdef CONFIG_PTP
+ {
+ pgd_t* new = (pgd_t *)__get_free_page(gfp);
+ unsigned long iee_addr = __phys_to_iee(__pa(new));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)new);
+ return new;
+ }
+#else
return (pgd_t *)__get_free_page(gfp);
+#endif
else
return kmem_cache_alloc(pgd_cache, gfp);
}
@@ -30,7 +60,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
if (PGD_SIZE == PAGE_SIZE)
+#ifdef CONFIG_PTP
+ {
+ unsigned long iee_addr = __phys_to_iee(__pa(pgd));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)pgd);
+ free_page((unsigned long)pgd);
+ }
+#else
free_page((unsigned long)pgd);
+#endif
else
kmem_cache_free(pgd_cache, pgd);
}
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 14fdf645edc8..230b2b883a51 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -22,6 +22,8 @@
#include <asm/smp.h>
#include <asm/sysreg.h>
+
+
#ifdef CONFIG_ARM64_64K_PAGES
#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
#elif defined(CONFIG_ARM64_16K_PAGES)
@@ -105,6 +107,19 @@ SYM_FUNC_START(cpu_do_suspend)
ret
SYM_FUNC_END(cpu_do_suspend)
+
+#ifdef CONFIG_IEE
+// SP_EL0 check failed.
+SYM_FUNC_START_LOCAL(sp_el0_check_failed)
+1:
+ nop
+ nop
+ nop
+ nop
+ b 1f
+SYM_FUNC_END(sp_el0_check_failed)
+#endif
+
/**
* cpu_do_resume - restore CPU register context
*
@@ -148,6 +163,13 @@ SYM_FUNC_START(cpu_do_resume)
msr sctlr_el1, x12
set_this_cpu_offset x13
msr sp_el0, x14
+#ifdef CONFIG_IEE
+ // tsk check.
+ ldr_this_cpu x2, __entry_task, x3
+ mrs x3, sp_el0
+ cmp x2, x3
+ b.ne sp_el0_check_failed
+#endif
/*
* Restore oslsr_el1 by writing oslar_el1
*/
@@ -190,6 +212,7 @@ SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
__idmap_cpu_set_reserved_ttbr1 x1, x3
offset_ttbr1 x0, x3
+
msr ttbr1_el1, x0
isb
@@ -452,6 +475,11 @@ SYM_FUNC_START(__cpu_setup)
orr tcr, tcr, #TCR_HA // hardware Access flag update
1:
#endif /* CONFIG_ARM64_HW_AFDBM */
+
+#ifdef CONFIG_IEE
+ orr tcr, tcr, #TCR_HPD1 // Hierarchical permission disables
+#endif
+
msr mair_el1, mair
msr tcr_el1, tcr
diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c
index 5139a28130c0..15d2a3faa048 100644
--- a/arch/arm64/mm/trans_pgd.c
+++ b/arch/arm64/mm/trans_pgd.c
@@ -25,6 +25,9 @@
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/kfence.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
static void *trans_alloc(struct trans_pgd_info *info)
{
@@ -65,10 +68,18 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp,
pte_t *src_ptep;
pte_t *dst_ptep;
unsigned long addr = start;
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
dst_ptep = trans_alloc(info);
if (!dst_ptep)
return -ENOMEM;
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(dst_ptep));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)dst_ptep);
+ #endif
pmd_populate_kernel(NULL, dst_pmdp, dst_ptep);
dst_ptep = pte_offset_kernel(dst_pmdp, start);
@@ -87,11 +98,19 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp,
pmd_t *dst_pmdp;
unsigned long next;
unsigned long addr = start;
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
if (pud_none(READ_ONCE(*dst_pudp))) {
dst_pmdp = trans_alloc(info);
if (!dst_pmdp)
return -ENOMEM;
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(dst_pmdp));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)dst_pmdp);
+ #endif
pud_populate(NULL, dst_pudp, dst_pmdp);
}
dst_pmdp = pmd_offset(dst_pudp, start);
@@ -123,11 +142,19 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp,
pud_t *src_pudp;
unsigned long next;
unsigned long addr = start;
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
if (p4d_none(READ_ONCE(*dst_p4dp))) {
dst_pudp = trans_alloc(info);
if (!dst_pudp)
return -ENOMEM;
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(dst_pudp));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)dst_pudp);
+ #endif
p4d_populate(NULL, dst_p4dp, dst_pudp);
}
dst_pudp = pud_offset(dst_p4dp, start);
@@ -212,6 +239,12 @@ int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp,
return -ENOMEM;
}
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(trans_pgd));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)trans_pgd);
+ #endif
+
rc = copy_page_tables(info, trans_pgd, start, end);
if (!rc)
*dst_pgdp = trans_pgd;
@@ -238,6 +271,9 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0,
int bits_mapped = PAGE_SHIFT - 4;
unsigned long level_mask, prev_level_entry, *levels[4];
int this_level, index, level_lsb, level_msb;
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
dst_addr &= PAGE_MASK;
prev_level_entry = pte_val(pfn_pte(pfn, PAGE_KERNEL_ROX));
@@ -247,12 +283,22 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0,
if (!levels[this_level])
return -ENOMEM;
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(levels[this_level]));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)levels[this_level]);
+ #endif
+
level_lsb = ARM64_HW_PGTABLE_LEVEL_SHIFT(this_level);
level_msb = min(level_lsb + bits_mapped, max_msb);
level_mask = GENMASK_ULL(level_msb, level_lsb);
index = (dst_addr & level_mask) >> level_lsb;
+ #ifdef CONFIG_PTP
+ set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry));
+ #else
*(levels[this_level] + index) = prev_level_entry;
+ #endif
pfn = virt_to_pfn(levels[this_level]);
prev_level_entry = pte_val(pfn_pte(pfn,
diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c
index 83092d93f36a..fb12e7d0660a 100644
--- a/drivers/firmware/efi/arm-runtime.c
+++ b/drivers/firmware/efi/arm-runtime.c
@@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void)
return 0;
}
+ #ifdef CONFIG_PTP
+ efi_memmap_unmap_after_init();
+ #else
efi_memmap_unmap();
+ #endif
mapsize = efi.memmap.desc_size * efi.memmap.nr_map;
diff --git a/drivers/firmware/efi/memmap.c b/drivers/firmware/efi/memmap.c
index a1180461a445..4c64b6f15717 100644
--- a/drivers/firmware/efi/memmap.c
+++ b/drivers/firmware/efi/memmap.c
@@ -105,6 +105,26 @@ void __init efi_memmap_unmap(void)
clear_bit(EFI_MEMMAP, &efi.flags);
}
+#ifdef CONFIG_PTP
+void __init efi_memmap_unmap_after_init(void)
+{
+ if (!efi_enabled(EFI_MEMMAP))
+ return;
+
+ if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) {
+ unsigned long size;
+
+ size = efi.memmap.desc_size * efi.memmap.nr_map;
+ early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size);
+ } else {
+ memunmap(efi.memmap.map);
+ }
+
+ efi.memmap.map = NULL;
+ clear_bit(EFI_MEMMAP, &efi.flags);
+}
+#endif
+
/**
* efi_memmap_init_late - Map efi.memmap with memremap()
* @phys_addr: Physical address of the new EFI memory map
diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c
index a5fbb6ed38ae..81428783b9da 100644
--- a/drivers/tty/serial/earlycon.c
+++ b/drivers/tty/serial/earlycon.c
@@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size)
{
void __iomem *base;
#ifdef CONFIG_FIX_EARLYCON_MEM
+ #ifdef CONFIG_PTP
+ __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO);
+ #else
set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK);
+ #endif
base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE);
base += paddr & ~PAGE_MASK;
#else
diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c
index 45b42d8f6453..b71072d6957e 100644
--- a/drivers/usb/early/ehci-dbgp.c
+++ b/drivers/usb/early/ehci-dbgp.c
@@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s)
* FIXME I don't have the bar size so just guess PAGE_SIZE is more
* than enough. 1K is the biggest I have seen.
*/
+ #ifdef CONFIG_PTP
+ __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE);
+ #else
set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK);
+ #endif
ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE);
ehci_bar += bar_val & ~PAGE_MASK;
dbgp_printk("ehci_bar: %p\n", ehci_bar);
diff --git a/fs/coredump.c b/fs/coredump.c
index 9d235fa14ab9..72be355903ca 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -53,6 +53,10 @@
#include <trace/events/sched.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
static bool dump_vma_snapshot(struct coredump_params *cprm);
static void free_vma_snapshot(struct coredump_params *cprm);
@@ -564,7 +568,11 @@ void do_coredump(const kernel_siginfo_t *siginfo)
*/
if (__get_dumpable(cprm.mm_flags) == SUID_DUMP_ROOT) {
/* Setuid core dump mode */
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(cred,GLOBAL_ROOT_UID);
+ #else
cred->fsuid = GLOBAL_ROOT_UID; /* Dump root private */
+ #endif
need_suid_safe = true;
}
diff --git a/fs/exec.c b/fs/exec.c
index 04fb89656cc3..3689c5f008ba 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -76,6 +76,14 @@
#include <trace/events/sched.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
+#ifdef CONFIG_IEE
+#include <asm/iee-token.h>
+#endif
+
static int bprm_creds_from_file(struct linux_binprm *bprm);
int suid_dumpable = 0;
@@ -1005,6 +1013,10 @@ static int exec_mmap(struct mm_struct *mm)
if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
local_irq_enable();
activate_mm(active_mm, mm);
+ #ifdef CONFIG_IEE
+ iee_set_token_mm(tsk, mm);
+ iee_set_token_pgd(tsk, mm->pgd);
+ #endif
if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
local_irq_enable();
lru_gen_add_mm(mm);
@@ -1618,12 +1630,20 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file)
if (mode & S_ISUID) {
bprm->per_clear |= PER_CLEAR_ON_SETID;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_euid(bprm->cred, vfsuid_into_kuid(vfsuid));
+ #else
bprm->cred->euid = vfsuid_into_kuid(vfsuid);
+ #endif
}
if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) {
bprm->per_clear |= PER_CLEAR_ON_SETID;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_egid(bprm->cred, vfsgid_into_kgid(vfsgid));
+ #else
bprm->cred->egid = vfsgid_into_kgid(vfsgid);
+ #endif
}
}
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 3e724cb7ef01..e32e136e4271 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -15,6 +15,10 @@
#include <linux/sunrpc/metrics.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
#include "flexfilelayout.h"
#include "../nfs4session.h"
#include "../nfs4idmap.h"
@@ -502,8 +506,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
rc = -ENOMEM;
if (!kcred)
goto out_err_free;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(kcred,uid);
+ iee_set_cred_fsgid(kcred,gid);
+ #else
kcred->fsuid = uid;
kcred->fsgid = gid;
+ #endif
cred = RCU_INITIALIZER(kcred);
if (lgr->range.iomode == IOMODE_READ)
diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c
index 25a7c771cfd8..b15ab8e33e0e 100644
--- a/fs/nfs/nfs4idmap.c
+++ b/fs/nfs/nfs4idmap.c
@@ -48,6 +48,10 @@
#include <linux/module.h>
#include <linux/user_namespace.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
#include "internal.h"
#include "netns.h"
#include "nfs4idmap.h"
@@ -226,8 +230,13 @@ int nfs_idmap_init(void)
goto failed_reg_legacy;
set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(cred,keyring);
+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING);
+ #else
cred->thread_keyring = keyring;
cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
+ #endif
id_resolver_cache = cred;
return 0;
diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c
index e6beaaf4f170..e89385fd81f1 100644
--- a/fs/nfsd/auth.c
+++ b/fs/nfsd/auth.c
@@ -2,6 +2,9 @@
/* Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> */
#include <linux/sched.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
#include "nfsd.h"
#include "auth.h"
@@ -32,22 +35,40 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp)
if (!new)
return -ENOMEM;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,rqstp->rq_cred.cr_uid);
+ iee_set_cred_fsgid(new,rqstp->rq_cred.cr_gid);
+ #else
new->fsuid = rqstp->rq_cred.cr_uid;
new->fsgid = rqstp->rq_cred.cr_gid;
+ #endif
rqgi = rqstp->rq_cred.cr_group_info;
if (flags & NFSEXP_ALLSQUASH) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,exp->ex_anon_uid);
+ iee_set_cred_fsgid(new,exp->ex_anon_gid);
+ #else
new->fsuid = exp->ex_anon_uid;
new->fsgid = exp->ex_anon_gid;
+ #endif
gi = groups_alloc(0);
if (!gi)
goto oom;
} else if (flags & NFSEXP_ROOTSQUASH) {
if (uid_eq(new->fsuid, GLOBAL_ROOT_UID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,exp->ex_anon_uid);
+ #else
new->fsuid = exp->ex_anon_uid;
+ #endif
if (gid_eq(new->fsgid, GLOBAL_ROOT_GID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsgid(new,exp->ex_anon_gid);
+ #else
new->fsgid = exp->ex_anon_gid;
+ #endif
gi = groups_alloc(rqgi->ngroups);
if (!gi)
@@ -67,18 +88,35 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp)
}
if (uid_eq(new->fsuid, INVALID_UID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,exp->ex_anon_uid);
+ #else
new->fsuid = exp->ex_anon_uid;
+ #endif
if (gid_eq(new->fsgid, INVALID_GID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsgid(new,exp->ex_anon_gid);
+ #else
new->fsgid = exp->ex_anon_gid;
+ #endif
set_groups(new, gi);
put_group_info(gi);
if (!uid_eq(new->fsuid, GLOBAL_ROOT_UID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,cap_drop_nfsd_set(new->cap_effective));
+ #else
new->cap_effective = cap_drop_nfsd_set(new->cap_effective);
+ #endif
else
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective,
+ new->cap_permitted));
+ #else
new->cap_effective = cap_raise_nfsd_set(new->cap_effective,
new->cap_permitted);
+ #endif
put_cred(override_creds(new));
put_cred(new);
return 0;
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 4039ffcf90ba..6e0dfa01e01e 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -41,6 +41,9 @@
#include "trace.h"
#include "xdr4cb.h"
#include "xdr4.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
#define NFSDDBG_FACILITY NFSDDBG_PROC
@@ -946,8 +949,13 @@ static const struct cred *get_backchannel_cred(struct nfs4_client *clp, struct r
if (!kcred)
return NULL;
- kcred->fsuid = ses->se_cb_sec.uid;
- kcred->fsgid = ses->se_cb_sec.gid;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_uid(kcred,ses->se_cb_sec.uid);
+ iee_set_cred_gid(kcred,ses->se_cb_sec.gid);
+ #else
+ kcred->uid = ses->se_cb_sec.uid;
+ kcred->gid = ses->se_cb_sec.gid;
+ #endif
return kcred;
}
}
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index 3509e73abe1f..798fe0352841 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -44,6 +44,10 @@
#include <linux/sunrpc/clnt.h>
#include <linux/nfsd/cld.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
#include "nfsd.h"
#include "state.h"
#include "vfs.h"
@@ -78,8 +82,13 @@ nfs4_save_creds(const struct cred **original_creds)
if (!new)
return -ENOMEM;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,GLOBAL_ROOT_UID);
+ iee_set_cred_fsgid(new,GLOBAL_ROOT_GID);
+ #else
new->fsuid = GLOBAL_ROOT_UID;
new->fsgid = GLOBAL_ROOT_GID;
+ #endif
*original_creds = override_creds(new);
put_cred(new);
return 0;
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index 355bf0db3235..6cb276dde4e9 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -16,6 +16,10 @@
#include "auth.h"
#include "trace.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
#define NFSDDBG_FACILITY NFSDDBG_FH
@@ -223,9 +227,14 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp)
error = nfserrno(-ENOMEM);
goto out;
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective,
+ new->cap_permitted));
+ #else
new->cap_effective =
cap_raise_nfsd_set(new->cap_effective,
new->cap_permitted);
+ #endif
put_cred(override_creds(new));
put_cred(new);
} else {
diff --git a/fs/open.c b/fs/open.c
index 54723fceb776..d83901dc50ff 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -35,6 +35,11 @@
#include <linux/mnt_idmapping.h>
#include <linux/filelock.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
+
#include "internal.h"
int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry,
@@ -414,17 +419,34 @@ static const struct cred *access_override_creds(void)
* routine.
*/
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(override_cred,override_cred->uid);
+ iee_set_cred_fsgid(override_cred,override_cred->gid);
+ #else
override_cred->fsuid = override_cred->uid;
override_cred->fsgid = override_cred->gid;
+ #endif
if (!issecure(SECURE_NO_SETUID_FIXUP)) {
/* Clear the capabilities if we switch to a non-root user */
kuid_t root_uid = make_kuid(override_cred->user_ns, 0);
if (!uid_eq(override_cred->uid, root_uid))
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = override_cred->cap_effective;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_effective(override_cred, tmp_cap);
+ } while (0);
+ #else
cap_clear(override_cred->cap_effective);
+ #endif
else
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(override_cred,override_cred->cap_permitted);
+ #else
override_cred->cap_effective =
override_cred->cap_permitted;
+ #endif
}
/*
@@ -444,7 +466,11 @@ static const struct cred *access_override_creds(void)
* expecting RCU freeing. But normal thread-synchronous
* cred accesses will keep things non-RCY.
*/
+ #ifdef CONFIG_CREDP
+ iee_set_cred_non_rcu(override_cred,1);
+ #else
override_cred->non_rcu = 1;
+ #endif
old_cred = override_creds(override_cred);
diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
index 033fc0458a3d..2afa31ead2b5 100644
--- a/fs/overlayfs/dir.c
+++ b/fs/overlayfs/dir.c
@@ -16,6 +16,10 @@
#include <linux/ratelimit.h>
#include "overlayfs.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
static unsigned short ovl_redirect_max = 256;
module_param_named(redirect_max, ovl_redirect_max, ushort, 0644);
MODULE_PARM_DESC(redirect_max,
@@ -593,8 +597,13 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
* create a new inode, so just use the ovl mounter's
* fs{u,g}id.
*/
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(override_cred,inode->i_uid);
+ iee_set_cred_fsgid(override_cred,inode->i_gid);
+ #else
override_cred->fsuid = inode->i_uid;
override_cred->fsgid = inode->i_gid;
+ #endif
err = security_dentry_create_files_as(dentry,
attr->mode, &dentry->d_name, old_cred,
override_cred);
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index 2c056d737c27..9ede99ddb04b 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -21,6 +21,10 @@
#include "overlayfs.h"
#include "params.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
MODULE_AUTHOR("Miklos Szeredi <miklos@szeredi.hu>");
MODULE_DESCRIPTION("Overlay filesystem");
MODULE_LICENSE("GPL");
@@ -1485,7 +1489,15 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc)
sb->s_export_op = &ovl_export_fid_operations;
/* Never override disk quota limits or use reserved space */
+ #ifdef CONFIG_CREDP
+ {
+ kernel_cap_t tmp = cred->cap_effective;
+ cap_lower(tmp, CAP_SYS_RESOURCE);
+ iee_set_cred_cap_effective(cred, tmp);
+ }
+ #else
cap_lower(cred->cap_effective, CAP_SYS_RESOURCE);
+ #endif
sb->s_magic = OVERLAYFS_SUPER_MAGIC;
sb->s_xattr = ofs->config.userxattr ? ovl_user_xattr_handlers :
diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c
index af7849e5974f..4ac2f0e65955 100644
--- a/fs/smb/client/cifs_spnego.c
+++ b/fs/smb/client/cifs_spnego.c
@@ -18,6 +18,10 @@
#include "cifs_spnego.h"
#include "cifs_debug.h"
#include "cifsproto.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
static const struct cred *spnego_cred;
/* create a new cifs key */
@@ -212,8 +216,13 @@ init_cifs_spnego(void)
* the results it looks up
*/
set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(cred,keyring);
+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING);
+ #else
cred->thread_keyring = keyring;
cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
+ #endif
spnego_cred = cred;
cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring));
diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c
index f5b6df82e857..9f0ff045836d 100644
--- a/fs/smb/client/cifsacl.c
+++ b/fs/smb/client/cifsacl.c
@@ -26,6 +26,10 @@
#include "cifs_fs_sb.h"
#include "cifs_unicode.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
/* security id for everyone/world system group */
static const struct cifs_sid sid_everyone = {
1, 1, {0, 0, 0, 0, 0, 1}, {0} };
@@ -491,8 +495,13 @@ init_cifs_idmap(void)
/* instruct request_key() to use this special keyring as a cache for
* the results it looks up */
set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(cred,keyring);
+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING);
+ #else
cred->thread_keyring = keyring;
cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
+ #endif
root_cred = cred;
cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring));
diff --git a/include/asm-generic/early_ioremap.h b/include/asm-generic/early_ioremap.h
index 9d0479f50f97..f501e0f965f8 100644
--- a/include/asm-generic/early_ioremap.h
+++ b/include/asm-generic/early_ioremap.h
@@ -17,6 +17,9 @@ extern void *early_memremap_ro(resource_size_t phys_addr,
extern void *early_memremap_prot(resource_size_t phys_addr,
unsigned long size, unsigned long prot_val);
extern void early_iounmap(void __iomem *addr, unsigned long size);
+#ifdef CONFIG_PTP
+extern void early_iounmap_after_init(void __iomem *addr, unsigned long size);
+#endif
extern void early_memunmap(void *addr, unsigned long size);
#if defined(CONFIG_GENERIC_EARLY_IOREMAP) && defined(CONFIG_MMU)
diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h
index 8cc7b09c1bc7..83158589a545 100644
--- a/include/asm-generic/fixmap.h
+++ b/include/asm-generic/fixmap.h
@@ -70,6 +70,24 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr)
__set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR)
#endif
+#ifdef CONFIG_PTP
+#ifndef clear_fixmap_init
+#define clear_fixmap_init(idx) \
+ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR)
+#endif
+
+#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \
+({ \
+ unsigned long ________addr; \
+ __iee_set_fixmap_pre_init(idx, phys, flags); \
+ ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \
+ ________addr; \
+})
+
+#define iee_set_fixmap_offset_pre_init(idx, phys) \
+ __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL)
+#endif
+
/* Return a pointer with offset calculated */
#define __set_fixmap_offset(idx, phys, flags) \
({ \
diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index c75d4a753849..506ff9662e02 100644
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -7,6 +7,10 @@
#define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO)
#define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
/**
* __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table
* @mm: the mm_struct of the current context
@@ -23,6 +27,13 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm)
if (!ptdesc)
return NULL;
+
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(ptdesc_address(ptdesc)));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)ptdesc_address(ptdesc));
+ #endif
+
return ptdesc_address(ptdesc);
}
@@ -46,6 +57,11 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
*/
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(pte));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)pte);
+ #endif
pagetable_free(virt_to_ptdesc(pte));
}
@@ -73,6 +89,13 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
return NULL;
}
+ #ifdef CONFIG_PTP
+ pte_t *pte = (pte_t *)page_address(ptdesc_page(ptdesc));
+ unsigned long iee_addr = __phys_to_iee(__pa(pte));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)pte);
+ #endif
+
return ptdesc_page(ptdesc);
}
@@ -103,9 +126,20 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
*/
static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
{
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
+
struct ptdesc *ptdesc = page_ptdesc(pte_page);
pagetable_pte_dtor(ptdesc);
+
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(page_address(pte_page)));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)page_address(pte_page));
+ #endif
+
pagetable_free(ptdesc);
}
@@ -145,10 +179,21 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
#ifndef __HAVE_ARCH_PMD_FREE
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
+
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
pagetable_pmd_dtor(ptdesc);
+
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(pmd));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)pmd);
+ #endif
+
pagetable_free(ptdesc);
}
#endif
@@ -190,7 +235,16 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
{
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
+
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(pud));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)pud);
+ #endif
pagetable_free(virt_to_ptdesc(pud));
}
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 200853042fc7..9d733afced53 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -346,6 +346,17 @@
KEEP(*(.dtb.init.rodata)) \
__dtb_end = .;
+#ifdef CONFIG_KOI
+#define KOI_DATA() \
+ . = ALIGN(PAGE_SIZE); \
+ __koi_data_start = .; \
+ *(.data..koi) \
+ . = ALIGN(PAGE_SIZE); \
+ __koi_data_end = .;
+#else
+#define KOI_DATA()
+#endif
+
/*
* .data section
*/
@@ -370,8 +381,8 @@
BRANCH_PROFILE() \
TRACE_PRINTKS() \
BPF_RAW_TP() \
- TRACEPOINT_STR()
-
+ TRACEPOINT_STR() \
+ KOI_DATA()
/*
* Data section helpers
*/
@@ -1093,6 +1104,14 @@
* They will fit only a subset of the architectures
*/
+#ifdef CONFIG_CREDP
+ #define CRED_DATA \
+ . = ALIGN(PAGE_SIZE); \
+ *(.iee.cred) \
+ . = ALIGN(PAGE_SIZE);
+#else
+ #define CRED_DATA
+#endif
/*
* Writeable data.
@@ -1110,6 +1129,7 @@
. = ALIGN(PAGE_SIZE); \
.data : AT(ADDR(.data) - LOAD_OFFSET) { \
INIT_TASK_DATA(inittask) \
+ CRED_DATA \
NOSAVE_DATA \
PAGE_ALIGNED_DATA(pagealigned) \
CACHELINE_ALIGNED_DATA(cacheline) \
diff --git a/include/linux/cred.h b/include/linux/cred.h
index e01c6d094a30..cceb4842b619 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -18,6 +18,10 @@
#include <linux/sched/user.h>
#include <linux/kabi.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-def.h>
+#endif
+
struct cred;
struct inode;
@@ -153,6 +157,22 @@ struct cred {
KABI_RESERVE(4)
} __randomize_layout;
+#ifdef CONFIG_CREDP
+extern unsigned long long iee_rw_gate(int flag, ...);
+static void iee_set_cred_non_rcu(struct cred *cred, int non_rcu)
+{
+ iee_rw_gate(IEE_OP_SET_CRED_NON_RCU,cred,non_rcu);
+ *(int *)(&(((struct rcu_head *)(cred->rcu.func))->next)) = non_rcu;
+}
+
+static bool noinline iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr)
+{
+ bool ret;
+ ret = iee_rw_gate(IEE_OP_SET_CRED_ATOP_USAGE,cred,flag,nr);
+ return ret;
+}
+#endif
+
extern void __put_cred(struct cred *);
extern void exit_creds(struct task_struct *);
extern int copy_creds(struct task_struct *, unsigned long);
@@ -189,7 +209,11 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred)
*/
static inline struct cred *get_new_cred_many(struct cred *cred, int nr)
{
+ #ifdef CONFIG_CREDP
+ iee_set_cred_atomic_op_usage(cred, AT_ADD, nr);
+ #else
atomic_long_add(nr, &cred->usage);
+ #endif
return cred;
}
@@ -202,7 +226,7 @@ static inline struct cred *get_new_cred_many(struct cred *cred, int nr)
*/
static inline struct cred *get_new_cred(struct cred *cred)
{
- return get_new_cred_many(cred, 1);
+ return get_new_cred_many(cred, 1); // XXXzgc atomic_inc -> get_new_cred_many
}
/**
@@ -224,7 +248,11 @@ static inline const struct cred *get_cred_many(const struct cred *cred, int nr)
struct cred *nonconst_cred = (struct cred *) cred;
if (!cred)
return cred;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_non_rcu(nonconst_cred,0);
+ #else
nonconst_cred->non_rcu = 0;
+ #endif
return get_new_cred_many(nonconst_cred, nr);
}
@@ -247,9 +275,19 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred)
struct cred *nonconst_cred = (struct cred *) cred;
if (!cred)
return NULL;
+ #ifdef CONFIG_CREDP
+ if (!iee_set_cred_atomic_op_usage(nonconst_cred,AT_INC_NOT_ZERO,0))
+ return NULL;
+ #else
if (!atomic_long_inc_not_zero(&nonconst_cred->usage))
return NULL;
+ #endif
+
+ #ifdef CONFIG_CREDP
+ iee_set_cred_non_rcu(nonconst_cred,0);
+ #else
nonconst_cred->non_rcu = 0;
+ #endif
return cred;
}
@@ -270,8 +308,13 @@ static inline void put_cred_many(const struct cred *_cred, int nr)
struct cred *cred = (struct cred *) _cred;
if (cred) {
+ #ifdef CONFIG_CREDP
+ if (iee_set_cred_atomic_op_usage(cred,AT_SUB_AND_TEST,nr))
+ __put_cred(cred);
+ #else
if (atomic_long_sub_and_test(nr, &cred->usage))
__put_cred(cred);
+ #endif
}
}
diff --git a/include/linux/efi.h b/include/linux/efi.h
index 9ed79128458c..970cc4f7068b 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -740,6 +740,15 @@ extern int __init __efi_memmap_init(struct efi_memory_map_data *data);
extern int __init efi_memmap_init_early(struct efi_memory_map_data *data);
extern int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size);
extern void __init efi_memmap_unmap(void);
+#ifdef CONFIG_PTP
+extern void __init efi_memmap_unmap_after_init(void);
+#endif
+extern int __init efi_memmap_install(struct efi_memory_map_data *data);
+extern int __init efi_memmap_split_count(efi_memory_desc_t *md,
+ struct range *range);
+extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap,
+ void *buf, struct efi_mem_range *mem);
+extern void __init efi_print_memmap(void);
#ifdef CONFIG_EFI_ESRT
extern void __init efi_esrt_init(void);
diff --git a/include/linux/iee-func.h b/include/linux/iee-func.h
new file mode 100644
index 000000000000..79171de67c2a
--- /dev/null
+++ b/include/linux/iee-func.h
@@ -0,0 +1,27 @@
+#ifndef _LINUX_IEE_FUNC_H
+#define _LINUX_IEE_FUNC_H
+
+#ifdef CONFIG_IEE
+// Declare the __entry_task.
+__attribute__((aligned(PAGE_SIZE))) DECLARE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task);
+
+extern unsigned long long iee_rw_gate(int flag, ...);
+extern u32 get_cpu_asid_bits(void);
+extern unsigned long arm64_mm_context_get(struct mm_struct *mm);
+extern void set_iee_page_valid(unsigned long addr);
+extern void set_iee_page_invalid(unsigned long addr);
+extern void iee_set_logical_mem_ro(unsigned long addr);
+extern void iee_set_logical_mem_rw(unsigned long addr);
+extern void iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm);
+extern void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd);
+extern void iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack);
+extern void iee_free_token(struct task_struct *tsk);
+extern unsigned long iee_read_token_stack(struct task_struct *tsk);
+extern void iee_set_token_page_valid(void *token, void *new);
+extern void iee_set_token_page_invalid(void *token);
+extern void iee_set_kernel_ppage(unsigned long addr);
+extern void iee_set_kernel_upage(unsigned long addr);
+extern void iee_write_in_byte(void *ptr, u64 data, int length);
+#endif
+
+#endif
\ No newline at end of file
diff --git a/include/linux/module.h b/include/linux/module.h
index 4db2878d9e42..ef8d51994017 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -606,6 +606,7 @@ struct module {
KABI_RESERVE(2)
KABI_RESERVE(3)
KABI_RESERVE(4)
+
} ____cacheline_aligned __randomize_layout;
#ifndef MODULE_ARCH_INIT
#define MODULE_ARCH_INIT {}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index f40411aa7b70..297becfbc8e3 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -773,6 +773,24 @@ struct task_struct_resvd {
struct task_struct *task;
};
+#if defined(CONFIG_IEE) || defined(CONFIG_KOI)
+struct task_token {
+#ifdef CONFIG_IEE
+ struct mm_struct *mm; /* VA */
+ pgd_t *pgd; /* Logical VA */
+ void *iee_stack; /* VA */
+ bool valid;
+ void *kernel_stack; /* VA */
+#endif
+#ifdef CONFIG_KOI
+ void *koi_kernel_stack; /* VA */
+ void *koi_stack; /* VA */
+ void *koi_stack_base; /* VA */
+ unsigned long current_ttbr1;
+#endif
+};
+#endif
+
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
@@ -795,6 +813,7 @@ struct task_struct {
randomized_struct_fields_start
void *stack;
+
refcount_t usage;
/* Per task flags (PF_*), defined further below: */
unsigned int flags;
diff --git a/init/main.c b/init/main.c
index 803332dd3d90..0f8d6e2744c2 100644
--- a/init/main.c
+++ b/init/main.c
@@ -102,6 +102,12 @@
#include <linux/randomize_kstack.h>
#include <net/net_namespace.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#include <asm/iee-si.h>
+#include <linux/stop_machine.h>
+#endif
+
#include <asm/io.h>
#include <asm/setup.h>
#include <asm/sections.h>
@@ -112,6 +118,10 @@
#include <kunit/test.h>
+#ifdef CONFIG_PTP
+extern void *bm_pte_addr;
+#endif
+
static int kernel_init(void *);
/*
@@ -880,6 +890,9 @@ void start_kernel(void)
{
char *command_line;
char *after_dashes;
+ #ifdef CONFIG_IEE
+ unsigned int cpu;
+ #endif
set_task_stack_end_magic(&init_task);
smp_setup_processor_id();
@@ -904,6 +917,16 @@ void start_kernel(void)
setup_command_line(command_line);
setup_nr_cpu_ids();
setup_per_cpu_areas();
+ #ifdef CONFIG_IEE
+ for_each_possible_cpu(cpu)
+ {
+ // Map the __entry_task to IEE.
+ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu]))));
+ // Set the __entry_task of cpu 0 readonly in lm.
+ if(cpu == smp_processor_id())
+ iee_set_logical_mem_ro((unsigned long)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu]));
+ }
+ #endif
smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
boot_cpu_hotplug_init();
@@ -1446,6 +1469,9 @@ static int __ref kernel_init(void *unused)
wait_for_completion(&kthreadd_done);
kernel_init_freeable();
+ #ifdef CONFIG_PTP
+ iee_set_logical_mem_ro((unsigned long)bm_pte_addr);
+ #endif
/* need to finish all async __init code before freeing the memory */
async_synchronize_full();
@@ -1462,7 +1488,7 @@ static int __ref kernel_init(void *unused)
* to finalize PTI.
*/
pti_finalize();
-
+
system_state = SYSTEM_RUNNING;
numa_default_policy();
diff --git a/kernel/cred.c b/kernel/cred.c
index c033a201c808..2e44530976d5 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -20,6 +20,11 @@
#include <linux/cn_proc.h>
#include <linux/uidgid.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#include <linux/iee-func.h>
+#endif
+
#if 0
#define kdebug(FMT, ...) \
printk("[%-5.5s%5u] " FMT "\n", \
@@ -34,6 +39,9 @@ do { \
#endif
static struct kmem_cache *cred_jar;
+#ifdef CONFIG_CREDP
+static struct kmem_cache *rcu_jar;
+#endif
/* init to 2 - one for init_task, one to ensure it is never freed */
static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) };
@@ -41,6 +49,32 @@ static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) };
/*
* The initial credentials for the initial task
*/
+#ifdef CONFIG_CREDP
+struct cred init_cred __section(".iee.cred") = {
+ .usage = ATOMIC_INIT(4),
+#ifdef CONFIG_DEBUG_CREDENTIALS
+ .subscribers = ATOMIC_INIT(2),
+ .magic = CRED_MAGIC,
+#endif
+ .uid = GLOBAL_ROOT_UID,
+ .gid = GLOBAL_ROOT_GID,
+ .suid = GLOBAL_ROOT_UID,
+ .sgid = GLOBAL_ROOT_GID,
+ .euid = GLOBAL_ROOT_UID,
+ .egid = GLOBAL_ROOT_GID,
+ .fsuid = GLOBAL_ROOT_UID,
+ .fsgid = GLOBAL_ROOT_GID,
+ .securebits = SECUREBITS_DEFAULT,
+ .cap_inheritable = CAP_EMPTY_SET,
+ .cap_permitted = CAP_FULL_SET,
+ .cap_effective = CAP_FULL_SET,
+ .cap_bset = CAP_FULL_SET,
+ .user = INIT_USER,
+ .user_ns = &init_user_ns,
+ .group_info = &init_groups,
+ .ucounts = &init_ucounts,
+};
+#else
struct cred init_cred = {
.usage = ATOMIC_INIT(4),
.uid = GLOBAL_ROOT_UID,
@@ -61,13 +95,43 @@ struct cred init_cred = {
.group_info = &init_groups,
.ucounts = &init_ucounts,
};
+#endif
+
+static inline void set_cred_subscribers(struct cred *cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+ atomic_set(&cred->subscribers, n);
+#endif
+}
+
+static inline int read_cred_subscribers(const struct cred *cred)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+ return atomic_read(&cred->subscribers);
+#else
+ return 0;
+#endif
+}
+
+static inline void alter_cred_subscribers(const struct cred *_cred, int n)
+{
+#ifdef CONFIG_DEBUG_CREDENTIALS
+ struct cred *cred = (struct cred *) _cred;
+
+ atomic_add(n, &cred->subscribers);
+#endif
+}
/*
* The RCU callback to actually dispose of a set of credentials
*/
static void put_cred_rcu(struct rcu_head *rcu)
{
+ #ifdef CONFIG_CREDP
+ struct cred *cred = *(struct cred **)(rcu + 1);
+ #else
struct cred *cred = container_of(rcu, struct cred, rcu);
+ #endif
kdebug("put_cred_rcu(%p)", cred);
@@ -86,6 +150,9 @@ static void put_cred_rcu(struct rcu_head *rcu)
if (cred->ucounts)
put_ucounts(cred->ucounts);
put_user_ns(cred->user_ns);
+ #ifdef CONFIG_CREDP
+ kmem_cache_free(rcu_jar, (struct rcu_head *)(cred->rcu.func));
+ #endif
kmem_cache_free(cred_jar, cred);
}
@@ -104,10 +171,22 @@ void __put_cred(struct cred *cred)
BUG_ON(cred == current->cred);
BUG_ON(cred == current->real_cred);
+ #ifdef CONFIG_CREDP
+ if (*(int *)(&(((struct rcu_head *)(cred->rcu.func))->next)))
+ #else
if (cred->non_rcu)
+ #endif
+ #ifdef CONFIG_CREDP
+ put_cred_rcu((struct rcu_head *)(cred->rcu.func));
+ #else
put_cred_rcu(&cred->rcu);
+ #endif
else
+ #ifdef CONFIG_CREDP
+ call_rcu((struct rcu_head *)(cred->rcu.func), put_cred_rcu);
+ #else
call_rcu(&cred->rcu, put_cred_rcu);
+ #endif
}
EXPORT_SYMBOL(__put_cred);
@@ -178,7 +257,18 @@ struct cred *cred_alloc_blank(void)
if (!new)
return NULL;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_rcu(new,kmem_cache_zalloc(rcu_jar, GFP_KERNEL));
+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new;
+ iee_set_cred_atomic_set_usage(new,1);
+ #else
atomic_long_set(&new->usage, 1);
+ #endif
+
+ #ifdef CONFIG_DEBUG_CREDENTIALS
+ new->magic = CRED_MAGIC;
+ #endif
+
if (security_cred_alloc_blank(new, GFP_KERNEL_ACCOUNT) < 0)
goto error;
@@ -213,13 +303,25 @@ struct cred *prepare_creds(void)
if (!new)
return NULL;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL));
+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new;
+ #endif
+
kdebug("prepare_creds() alloc %p", new);
old = task->cred;
+ #ifdef CONFIG_CREDP
+ iee_copy_cred(old,new);
+
+ iee_set_cred_non_rcu(new,0);
+ iee_set_cred_atomic_set_usage(new,1);
+ #else
memcpy(new, old, sizeof(struct cred));
new->non_rcu = 0;
atomic_long_set(&new->usage, 1);
+ #endif
get_group_info(new->group_info);
get_uid(new->user);
get_user_ns(new->user_ns);
@@ -232,10 +334,18 @@ struct cred *prepare_creds(void)
#endif
#ifdef CONFIG_SECURITY
+#ifdef CONFIG_CREDP
+ iee_set_cred_security(new,NULL);
+#else
new->security = NULL;
+#endif
#endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_ucounts(new, get_ucounts(new->ucounts));
+ #else
new->ucounts = get_ucounts(new->ucounts);
+ #endif
if (!new->ucounts)
goto error;
@@ -265,15 +375,30 @@ struct cred *prepare_exec_creds(void)
#ifdef CONFIG_KEYS
/* newly exec'd tasks don't get a thread keyring */
key_put(new->thread_keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(new,NULL);
+ #else
new->thread_keyring = NULL;
+ #endif
/* inherit the session keyring; new process keyring */
key_put(new->process_keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_process_keyring(new,NULL);
+ #else
new->process_keyring = NULL;
+ #endif
#endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,new->euid);
+ iee_set_cred_suid(new,new->euid);
+ iee_set_cred_fsgid(new,new->egid);
+ iee_set_cred_sgid(new,new->egid);
+ #else
new->suid = new->fsuid = new->euid;
new->sgid = new->fsgid = new->egid;
+ #endif
return new;
}
@@ -327,7 +452,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
* had one */
if (new->thread_keyring) {
key_put(new->thread_keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(new,NULL);
+ #else
new->thread_keyring = NULL;
+ #endif
if (clone_flags & CLONE_THREAD)
install_thread_keyring_to_cred(new);
}
@@ -337,7 +466,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags)
*/
if (!(clone_flags & CLONE_THREAD)) {
key_put(new->process_keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_process_keyring(new,NULL);
+ #else
new->process_keyring = NULL;
+ #endif
}
#endif
@@ -594,7 +727,11 @@ int set_cred_ucounts(struct cred *new)
if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid)))
return -EAGAIN;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_ucounts(new, new_ucounts);
+ #else
new->ucounts = new_ucounts;
+ #endif
put_ucounts(old_ucounts);
return 0;
@@ -606,8 +743,21 @@ int set_cred_ucounts(struct cred *new)
void __init cred_init(void)
{
/* allocate a slab in which we can store credentials */
+ #ifdef CONFIG_CREDP
cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE, NULL);
+ rcu_jar = kmem_cache_create("rcu_jar", sizeof(struct rcu_head) + sizeof(struct cred *), 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
+ // Map init_cred
+ *((struct rcu_head **)(&(init_cred.rcu.func))) = (struct rcu_head *)kmem_cache_zalloc(rcu_jar, GFP_KERNEL);
+ *(struct cred **)(((struct rcu_head *)(init_cred.rcu.func)) + 1) = &init_cred;
+ set_iee_page_valid(__phys_to_iee(__pa_symbol(&init_cred)));
+ iee_set_logical_mem_ro((unsigned long)&init_cred);
+ iee_set_logical_mem_ro((unsigned long)__va(__pa_symbol(&init_cred)));
+ #else
+ cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
+ #endif
}
/**
@@ -638,29 +788,56 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon)
if (!new)
return NULL;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL));
+ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new;
+ #endif
+
kdebug("prepare_kernel_cred() alloc %p", new);
old = get_task_cred(daemon);
+ #ifdef CONFIG_CREDP
+ iee_copy_cred(old,new);
+ iee_set_cred_non_rcu(new,0);
+ iee_set_cred_atomic_set_usage(new,1);
+ #else
*new = *old;
new->non_rcu = 0;
atomic_long_set(&new->usage, 1);
+ #endif
get_uid(new->user);
get_user_ns(new->user_ns);
get_group_info(new->group_info);
#ifdef CONFIG_KEYS
+#ifdef CONFIG_CREDP
+ iee_set_cred_session_keyring(new,NULL);
+ iee_set_cred_process_keyring(new,NULL);
+ iee_set_cred_thread_keyring(new,NULL);
+ iee_set_cred_request_key_auth(new,NULL);
+ iee_set_cred_jit_keyring(new,KEY_REQKEY_DEFL_THREAD_KEYRING);
+#else
new->session_keyring = NULL;
new->process_keyring = NULL;
new->thread_keyring = NULL;
new->request_key_auth = NULL;
new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
#endif
+#endif
#ifdef CONFIG_SECURITY
+#ifdef CONFIG_CREDP
+ iee_set_cred_security(new,NULL);
+#else
new->security = NULL;
#endif
+#endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_ucounts(new, get_ucounts(new->ucounts));
+ #else
new->ucounts = get_ucounts(new->ucounts);
+ #endif
if (!new->ucounts)
goto error;
@@ -727,8 +904,13 @@ int set_create_files_as(struct cred *new, struct inode *inode)
{
if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
return -EINVAL;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,inode->i_uid);
+ iee_set_cred_fsgid(new,inode->i_gid);
+ #else
new->fsuid = inode->i_uid;
new->fsgid = inode->i_gid;
+ #endif
return security_kernel_create_files_as(new, inode);
}
EXPORT_SYMBOL(set_create_files_as);
diff --git a/kernel/exit.c b/kernel/exit.c
index 21a59a6e1f2e..d21a109f0497 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -74,6 +74,10 @@
#include <asm/unistd.h>
#include <asm/mmu_context.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#endif
+
/*
* The default value should be high enough to not crash a system that randomly
* crashes its kernel from time to time, but low enough to at least not permit
@@ -558,6 +562,10 @@ static void exit_mm(void)
smp_mb__after_spinlock();
local_irq_disable();
current->mm = NULL;
+ #ifdef CONFIG_IEE
+ iee_set_token_mm(current, NULL);
+ iee_set_token_pgd(current, NULL);
+ #endif
membarrier_update_current_mm(NULL);
enter_lazy_tlb(mm, current);
local_irq_enable();
diff --git a/kernel/fork.c b/kernel/fork.c
index e033388b11bd..c93e18a4f0b3 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -115,6 +115,10 @@
#define CREATE_TRACE_POINTS
#include <trace/events/task.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#endif
+
/*
* Minimum number of threads to boot the kernel
*/
@@ -128,14 +132,14 @@
/*
* Protected counters by write_lock_irq(&tasklist_lock)
*/
-unsigned long total_forks; /* Handle normal Linux uptimes. */
-int nr_threads; /* The idle threads do not count.. */
+unsigned long total_forks; /* Handle normal Linux uptimes. */
+int nr_threads; /* The idle threads do not count.. */
-static int max_threads; /* tunable limit on nr_threads */
+static int max_threads; /* tunable limit on nr_threads */
-#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
+#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
-static const char * const resident_page_types[] = {
+static const char *const resident_page_types[] = {
NAMED_ARRAY_INDEX(MM_FILEPAGES),
NAMED_ARRAY_INDEX(MM_ANONPAGES),
NAMED_ARRAY_INDEX(MM_SWAPENTS),
@@ -144,7 +148,7 @@ static const char * const resident_page_types[] = {
DEFINE_PER_CPU(unsigned long, process_counts) = 0;
-__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
+__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
#ifdef CONFIG_PROVE_RCU
int lockdep_tasklist_lock_is_held(void)
@@ -159,7 +163,7 @@ int nr_processes(void)
int cpu;
int total = 0;
- for_each_possible_cpu(cpu)
+ for_each_possible_cpu (cpu)
total += per_cpu(process_counts, cpu);
return total;
@@ -190,7 +194,7 @@ static inline void free_task_struct(struct task_struct *tsk)
* Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
* kmemcache based allocator.
*/
-# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+#if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
# ifdef CONFIG_VMAP_STACK
/*
@@ -311,8 +315,8 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node)
* so memcg accounting is performed manually on assigning/releasing
* stacks to tasks. Drop __GFP_ACCOUNT.
*/
- stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
- VMALLOC_START, VMALLOC_END,
+ stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START,
+ VMALLOC_END,
THREADINFO_GFP & ~__GFP_ACCOUNT,
PAGE_KERNEL,
0, node, __builtin_return_address(0));
@@ -410,9 +414,10 @@ static void free_thread_stack(struct task_struct *tsk)
void thread_stack_cache_init(void)
{
- thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
- THREAD_SIZE, THREAD_SIZE, 0, 0,
- THREAD_SIZE, NULL);
+ thread_stack_cache =
+ kmem_cache_create_usercopy("thread_stack", THREAD_SIZE,
+ THREAD_SIZE, 0, 0, THREAD_SIZE,
+ NULL);
BUG_ON(thread_stack_cache == NULL);
}
@@ -502,7 +507,8 @@ struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
{
- struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
+ struct vm_area_struct *new =
+ kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
if (!new)
return NULL;
@@ -602,8 +608,15 @@ void put_task_stack(struct task_struct *tsk)
}
#endif
+#ifdef CONFIG_KOI
+extern s64 koi_offset;
+#endif
+
void free_task(struct task_struct *tsk)
{
+ #ifdef CONFIG_IEE
+ void *iee_stack;
+ #endif
#ifdef CONFIG_SECCOMP
WARN_ON_ONCE(tsk->seccomp.filter);
#endif
@@ -633,6 +646,45 @@ void free_task(struct task_struct *tsk)
if (dynamic_affinity_enabled())
sched_prefer_cpus_free(tsk);
#endif
+#ifdef CONFIG_IEE
+ // Free iee stack.
+ iee_stack = (void *)iee_read_token_stack(tsk);
+ if (iee_stack) {
+ iee_set_kernel_ppage(
+ (unsigned long)(iee_stack - PAGE_SIZE * 4));
+ free_pages((unsigned long)(iee_stack - PAGE_SIZE * 4), 3);
+ }
+ // Free task_token.
+ // Empty the token
+ iee_free_token(tsk);
+
+#ifdef CONFIG_KOI
+ // Free koi stack.
+ unsigned long koi_stack = iee_rw_gate(IEE_READ_KOI_STACK_BASE, current);
+ if (koi_stack != 0)
+ free_pages(koi_stack, 2);
+#endif
+#else
+#ifdef CONFIG_KOI
+// free koi stack
+ struct task_token *token = (struct task_token *)((unsigned long)current + koi_offset);
+ unsigned long flags;
+ local_irq_save(flags);
+ asm volatile(
+ "at s1e1r, %0\n"
+ "isb\n"
+ :
+ :"r"(token));
+ unsigned long res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+ if (!(res & 0x1)) {
+ unsigned long koi_stack = token->koi_stack_base;
+ if (koi_stack != 0)
+ free_pages(koi_stack, 2);
+ }
+#endif
+#endif
+
#ifdef CONFIG_QOS_SCHED_SMART_GRID
if (smart_grid_enabled())
sched_grid_qos_free(tsk);
@@ -657,7 +709,7 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)
#ifdef CONFIG_MMU
static __latent_entropy int dup_mmap(struct mm_struct *mm,
- struct mm_struct *oldmm)
+ struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp;
int retval;
@@ -773,7 +825,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */
vma_interval_tree_insert_after(tmp, mpnt,
- &mapping->i_mmap);
+ &mapping->i_mmap);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
@@ -842,7 +894,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
mmap_write_unlock(oldmm);
return 0;
}
-#define mm_alloc_pgd(mm) (0)
+#define mm_alloc_pgd(mm) (0)
#define mm_free_pgd(mm)
#endif /* CONFIG_MMU */
@@ -850,20 +902,22 @@ static void check_mm(struct mm_struct *mm)
{
int i;
- BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
- "Please make sure 'struct resident_page_types[]' is updated as well");
+ BUILD_BUG_ON_MSG(
+ ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
+ "Please make sure 'struct resident_page_types[]' is updated as well");
for (i = 0; i < NR_MM_COUNTERS; i++) {
long x = mm_counter_sum(mm, i);
if (unlikely(x))
- pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
- mm, resident_page_types[i], x);
+ pr_alert(
+ "BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
+ mm, resident_page_types[i], x);
}
if (mm_pgtables_bytes(mm))
pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
- mm_pgtables_bytes(mm));
+ mm_pgtables_bytes(mm));
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
@@ -1014,14 +1068,6 @@ void __put_task_struct(struct task_struct *tsk)
}
EXPORT_SYMBOL_GPL(__put_task_struct);
-void __put_task_struct_rcu_cb(struct rcu_head *rhp)
-{
- struct task_struct *task = container_of(rhp, struct task_struct, rcu);
-
- __put_task_struct(task);
-}
-EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);
-
void __init __weak arch_task_cache_init(void) { }
/*
@@ -1039,8 +1085,8 @@ static void set_max_threads(unsigned int max_threads_suggested)
if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
threads = MAX_THREADS;
else
- threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
- (u64) THREAD_SIZE * 8UL);
+ threads = div64_u64((u64)nr_pages * (u64)PAGE_SIZE,
+ (u64)THREAD_SIZE * 8UL);
if (threads > max_threads_suggested)
threads = max_threads_suggested;
@@ -1075,17 +1121,24 @@ void __init fork_init(void)
int i;
#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
#ifndef ARCH_MIN_TASKALIGN
-#define ARCH_MIN_TASKALIGN 0
+#define ARCH_MIN_TASKALIGN 0
#endif
int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
unsigned long useroffset, usersize;
/* create a slab on which task_structs can be allocated */
task_struct_whitelist(&useroffset, &usersize);
+ #ifdef CONFIG_IEE
task_struct_cachep = kmem_cache_create_usercopy("task_struct",
arch_task_struct_size, align,
- SLAB_PANIC|SLAB_ACCOUNT,
+ SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE,
useroffset, usersize, NULL);
+ #else
+ task_struct_cachep =
+ kmem_cache_create_usercopy("task_struct", arch_task_struct_size,
+ align, SLAB_PANIC | SLAB_ACCOUNT,
+ useroffset, usersize, NULL);
+ #endif
#endif
/* do the arch specific task caches init */
@@ -1093,8 +1146,8 @@ void __init fork_init(void)
set_max_threads(MAX_THREADS);
- init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
- init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
+ init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads / 2;
+ init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads / 2;
init_task.signal->rlim[RLIMIT_SIGPENDING] =
init_task.signal->rlim[RLIMIT_NPROC];
@@ -1107,8 +1160,8 @@ void __init fork_init(void)
set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY);
#ifdef CONFIG_VMAP_STACK
- cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
- NULL, free_vm_stack_cache);
+ cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", NULL,
+ free_vm_stack_cache);
#endif
scs_init();
@@ -1118,7 +1171,7 @@ void __init fork_init(void)
}
int __weak arch_dup_task_struct(struct task_struct *dst,
- struct task_struct *src)
+ struct task_struct *src)
{
*dst = *src;
return 0;
@@ -1129,14 +1182,14 @@ void set_task_stack_end_magic(struct task_struct *tsk)
unsigned long *stackend;
stackend = end_of_stack(tsk);
- *stackend = STACK_END_MAGIC; /* for overflow detection */
+ *stackend = STACK_END_MAGIC; /* for overflow detection */
}
static bool dup_resvd_task_struct(struct task_struct *dst,
struct task_struct *orig, int node)
{
- dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd),
- GFP_KERNEL, node);
+ dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd), GFP_KERNEL,
+ node);
if (!dst->_resvd)
return false;
@@ -1309,7 +1362,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
}
static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
- struct user_namespace *user_ns)
+ struct user_namespace *user_ns)
{
mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
@@ -1425,8 +1478,8 @@ EXPORT_SYMBOL_GPL(mmput);
#ifdef CONFIG_MMU
static void mmput_async_fn(struct work_struct *work)
{
- struct mm_struct *mm = container_of(work, struct mm_struct,
- async_put_work);
+ struct mm_struct *mm =
+ container_of(work, struct mm_struct, async_put_work);
__mmput(mm);
}
@@ -1602,13 +1655,12 @@ struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
struct mm_struct *mm;
int err;
- err = down_read_killable(&task->signal->exec_update_lock);
+ err = down_read_killable(&task->signal->exec_update_lock);
if (err)
return ERR_PTR(err);
mm = get_task_mm(task);
- if (mm && mm != current->mm &&
- !ptrace_may_access(task, mode)) {
+ if (mm && mm != current->mm && !ptrace_may_access(task, mode)) {
mmput(mm);
mm = ERR_PTR(-EACCES);
}
@@ -1631,7 +1683,7 @@ static void complete_vfork_done(struct task_struct *tsk)
}
static int wait_for_vfork_done(struct task_struct *child,
- struct completion *vfork)
+ struct completion *vfork)
{
unsigned int state = TASK_UNINTERRUPTIBLE|TASK_KILLABLE|TASK_FREEZABLE;
int killed;
@@ -1682,8 +1734,8 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
* not set up a proper pointer then tough luck.
*/
put_user(0, tsk->clear_child_tid);
- do_futex(tsk->clear_child_tid, FUTEX_WAKE,
- 1, NULL, NULL, 0, 0);
+ do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1, NULL,
+ NULL, 0, 0);
}
tsk->clear_child_tid = NULL;
}
@@ -1767,6 +1819,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
#endif
tsk->mm = NULL;
+#ifdef CONFIG_IEE
+ iee_set_token_mm(tsk, NULL);
+ iee_set_token_pgd(tsk, NULL);
+#endif
tsk->active_mm = NULL;
/*
@@ -1798,6 +1854,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
}
tsk->mm = mm;
+#ifdef CONFIG_IEE
+ iee_set_token_mm(tsk, mm);
+ iee_set_token_pgd(tsk, mm->pgd);
+#endif
tsk->active_mm = mm;
sched_mm_cid_fork(tsk);
return 0;
@@ -2015,8 +2075,8 @@ static inline void init_task_pid_links(struct task_struct *task)
INIT_HLIST_NODE(&task->pid_links[type]);
}
-static inline void
-init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
+static inline void init_task_pid(struct task_struct *task, enum pid_type type,
+ struct pid *pid)
{
if (type == PIDTYPE_PID)
task->thread_pid = pid;
@@ -2277,6 +2337,12 @@ static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
mutex_unlock(&oom_adj_mutex);
}
+#if defined(CONFIG_KOI) && !defined(CONFIG_IEE)
+extern s64 koi_offset;
+extern int koi_add_page_mapping(unsigned long dst, unsigned long src);
+#endif
+
+
#ifdef CONFIG_RV
static void rv_task_fork(struct task_struct *p)
{
@@ -2309,15 +2375,21 @@ __latent_entropy struct task_struct *copy_process(
struct file *pidfile = NULL;
const u64 clone_flags = args->flags;
struct nsproxy *nsp = current->nsproxy;
+ #ifdef CONFIG_IEE
+ gfp_t gfp;
+ void *pstack;
+ #endif
/*
* Don't allow sharing the root directory with processes in a different
* namespace
*/
- if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
+ if ((clone_flags & (CLONE_NEWNS | CLONE_FS)) ==
+ (CLONE_NEWNS | CLONE_FS))
return ERR_PTR(-EINVAL);
- if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
+ if ((clone_flags & (CLONE_NEWUSER | CLONE_FS)) ==
+ (CLONE_NEWUSER | CLONE_FS))
return ERR_PTR(-EINVAL);
/*
@@ -2342,7 +2414,7 @@ __latent_entropy struct task_struct *copy_process(
* from creating siblings.
*/
if ((clone_flags & CLONE_PARENT) &&
- current->signal->flags & SIGNAL_UNKILLABLE)
+ current->signal->flags & SIGNAL_UNKILLABLE)
return ERR_PTR(-EINVAL);
/*
@@ -2387,6 +2459,15 @@ __latent_entropy struct task_struct *copy_process(
p = dup_task_struct(current, node);
if (!p)
goto fork_out;
+ #ifdef CONFIG_IEE
+ // Alloc iee stack.
+ gfp = GFP_KERNEL;
+ pstack = (void *)__get_free_pages(gfp, 3);
+ iee_set_kernel_upage((unsigned long)pstack);
+ // Init token.
+ iee_init_token(p, NULL, pstack + PAGE_SIZE * 4);
+ #endif
+
p->flags &= ~PF_KTHREAD;
if (args->kthread)
p->flags |= PF_KTHREAD;
@@ -2408,7 +2489,8 @@ __latent_entropy struct task_struct *copy_process(
/*
* Clear TID on mm_release()?
*/
- p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
+ p->clear_child_tid =
+ (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
ftrace_graph_init_task(p);
@@ -2519,10 +2601,10 @@ __latent_entropy struct task_struct *copy_process(
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
memset(&p->irqtrace, 0, sizeof(p->irqtrace));
- p->irqtrace.hardirq_disable_ip = _THIS_IP_;
- p->irqtrace.softirq_enable_ip = _THIS_IP_;
- p->softirqs_enabled = 1;
- p->softirq_context = 0;
+ p->irqtrace.hardirq_disable_ip = _THIS_IP_;
+ p->irqtrace.softirq_enable_ip = _THIS_IP_;
+ p->softirqs_enabled = 1;
+ p->softirq_context = 0;
#endif
p->pagefault_disabled = 0;
@@ -2535,8 +2617,8 @@ __latent_entropy struct task_struct *copy_process(
p->blocked_on = NULL; /* not blocked yet */
#endif
#ifdef CONFIG_BCACHE
- p->sequential_io = 0;
- p->sequential_io_avg = 0;
+ p->sequential_io = 0;
+ p->sequential_io_avg = 0;
#endif
#ifdef CONFIG_BPF_SYSCALL
RCU_INIT_POINTER(p->bpf_storage, NULL);
@@ -2623,7 +2705,7 @@ __latent_entropy struct task_struct *copy_process(
/*
* sigaltstack should be cleared when sharing the same VM
*/
- if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
+ if ((clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM)
sas_ss_reset(p);
/*
@@ -2702,7 +2784,7 @@ __latent_entropy struct task_struct *copy_process(
write_lock_irq(&tasklist_lock);
/* CLONE_PARENT re-uses the old parent */
- if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
+ if (clone_flags & (CLONE_PARENT | CLONE_THREAD)) {
p->real_parent = current->real_parent;
p->parent_exec_id = current->parent_exec_id;
if (clone_flags & CLONE_THREAD)
@@ -2766,8 +2848,9 @@ __latent_entropy struct task_struct *copy_process(
* tasklist_lock with adding child to the process tree
* for propagate_has_child_subreaper optimization.
*/
- p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
- p->real_parent->signal->is_child_subreaper;
+ p->signal->has_child_subreaper =
+ p->real_parent->signal->has_child_subreaper ||
+ p->real_parent->signal->is_child_subreaper;
list_add_tail(&p->sibling, &p->real_parent->children);
list_add_tail_rcu(&p->tasks, &init_task.tasks);
attach_pid(p, PIDTYPE_TGID);
@@ -2918,8 +3001,8 @@ struct task_struct * __init fork_idle(int cpu)
*/
struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
{
- unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
- CLONE_IO;
+ unsigned long flags = CLONE_FS | CLONE_FILES | CLONE_SIGHAND |
+ CLONE_THREAD | CLONE_IO;
struct kernel_clone_args args = {
.flags = ((lower_32_bits(flags) | CLONE_VM |
CLONE_UNTRACED) & ~CSIGNAL),
@@ -3083,8 +3166,8 @@ SYSCALL_DEFINE0(fork)
SYSCALL_DEFINE0(vfork)
{
struct kernel_clone_args args = {
- .flags = CLONE_VFORK | CLONE_VM,
- .exit_signal = SIGCHLD,
+ .flags = CLONE_VFORK | CLONE_VM,
+ .exit_signal = SIGCHLD,
};
return kernel_clone(&args);
@@ -3094,35 +3177,30 @@ SYSCALL_DEFINE0(vfork)
#ifdef __ARCH_WANT_SYS_CLONE
#ifdef CONFIG_CLONE_BACKWARDS
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
- int __user *, parent_tidptr,
- unsigned long, tls,
- int __user *, child_tidptr)
+ int __user *, parent_tidptr, unsigned long, tls, int __user *,
+ child_tidptr)
#elif defined(CONFIG_CLONE_BACKWARDS2)
SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
- int __user *, parent_tidptr,
- int __user *, child_tidptr,
- unsigned long, tls)
-#elif defined(CONFIG_CLONE_BACKWARDS3)
-SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
- int, stack_size,
- int __user *, parent_tidptr,
- int __user *, child_tidptr,
+ int __user *, parent_tidptr, int __user *, child_tidptr,
unsigned long, tls)
+#elif defined(CONFIG_CLONE_BACKWARDS3)
+SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, int,
+ stack_size, int __user *, parent_tidptr, int __user *,
+ child_tidptr, unsigned long, tls)
#else
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
- int __user *, parent_tidptr,
- int __user *, child_tidptr,
- unsigned long, tls)
+ int __user *, parent_tidptr, int __user *, child_tidptr,
+ unsigned long, tls)
#endif
{
struct kernel_clone_args args = {
- .flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
- .pidfd = parent_tidptr,
- .child_tid = child_tidptr,
- .parent_tid = parent_tidptr,
- .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
- .stack = newsp,
- .tls = tls,
+ .flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
+ .pidfd = parent_tidptr,
+ .child_tid = child_tidptr,
+ .parent_tid = parent_tidptr,
+ .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
+ .stack = newsp,
+ .tls = tls,
};
return kernel_clone(&args);
@@ -3178,21 +3256,21 @@ noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
return -EINVAL;
*kargs = (struct kernel_clone_args){
- .flags = args.flags,
- .pidfd = u64_to_user_ptr(args.pidfd),
- .child_tid = u64_to_user_ptr(args.child_tid),
- .parent_tid = u64_to_user_ptr(args.parent_tid),
- .exit_signal = args.exit_signal,
- .stack = args.stack,
- .stack_size = args.stack_size,
- .tls = args.tls,
- .set_tid_size = args.set_tid_size,
- .cgroup = args.cgroup,
+ .flags = args.flags,
+ .pidfd = u64_to_user_ptr(args.pidfd),
+ .child_tid = u64_to_user_ptr(args.child_tid),
+ .parent_tid = u64_to_user_ptr(args.parent_tid),
+ .exit_signal = args.exit_signal,
+ .stack = args.stack,
+ .stack_size = args.stack_size,
+ .tls = args.tls,
+ .set_tid_size = args.set_tid_size,
+ .cgroup = args.cgroup,
};
if (args.set_tid &&
- copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
- (kargs->set_tid_size * sizeof(pid_t))))
+ copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
+ (kargs->set_tid_size * sizeof(pid_t))))
return -EFAULT;
kargs->set_tid = kset_tid;
@@ -3287,7 +3365,8 @@ SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
}
#endif
-void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
+void walk_process_tree(struct task_struct *top, proc_visitor visitor,
+ void *data)
{
struct task_struct *leader, *parent, *child;
int res;
@@ -3295,8 +3374,8 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data
read_lock(&tasklist_lock);
leader = top = top->group_leader;
down:
- for_each_thread(leader, parent) {
- list_for_each_entry(child, &parent->children, sibling) {
+ for_each_thread (leader, parent) {
+ list_for_each_entry (child, &parent->children, sibling) {
res = visitor(child, data);
if (res) {
if (res < 0)
@@ -3304,8 +3383,7 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data
leader = child;
goto down;
}
-up:
- ;
+ up:;
}
}
@@ -3382,11 +3460,11 @@ void __init proc_caches_init(void)
*/
static int check_unshare_flags(unsigned long unshare_flags)
{
- if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
- CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
- CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
- CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP|
- CLONE_NEWTIME))
+ if (unshare_flags &
+ ~(CLONE_THREAD | CLONE_FS | CLONE_NEWNS | CLONE_SIGHAND | CLONE_VM |
+ CLONE_FILES | CLONE_SYSVSEM | CLONE_NEWUTS | CLONE_NEWIPC |
+ CLONE_NEWNET | CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWCGROUP |
+ CLONE_NEWTIME))
return -EINVAL;
/*
* Not implemented, but pretend it works if there is nothing
@@ -3497,7 +3575,7 @@ int ksys_unshare(unsigned long unshare_flags)
* to a new ipc namespace, the semaphore arrays from the old
* namespace are unreachable.
*/
- if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
+ if (unshare_flags & (CLONE_NEWIPC | CLONE_SYSVSEM))
do_sysvsem = 1;
err = unshare_fs(unshare_flags, &new_fs);
if (err)
@@ -3508,8 +3586,8 @@ int ksys_unshare(unsigned long unshare_flags)
err = unshare_userns(unshare_flags, &new_cred);
if (err)
goto bad_unshare_cleanup_fd;
- err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
- new_cred, new_fs);
+ err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_cred,
+ new_fs);
if (err)
goto bad_unshare_cleanup_cred;
@@ -3606,8 +3684,8 @@ int unshare_files(void)
return 0;
}
-int sysctl_max_threads(struct ctl_table *table, int write,
- void *buffer, size_t *lenp, loff_t *ppos)
+int sysctl_max_threads(struct ctl_table *table, int write, void *buffer,
+ size_t *lenp, loff_t *ppos)
{
struct ctl_table t;
int ret;
diff --git a/kernel/groups.c b/kernel/groups.c
index 9b43da22647d..8045812e8a3c 100644
--- a/kernel/groups.c
+++ b/kernel/groups.c
@@ -11,6 +11,9 @@
#include <linux/user_namespace.h>
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
struct group_info *groups_alloc(int gidsetsize)
{
@@ -119,7 +122,11 @@ void set_groups(struct cred *new, struct group_info *group_info)
{
put_group_info(new->group_info);
get_group_info(group_info);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_group_info(new,group_info);
+ #else
new->group_info = group_info;
+ #endif
}
EXPORT_SYMBOL(set_groups);
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 1eea53050bab..317eac6eb2f2 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -30,6 +30,10 @@
#include <linux/sched/isolation.h>
#include <trace/events/sched.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#endif
+
static DEFINE_SPINLOCK(kthread_create_lock);
static LIST_HEAD(kthread_create_list);
@@ -1429,6 +1433,10 @@ void kthread_use_mm(struct mm_struct *mm)
tsk->active_mm = mm;
tsk->mm = mm;
membarrier_update_current_mm(mm);
+ #ifdef CONFIG_IEE
+ iee_set_token_mm(tsk, mm);
+ iee_set_token_pgd(tsk, mm->pgd);
+ #endif
switch_mm_irqs_off(active_mm, mm, tsk);
local_irq_enable();
task_unlock(tsk);
@@ -1473,7 +1481,12 @@ void kthread_unuse_mm(struct mm_struct *mm)
local_irq_disable();
tsk->mm = NULL;
membarrier_update_current_mm(NULL);
+ #ifdef CONFIG_IEE
+ iee_set_token_mm(tsk, mm);
+ iee_set_token_pgd(tsk, NULL);
+ #endif
mmgrab_lazy_tlb(mm);
+
/* active_mm is still 'mm' */
enter_lazy_tlb(mm, tsk);
local_irq_enable();
diff --git a/kernel/smpboot.c b/kernel/smpboot.c
index f47d8f375946..60c7d365c0e1 100644
--- a/kernel/smpboot.c
+++ b/kernel/smpboot.c
@@ -16,6 +16,10 @@
#include <linux/kthread.h>
#include <linux/smpboot.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#endif
+
#include "smpboot.h"
#ifdef CONFIG_SMP
@@ -57,6 +61,11 @@ static __always_inline void idle_init(unsigned int cpu)
pr_err("SMP: fork_idle() failed for CPU %u\n", cpu);
else
per_cpu(idle_threads, cpu) = tsk;
+ #ifdef CONFIG_IEE
+ // Set the secondary __entry_task.
+ *(struct task_struct **)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu]) = tsk;
+ iee_set_logical_mem_ro((unsigned long)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu]));
+ #endif
}
}
diff --git a/kernel/sys.c b/kernel/sys.c
index 44b575990333..fbc47f83af50 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -75,6 +75,10 @@
#include <asm/io.h>
#include <asm/unistd.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
#include "uid16.h"
#ifndef SET_UNALIGN_CTL
@@ -395,7 +399,11 @@ long __sys_setregid(gid_t rgid, gid_t egid)
if (gid_eq(old->gid, krgid) ||
gid_eq(old->egid, krgid) ||
ns_capable_setid(old->user_ns, CAP_SETGID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_gid(new,krgid);
+ #else
new->gid = krgid;
+ #endif
else
goto error;
}
@@ -404,15 +412,27 @@ long __sys_setregid(gid_t rgid, gid_t egid)
gid_eq(old->egid, kegid) ||
gid_eq(old->sgid, kegid) ||
ns_capable_setid(old->user_ns, CAP_SETGID))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_egid(new,kegid);
+ #else
new->egid = kegid;
+ #endif
else
goto error;
}
if (rgid != (gid_t) -1 ||
(egid != (gid_t) -1 && !gid_eq(kegid, old->gid)))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_sgid(new,new->egid);
+ #else
new->sgid = new->egid;
+ #endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsgid(new,new->egid);
+ #else
new->fsgid = new->egid;
+ #endif
retval = security_task_fix_setgid(new, old, LSM_SETID_RE);
if (retval < 0)
@@ -454,9 +474,25 @@ long __sys_setgid(gid_t gid)
retval = -EPERM;
if (ns_capable_setid(old->user_ns, CAP_SETGID))
+ #ifdef CONFIG_CREDP
+ {
+ iee_set_cred_fsgid(new,kgid);
+ iee_set_cred_sgid(new,kgid);
+ iee_set_cred_egid(new,kgid);
+ iee_set_cred_gid(new,kgid);
+ }
+ #else
new->gid = new->egid = new->sgid = new->fsgid = kgid;
+ #endif
else if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->sgid))
+ #ifdef CONFIG_CREDP
+ {
+ iee_set_cred_fsgid(new,kgid);
+ iee_set_cred_egid(new,kgid);
+ }
+ #else
new->egid = new->fsgid = kgid;
+ #endif
else
goto error;
@@ -488,7 +524,11 @@ static int set_user(struct cred *new)
return -EAGAIN;
free_uid(new->user);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_user(new,new_user);
+ #else
new->user = new_user;
+ #endif
return 0;
}
@@ -549,7 +589,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid)
retval = -EPERM;
if (ruid != (uid_t) -1) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_uid(new,kruid);
+ #else
new->uid = kruid;
+ #endif
if (!uid_eq(old->uid, kruid) &&
!uid_eq(old->euid, kruid) &&
!ns_capable_setid(old->user_ns, CAP_SETUID))
@@ -557,7 +601,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid)
}
if (euid != (uid_t) -1) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_euid(new,keuid);
+ #else
new->euid = keuid;
+ #endif
if (!uid_eq(old->uid, keuid) &&
!uid_eq(old->euid, keuid) &&
!uid_eq(old->suid, keuid) &&
@@ -572,8 +620,16 @@ long __sys_setreuid(uid_t ruid, uid_t euid)
}
if (ruid != (uid_t) -1 ||
(euid != (uid_t) -1 && !uid_eq(keuid, old->uid)))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_suid(new,new->euid);
+ #else
new->suid = new->euid;
+ #endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,new->euid);
+ #else
new->fsuid = new->euid;
+ #endif
retval = security_task_fix_setuid(new, old, LSM_SETID_RE);
if (retval < 0)
@@ -626,7 +682,12 @@ long __sys_setuid(uid_t uid)
retval = -EPERM;
if (ns_capable_setid(old->user_ns, CAP_SETUID)) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_uid(new,kuid);
+ iee_set_cred_suid(new,kuid);
+ #else
new->suid = new->uid = kuid;
+ #endif
if (!uid_eq(kuid, old->uid)) {
retval = set_user(new);
if (retval < 0)
@@ -636,7 +697,12 @@ long __sys_setuid(uid_t uid)
goto error;
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_euid(new,kuid);
+ iee_set_cred_fsuid(new,kuid);
+ #else
new->fsuid = new->euid = kuid;
+ #endif
retval = security_task_fix_setuid(new, old, LSM_SETID_ID);
if (retval < 0)
@@ -710,7 +776,11 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid)
return -ENOMEM;
if (ruid != (uid_t) -1) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_uid(new,kruid);
+ #else
new->uid = kruid;
+ #endif
if (!uid_eq(kruid, old->uid)) {
retval = set_user(new);
if (retval < 0)
@@ -718,10 +788,22 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid)
}
}
if (euid != (uid_t) -1)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_euid(new,keuid);
+ #else
new->euid = keuid;
+ #endif
if (suid != (uid_t) -1)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_suid(new,ksuid);
+ #else
new->suid = ksuid;
+ #endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,new->euid);
+ #else
new->fsuid = new->euid;
+ #endif
retval = security_task_fix_setuid(new, old, LSM_SETID_RES);
if (retval < 0)
@@ -810,12 +892,29 @@ long __sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid)
return -ENOMEM;
if (rgid != (gid_t) -1)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_gid(new,krgid);
+ #else
new->gid = krgid;
+ #endif
if (egid != (gid_t) -1)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_egid(new,kegid);
+ #else
new->egid = kegid;
+ #endif
if (sgid != (gid_t) -1)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_sgid(new,ksgid);
+ #else
new->sgid = ksgid;
+ #endif
+
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsgid(new,new->egid);
+ #else
new->fsgid = new->egid;
+ #endif
retval = security_task_fix_setgid(new, old, LSM_SETID_RES);
if (retval < 0)
@@ -882,7 +981,11 @@ long __sys_setfsuid(uid_t uid)
uid_eq(kuid, old->suid) || uid_eq(kuid, old->fsuid) ||
ns_capable_setid(old->user_ns, CAP_SETUID)) {
if (!uid_eq(kuid, old->fsuid)) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,kuid);
+ #else
new->fsuid = kuid;
+ #endif
if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0)
goto change_okay;
}
@@ -926,7 +1029,11 @@ long __sys_setfsgid(gid_t gid)
gid_eq(kgid, old->sgid) || gid_eq(kgid, old->fsgid) ||
ns_capable_setid(old->user_ns, CAP_SETGID)) {
if (!gid_eq(kgid, old->fsgid)) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsgid(new,kgid);
+ #else
new->fsgid = kgid;
+ #endif
if (security_task_fix_setgid(new,old,LSM_SETID_FS) == 0)
goto change_okay;
}
diff --git a/kernel/umh.c b/kernel/umh.c
index 1b13c5d34624..32f5c88e10bf 100644
--- a/kernel/umh.c
+++ b/kernel/umh.c
@@ -32,6 +32,10 @@
#include <trace/events/module.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
static kernel_cap_t usermodehelper_bset = CAP_FULL_SET;
static kernel_cap_t usermodehelper_inheritable = CAP_FULL_SET;
static DEFINE_SPINLOCK(umh_sysctl_lock);
@@ -91,9 +95,15 @@ static int call_usermodehelper_exec_async(void *data)
goto out;
spin_lock(&umh_sysctl_lock);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_bset(new,cap_intersect(usermodehelper_bset, new->cap_bset));
+ iee_set_cred_cap_inheritable(new,cap_intersect(usermodehelper_inheritable,
+ new->cap_inheritable));
+ #else
new->cap_bset = cap_intersect(usermodehelper_bset, new->cap_bset);
new->cap_inheritable = cap_intersect(usermodehelper_inheritable,
new->cap_inheritable);
+ #endif
spin_unlock(&umh_sysctl_lock);
if (sub_info->init) {
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index 1d8e47bed3f1..9f1921025539 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -22,6 +22,10 @@
#include <linux/bsearch.h>
#include <linux/sort.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
static struct kmem_cache *user_ns_cachep __read_mostly;
static DEFINE_MUTEX(userns_state_mutex);
@@ -45,6 +49,19 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
/* Start with the same capabilities as init but useless for doing
* anything as the capabilities are bound to the new user namespace.
*/
+ #ifdef CONFIG_CREDP
+ iee_set_cred_securebits(cred,SECUREBITS_DEFAULT);
+ iee_set_cred_cap_inheritable(cred,CAP_EMPTY_SET);
+ iee_set_cred_cap_permitted(cred,CAP_FULL_SET);
+ iee_set_cred_cap_effective(cred,CAP_FULL_SET);
+ iee_set_cred_cap_ambient(cred,CAP_EMPTY_SET);
+ iee_set_cred_cap_bset(cred,CAP_FULL_SET);
+#ifdef CONFIG_KEYS
+ key_put(cred->request_key_auth);
+ iee_set_cred_request_key_auth(cred,NULL);
+#endif
+ iee_set_cred_user_ns(cred,user_ns);
+ #else
cred->securebits = SECUREBITS_DEFAULT;
cred->cap_inheritable = CAP_EMPTY_SET;
cred->cap_permitted = CAP_FULL_SET;
@@ -57,6 +74,7 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns)
#endif
/* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */
cred->user_ns = user_ns;
+ #endif
}
static unsigned long enforced_nproc_rlimit(void)
diff --git a/mm/Kconfig b/mm/Kconfig
index 45d4139c959c..eb9d41768c15 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -530,6 +530,18 @@ config NUMA_KEEP_MEMINFO
config MEMORY_ISOLATION
bool
+# Config for kernel module isolation
+config KOI
+ depends on ARM64
+ depends on ARM64_VA_BITS_48
+ depends on ARM64_4K_PAGES
+ def_bool n
+
+# Configs for pgtable isolation
+config PTP
+ depends on IEE
+ def_bool y
+
# IORESOURCE_SYSTEM_RAM regions in the kernel resource tree that are marked
# IORESOURCE_EXCLUSIVE cannot be mapped to user space, for example, via
# /dev/mem.
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index d25d99cb5f2b..2ea51f559d4e 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -44,6 +44,7 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr
if (!folio)
return;
+
if (ptep_clear_young_notify(vma, addr, pte))
folio_set_young(folio);
diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c
index 13f0d1192707..60dc95c5b286 100644
--- a/mm/debug_vm_pgtable.c
+++ b/mm/debug_vm_pgtable.c
@@ -452,7 +452,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args)
* X86 defined pmd_set_huge() verifies that the given
* PMD is not a populated non-leaf entry.
*/
+ #ifdef CONFIG_PTP
+ set_pmd(args->pmdp, __pmd(0));
+ #else
WRITE_ONCE(*args->pmdp, __pmd(0));
+ #endif
WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot));
WARN_ON(!pmd_clear_huge(args->pmdp));
pmd = READ_ONCE(*args->pmdp);
@@ -472,7 +476,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args)
* X86 defined pud_set_huge() verifies that the given
* PUD is not a populated non-leaf entry.
*/
+ #ifdef CONFIG_PTP
+ set_pud(args->pudp, __pud(0));
+ #else
WRITE_ONCE(*args->pudp, __pud(0));
+ #endif
WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot));
WARN_ON(!pud_clear_huge(args->pudp));
pud = READ_ONCE(*args->pudp);
@@ -511,7 +519,11 @@ static void __init pud_clear_tests(struct pgtable_debug_args *args)
pr_debug("Validating PUD clear\n");
pud = __pud(pud_val(pud) | RANDOM_ORVALUE);
+ #ifdef CONFIG_PTP
+ set_pud(args->pudp, pud);
+ #else
WRITE_ONCE(*args->pudp, pud);
+ #endif
pud_clear(args->pudp);
pud = READ_ONCE(*args->pudp);
WARN_ON(!pud_none(pud));
@@ -548,7 +560,11 @@ static void __init p4d_clear_tests(struct pgtable_debug_args *args)
pr_debug("Validating P4D clear\n");
p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE);
+ #ifdef CONFIG_PTP
+ set_p4d(args->p4dp, p4d);
+ #else
WRITE_ONCE(*args->p4dp, p4d);
+ #endif
p4d_clear(args->p4dp);
p4d = READ_ONCE(*args->p4dp);
WARN_ON(!p4d_none(p4d));
@@ -582,7 +598,11 @@ static void __init pgd_clear_tests(struct pgtable_debug_args *args)
pr_debug("Validating PGD clear\n");
pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE);
+ #ifdef CONFIG_PTP
+ set_pgd(args->pgdp, pgd);
+ #else
WRITE_ONCE(*args->pgdp, pgd);
+ #endif
pgd_clear(args->pgdp);
pgd = READ_ONCE(*args->pgdp);
WARN_ON(!pgd_none(pgd));
@@ -650,7 +670,11 @@ static void __init pmd_clear_tests(struct pgtable_debug_args *args)
pr_debug("Validating PMD clear\n");
pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE);
+ #ifdef CONFIG_PTP
+ set_pmd(args->pmdp, pmd);
+ #else
WRITE_ONCE(*args->pmdp, pmd);
+ #endif
pmd_clear(args->pmdp);
pmd = READ_ONCE(*args->pmdp);
WARN_ON(!pmd_none(pmd));
diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c
index ce06b2884789..a039c7a50ec5 100644
--- a/mm/early_ioremap.c
+++ b/mm/early_ioremap.c
@@ -147,7 +147,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
if (after_paging_init)
__late_set_fixmap(idx, phys_addr, prot);
else
+ #ifdef CONFIG_PTP
+ __iee_set_fixmap_pre_init(idx, phys_addr, prot);
+ #else
__early_set_fixmap(idx, phys_addr, prot);
+ #endif
phys_addr += PAGE_SIZE;
--idx;
--nrpages;
@@ -199,13 +203,66 @@ void __init early_iounmap(void __iomem *addr, unsigned long size)
if (after_paging_init)
__late_clear_fixmap(idx);
else
+ #ifdef CONFIG_PTP
+ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR);
+ #else
__early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR);
+ #endif
--idx;
--nrpages;
}
prev_map[slot] = NULL;
}
+#ifdef CONFIG_PTP
+void __init early_iounmap_after_init(void __iomem *addr, unsigned long size)
+{
+ unsigned long virt_addr;
+ unsigned long offset;
+ unsigned int nrpages;
+ enum fixed_addresses idx;
+ int i, slot;
+
+ slot = -1;
+ for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
+ if (prev_map[i] == addr) {
+ slot = i;
+ break;
+ }
+ }
+
+ if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n",
+ addr, size))
+ return;
+
+ if (WARN(prev_size[slot] != size,
+ "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
+ addr, size, slot, prev_size[slot]))
+ return;
+
+ WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n",
+ addr, size, slot);
+
+ virt_addr = (unsigned long)addr;
+ if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)))
+ return;
+
+ offset = offset_in_page(virt_addr);
+ nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
+
+ idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
+ while (nrpages > 0) {
+ if (after_paging_init)
+ __late_clear_fixmap(idx);
+ else
+ __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR);
+ --idx;
+ --nrpages;
+ }
+ prev_map[slot] = NULL;
+}
+#endif
+
/* Remap an IO device */
void __init __iomem *
early_ioremap(resource_size_t phys_addr, unsigned long size)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 763bb25e4f99..80bb2c0abeda 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -39,6 +39,10 @@
#include <linux/memory-tiers.h>
#include <linux/compat.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"
@@ -2489,6 +2493,10 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
unsigned long addr;
pte_t *pte;
int i;
+ #ifdef CONFIG_PTP
+ pte_t *ptep;
+ unsigned long iee_addr;
+ #endif
/*
* Leave pmd empty until pte is filled note that it is fine to delay
@@ -2501,7 +2509,14 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
pgtable = pgtable_trans_huge_withdraw(mm, pmd);
- pmd_populate(mm, &_pmd, pgtable);
+ #ifdef CONFIG_PTP
+ ptep = (pte_t *)page_address(pgtable);
+ iee_addr = __phys_to_iee(__pa(ptep));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)ptep);
+ #endif
+ //pmd_populate(mm, &_pmd, pgtable);
+ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE);
pte = pte_offset_map(&_pmd, haddr);
VM_BUG_ON(!pte);
@@ -2534,6 +2549,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr;
pte_t *pte;
int i;
+ #ifdef CONFIG_PTP
+ pte_t *ptep;
+ unsigned long iee_addr;
+ #endif
VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
@@ -2671,7 +2690,14 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
* This's critical for some architectures (Power).
*/
pgtable = pgtable_trans_huge_withdraw(mm, pmd);
- pmd_populate(mm, &_pmd, pgtable);
+ #ifdef CONFIG_PTP
+ ptep = (pte_t *)page_to_virt(pgtable);
+ iee_addr = __phys_to_iee(__pa(ptep));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)ptep);
+ #endif
+ //pmd_populate(mm, &_pmd, pgtable);
+ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE);
pte = pte_offset_map(&_pmd, haddr);
VM_BUG_ON(!pte);
diff --git a/mm/init-mm.c b/mm/init-mm.c
index 24c809379274..07d060fca6f0 100644
--- a/mm/init-mm.c
+++ b/mm/init-mm.c
@@ -55,3 +55,20 @@ void setup_initial_init_mm(void *start_code, void *end_code,
init_mm.end_data = (unsigned long)end_data;
init_mm.brk = (unsigned long)brk;
}
+
+#ifdef CONFIG_KOI
+/*
+ * This is used to init ko_mm when creating pgtable for a ko to be isolated
+ * the ko_mm belongs to a specific ko, pgdp is allocated by koi_pgd_alloc
+ */
+void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp) {
+ ko_mm->mm_rb = RB_ROOT;
+ ko_mm->pgd = pgdp;
+ ko_mm->mm_users = (atomic_t)ATOMIC_INIT(2);
+ ko_mm->mm_count = (atomic_t)ATOMIC_INIT(1);
+ ko_mm->mmap_lock = (struct rw_semaphore)__RWSEM_INITIALIZER(ko_mm->mmap_lock);
+ ko_mm->page_table_lock = __SPIN_LOCK_UNLOCKED(ko_mm.page_table_lock);
+ ko_mm->arg_lock = __SPIN_LOCK_UNLOCKED(ko_mm->arg_lock);
+ ko_mm->mmlist = (struct list_head)LIST_HEAD_INIT(ko_mm->mmlist);
+}
+#endif
diff --git a/mm/memory.c b/mm/memory.c
index 4ef917a182f9..28da89a19e30 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -80,6 +80,10 @@
#include <linux/userswap.h>
#include <linux/dynamic_pool.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
#include <trace/events/kmem.h>
#include <asm/io.h>
@@ -5872,6 +5876,11 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
spin_lock(&mm->page_table_lock);
if (!p4d_present(*p4d)) {
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(new));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)new);
+ #endif
mm_inc_nr_puds(mm);
smp_wmb(); /* See comment in pmd_install() */
p4d_populate(mm, p4d, new);
@@ -5896,6 +5905,11 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
ptl = pud_lock(mm, pud);
if (!pud_present(*pud)) {
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr = __phys_to_iee(__pa(new));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)new);
+ #endif
mm_inc_nr_pmds(mm);
smp_wmb(); /* See comment in pmd_install() */
pud_populate(mm, pud, new);
diff --git a/mm/slub.c b/mm/slub.c
index ee3e32cdb7fd..20a45a7feed5 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -42,6 +42,11 @@
#include <kunit/test-bug.h>
#include <linux/sort.h>
+#ifdef CONFIG_IEE
+#include <linux/iee-func.h>
+#include <asm/iee-access.h>
+#endif
+
#include <linux/debugfs.h>
#include <trace/events/kmem.h>
@@ -317,6 +322,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
/*
* Tracking user of a slab.
*/
+#ifndef CONFIG_IEE
#define TRACK_ADDRS_COUNT 16
struct track {
unsigned long addr; /* Called from address */
@@ -329,6 +335,7 @@ struct track {
};
enum track_item { TRACK_ALLOC, TRACK_FREE };
+#endif
#ifdef SLAB_SUPPORTS_SYSFS
static int sysfs_slab_add(struct kmem_cache *);
@@ -379,7 +386,9 @@ static struct workqueue_struct *flushwq;
* freeptr_t represents a SLUB freelist pointer, which might be encoded
* and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
*/
+#ifndef CONFIG_IEE
typedef struct { unsigned long v; } freeptr_t;
+#endif
/*
* Returns freelist pointer (ptr). With hardening, this is obfuscated
@@ -464,7 +473,14 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
#endif
freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr));
+ else
+ *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
+ #else
*(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
+ #endif
}
/* Loop over all objects in a slab */
@@ -809,7 +825,34 @@ static void set_track_update(struct kmem_cache *s, void *object,
depot_stack_handle_t handle)
{
struct track *p = get_track(s, object, alloc);
+#ifdef CONFIG_IEE
+ struct track tmp;
+#endif
+#ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ {
+ tmp = *p;
+ #ifdef CONFIG_STACKDEPOT
+ tmp.handle = handle;
+ #endif
+ tmp.addr = addr;
+ tmp.cpu = smp_processor_id();
+ tmp.pid = current->pid;
+ tmp.when = jiffies;
+ iee_set_track(p,&tmp);
+ }
+ else
+ {
+ #ifdef CONFIG_STACKDEPOT
+ p->handle = handle;
+ #endif
+ p->addr = addr;
+ p->cpu = smp_processor_id();
+ p->pid = current->pid;
+ p->when = jiffies;
+ }
+#else
#ifdef CONFIG_STACKDEPOT
p->handle = handle;
#endif
@@ -817,6 +860,7 @@ static void set_track_update(struct kmem_cache *s, void *object,
p->cpu = smp_processor_id();
p->pid = current->pid;
p->when = jiffies;
+#endif
}
static __always_inline void set_track(struct kmem_cache *s, void *object,
@@ -835,7 +879,14 @@ static void init_tracking(struct kmem_cache *s, void *object)
return;
p = get_track(s, object, TRACK_ALLOC);
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ iee_memset(p, 0, 2*sizeof(struct track));
+ else
+ memset(p, 0, 2*sizeof(struct track));
+ #else
memset(p, 0, 2*sizeof(struct track));
+ #endif
}
static void print_track(const char *s, struct track *t, unsigned long pr_time)
@@ -1045,7 +1096,14 @@ static void init_object(struct kmem_cache *s, void *object, u8 val)
unsigned int poison_size = s->object_size;
if (s->flags & SLAB_RED_ZONE) {
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ iee_memset(p - s->red_left_pad, val, s->red_left_pad);
+ else
+ memset(p - s->red_left_pad, val, s->red_left_pad);
+ #else
memset(p - s->red_left_pad, val, s->red_left_pad);
+ #endif
if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
/*
@@ -1058,12 +1116,34 @@ static void init_object(struct kmem_cache *s, void *object, u8 val)
}
if (s->flags & __OBJECT_POISON) {
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ {
+ iee_memset(p, POISON_FREE, poison_size - 1);
+ iee_memset(&p[poison_size - 1], POISON_END, 1);
+ }
+ else
+ {
+ memset(p, POISON_FREE, poison_size - 1);
+ p[poison_size - 1] = POISON_END;
+ }
+ #else
memset(p, POISON_FREE, poison_size - 1);
p[poison_size - 1] = POISON_END;
+ #endif
}
- if (s->flags & SLAB_RED_ZONE)
+ if (s->flags & SLAB_RED_ZONE) {
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ iee_memset(p + poison_size, val, s->inuse - poison_size);
+ else
+ memset(p + poison_size, val, s->inuse - poison_size);
+ #else
memset(p + poison_size, val, s->inuse - poison_size);
+ #endif
+
+ }
}
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
@@ -1433,7 +1513,14 @@ void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
return;
metadata_access_enable();
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ iee_memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
+ else
+ memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
+ #else
memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
+ #endif
metadata_access_disable();
}
@@ -2015,6 +2102,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
void *start, *p, *next;
int idx;
bool shuffle;
+ #ifdef CONFIG_IEE
+ unsigned int order;
+ #endif
flags &= gfp_allowed_mask;
@@ -2029,6 +2119,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
slab = alloc_slab_page(alloc_gfp, node, oo);
+ #ifdef CONFIG_IEE
+ order = oo_order(oo);
+ #endif
if (unlikely(!slab)) {
oo = s->min;
alloc_gfp = flags;
@@ -2037,6 +2130,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
* Try a lower order alloc if possible
*/
slab = alloc_slab_page(alloc_gfp, node, oo);
+ #ifdef CONFIG_IEE
+ order = oo_order(oo);
+ #endif
if (unlikely(!slab))
return NULL;
stat(s, ORDER_FALLBACK);
@@ -2046,6 +2142,45 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
slab->inuse = 0;
slab->frozen = 0;
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ {
+ int i;
+ for(i = 0; i < (0x1 << order); i++)
+ {
+ unsigned long iee_addr = __phys_to_iee(page_to_phys(folio_page(slab_folio(slab), i)));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)page_address(folio_page(slab_folio(slab), i)));
+ }
+ }
+
+ // If the page belongs to a task_struct, alloc token for it and set iee&lm va.
+ if(strcmp(s->name, "task_struct") == 0)
+ {
+ int i;
+ for(i = 0; i < (0x1 << order); i++)
+ {
+ void *token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(slab_folio(slab), i)));
+ // Get lm va of the page.
+ void *alloc_token = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+ iee_set_token_page_valid(token_addr, alloc_token);
+ set_iee_page_valid(__phys_to_iee(__pa(alloc_token)));
+ iee_set_logical_mem_ro((unsigned long)alloc_token);
+ }
+ }
+ #else
+ #ifdef CONFIG_KOI
+ if (strcmp(s->name, "task_struct") == 0) {
+ int i;
+ for (i = 0; i < (0x1 << order); i++) {
+ void *token_addr = __phys_to_virt(page_to_phys(page + i)) + koi_offset;
+ void *alloc_token = __get_free_page(GFP_KERNEL | __GFP_ZERO);
+ koi_add_page_mapping(token_addr, alloc_token);
+ }
+ }
+ #endif
+ #endif
+
account_slab(slab, oo_order(oo), s, flags);
slab->slab_cache = s;
@@ -2098,6 +2233,67 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)
__folio_clear_slab(folio);
mm_account_reclaimed_pages(pages);
unaccount_slab(slab, order, s);
+
+ #ifdef CONFIG_IEE
+ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0)
+ {
+ int i;
+ for(i = 0; i < (0x1 << order); i++)
+ {
+ unsigned long iee_addr = __phys_to_iee(page_to_phys(folio_page(folio, i)));
+ set_iee_page_invalid(iee_addr);
+ iee_set_logical_mem_rw((unsigned long)page_address(folio_page(folio, i)));
+ }
+ }
+ // If the page containing this token is empty, free it and restore iee&lm va.
+ if(strcmp(s->name, "task_struct") == 0)
+ {
+ int i;
+ for(i = 0; i < (0x1 << order); i++)
+ {
+ void *token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(folio, i)));
+ unsigned long flags;
+ unsigned long res;
+ local_irq_save(flags);
+ asm volatile("at s1e1r, %0"::"r"(token_addr));
+ isb();
+ res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+ if(!(res & 0x1))
+ {
+ // Get lm va of the page.
+ void *token_page = __va(res & PTE_ADDR_MASK);
+ iee_set_token_page_invalid(token_addr);
+ set_iee_page_invalid(__phys_to_iee(__pa(token_page)));
+ iee_set_logical_mem_rw((unsigned long)token_page);
+ free_page((unsigned long)token_page);
+ }
+ }
+ }
+ #else
+ #ifdef CONFIG_KOI
+ if(strcmp(s->name, "task_struct") == 0)
+ {
+ int i;
+ for(i = 0; i < (0x1 << order); i++)
+ {
+ void *token_addr = __phys_to_virt(page_to_phys(page + i)) + koi_offset;
+ unsigned long flags;
+ local_irq_save(flags);
+ asm volatile("at s1e1r, %0"::"r"(token_addr));
+ isb();
+ unsigned long res = read_sysreg(par_el1);
+ local_irq_restore(flags);
+ if(!(res & 0x1))
+ {
+ koi_remove_page_mapping(token_addr);
+ free_page(__va(res & PTE_ADDR_MASK));
+ }
+ }
+ }
+ #endif
+ #endif
+
__free_pages(&folio->page, order);
}
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index a2cbe44c48e1..7cf05d293312 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -28,6 +28,10 @@
#include <linux/vmalloc.h>
#include <linux/sched.h>
+#ifdef CONFIG_PTP
+#include <linux/iee-func.h>
+#endif
+
#include <asm/dma.h>
#include <asm/pgalloc.h>
@@ -146,6 +150,9 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct page *reuse)
{
pte_t *pte = pte_offset_kernel(pmd, addr);
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
if (pte_none(ptep_get(pte))) {
pte_t entry;
void *p;
@@ -167,6 +174,11 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
get_page(reuse);
p = page_to_virt(reuse);
}
+#ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(p));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)p);
+#endif
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
set_pte_at(&init_mm, addr, pte, entry);
}
@@ -176,11 +188,20 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
{
void *p = vmemmap_alloc_block(size, node);
+ #ifdef CONFIG_PTP
+ unsigned long iee_addr;
+ #endif
if (!p)
return NULL;
memset(p, 0, size);
+ #ifdef CONFIG_PTP
+ iee_addr = __phys_to_iee(__pa(p));
+ set_iee_page_valid(iee_addr);
+ iee_set_logical_mem_ro((unsigned long)p);
+ #endif
+
return p;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index e6058942a084..27a006728009 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3431,7 +3431,7 @@ static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private)
if (WARN_ON_ONCE(pfn_valid(pfn)))
return -EINVAL;
-
+
ptent = pte_mkspecial(pfn_pte(pfn, data->prot));
set_pte_at(&init_mm, addr, pte, ptent);
diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c
index c42ddd85ff1f..4714b4f2be08 100644
--- a/net/dns_resolver/dns_key.c
+++ b/net/dns_resolver/dns_key.c
@@ -34,6 +34,10 @@
#include <keys/user-type.h>
#include "internal.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
MODULE_DESCRIPTION("DNS Resolver");
MODULE_AUTHOR("Wang Lei");
MODULE_LICENSE("GPL");
@@ -365,8 +369,13 @@ static int __init init_dns_resolver(void)
/* instruct request_key() to use this special keyring as a cache for
* the results it looks up */
set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(cred,keyring);
+ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING);
+ #else
cred->thread_keyring = keyring;
cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
+ #endif
dns_resolver_cache = cred;
kdebug("DNS resolver keyring: %d\n", key_serial(keyring));
diff --git a/security/commoncap.c b/security/commoncap.c
index bc0521104197..d7d3b7cc13e8 100644
--- a/security/commoncap.c
+++ b/security/commoncap.c
@@ -26,6 +26,10 @@
#include <linux/personality.h>
#include <linux/mnt_idmapping.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
/*
* If a non-root user executes a setuid-root binary in
* !secure(SECURE_NOROOT) mode, then we raise capabilities.
@@ -266,6 +270,15 @@ int cap_capset(struct cred *new,
if (!cap_issubset(*effective, *permitted))
return -EPERM;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,*effective);
+ iee_set_cred_cap_inheritable(new,*inheritable);
+ iee_set_cred_cap_permitted(new,*permitted);
+
+ iee_set_cred_cap_ambient(new,cap_intersect(new->cap_ambient,
+ cap_intersect(*permitted,
+ *inheritable)));
+ #else
new->cap_effective = *effective;
new->cap_inheritable = *inheritable;
new->cap_permitted = *permitted;
@@ -277,6 +290,7 @@ int cap_capset(struct cred *new,
new->cap_ambient = cap_intersect(new->cap_ambient,
cap_intersect(*permitted,
*inheritable));
+ #endif
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EINVAL;
return 0;
@@ -601,9 +615,16 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps,
* pP' = (X & fP) | (pI & fI)
* The addition of pA' is handled later.
*/
+#ifdef CONFIG_CREDP
+ kernel_cap_t temp = new->cap_permitted;
+ temp.val = (new->cap_bset.val & caps->permitted.val) |
+ (new->cap_inheritable.val & caps->inheritable.val);
+ iee_set_cred_cap_permitted(new,temp);
+#else
new->cap_permitted.val =
(new->cap_bset.val & caps->permitted.val) |
(new->cap_inheritable.val & caps->inheritable.val);
+#endif
if (caps->permitted.val & ~new->cap_permitted.val)
/* insufficient to execute correctly */
@@ -726,7 +747,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file,
int rc = 0;
struct cpu_vfs_cap_data vcaps;
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = bprm->cred->cap_permitted;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_permitted(bprm->cred, tmp_cap);
+ } while (0);
+ #else
cap_clear(bprm->cred->cap_permitted);
+ #endif
if (!file_caps_enabled)
return 0;
@@ -757,7 +786,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file,
out:
if (rc)
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = bprm->cred->cap_permitted;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_permitted(bprm->cred, tmp_cap);
+ } while (0);
+ #else
cap_clear(bprm->cred->cap_permitted);
+ #endif
return rc;
}
@@ -809,8 +846,13 @@ static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap,
*/
if (__is_eff(root_uid, new) || __is_real(root_uid, new)) {
/* pP' = (cap_bset & ~0) | (pI & ~0) */
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_permitted(new,cap_combine(old->cap_bset,
+ old->cap_inheritable));
+ #else
new->cap_permitted = cap_combine(old->cap_bset,
old->cap_inheritable);
+ #endif
}
/*
* If only the real uid is 0, we do not set the effective bit.
@@ -919,34 +961,71 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file)
/* downgrade; they get no more than they had, and maybe less */
if (!ns_capable(new->user_ns, CAP_SETUID) ||
(bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_euid(new,new->uid);
+ iee_set_cred_egid(new,new->gid);
+ #else
new->euid = new->uid;
new->egid = new->gid;
+ #endif
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_permitted(new,cap_intersect(new->cap_permitted,
+ old->cap_permitted));
+ #else
new->cap_permitted = cap_intersect(new->cap_permitted,
old->cap_permitted);
+ #endif
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_fsuid(new,new->euid);
+ iee_set_cred_suid(new,new->euid);
+ iee_set_cred_fsgid(new,new->egid);
+ iee_set_cred_sgid(new,new->egid);
+ #else
new->suid = new->fsuid = new->euid;
new->sgid = new->fsgid = new->egid;
+ #endif
/* File caps or setid cancels ambient. */
if (has_fcap || is_setid)
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = new->cap_ambient;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_ambient(new, tmp_cap);
+ } while (0);
+ #else
cap_clear(new->cap_ambient);
+ #endif
/*
* Now that we've computed pA', update pP' to give:
* pP' = (X & fP) | (pI & fI) | pA'
*/
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_permitted(new,cap_combine(new->cap_permitted, new->cap_ambient));
+ #else
new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient);
+ #endif
/*
* Set pE' = (fE ? pP' : pA'). Because pA' is zero if fE is set,
* this is the same as pE' = (fE ? pP' : 0) | pA'.
*/
if (effective)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,new->cap_permitted);
+ #else
new->cap_effective = new->cap_permitted;
+ #endif
else
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,new->cap_ambient);
+ #else
new->cap_effective = new->cap_ambient;
+ #endif
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EPERM;
@@ -957,7 +1036,11 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file)
return ret;
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS));
+ #else
new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
+ #endif
if (WARN_ON(!cap_ambient_invariant_ok(new)))
return -EPERM;
@@ -1092,8 +1175,21 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old)
!uid_eq(new->euid, root_uid) &&
!uid_eq(new->suid, root_uid))) {
if (!issecure(SECURE_KEEP_CAPS)) {
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = new->cap_permitted;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_permitted(new, tmp_cap);
+ } while (0);
+ do {
+ kernel_cap_t tmp_cap = new->cap_effective;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_effective(new, tmp_cap);
+ } while (0);
+ #else
cap_clear(new->cap_permitted);
cap_clear(new->cap_effective);
+ #endif
}
/*
@@ -1101,12 +1197,32 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old)
* by exec to drop capabilities. We should make sure that
* this remains the case.
*/
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = new->cap_ambient;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_ambient(new, tmp_cap);
+ } while (0);
+ #else
cap_clear(new->cap_ambient);
+ #endif
}
if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid))
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = new->cap_effective;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_effective(new, tmp_cap);
+ } while (0);
+ #else
cap_clear(new->cap_effective);
+ #endif
if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,new->cap_permitted);
+ #else
new->cap_effective = new->cap_permitted;
+ #endif
}
/**
@@ -1142,13 +1258,22 @@ int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags)
if (!issecure(SECURE_NO_SETUID_FIXUP)) {
kuid_t root_uid = make_kuid(old->user_ns, 0);
if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,cap_drop_fs_set(new->cap_effective));
+ #else
new->cap_effective =
cap_drop_fs_set(new->cap_effective);
+ #endif
if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid))
+ #ifdef CONFIG_CREDP
+ iee_set_cred_cap_effective(new,cap_raise_fs_set(new->cap_effective,
+ new->cap_permitted));
+ #else
new->cap_effective =
cap_raise_fs_set(new->cap_effective,
new->cap_permitted);
+ #endif
}
break;
@@ -1243,7 +1368,15 @@ static int cap_prctl_drop(unsigned long cap)
new = prepare_creds();
if (!new)
return -ENOMEM;
+ #ifdef CONFIG_CREDP
+ {
+ kernel_cap_t tmp = new->cap_bset;
+ cap_lower(tmp, cap);
+ iee_set_cred_cap_bset(new, tmp);
+ }
+ #else
cap_lower(new->cap_bset, cap);
+ #endif
return commit_creds(new);
}
@@ -1319,7 +1452,11 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
new = prepare_creds();
if (!new)
return -ENOMEM;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_securebits(new,arg2);
+ #else
new->securebits = arg2;
+ #endif
return commit_creds(new);
case PR_GET_SECUREBITS:
@@ -1338,9 +1475,17 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
if (!new)
return -ENOMEM;
if (arg2)
+ #ifdef CONFIG_CREDP
+ iee_set_cred_securebits(new,new->securebits | issecure_mask(SECURE_KEEP_CAPS));
+ #else
new->securebits |= issecure_mask(SECURE_KEEP_CAPS);
+ #endif
else
+ #ifdef CONFIG_CREDP
+ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS));
+ #else
new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
+ #endif
return commit_creds(new);
case PR_CAP_AMBIENT:
@@ -1351,7 +1496,15 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
new = prepare_creds();
if (!new)
return -ENOMEM;
+ #ifdef CONFIG_CREDP
+ do {
+ kernel_cap_t tmp_cap = new->cap_ambient;
+ tmp_cap.val = 0;
+ iee_set_cred_cap_ambient(new, tmp_cap);
+ } while (0);
+ #else
cap_clear(new->cap_ambient);
+ #endif
return commit_creds(new);
}
@@ -1375,9 +1528,25 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
if (!new)
return -ENOMEM;
if (arg2 == PR_CAP_AMBIENT_RAISE)
+ #ifdef CONFIG_CREDP
+ {
+ kernel_cap_t tmp = new->cap_ambient;
+ cap_raise(tmp, arg3);
+ iee_set_cred_cap_ambient(new, tmp);
+ }
+ #else
cap_raise(new->cap_ambient, arg3);
+ #endif
else
+ #ifdef CONFIG_CREDP
+ {
+ kernel_cap_t tmp = new->cap_ambient;
+ cap_lower(tmp, arg3);
+ iee_set_cred_cap_ambient(new, tmp);
+ }
+ #else
cap_lower(new->cap_ambient, arg3);
+ #endif
return commit_creds(new);
}
diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c
index 19be69fa4d05..6cb164dfc19b 100644
--- a/security/keys/keyctl.c
+++ b/security/keys/keyctl.c
@@ -23,6 +23,9 @@
#include <linux/uaccess.h>
#include <keys/request_key_auth-type.h>
#include "internal.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
#define KEY_MAX_DESC_SIZE 4096
@@ -1155,7 +1158,11 @@ static int keyctl_change_reqkey_auth(struct key *key)
return -ENOMEM;
key_put(new->request_key_auth);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_request_key_auth(new,key_get(key));
+ #else
new->request_key_auth = key_get(key);
+ #endif
return commit_creds(new);
}
@@ -1432,7 +1439,11 @@ long keyctl_set_reqkey_keyring(int reqkey_defl)
}
set:
+ #ifdef CONFIG_CREDP
+ iee_set_cred_jit_keyring(new,reqkey_defl);
+ #else
new->jit_keyring = reqkey_defl;
+ #endif
commit_creds(new);
return old_setting;
error:
@@ -1644,9 +1655,17 @@ long keyctl_session_to_parent(void)
cred = cred_alloc_blank();
if (!cred)
goto error_keyring;
+ #ifdef CONFIG_CREDP
+ newwork = (struct rcu_head *)(cred->rcu.func);
+ #else
newwork = &cred->rcu;
+ #endif
+ #ifdef CONFIG_CREDP
+ iee_set_cred_session_keyring(cred,key_ref_to_ptr(keyring_r));
+ #else
cred->session_keyring = key_ref_to_ptr(keyring_r);
+ #endif
keyring_r = NULL;
init_task_work(newwork, key_change_session_keyring);
@@ -1705,7 +1724,11 @@ long keyctl_session_to_parent(void)
write_unlock_irq(&tasklist_lock);
rcu_read_unlock();
if (oldwork)
+ #ifdef CONFIG_CREDP
+ put_cred(*(struct cred **)(oldwork + 1));
+ #else
put_cred(container_of(oldwork, struct cred, rcu));
+ #endif
if (newwork)
put_cred(cred);
return ret;
diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
index b5d5333ab330..aaa2a2347d84 100644
--- a/security/keys/process_keys.c
+++ b/security/keys/process_keys.c
@@ -19,6 +19,10 @@
#include <keys/request_key_auth-type.h>
#include "internal.h"
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
+
/* Session keyring create vs join semaphore */
static DEFINE_MUTEX(key_session_mutex);
@@ -232,7 +236,11 @@ int install_thread_keyring_to_cred(struct cred *new)
if (IS_ERR(keyring))
return PTR_ERR(keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_thread_keyring(new,keyring);
+ #else
new->thread_keyring = keyring;
+ #endif
return 0;
}
@@ -279,7 +287,11 @@ int install_process_keyring_to_cred(struct cred *new)
if (IS_ERR(keyring))
return PTR_ERR(keyring);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_process_keyring(new,keyring);
+ #else
new->process_keyring = keyring;
+ #endif
return 0;
}
@@ -338,7 +350,11 @@ int install_session_keyring_to_cred(struct cred *cred, struct key *keyring)
/* install the keyring */
old = cred->session_keyring;
+ #ifdef CONFIG_CREDP
+ iee_set_cred_session_keyring(cred,keyring);
+ #else
cred->session_keyring = keyring;
+ #endif
if (old)
key_put(old);
@@ -911,7 +927,11 @@ long join_session_keyring(const char *name)
void key_change_session_keyring(struct callback_head *twork)
{
const struct cred *old = current_cred();
+ #ifdef CONFIG_CREDP
+ struct cred *new = *(struct cred **)(twork + 1);
+ #else
struct cred *new = container_of(twork, struct cred, rcu);
+ #endif
if (unlikely(current->flags & PF_EXITING)) {
put_cred(new);
@@ -925,6 +945,38 @@ void key_change_session_keyring(struct callback_head *twork)
return;
}
+ /* If get_ucounts fails more bits are needed in the refcount */
+ if (unlikely(!get_ucounts(old->ucounts))) {
+ WARN_ONCE(1, "In %s get_ucounts failed\n", __func__);
+ put_cred(new);
+ return;
+ }
+
+ #ifdef CONFIG_CREDP
+ iee_set_cred_uid(new,old-> uid);
+ iee_set_cred_euid(new,old-> euid);
+ iee_set_cred_suid(new,old-> suid);
+ iee_set_cred_fsuid(new,old->fsuid);
+ iee_set_cred_gid(new,old-> gid);
+ iee_set_cred_egid(new,old-> egid);
+ iee_set_cred_sgid(new,old-> sgid);
+ iee_set_cred_fsgid(new,old->fsgid);
+ iee_set_cred_user(new,get_uid(old->user));
+ iee_set_cred_ucounts(new, old->ucounts);
+ iee_set_cred_user_ns(new,get_user_ns(old->user_ns));
+ iee_set_cred_group_info(new,get_group_info(old->group_info));
+
+ iee_set_cred_securebits(new,old->securebits);
+ iee_set_cred_cap_inheritable(new,old->cap_inheritable);
+ iee_set_cred_cap_permitted(new,old->cap_permitted);
+ iee_set_cred_cap_effective(new,old->cap_effective);
+ iee_set_cred_cap_ambient(new,old->cap_ambient);
+ iee_set_cred_cap_bset(new,old->cap_bset);
+
+ iee_set_cred_jit_keyring(new,old->jit_keyring);
+ iee_set_cred_thread_keyring(new,key_get(old->thread_keyring));
+ iee_set_cred_process_keyring(new,key_get(old->process_keyring));
+ #else
new-> uid = old-> uid;
new-> euid = old-> euid;
new-> suid = old-> suid;
@@ -948,6 +1000,7 @@ void key_change_session_keyring(struct callback_head *twork)
new->jit_keyring = old->jit_keyring;
new->thread_keyring = key_get(old->thread_keyring);
new->process_keyring = key_get(old->process_keyring);
+ #endif
security_transfer_creds(new, old);
diff --git a/security/security.c b/security/security.c
index 407b51719f79..74ffd7ea3f37 100644
--- a/security/security.c
+++ b/security/security.c
@@ -30,6 +30,9 @@
#include <linux/string.h>
#include <linux/msg.h>
#include <net/flow.h>
+#ifdef CONFIG_CREDP
+#include <asm/iee-cred.h>
+#endif
/* How many LSMs were built into the kernel? */
#define LSM_COUNT (__end_lsm_info - __start_lsm_info)
@@ -570,11 +573,19 @@ EXPORT_SYMBOL(unregister_blocking_lsm_notifier);
static int lsm_cred_alloc(struct cred *cred, gfp_t gfp)
{
if (blob_sizes.lbs_cred == 0) {
+ #ifdef CONFIG_CREDP
+ iee_set_cred_security(cred,NULL);
+ #else
cred->security = NULL;
+ #endif
return 0;
}
+ #ifdef CONFIG_CREDP
+ iee_set_cred_security(cred,kzalloc(blob_sizes.lbs_cred, gfp));
+ #else
cred->security = kzalloc(blob_sizes.lbs_cred, gfp);
+ #endif
if (cred->security == NULL)
return -ENOMEM;
return 0;
@@ -2950,7 +2961,11 @@ void security_cred_free(struct cred *cred)
call_void_hook(cred_free, cred);
kfree(cred->security);
+ #ifdef CONFIG_CREDP
+ iee_set_cred_security(cred,NULL);
+ #else
cred->security = NULL;
+ #endif
}
/**
--
2.34.1
Reference in New Issue View Git Blame Copy Permalink