From 5460d7d1997a16ca73ae84ba6c42525a7b419481 Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Sun, 28 Sep 2025 19:28:24 +0800 Subject: [PATCH 1/8] Date: Sun Sep 27 19:28:10 2025 HAOC: Add support for x86 Isolated Execution Environment community inclusion category: feature ------------------------------------------------- Support Isolated Execution Environment for x86. IEE depends on CR0.wp --- arch/x86/Kconfig | 5 +- arch/x86/include/asm/haoc/haoc-def.h | 21 ++++ arch/x86/include/asm/haoc/haoc.h | 20 ++++ arch/x86/include/asm/haoc/iee-access.h | 37 ++++++ arch/x86/include/asm/haoc/iee-func.h | 16 +++ arch/x86/include/asm/haoc/iee.h | 33 ++++++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/asm-offsets.c | 7 ++ arch/x86/kernel/cpu/common.c | 9 ++ arch/x86/kernel/haoc/Makefile | 2 + arch/x86/kernel/haoc/haoc.c | 19 +++ arch/x86/kernel/haoc/iee/Makefile | 2 + arch/x86/kernel/haoc/iee/iee-func.c | 20 ++++ arch/x86/kernel/haoc/iee/iee-gate.S | 73 ++++++++++++ arch/x86/kernel/haoc/iee/iee-init.c | 158 +++++++++++++++++++++++++ arch/x86/kernel/haoc/iee/iee.c | 32 +++++ arch/x86/mm/init_64.c | 12 ++ 17 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 arch/x86/include/asm/haoc/haoc-def.h create mode 100644 arch/x86/include/asm/haoc/haoc.h create mode 100644 arch/x86/include/asm/haoc/iee-access.h create mode 100644 arch/x86/include/asm/haoc/iee-func.h create mode 100644 arch/x86/include/asm/haoc/iee.h create mode 100644 arch/x86/kernel/haoc/Makefile create mode 100644 arch/x86/kernel/haoc/haoc.c create mode 100644 arch/x86/kernel/haoc/iee/Makefile create mode 100644 arch/x86/kernel/haoc/iee/iee-func.c create mode 100644 arch/x86/kernel/haoc/iee/iee-gate.S create mode 100644 arch/x86/kernel/haoc/iee/iee-init.c create mode 100644 arch/x86/kernel/haoc/iee/iee.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5f86dfa16658..7ea816a0b77b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1551,7 +1551,10 @@ config AMD_MEM_ENCRYPT Say yes to enable support for the encryption of system memory. This requires an AMD processor that supports Secure Memory Encryption (SME). - + +config IEE + depends on X86_64 + def_bool n # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h new file mode 100644 index 000000000000..55b5d37ff762 --- /dev/null +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#ifndef _LINUX_HAOC_DEF_H +#define _LINUX_HAOC_DEF_H + +enum { + IEE_OP_MEMCPY, + IEE_OP_MEMSET, + IEE_OP_SET_FREEPTR, + IEE_OP_TEST_CLEAR_BIT, + IEE_FLAG_END +}; + +#endif diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h new file mode 100644 index 000000000000..e6d1193c4f1a --- /dev/null +++ b/arch/x86/include/asm/haoc/haoc.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#ifndef _LINUX_HAOC_H +#define _LINUX_HAOC_H + +#include + +void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); +void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); +void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); +unsigned long _iee_test_and_clear_bit(unsigned long __unused, + long nr, unsigned long *addr); +#endif diff --git a/arch/x86/include/asm/haoc/iee-access.h b/arch/x86/include/asm/haoc/iee-access.h new file mode 100644 index 000000000000..7ca29a8d8b19 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-access.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#ifndef _LINUX_IEE_ACCESS_H +#define _LINUX_IEE_ACCESS_H + +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static inline void iee_memcpy(void *dst, const void *src, size_t n) +{ + iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); +} + +static inline void iee_memset(void *ptr, int data, size_t n) +{ + iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); +} + +static inline void iee_set_freeptr(void **pptr, void *ptr) +{ + iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); +} + +static inline unsigned long iee_test_and_clear_bit(long nr, unsigned long *addr) +{ + return iee_rw_gate(IEE_OP_TEST_CLEAR_BIT, nr, addr); +} + +#endif diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h new file mode 100644 index 000000000000..42455aa11615 --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#ifndef _LINUX_IEE_FUNC_H +#define _LINUX_IEE_FUNC_H + +extern void set_iee_page(unsigned long addr, unsigned int order); +extern void unset_iee_page(unsigned long addr, unsigned int order); + +#endif /* _LINUX_IEE_FUNC_H */ diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h new file mode 100644 index 000000000000..95472e0d6a6a --- /dev/null +++ b/arch/x86/include/asm/haoc/iee.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#ifndef _LINUX_IEE_H +#define _LINUX_IEE_H + +#include + +extern unsigned long IEE_OFFSET; +#define __iee_pa(x) (__pa(x - IEE_OFFSET)) +#define __phys_to_iee(x) ((void *)(__va(x) + IEE_OFFSET)) +#define __page_to_phys(x) (page_to_pfn(x) << PAGE_SHIFT) +#define __page_to_iee(x) ((unsigned long)(__phys_to_iee(__page_to_phys(x)))) +#define __slab_to_iee(x) (__page_to_iee(folio_page(slab_folio(x), 0))) +#define __addr_to_iee(x) (__phys_to_iee(__pa(x))) + +#define IEE_DATA_ORDER (PMD_SHIFT - PAGE_SHIFT) +#define IEE_STACK_ORDER 0 +struct iee_stack { + void *stack; +}; + +DECLARE_PER_CPU(struct iee_stack, iee_stacks); + +extern void *alloc_low_pages(unsigned int num); +extern void iee_init(void); +#endif diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index f4717d0da6c7..84a58b289edd 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -158,3 +158,4 @@ ifeq ($(CONFIG_X86_64),y) endif obj-$(CONFIG_HYGON_CSV) += csv.o +obj-$(CONFIG_IEE) += haoc/ diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 50383bc46dd7..acb79f30cede 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -25,6 +25,9 @@ #include #endif +#ifdef CONFIG_IEE +#include +#endif #ifdef CONFIG_X86_32 # include "asm-offsets_32.c" #else @@ -127,4 +130,8 @@ static void __used common(void) OFFSET(ARIA_CTX_rounds, aria_ctx, rounds); #endif +#ifdef CONFIG_IEE + /* Offset for fields in iee_stack */ + OFFSET(IEE_STACK, iee_stack, stack); +#endif } diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 46c2a0d7b4ea..f6c4b88f6301 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -611,7 +611,16 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) else wrmsrl(MSR_IA32_S_CET, 0); + #ifndef CONFIG_IEE + /* + * NOTE: IEE relies on CR0.WP (Write Protection). + * According to Intel SDM Vol.3(Section 2.5): + * This flag must be set before software can set CR4.CET, + * and it cannot be cleared as long as CR4.CET = 1. + * Therefore, IEE does not enable CR4.CET during kernel boot. + */ cr4_set_bits(X86_CR4_CET); + #endif if (kernel_ibt && ibt_selftest()) { pr_err("IBT selftest: Failed!\n"); diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile new file mode 100644 index 000000000000..eada289a033f --- /dev/null +++ b/arch/x86/kernel/haoc/Makefile @@ -0,0 +1,2 @@ +obj-y += haoc.o +obj-y += iee/ diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c new file mode 100644 index 000000000000..4676f3c0454e --- /dev/null +++ b/arch/x86/kernel/haoc/haoc.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#include + +typedef void (*iee_func)(void); +iee_func iee_funcs[] = { + (iee_func)_iee_memcpy, + (iee_func)_iee_memset, + (iee_func)_iee_set_freeptr, + (iee_func)_iee_test_and_clear_bit, + NULL +}; diff --git a/arch/x86/kernel/haoc/iee/Makefile b/arch/x86/kernel/haoc/iee/Makefile new file mode 100644 index 000000000000..48c68356002d --- /dev/null +++ b/arch/x86/kernel/haoc/iee/Makefile @@ -0,0 +1,2 @@ +obj-y += iee-gate.o iee-init.o iee.o iee-func.o +ccflags-y += -I$(srctree)/mm diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c new file mode 100644 index 000000000000..7b669401c624 --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#include + +void set_iee_page(unsigned long addr, unsigned int order) +{ + set_memory_ro(addr, 1 << order); +} + +void unset_iee_page(unsigned long addr, unsigned int order) +{ + set_memory_rw(addr, 1 << order); +} diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S new file mode 100644 index 000000000000..8d945acd214e --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#include +#include +#include +#include +#include + +#define X86_CR4_SMEP_SMAP (X86_CR4_SMEP | X86_CR4_SMAP) + +/* + * scratch_reg would be changed, + * caller should dertimine if scratch_reg should be saved and restored. + */ +.macro DISABLE_WP scratch_reg:req + /* Disable write protection*/ + movq %cr0, %\scratch_reg + andq $(~X86_CR0_WP), %\scratch_reg + movq %\scratch_reg, %cr0 +.endm + +.macro ENABLE_WP scratch_reg:req + /* Enable write protection */ + movq %cr0, %\scratch_reg +1: + orq $X86_CR0_WP, %\scratch_reg + movq %\scratch_reg, %cr0 + testq $X86_CR0_WP, %\scratch_reg + je 1b +.endm + +/* + * IEE memory access gate. + * Kernel calls the gate to modify IEE-protected memory. + */ + +SYM_FUNC_START(iee_rw_gate) + /* save Interrupt flag */ + pushfq + /* close irq*/ + cli + + pushq %r12 + + DISABLE_WP r12 + + /* switch to iee stack */ + movq %rsp, %r12 + movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp + + /* call iee func */ + leaq iee_funcs(%rip), %rax + call *(%rax, %rdi, 8) + + /* switch to kernel stack */ + movq %r12, %rsp + + ENABLE_WP r12 + + popq %r12 + + /* restore irq*/ + popfq + + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(iee_rw_gate) diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c new file mode 100644 index 000000000000..28a588da2e29 --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#include +#include +#include +#include +#include +#include +#include + +/* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ +unsigned long IEE_OFFSET = 0x200000000000; +bool iee_init_done; +DEFINE_PER_CPU(struct iee_stack, iee_stacks); + +static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, unsigned long end) +{ + void *p; + pmd_t *pmd; + unsigned long pmd_next; + phys_addr_t phys; + pgprot_t pgprot_shadow_pmd; + + addr = ALIGN_DOWN(addr, PMD_SIZE); + phys = __iee_pa(addr); + pgprot_shadow_pmd = __pgprot(pgprot_val(PAGE_KERNEL_LARGE) & (~__RW) & (~___D)); + + if (pud_none(*pud)) { + p = alloc_low_pages(1); + pud_populate(&init_mm, pud, p); + } + + pmd = pmd_offset(pud, addr); + do { + pmd_next = pmd_addr_end(addr, end); + set_pmd(pmd, __pmd(phys | pgprot_val(pgprot_shadow_pmd))); + phys += pmd_next - addr; + } while (pmd++, addr = pmd_next, addr != end); +} + +static void __init _iee_mapping_populate_p4d(p4d_t *p4d, unsigned long addr, unsigned long end) +{ + void *p; + pud_t *pud; + unsigned long pud_next; + + if (p4d_none(*p4d)) { + p = alloc_low_pages(1); + p4d_populate(&init_mm, p4d, p); + } + + pud = pud_offset(p4d, addr); + do { + pud_next = pud_addr_end(addr, end); + pr_info("IEE: iee_populate_pud(%#010lx, %#010lx)\n", + addr, pud_next); + _iee_mapping_populate_pud(pud, addr, pud_next); + } while (pud++, addr = pud_next, addr != end); +} + +static void __init _iee_mapping_populate_pgd(pgd_t *pgd, unsigned long addr, unsigned long end) +{ + void *p; + p4d_t *p4d; + unsigned long p4d_next; + + if (pgd_none(*pgd)) { + p = alloc_low_pages(1); + pgd_populate(&init_mm, pgd, p); + } + + p4d = p4d_offset(pgd, addr); + do { + p4d_next = p4d_addr_end(addr, end); + pr_info("IEE: iee_populate_p4d(%#010lx, %#010lx)\n", + addr, p4d_next); + _iee_mapping_populate_p4d(p4d, addr, p4d_next); + } while (p4d++, addr = p4d_next, addr != end); +} + +static void __init _iee_init_mapping(phys_addr_t start_paddr, phys_addr_t end_paddr) +{ + unsigned long addr = (unsigned long)__phys_to_iee(start_paddr); + unsigned long end = (unsigned long)__phys_to_iee(end_paddr); + unsigned long pgd_next; + + pgd_t *pgd = pgd_offset_k(addr); + + spin_lock(&pgd_lock); + do { + pgd_next = pgd_addr_end(addr, end); + pr_info("IEE: iee_populate_pgd(%#010lx, %#010lx)\n", + addr, pgd_next); + _iee_mapping_populate_pgd(pgd, addr, pgd_next); + } while (pgd++, addr = pgd_next, addr != end); + spin_unlock(&pgd_lock); +} + +static void __init _iee_mapping_init(void) +{ + struct memblock_region *r; + unsigned long start_pfn, end_pfn; + phys_addr_t start_paddr, end_paddr; + unsigned long nr_pages = 0; + + for_each_mem_region(r) { + start_pfn = memblock_region_memory_base_pfn(r); + end_pfn = memblock_region_memory_end_pfn(r); + + start_paddr = PFN_PHYS(start_pfn); + end_paddr = PFN_PHYS(end_pfn); + + nr_pages += end_pfn - start_pfn; + + pr_info("IEE: mapping iee mapping [mem %#010lx-%#010lx]\n", + (unsigned long)start_paddr, (unsigned long)end_paddr); + + _iee_init_mapping(start_paddr, end_paddr); + } + pr_info("IEE: IEE shadow mapping init done"); +} + +static void __init _iee_stack_init(void) +{ + int cpu; + struct iee_stack *iee_stack; + void *stack_base; + struct page *page; + + for_each_possible_cpu(cpu) { + stack_base = (void *)page_address(alloc_pages(GFP_KERNEL, IEE_STACK_ORDER)); + iee_stack = per_cpu_ptr(&iee_stacks, cpu); + page = alloc_pages(GFP_KERNEL, IEE_STACK_ORDER); + iee_stack->stack = (void *)page_address(page) + PAGE_SIZE * (1 << IEE_STACK_ORDER); + pr_info("IEE: cpu %d, iee_stack 0x%lx", cpu, (unsigned long)iee_stack->stack); + set_memory_ro((unsigned long)stack_base, (1 << IEE_STACK_ORDER)); + } +} + +static void __init _iee_offset_init(void) +{ + if (pgtable_l5_enabled()) + IEE_OFFSET = 0x40000000000000; +} + +void __init iee_init(void) +{ + _iee_offset_init(); + _iee_mapping_init(); + _iee_stack_init(); +} diff --git a/arch/x86/kernel/haoc/iee/iee.c b/arch/x86/kernel/haoc/iee/iee.c new file mode 100644 index 000000000000..35c54c3352ac --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HAOC feature support + * + * Copyright (C) 2025 ZGCLAB + * Authors: Shu Hang + * Hu Bing + */ + +#include + +void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n) +{ + memcpy(dst, src, n); +} + +void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n) +{ + memset(ptr, data, n); +} + +void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr) +{ + *pptr = ptr; +} + +unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, unsigned long *addr) +{ + kcsan_mb(); + instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_and_clear_bit(nr, addr); +} diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index a0106e1c1bbb..5b151afeef7c 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -56,6 +56,9 @@ #include #include +#ifdef CONFIG_IEE +#include +#endif #include "mm_internal.h" #include "ident_map.c" @@ -1372,6 +1375,15 @@ void __init mem_init(void) if (get_gate_vma(&init_mm)) kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); + #ifdef CONFIG_IEE + /* + * Split the linear mapping region of the kernel address space into two equally-sized parts. + * The lower region retains the original linear mapping. + * The upper region becomes the IEE linear mapping area. + * Note that the IEE mapping region is mapped with read-only permissions. + */ + iee_init(); + #endif preallocate_vmalloc_pages(); } -- Gitee From 9c929eb12389367de183e0b4e520fa53ada05661 Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Mon, 29 Sep 2025 10:54:17 +0800 Subject: [PATCH 2/8] HAOC: Add kernel command line support for x86 IEE. ----------------------------------- Use kernel command line haoc to control if HAOC should be enabled. eg: haoc=on to enable haoc protection. --- arch/x86/Kconfig | 3 ++- arch/x86/include/asm/haoc/iee-access.h | 21 +++++++++++++++++---- arch/x86/include/asm/haoc/iee.h | 1 + arch/x86/kernel/cpu/common.c | 26 +++++++++++++++++--------- arch/x86/kernel/haoc/iee/iee-init.c | 7 +++++++ arch/x86/mm/init_64.c | 3 ++- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7ea816a0b77b..d76f72fbe9f7 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1554,7 +1554,8 @@ config AMD_MEM_ENCRYPT config IEE depends on X86_64 - def_bool n + def_bool y + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/include/asm/haoc/iee-access.h b/arch/x86/include/asm/haoc/iee-access.h index 7ca29a8d8b19..de0168e821a9 100644 --- a/arch/x86/include/asm/haoc/iee-access.h +++ b/arch/x86/include/asm/haoc/iee-access.h @@ -10,28 +10,41 @@ #ifndef _LINUX_IEE_ACCESS_H #define _LINUX_IEE_ACCESS_H +#include #include extern unsigned long long iee_rw_gate(int flag, ...); static inline void iee_memcpy(void *dst, const void *src, size_t n) { - iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); + if (haoc_enabled) + iee_rw_gate(IEE_OP_MEMCPY, dst, src, n); + else + memcpy(dst, src, n); } static inline void iee_memset(void *ptr, int data, size_t n) { - iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); + if (haoc_enabled) + iee_rw_gate(IEE_OP_MEMSET, ptr, data, n); + else + memset(ptr, data, n); } static inline void iee_set_freeptr(void **pptr, void *ptr) { - iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); + if (haoc_enabled) + iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); + else + *pptr = ptr; } static inline unsigned long iee_test_and_clear_bit(long nr, unsigned long *addr) { - return iee_rw_gate(IEE_OP_TEST_CLEAR_BIT, nr, addr); + if (haoc_enabled) + return iee_rw_gate(IEE_OP_TEST_CLEAR_BIT, nr, addr); + else + return test_and_clear_bit(nr, addr); } #endif diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h index 95472e0d6a6a..899013c4d433 100644 --- a/arch/x86/include/asm/haoc/iee.h +++ b/arch/x86/include/asm/haoc/iee.h @@ -30,4 +30,5 @@ DECLARE_PER_CPU(struct iee_stack, iee_stacks); extern void *alloc_low_pages(unsigned int num); extern void iee_init(void); +extern bool haoc_enabled; #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index f6c4b88f6301..01c6fb80fac1 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -66,6 +66,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #include "cpu.h" @@ -596,6 +599,20 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) if (!IS_ENABLED(CONFIG_X86_CET)) return; +#ifdef CONFIG_IEE + if (haoc_enabled) { + /* + * NOTE: IEE relies on CR0.WP (Write Protection). + * According to Intel SDM Vol.3(Section 2.5): + * This flag must be set before software can set CR4.CET, + * and it cannot be cleared as long as CR4.CET = 1. + * Therefore, IEE does not enable CET during kernel boot. + */ + pr_info("CET disabled because of the contradiction with IEE"); + return; + } +#endif + kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT); user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) && IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK); @@ -611,16 +628,7 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) else wrmsrl(MSR_IA32_S_CET, 0); - #ifndef CONFIG_IEE - /* - * NOTE: IEE relies on CR0.WP (Write Protection). - * According to Intel SDM Vol.3(Section 2.5): - * This flag must be set before software can set CR4.CET, - * and it cannot be cleared as long as CR4.CET = 1. - * Therefore, IEE does not enable CR4.CET during kernel boot. - */ cr4_set_bits(X86_CR4_CET); - #endif if (kernel_ibt && ibt_selftest()) { pr_err("IBT selftest: Failed!\n"); diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 28a588da2e29..358ad8433bc5 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -156,3 +156,10 @@ void __init iee_init(void) _iee_mapping_init(); _iee_stack_init(); } + +bool __ro_after_init haoc_enabled; +static int __init parse_haoc_enabled(char *str) +{ + return kstrtobool(str, &haoc_enabled); +} +early_param("haoc", parse_haoc_enabled); diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 5b151afeef7c..b6c1983c493c 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1382,7 +1382,8 @@ void __init mem_init(void) * The upper region becomes the IEE linear mapping area. * Note that the IEE mapping region is mapped with read-only permissions. */ - iee_init(); + if (haoc_enabled) + iee_init(); #endif preallocate_vmalloc_pages(); } -- Gitee From 261ef21dd95b56eb1dff0c3702334297ec2a1ad0 Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Mon, 29 Sep 2025 11:22:40 +0800 Subject: [PATCH 3/8] Remove "def_bool = y" for IEE in Kconfig and edit x86_64_defconfig to enable IEE by default. --- arch/x86/Kconfig | 6 +++++- arch/x86/configs/x86_64_defconfig | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d76f72fbe9f7..c5b79b76685b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1553,8 +1553,12 @@ config AMD_MEM_ENCRYPT Encryption (SME). config IEE + bool "Isolated Execution Environment Framework(IEE)" + help + Support for Isolated Execution Environment Framework. Foundation of HAOC. + Could isolate kernel critical data and enforce all write access made and + verified in IEE APIs. depends on X86_64 - def_bool y # Common NUMA Features config NUMA diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 409e9182bd29..8ccbbb74a264 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -33,6 +33,7 @@ CONFIG_PARAVIRT=y CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y CONFIG_X86_MSR=y CONFIG_X86_CPUID=y +CONFIG_IEE=y CONFIG_NUMA=y CONFIG_X86_CHECK_BIOS_CORRUPTION=y # CONFIG_MTRR_SANITIZER is not set -- Gitee From d948ec65ada011ac818366922538c5c5ab6933aa Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Thu, 9 Oct 2025 18:15:15 +0800 Subject: [PATCH 4/8] Haoc: Add support for x86 Sensitive Instruction Protection(IEE_SIP) --- arch/x86/Kconfig | 5 +- arch/x86/boot/compressed/ident_map_64.c | 21 ++++ arch/x86/boot/compressed/pgtable_64.c | 7 ++ arch/x86/include/asm/desc.h | 19 ++++ arch/x86/include/asm/haoc/haoc-def.h | 8 ++ arch/x86/include/asm/haoc/iee-si.h | 43 ++++++++ arch/x86/include/asm/haoc/iee-token.h | 55 ++++++++++ arch/x86/include/asm/haoc/iee.h | 1 + arch/x86/include/asm/special_insns.h | 18 +++ arch/x86/kernel/cpu/common.c | 58 ++++++++++ arch/x86/kernel/haoc/iee/Makefile | 1 + arch/x86/kernel/haoc/iee/iee-gate.S | 46 ++++++++ arch/x86/kernel/haoc/iee/iee-init.c | 21 +++- arch/x86/kernel/haoc/iee/iee-si.c | 115 +++++++++++++++++++ arch/x86/kernel/haoc/iee/iee-token.c | 140 ++++++++++++++++++++++++ arch/x86/kernel/head64.c | 7 ++ arch/x86/kernel/vmlinux.lds.S | 22 ++++ 17 files changed, 585 insertions(+), 2 deletions(-) create mode 100644 arch/x86/include/asm/haoc/iee-si.h create mode 100644 arch/x86/include/asm/haoc/iee-token.h create mode 100644 arch/x86/kernel/haoc/iee/iee-si.c create mode 100644 arch/x86/kernel/haoc/iee/iee-token.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index c5b79b76685b..f946792fd1ef 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1559,7 +1559,10 @@ config IEE Could isolate kernel critical data and enforce all write access made and verified in IEE APIs. depends on X86_64 - +config IEE_SIP + depends on IEE + def_bool y + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 8ecb4d40e20d..848a958c639f 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -180,7 +180,14 @@ void initialize_identity_maps(void *rmode) sev_prep_identity_maps(top_level_pgt); /* Load the new page-table. */ + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr3_early(top_level_pgt); + else + write_cr3(top_level_pgt); + #else write_cr3(top_level_pgt); + #endif /* * Now that the required page table mappings are established and a @@ -224,7 +231,14 @@ static pte_t *split_large_pmd(struct x86_mapping_info *info, pmd = __pmd((unsigned long)pte | info->kernpg_flag); set_pmd(pmdp, pmd); /* Flush TLB to establish the new PMD */ + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr3_early(top_level_pgt); + else + write_cr3(top_level_pgt); + #else write_cr3(top_level_pgt); + #endif return pte + pte_index(__address); } @@ -325,7 +339,14 @@ static int set_clr_page_flags(struct x86_mapping_info *info, snp_set_page_private(__pa(address & PAGE_MASK)); /* Flush TLB after changing encryption attribute */ + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr3_early(top_level_pgt); + else + write_cr3(top_level_pgt); + #else write_cr3(top_level_pgt); + #endif return 0; } diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 15354673d3aa..c160cee8f071 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -197,7 +197,14 @@ asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable) * Move the top level page table out of trampoline memory. */ memcpy(pgtable, trampoline_32bit, PAGE_SIZE); + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr3_early((unsigned long)pgtable); + else + native_write_cr3((unsigned long)pgtable); + #else native_write_cr3((unsigned long)pgtable); + #endif /* Restore trampoline memory */ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE); diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index ab97b22ac04a..4d3dd0eec916 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -13,6 +13,11 @@ #include #include +#ifdef CONFIG_IEE_SIP +#include +extern bool haoc_enabled; +#endif + static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info) { desc->limit0 = info->limit & 0x0ffff; @@ -210,9 +215,23 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) asm volatile("lgdt %0"::"m" (*dtr)); } +#ifdef CONFIG_IEE_SIP +static __always_inline void iee_load_idt_early(const struct desc_ptr *dtr) +{ + asm volatile("lidt %0"::"m" (*dtr)); +} +#endif + static __always_inline void native_load_idt(const struct desc_ptr *dtr) { + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_load_idt((void *)dtr); + else + asm volatile("lidt %0"::"m" (*dtr)); + #else asm volatile("lidt %0"::"m" (*dtr)); + #endif } static inline void native_store_gdt(struct desc_ptr *dtr) diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index 55b5d37ff762..9abddba274c9 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -18,4 +18,12 @@ enum { IEE_FLAG_END }; +#ifdef CONFIG_IEE_SIP +#define IEE_SIP_TEST 0 +#define IEE_WRITE_CR0 1 +#define IEE_WRITE_CR3 2 +#define IEE_WRITE_CR4 3 +#define IEE_LOAD_IDT 4 +#endif + #endif diff --git a/arch/x86/include/asm/haoc/iee-si.h b/arch/x86/include/asm/haoc/iee-si.h new file mode 100644 index 000000000000..3baf1813f19d --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-si.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_SI_H +#define _LINUX_IEE_SI_H +#include +#include + +#define __iee_si_code __section(".iee.si_text") +#define __iee_si_data __section(".iee.si_data") + +extern unsigned long cr4_pinned_mask; +extern struct static_key_false cr_pinning; +extern unsigned long cr4_pinned_bits; + +extern unsigned long __iee_si_text_start[]; +extern unsigned long __iee_si_text_end[]; +extern unsigned long __iee_si_data_start[]; +extern unsigned long __iee_si_data_end[]; +extern void iee_sip_init(void); + +extern void iee_rwx_gate(int flag, ...); + +static inline void iee_sip_test(void) +{ + iee_rwx_gate(IEE_SIP_TEST); +} +static inline void iee_write_cr0(unsigned long val) +{ + iee_rwx_gate(IEE_WRITE_CR0, val); +} +static inline void iee_write_cr3(unsigned long val) +{ + iee_rwx_gate(IEE_WRITE_CR3, val); +} +static inline void iee_write_cr4(unsigned long val) +{ + iee_rwx_gate(IEE_WRITE_CR4, val); +} +static inline void iee_load_idt(void *ptr) +{ + iee_rwx_gate(IEE_LOAD_IDT, ptr); +} + +#endif \ No newline at end of file diff --git a/arch/x86/include/asm/haoc/iee-token.h b/arch/x86/include/asm/haoc/iee-token.h new file mode 100644 index 000000000000..a96638f3260c --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-token.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IEE_TOKEN_H +#define _LINUX_IEE_TOKEN_H +#include +#include +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +extern struct kmem_cache *task_struct_cachep; + +extern void iee_set_token_page_valid(unsigned long token, unsigned long new, + unsigned int order); +extern void iee_set_token_page_invalid(unsigned long token_addr, + unsigned long __unused, + unsigned int order); +extern struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, + struct slab *slab, + unsigned int order); + +struct task_token { + pgd_t *pgd; + bool valid; +}; + +static inline void iee_verify_token_pgd(struct task_struct *tsk) +{ + struct task_token *token; + + if (tsk == &init_task) + return; + + token = (struct task_token *)__addr_to_iee(tsk); + if (token->pgd != tsk->mm->pgd) + panic("IEE Pgd Error: tsk_pgd: 0x%lx, token_pgd: 0x%lx", + (unsigned long)tsk->mm->pgd, (unsigned long)token->pgd); +} + +static inline void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) +{ + iee_rw_gate(IEE_OP_SET_TOKEN_PGD, tsk, pgd); +} + +static inline void iee_invalidate_token(struct task_struct *tsk) +{ + iee_rw_gate(IEE_OP_INVALIDATE_TOKEN, tsk); +} + +static inline void iee_validate_token(struct task_struct *tsk) +{ + iee_rw_gate(IEE_OP_VALIDATE_TOKEN, tsk); +} + +#endif \ No newline at end of file diff --git a/arch/x86/include/asm/haoc/iee.h b/arch/x86/include/asm/haoc/iee.h index 899013c4d433..09b939a4b0c6 100644 --- a/arch/x86/include/asm/haoc/iee.h +++ b/arch/x86/include/asm/haoc/iee.h @@ -31,4 +31,5 @@ DECLARE_PER_CPU(struct iee_stack, iee_stacks); extern void *alloc_low_pages(unsigned int num); extern void iee_init(void); extern bool haoc_enabled; +extern bool iee_init_done; #endif diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 1c5513b04f03..de9508e4d912 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -9,6 +9,10 @@ #include #include #include +#ifdef CONFIG_IEE_SIP +#include +extern bool haoc_enabled; +#endif /* * The compiler should not reorder volatile asm statements with respect to each @@ -49,9 +53,23 @@ static inline unsigned long __native_read_cr3(void) return val; } +#ifdef CONFIG_IEE_SIP +static inline void iee_write_cr3_early(unsigned long val) +{ + asm volatile("mov %0,%%cr3" : : "r" (val) : "memory"); +} +#endif + static inline void native_write_cr3(unsigned long val) { + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr3(val); + else + asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); + #else asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); + #endif } static inline unsigned long native_read_cr4(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 01c6fb80fac1..78f57964f3c8 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -69,6 +69,9 @@ #ifdef CONFIG_IEE #include #endif +#ifdef CONFIG_IEE_SIP +#include +#endif #include "cpu.h" @@ -407,14 +410,42 @@ static __always_inline void setup_umip(struct cpuinfo_x86 *c) } /* These bits should not change their value after CPU init is finished. */ +#ifdef CONFIG_IEE_SIP +unsigned long cr4_pinned_mask = + X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | + X86_CR4_FSGSBASE | X86_CR4_CET; +DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); +unsigned long cr4_pinned_bits __ro_after_init; +#else static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE | X86_CR4_CET; static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); static unsigned long cr4_pinned_bits __ro_after_init; +#endif void native_write_cr0(unsigned long val) { + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr0(val); + else{ + unsigned long bits_missing = 0; + + set_register: + asm volatile("mov %0,%%cr0": "+r" (val) : : "memory"); + + if (static_branch_likely(&cr_pinning)) { + if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { + bits_missing = X86_CR0_WP; + val |= bits_missing; + goto set_register; + } + /* Warn after we've set the missing bits. */ + WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); + } + } + #else unsigned long bits_missing = 0; set_register: @@ -429,11 +460,33 @@ void native_write_cr0(unsigned long val) /* Warn after we've set the missing bits. */ WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); } + #endif } EXPORT_SYMBOL(native_write_cr0); void __no_profile native_write_cr4(unsigned long val) { + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_write_cr4(val); + else{ + unsigned long bits_changed = 0; + + set_register: + asm volatile("mov %0,%%cr4": "+r" (val) : : "memory"); + + if (static_branch_likely(&cr_pinning)) { + if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) { + bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits; + val = (val & ~cr4_pinned_mask) | cr4_pinned_bits; + goto set_register; + } + /* Warn after we've corrected the changed bits. */ + WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", + bits_changed); + } + } + #else unsigned long bits_changed = 0; set_register: @@ -449,6 +502,7 @@ void __no_profile native_write_cr4(unsigned long val) WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n", bits_changed); } + #endif } #if IS_MODULE(CONFIG_LKDTM) EXPORT_SYMBOL_GPL(native_write_cr4); @@ -2546,4 +2600,8 @@ void __init arch_cpu_finalize_init(void) * hypercalls work when the SWIOTLB bounce buffers are decrypted. */ mem_encrypt_init(); + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_sip_init(); + #endif } diff --git a/arch/x86/kernel/haoc/iee/Makefile b/arch/x86/kernel/haoc/iee/Makefile index 48c68356002d..33c902b314ca 100644 --- a/arch/x86/kernel/haoc/iee/Makefile +++ b/arch/x86/kernel/haoc/iee/Makefile @@ -1,2 +1,3 @@ obj-y += iee-gate.o iee-init.o iee.o iee-func.o +obj-$(CONFIG_IEE_SIP) += iee-si.o ccflags-y += -I$(srctree)/mm diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 8d945acd214e..974cdd707b25 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -71,3 +71,49 @@ SYM_FUNC_START(iee_rw_gate) jmp __x86_return_thunk /* ret */ SYM_FUNC_END(iee_rw_gate) + +#ifdef CONFIG_IEE_SIP +SYM_FUNC_START(iee_rwx_gate) + pushq %r12 + + /* save Interrupt flag*/ + pushfq + /* close irq */ + cli + + /* set SMEP=0 to enable supervisor-mode exec user-mode insn */ + movq %cr4, %rax /* rax -> cr4 */ + andq $(~X86_CR4_SMEP), %rax + movq %rax, %cr4 + + DISABLE_WP r12 + + movq %rsp, %r12 + /* If iee hasn't been initialized, skip stack switch. */ + cmpb $0, iee_init_done(%rip) + jz 2f + + /* switch to iee stack */ + movq PER_CPU_VAR(iee_stacks) + IEE_STACK, %rsp + +2: call _iee_si_handler + /* switch to kernel stack. If iee hasn't been initialized, skip switch*/ + movq %r12, %rsp + + ENABLE_WP r12 + + /* set SMEP=1 to disable supervisor-mode exec user-mode insn */ + movq %cr4, %rax /* rax -> cr4 */ +1: orq $X86_CR4_SMEP_SMAP, %rax + movq %rax, %cr4 + andq $X86_CR4_SMEP_SMAP, %rax + cmpq $X86_CR4_SMEP_SMAP, %rax + jnz 1 + + /* restore irq*/ + popfq + + popq %r12 + jmp __x86_return_thunk /* ret */ +SYM_FUNC_END(iee_rwx_gate) +#endif \ No newline at end of file diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 358ad8433bc5..9d9366609390 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -14,10 +14,17 @@ #include #include #include +#ifdef CONFIG_IEE_SIP +#include +#endif /* IEE_OFFSET = pgtable_l5_enabled() ? 0x40000000000000 : 0x200000000000; */ unsigned long IEE_OFFSET = 0x200000000000; +#ifdef CONFIG_IEE_SIP +bool iee_init_done __iee_si_data; +#else bool iee_init_done; +#endif DEFINE_PER_CPU(struct iee_stack, iee_stacks); static void __init _iee_mapping_populate_pud(pud_t *pud, unsigned long addr, unsigned long end) @@ -158,8 +165,20 @@ void __init iee_init(void) } bool __ro_after_init haoc_enabled; +#ifdef CONFIG_IEE_SIP +extern unsigned long cr4_pinned_mask; +#endif static int __init parse_haoc_enabled(char *str) { - return kstrtobool(str, &haoc_enabled); + int ret = kstrtobool(str, &haoc_enabled); + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + { + cr4_pinned_mask = + X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | + X86_CR4_FSGSBASE | X86_CR4_CET; + } + #endif + return ret; } early_param("haoc", parse_haoc_enabled); diff --git a/arch/x86/kernel/haoc/iee/iee-si.c b/arch/x86/kernel/haoc/iee/iee-si.c new file mode 100644 index 000000000000..214c7617cfed --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee-si.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +unsigned long __iee_si_code notrace _iee_si_handler(int flag, ...) +{ + va_list pArgs; + unsigned long val; + + va_start(pArgs, flag); + switch (flag) { + case IEE_SIP_TEST: + break; + case IEE_WRITE_CR0: { + val = va_arg(pArgs, u64); + unsigned long bits_missing = 0; + set_register_cr0: + asm volatile("mov %0,%%cr0" : "+r"(val) : : "memory"); + if (static_branch_likely(&cr_pinning)) { + if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) { + bits_missing = X86_CR0_WP; + val |= bits_missing; + goto set_register_cr0; + } + /* Warn after we've set the missing bits. */ + WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n"); + } + break; + } + case IEE_WRITE_CR3: { + val = va_arg(pArgs, u64); + asm volatile("mov %0,%%cr3" : : "r"(val) : "memory"); + break; + } + case IEE_WRITE_CR4: { + val = va_arg(pArgs, u64); + val &= ~(X86_CR4_SMEP); + asm volatile("mov %0,%%cr4" : "+r" (val) : : "memory"); + break; + } + case IEE_LOAD_IDT: { + const struct desc_ptr *new_val; + + new_val = va_arg(pArgs, const struct desc_ptr*); + asm volatile("lidt %0"::"m" (*new_val)); + break; + } + } + va_end(pArgs); + return 0; +} + +static void __init _iee_set_kernel_upage(unsigned long addr) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + pgd_t pgd = READ_ONCE(*pgdp); + + pgd = __pgd((pgd_val(pgd) | _USR) & ~___G); + set_pgd(pgdp, pgd); + + p4d_t *p4dp = p4d_offset(pgdp, addr); + p4d_t p4d = READ_ONCE(*p4dp); + + p4d = __p4d((p4d_val(p4d) | _USR) & ~___G); + set_p4d(p4dp, p4d); + + pud_t *pudp = pud_offset(p4dp, addr); + pud_t pud = READ_ONCE(*pudp); + + pud = __pud((pud_val(pud) | _USR) & ~___G); + set_pud(pudp, pud); + + pmd_t *pmdp = pmd_offset(pudp, addr); + pmd_t pmd = READ_ONCE(*pmdp); + + pmd = __pmd((pmd_val(pmd) | _USR) & ~___G); + set_pmd(pmdp, pmd); + + pte_t *ptep = pte_offset_kernel(pmdp, addr); + pte_t pte = READ_ONCE(*ptep); + + pte = __pte((pte_val(pte) | _USR) & ~___G); + set_pte(ptep, pte); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); +} + +void __init iee_sip_init(void) +{ + unsigned long addr, start, end; + int num_pages; + /* Map .iee.text as U RWX pages */ + start = (unsigned long)__iee_si_text_start; + end = (unsigned long)__iee_si_text_end; + pr_info("IEE: mapping .iee.text:[0x%lx, 0x%lx] as U pages...", start, end); + addr = start; + for ( ; addr < end; addr += PAGE_SIZE) { + set_memory_4k(addr, 1); + _iee_set_kernel_upage(addr); + set_memory_4k((unsigned long)__va(__pa(addr)), 1); + _iee_set_kernel_upage((unsigned long)__va(__pa(addr))); + } + iee_init_done = true; + /* Map .iee.data as RO pages */ + start = (unsigned long)__iee_si_data_start; + end = (unsigned long)__iee_si_data_end; + num_pages = (end - start) / PAGE_SIZE; + set_memory_ro(start, num_pages); + /* All initialization is done. Do some simple tests. */ + pr_info("IEE: testing iee_exec_entry si_test..."); + iee_sip_test(); +} \ No newline at end of file diff --git a/arch/x86/kernel/haoc/iee/iee-token.c b/arch/x86/kernel/haoc/iee/iee-token.c new file mode 100644 index 000000000000..b39ba03d9c56 --- /dev/null +++ b/arch/x86/kernel/haoc/iee/iee-token.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "slab.h" + +void iee_set_token_page_valid(unsigned long token, unsigned long token_page, + unsigned int order) +{ + set_memory_4k(token, 1 << order); + set_memory_4k(token_page, 1 << order); + + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, token); + p4d_t *p4dp = p4d_offset(pgdp, token); + pud_t *pudp = pud_offset(p4dp, token); + pmd_t *token_pmdp = pmd_offset(pudp, token); + pte_t *token_ptep = pte_offset_kernel(token_pmdp, token); + + if (!token_page) + panic("Token of task_struct was unset.\n"); + + pgdp = pgd_offset_pgd(pgdir, token_page); + p4dp = p4d_offset(pgdp, token_page); + pudp = pud_offset(p4dp, token_page); + pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); + pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | + (__phys_to_pfn(__pa(token_page + i * PAGE_SIZE)) + << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte((pte_val(pte) & ~__RW) & ~___D); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } + + flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); + flush_tlb_kernel_range(token_page, + token_page + (PAGE_SIZE * (1 << order))); +} + +void iee_set_token_page_invalid(unsigned long token, unsigned long __unused, + unsigned int order) +{ + pgd_t *pgdir = swapper_pg_dir; + pgd_t *pgdp = pgd_offset_pgd(pgdir, token); + p4d_t *p4dp = p4d_offset(pgdp, token); + pud_t *pudp = pud_offset(p4dp, token); + pmd_t *token_pmdp = pmd_offset(pudp, token); + pte_t *token_ptep = pte_offset_kernel(token_pmdp, token); + unsigned long token_page = + (unsigned long)page_address(pte_page(*token_ptep)); + + if (!token_page) + panic("Token of task_struct was unset.\n"); + + pgdp = pgd_offset_pgd(pgdir, token_page); + p4dp = p4d_offset(pgdp, token_page); + pudp = pud_offset(p4dp, token_page); + pmd_t *token_page_pmdp = pmd_offset(pudp, token_page); + pte_t *token_page_ptep = pte_offset_kernel(token_page_pmdp, token_page); + + for (int i = 0; i < (0x1 << order); i++) { + pte_t pte = READ_ONCE(*token_ptep); + + pte = __pte((pte_val(pte) & ~PTE_PFN_MASK) | + (__phys_to_pfn(__iee_pa(token + i * PAGE_SIZE)) + << PAGE_SHIFT)); + WRITE_ONCE(*token_ptep, pte); + pte = READ_ONCE(*token_page_ptep); + pte = __pte(pte_val(pte) | ___D | __RW); + WRITE_ONCE(*token_page_ptep, pte); + token_ptep++; + token_page_ptep++; + } + free_pages(token_page, order); + flush_tlb_kernel_range(token, token + (PAGE_SIZE * (1 << order))); + flush_tlb_kernel_range(token_page, + token_page + (PAGE_SIZE * (1 << order))); +} + +struct slab *iee_alloc_task_token_slab(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ + if (!slab || s != task_struct_cachep) + return slab; + + struct folio *folio = slab_folio(slab); + unsigned long token_addr = __slab_to_iee(slab); + unsigned long alloc_token = + __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + + /* Allocation of task_struct and token pages must be done at the same time. */ + if (!alloc_token) { + /* Failed on allocation of token page. Free the allocated ones, + * return and try smaller order. + */ + __slab_clear_pfmemalloc(slab); + folio->mapping = NULL; + /* Make the mapping reset visible before clearing the flag */ + smp_wmb(); + __folio_clear_slab(folio); + __free_pages((struct page *)folio, order); + return NULL; + } + + /* Map allocated token pages to token addresses. */ + iee_set_token_page_valid(token_addr, alloc_token, order); + return slab; +} + +void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, + pgd_t *pgd) +{ + struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + + token->pgd = pgd; +} + +void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk) +{ + struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + + token->pgd = NULL; + token->valid = false; +} + +void _iee_validate_token(unsigned long __unused, struct task_struct *tsk) +{ + struct task_token *token = (struct task_token *)__addr_to_iee(tsk); + + if (token->valid) + pr_err("IEE: validate token for multiple times."); + token->valid = true; +} \ No newline at end of file diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 1defe865de67..f78dcb4f11fa 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -720,7 +720,14 @@ static void startup_64_load_idt(unsigned long physbase) } desc->address = (unsigned long)idt; + #ifdef CONFIG_IEE_SIP + if(haoc_enabled) + iee_load_idt_early(desc); + else + native_load_idt(desc); + #else native_load_idt(desc); + #endif } /* This is used when running on kernel addresses */ diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index c57d5df1abc6..f58362f004e5 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -99,6 +99,22 @@ jiffies = jiffies_64; #endif +#ifdef CONFIG_IEE_SIP +#define IEE_SI_TEXT \ +. = ALIGN(PAGE_SIZE); \ +__iee_si_text_start = .; \ +*(.iee.si_text) \ +. = ALIGN(PAGE_SIZE); \ +__iee_si_text_end = .; + +#define IEE_SI_DATA \ + . = ALIGN(PAGE_SIZE); \ + __iee_si_data_start = .; \ +*(.iee.si_data) \ +. = ALIGN(PAGE_SIZE); \ +__iee_si_data_end = .; +#endif + PHDRS { text PT_LOAD FLAGS(5); /* R_E */ data PT_LOAD FLAGS(6); /* RW_ */ @@ -131,6 +147,9 @@ SECTIONS SCHED_TEXT LOCK_TEXT KPROBES_TEXT +#ifdef CONFIG_IEE_SIP + IEE_SI_TEXT +#endif SOFTIRQENTRY_TEXT #ifdef CONFIG_RETPOLINE *(.text..__x86.indirect_thunk) @@ -181,6 +200,9 @@ SECTIONS CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES) DATA_DATA +#ifdef CONFIG_IEE_SIP + IEE_SI_DATA +#endif CONSTRUCTORS /* rarely changed data like cpu maps */ -- Gitee From f38713fc30ae865c88c56576eb97c10ec7619dcd Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Sat, 11 Oct 2025 14:59:43 +0800 Subject: [PATCH 5/8] Add SMEP and SMAP check for IEE_SIP --- arch/x86/kernel/setup.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 93dc119c8e2e..3f556e286d09 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1011,6 +1011,27 @@ void __init setup_arch(char **cmdline_p) parse_early_param(); +#ifdef CONFIG_IEE_SIP + /* + * Perform a one-time check for IEE_SIP prerequisites. This must be done + * early in setup_arch() before any code might rely on these features. + * At this point, CPU features (from early_cpu_init) and kernel command + * line (from parse_early_param) are both available. + */ + if (haoc_enabled) { + bool smep_ok = cpu_feature_enabled(X86_FEATURE_SMEP); + bool smap_ok = cpu_feature_enabled(X86_FEATURE_SMAP); + if (smep_ok && smap_ok) { + pr_info("IEE_SIP: Feature is active. CPU supports SMEP and SMAP.\n"); + } else { + // Fail-fast: The user wants the feature, but the hardware + // doesn't support it. This is a fatal configuration error. + panic("IEE_SIP: FATAL: Feature enabled via 'haoc=on' but hardware is missing support (SMEP:%d, SMAP:%d).\n", + smep_ok, smap_ok); + } + } +#endif + if (efi_enabled(EFI_BOOT)) efi_memblock_x86_reserve_range(); -- Gitee From 1040e85fee470e5dbb9b52b7ef6ddc69431cb8cf Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Sun, 12 Oct 2025 15:07:59 +0800 Subject: [PATCH 6/8] HAOC: Support pointer protection for x86 IEE (IEE_PTRP) Support pointer protection for x86 IEE (IEE_PTRP). --- arch/x86/Kconfig | 4 ++ arch/x86/include/asm/haoc/haoc-def.h | 5 +++ arch/x86/include/asm/haoc/haoc.h | 7 ++++ arch/x86/include/asm/haoc/iee-func.h | 4 +- arch/x86/kernel/haoc/haoc.c | 5 +++ arch/x86/kernel/haoc/iee/Makefile | 1 + arch/x86/kernel/haoc/iee/iee-func.c | 60 ++++++++++++++++++++++++++++ fs/exec.c | 8 ++++ kernel/exit.c | 7 ++++ kernel/fork.c | 23 +++++++++++ kernel/kthread.c | 11 +++++ kernel/sched/core.c | 7 ++++ mm/slub.c | 40 +++++++++++++++++++ 13 files changed, 181 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f946792fd1ef..3b27d0fbdb06 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1562,6 +1562,10 @@ config IEE config IEE_SIP depends on IEE def_bool y + +config IEE_PTRP + depends on IEE + def_bool y # Common NUMA Features config NUMA diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index 9abddba274c9..1eb801149521 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -15,6 +15,11 @@ enum { IEE_OP_MEMSET, IEE_OP_SET_FREEPTR, IEE_OP_TEST_CLEAR_BIT, +#ifdef CONFIG_IEE_PTRP + IEE_OP_SET_TOKEN_PGD, + IEE_OP_INVALIDATE_TOKEN, + IEE_OP_VALIDATE_TOKEN, +#endif IEE_FLAG_END }; diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index e6d1193c4f1a..8b24a71ec67c 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -11,10 +11,17 @@ #define _LINUX_HAOC_H #include +#include void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, unsigned long *addr); +#ifdef CONFIG_IEE_PTRP +void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, + pgd_t *pgd); +void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); +void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); +#endif #endif diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h index 42455aa11615..523ebecf7f38 100644 --- a/arch/x86/include/asm/haoc/iee-func.h +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -10,7 +10,9 @@ #ifndef _LINUX_IEE_FUNC_H #define _LINUX_IEE_FUNC_H +#include extern void set_iee_page(unsigned long addr, unsigned int order); extern void unset_iee_page(unsigned long addr, unsigned int order); - +extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); +extern unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order); #endif /* _LINUX_IEE_FUNC_H */ diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 4676f3c0454e..79ebc303ae99 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -15,5 +15,10 @@ iee_func iee_funcs[] = { (iee_func)_iee_memset, (iee_func)_iee_set_freeptr, (iee_func)_iee_test_and_clear_bit, +#ifdef CONFIG_IEE_PTRP + (iee_func)_iee_set_token_pgd, + (iee_func)_iee_invalidate_token, + (iee_func)_iee_validate_token, +#endif NULL }; diff --git a/arch/x86/kernel/haoc/iee/Makefile b/arch/x86/kernel/haoc/iee/Makefile index 33c902b314ca..6c13071f54d3 100644 --- a/arch/x86/kernel/haoc/iee/Makefile +++ b/arch/x86/kernel/haoc/iee/Makefile @@ -1,3 +1,4 @@ obj-y += iee-gate.o iee-init.o iee.o iee-func.o obj-$(CONFIG_IEE_SIP) += iee-si.o +obj-$(CONFIG_IEE_PTRP) += iee-token.o ccflags-y += -I$(srctree)/mm diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index 7b669401c624..c18dbecb696d 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -8,6 +8,11 @@ */ #include +#include +#include "slab.h" +#ifdef CONFIG_IEE_PTRP +#include +#endif void set_iee_page(unsigned long addr, unsigned int order) { @@ -18,3 +23,58 @@ void unset_iee_page(unsigned long addr, unsigned int order) { set_memory_rw(addr, 1 << order); } + +struct iee_free_slab_work { + struct work_struct work; + struct kmem_cache *s; + struct slab *slab; +}; + +void iee_free_slab(struct kmem_cache *s, struct slab *slab, + void (*do_free_slab)(struct work_struct *work)) +{ + struct iee_free_slab_work *iee_free_slab_work = + kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); + + iee_free_slab_work->s = s; + iee_free_slab_work->slab = slab; + INIT_WORK(&iee_free_slab_work->work, do_free_slab); + schedule_work(&iee_free_slab_work->work); +} + +#ifdef CONFIG_IEE_PTRP +static void iee_free_task_struct_slab(struct work_struct *work) +{ + struct iee_free_slab_work *iee_free_slab_work = + container_of(work, struct iee_free_slab_work, work); + struct slab *slab = iee_free_slab_work->slab; + struct folio *folio = slab_folio(slab); + unsigned int order = folio_order(folio); + unsigned long token = __slab_to_iee(slab); + // Free token. + iee_set_token_page_invalid(token, 0, order); + __free_pages(&folio->page, order); + kfree(iee_free_slab_work); +} +#endif + +bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, + unsigned int order) +{ +#ifdef CONFIG_IEE_PTRP + if (s == task_struct_cachep) { + iee_free_slab(s, slab, iee_free_task_struct_slab); + return true; + } +#endif + return false; +} + +unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) +{ +#ifdef CONFIG_IEE_PTRP + if (strcmp(s->name, "task_struct") == 0) + return IEE_DATA_ORDER; +#endif + return order; +} \ No newline at end of file diff --git a/fs/exec.c b/fs/exec.c index 588ddf9b5d56..006c43f03523 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -77,6 +77,10 @@ #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif + static int bprm_creds_from_file(struct linux_binprm *bprm); int suid_dumpable = 0; @@ -1025,6 +1029,10 @@ static int exec_mmap(struct mm_struct *mm) if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); activate_mm(active_mm, mm); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(tsk, mm->pgd); +#endif if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); lru_gen_add_mm(mm); diff --git a/kernel/exit.c b/kernel/exit.c index 2a4c8b44baf3..1d767f67752c 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -79,6 +79,9 @@ #include #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif extern int sysctl_vm_memory_qos; unsigned long sysctl_async_mem_free_pages = ULONG_MAX; @@ -571,6 +574,10 @@ static void exit_mm(void) smp_mb__after_spinlock(); local_irq_disable(); current->mm = NULL; +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(current, NULL); +#endif membarrier_update_current_mm(NULL); enter_lazy_tlb(mm, current); local_irq_enable(); diff --git a/kernel/fork.c b/kernel/fork.c index 42e6732742b1..b0a722987503 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -114,6 +114,9 @@ #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif /* * Minimum number of threads to boot the kernel @@ -181,7 +184,11 @@ void __weak arch_release_task_struct(struct task_struct *tsk) } #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR +#ifdef CONFIG_IEE_PTRP +struct kmem_cache *task_struct_cachep; +#else static struct kmem_cache *task_struct_cachep; +#endif static inline struct task_struct *alloc_task_struct_node(int node) { @@ -639,6 +646,10 @@ void free_task(struct task_struct *tsk) if (tsk->flags & PF_KTHREAD) free_kthread_struct(tsk); bpf_task_storage_free(tsk); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_invalidate_token(tsk); +#endif free_task_struct(tsk); } EXPORT_SYMBOL(free_task); @@ -1739,6 +1750,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) #endif tsk->mm = NULL; +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(tsk, NULL); +#endif tsk->active_mm = NULL; /* @@ -1760,6 +1775,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) } tsk->mm = mm; +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(tsk, mm->pgd); +#endif tsk->active_mm = mm; sched_mm_cid_fork(tsk); return 0; @@ -2347,6 +2366,10 @@ __latent_entropy struct task_struct *copy_process( p = dup_task_struct(current, node); if (!p) goto fork_out; +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_validate_token(p); +#endif p->flags &= ~PF_KTHREAD; if (args->kthread) p->flags |= PF_KTHREAD; diff --git a/kernel/kthread.c b/kernel/kthread.c index 980e6b325b7d..0caf7249eda5 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -29,6 +29,9 @@ #include #include #include +#ifdef CONFIG_IEE_PTRP +#include +#endif static DEFINE_SPINLOCK(kthread_create_lock); @@ -1457,6 +1460,10 @@ void kthread_use_mm(struct mm_struct *mm) tsk->active_mm = mm; tsk->mm = mm; membarrier_update_current_mm(mm); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(tsk, mm->pgd); +#endif switch_mm_irqs_off(active_mm, mm, tsk); local_irq_enable(); task_unlock(tsk); @@ -1501,6 +1508,10 @@ void kthread_unuse_mm(struct mm_struct *mm) local_irq_disable(); tsk->mm = NULL; membarrier_update_current_mm(NULL); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_set_token_pgd(tsk, NULL); +#endif mmgrab_lazy_tlb(mm); /* active_mm is still 'mm' */ enter_lazy_tlb(mm, tsk); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c7275db331ce..080ba08b7314 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -104,6 +104,9 @@ #include "../workqueue_internal.h" #include "../../io_uring/io-wq.h" #include "../smpboot.h" +#ifdef CONFIG_IEE_PTRP +#include +#endif EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); @@ -5587,6 +5590,10 @@ context_switch(struct rq *rq, struct task_struct *prev, * case 'prev->active_mm == next->mm' through * finish_task_switch()'s mmdrop(). */ +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + iee_verify_token_pgd(next); +#endif switch_mm_irqs_off(prev->active_mm, next->mm, next); lru_gen_use_mm(next->mm); diff --git a/mm/slub.c b/mm/slub.c index 7cfa8cbddb76..105e86b5a3d4 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -44,6 +44,12 @@ #include #include +#ifdef CONFIG_IEE +#include +#endif +#ifdef CONFIG_IEE_PTRP +#include +#endif #include "internal.h" @@ -165,6 +171,17 @@ * options set. This moves slab handling out of * the fast path and disables lockless freelists. */ +#ifdef CONFIG_IEE +void __weak iee_allocate_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order) {} +bool __weak iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order) +{ + return false; +} +unsigned int __weak iee_calculate_order(struct kmem_cache *s, unsigned int order) +{ + return order; +} +#endif /* * We could simply use migrate_disable()/enable() but as long as it's a @@ -2031,6 +2048,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; slab = alloc_slab_page(alloc_gfp, node, oo); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); +#endif if (unlikely(!slab)) { oo = s->min; alloc_gfp = flags; @@ -2039,6 +2060,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * Try a lower order alloc if possible */ slab = alloc_slab_page(alloc_gfp, node, oo); +#ifdef CONFIG_IEE_PTRP + if(haoc_enabled) + slab = iee_alloc_task_token_slab(s, slab, oo_order(oo)); +#endif if (unlikely(!slab)) return NULL; stat(s, ORDER_FALLBACK); @@ -2048,6 +2073,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) slab->inuse = 0; slab->frozen = 0; +#ifdef CONFIG_IEE + if(haoc_enabled) + iee_allocate_slab_data(s, slab, oo_order(oo)); +#endif account_slab(slab, oo_order(oo), s, flags); slab->slab_cache = s; @@ -2100,6 +2129,13 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) __folio_clear_slab(folio); mm_account_reclaimed_pages(pages); unaccount_slab(slab, order, s); +#ifdef CONFIG_IEE + if(haoc_enabled) + { + if (iee_free_slab_data(s, slab, order)) + return; + } +#endif __free_pages(&folio->page, order); } @@ -4490,6 +4526,10 @@ static int calculate_sizes(struct kmem_cache *s) s->size = size; s->reciprocal_size = reciprocal_value(size); order = calculate_order(size); + #ifdef CONFIG_IEE + if(haoc_enabled) + order = iee_calculate_order(s, order); + #endif if ((int)order < 0) return 0; -- Gitee From 9ab154c82af4fc3bd865b93d84ef15a594f31d22 Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Mon, 13 Oct 2025 15:52:21 +0800 Subject: [PATCH 7/8] HAOC: Add support for x86 CRED Protection (CREDP). --- arch/x86/Kconfig | 3 + arch/x86/include/asm/haoc/haoc-def.h | 36 ++++ arch/x86/include/asm/haoc/haoc.h | 52 ++++- arch/x86/include/asm/haoc/iee-cred.h | 285 +++++++++++++++++++++++++ arch/x86/include/asm/haoc/iee-func.h | 6 + arch/x86/kernel/haoc/Makefile | 1 + arch/x86/kernel/haoc/credp/Makefile | 3 + arch/x86/kernel/haoc/credp/credp.c | 204 ++++++++++++++++++ arch/x86/kernel/haoc/haoc.c | 31 +++ arch/x86/kernel/haoc/iee/iee-func.c | 11 + arch/x86/kernel/haoc/iee/iee-gate.S | 1 + arch/x86/kernel/haoc/iee/iee-init.c | 1 + arch/x86/kernel/vmlinux.lds.S | 9 + fs/coredump.c | 8 + fs/exec.c | 11 + fs/nfs/flexfilelayout/flexfilelayout.c | 9 + fs/nfs/nfs4idmap.c | 9 + fs/nfsd/auth.c | 38 ++++ fs/nfsd/nfs4callback.c | 8 + fs/nfsd/nfs4recover.c | 9 + fs/nfsd/nfsfh.c | 8 + fs/open.c | 25 +++ fs/overlayfs/dir.c | 8 + fs/overlayfs/super.c | 12 ++ fs/smb/client/cifs_spnego.c | 8 + fs/smb/client/cifsacl.c | 8 + include/asm-generic/vmlinux.lds.h | 9 + include/linux/cred.h | 64 ++++++ kernel/cred.c | 179 ++++++++++++++++ kernel/groups.c | 7 + kernel/sys.c | 105 +++++++++ kernel/umh.c | 9 + kernel/user_namespace.c | 17 ++ mm/slab.h | 16 +- mm/slub.c | 107 +++++++++- net/dns_resolver/dns_key.c | 8 + net/sunrpc/auth.c | 28 +++ security/commoncap.c | 179 +++++++++++++++- security/keys/keyctl.c | 32 ++- security/keys/process_keys.c | 49 +++++ security/security.c | 15 ++ 41 files changed, 1620 insertions(+), 8 deletions(-) create mode 100644 arch/x86/include/asm/haoc/iee-cred.h create mode 100644 arch/x86/kernel/haoc/credp/Makefile create mode 100644 arch/x86/kernel/haoc/credp/credp.c diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 3b27d0fbdb06..989af2993ab5 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1567,6 +1567,9 @@ config IEE_PTRP depends on IEE def_bool y +config CREDP + depends on IEE + def_bool y # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/arch/x86/include/asm/haoc/haoc-def.h b/arch/x86/include/asm/haoc/haoc-def.h index 1eb801149521..e812fb44c660 100644 --- a/arch/x86/include/asm/haoc/haoc-def.h +++ b/arch/x86/include/asm/haoc/haoc-def.h @@ -19,6 +19,37 @@ enum { IEE_OP_SET_TOKEN_PGD, IEE_OP_INVALIDATE_TOKEN, IEE_OP_VALIDATE_TOKEN, +#endif +#ifdef CONFIG_CREDP + IEE_OP_COPY_CRED, + IEE_OP_SET_CRED_UID, + IEE_OP_SET_CRED_GID, + IEE_OP_SET_CRED_SUID, + IEE_OP_SET_CRED_SGID, + IEE_OP_SET_CRED_EUID, + IEE_OP_SET_CRED_EGID, + IEE_OP_SET_CRED_FSUID, + IEE_OP_SET_CRED_FSGID, + IEE_OP_SET_CRED_USER, + IEE_OP_SET_CRED_USER_NS, + IEE_OP_SET_CRED_GROUP_INFO, + IEE_OP_SET_CRED_SECUREBITS, + IEE_OP_SET_CRED_CAP_INHER, + IEE_OP_SET_CRED_CAP_PERM, + IEE_OP_SET_CRED_CAP_EFFECT, + IEE_OP_SET_CRED_CAP_BSET, + IEE_OP_SET_CRED_CAP_AMBIENT, + IEE_OP_SET_CRED_JIT_KEYRING, + IEE_OP_SET_CRED_SESS_KEYRING, + IEE_OP_SET_CRED_PROC_KEYRING, + IEE_OP_SET_CRED_THREAD_KEYRING, + IEE_OP_SET_CRED_REQ_KEYRING, + IEE_OP_SET_CRED_NON_RCU, + IEE_OP_SET_CRED_ATSET_USAGE, + IEE_OP_SET_CRED_ATOP_USAGE, + IEE_OP_SET_CRED_SECURITY, + IEE_OP_SET_CRED_RCU, + IEE_OP_SET_CRED_UCOUNTS, #endif IEE_FLAG_END }; @@ -31,4 +62,9 @@ enum { #define IEE_LOAD_IDT 4 #endif +#ifdef CONFIG_CREDP +#define AT_ADD 1 +#define AT_INC_NOT_ZERO 2 +#define AT_SUB_AND_TEST 3 +#endif #endif diff --git a/arch/x86/include/asm/haoc/haoc.h b/arch/x86/include/asm/haoc/haoc.h index 8b24a71ec67c..a10917e07885 100644 --- a/arch/x86/include/asm/haoc/haoc.h +++ b/arch/x86/include/asm/haoc/haoc.h @@ -13,9 +13,9 @@ #include #include -void _iee_memcpy(unsigned long __unused, void *dst, void *src, size_t n); -void _iee_memset(unsigned long __unused, void *ptr, int data, size_t n); -void _iee_set_freeptr(unsigned long __unused, void **pptr, void *ptr); +void _iee_memcpy(unsigned long __unused,void *dst,void *src, size_t n); +void _iee_memset(unsigned long __unused,void *ptr, int data, size_t n); +void _iee_set_freeptr(unsigned long __unused,void **pptr,void *ptr); unsigned long _iee_test_and_clear_bit(unsigned long __unused, long nr, unsigned long *addr); #ifdef CONFIG_IEE_PTRP @@ -24,4 +24,50 @@ void _iee_set_token_pgd(unsigned long __unused, struct task_struct *tsk, void _iee_invalidate_token(unsigned long __unused, struct task_struct *tsk); void _iee_validate_token(unsigned long __unused, struct task_struct *tsk); #endif + +#ifdef CONFIG_CREDP +#include + +void _iee_copy_cred(unsigned long __unused, struct cred *old, struct cred *new); +void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid); +void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid); +void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid); +void _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid); +void _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid); +void _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid); +void _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid); +void _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid); +void _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user); +void _iee_set_cred_user_ns(unsigned long __unused, + struct cred *cred, struct user_namespace *user_ns); +void _iee_set_cred_group_info(unsigned long __unused, + struct cred *cred, struct group_info *group_info); +void _iee_set_cred_securebits(unsigned long __unused, + struct cred *cred, unsigned int securebits); +void _iee_set_cred_cap_inheritable(unsigned long __unused, + struct cred *cred, kernel_cap_t cap_inheritable); +void _iee_set_cred_cap_permitted(unsigned long __unused, + struct cred *cred, kernel_cap_t cap_permitted); +void _iee_set_cred_cap_effective(unsigned long __unused, + struct cred *cred, kernel_cap_t cap_effective); +void _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset); +void _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient); +void _iee_set_cred_jit_keyring(unsigned long __unused, + struct cred *cred, unsigned char jit_keyring); +void _iee_set_cred_session_keyring(unsigned long __unused, + struct cred *cred, struct key *session_keyring); +void _iee_set_cred_process_keyring(unsigned long __unused, + struct cred *cred, struct key *process_keyring); +void _iee_set_cred_thread_keyring(unsigned long __unused, + struct cred *cred, struct key *thread_keyring); +void _iee_set_cred_request_key_auth(unsigned long __unused, + struct cred *cred, struct key *request_key_auth); +void _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu); +void _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i); +unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, + struct cred *cred, int flag, int nr); +void _iee_set_cred_security(unsigned long __unused, struct cred *cred,void *security); +void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu); +void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts); +#endif #endif diff --git a/arch/x86/include/asm/haoc/iee-cred.h b/arch/x86/include/asm/haoc/iee-cred.h new file mode 100644 index 000000000000..7ddab8c1453c --- /dev/null +++ b/arch/x86/include/asm/haoc/iee-cred.h @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _LINUX_IEE_CRED_H +#define _LINUX_IEE_CRED_H + +#include +#include + +extern unsigned long long iee_rw_gate(int flag, ...); + +static void __maybe_unused iee_copy_cred(const struct cred *old, struct cred *new) +{ + if(!haoc_enabled) + { + memcpy(new, old, sizeof(struct cred)); + return; + } + iee_rw_gate(IEE_OP_COPY_CRED, old, new); +} + +static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) +{ + if(!haoc_enabled) + { + cred->uid = uid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_UID, cred, uid); +} + +static void __maybe_unused iee_set_cred_gid(struct cred *cred, kgid_t gid) +{ + if(!haoc_enabled) + { + cred->gid = gid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_GID, cred, gid); +} + +static void __maybe_unused iee_set_cred_suid(struct cred *cred, kuid_t suid) +{ + if(!haoc_enabled) + { + cred->suid = suid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_SUID, cred, suid); +} + +static void __maybe_unused iee_set_cred_sgid(struct cred *cred, kgid_t sgid) +{ + if(!haoc_enabled) + { + cred->sgid = sgid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_SGID, cred, sgid); +} + +static void __maybe_unused iee_set_cred_euid(struct cred *cred, kuid_t euid) +{ + if(!haoc_enabled) + { + cred->euid = euid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_EUID, cred, euid); +} + +static void __maybe_unused iee_set_cred_egid(struct cred *cred, kgid_t egid) +{ + if(!haoc_enabled) + { + cred->egid = egid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_EGID, cred, egid); +} + +static void __maybe_unused iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid) +{ + if(!haoc_enabled) + { + cred->fsuid = fsuid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_FSUID, cred, fsuid); +} + +static void __maybe_unused iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid) +{ + if(!haoc_enabled) + { + cred->fsgid = fsgid; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_FSGID, cred, fsgid); +} + +static void __maybe_unused iee_set_cred_user(struct cred *cred, struct user_struct *user) +{ + if(!haoc_enabled) + { + cred->user = user; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_USER, cred, user); +} + +static void __maybe_unused iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) +{ + if(!haoc_enabled) + { + cred->user_ns = user_ns; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_USER_NS, cred, user_ns); +} + +static void __maybe_unused iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts) +{ + if(!haoc_enabled) + { + cred->ucounts = ucounts; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS, cred, ucounts); +} + +static void __maybe_unused iee_set_cred_group_info(struct cred *cred, struct group_info *group_info) +{ + if(!haoc_enabled) + { + cred->group_info = group_info; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO, cred, group_info); +} + +static void __maybe_unused iee_set_cred_securebits(struct cred *cred, + unsigned int securebits) +{ + if(!haoc_enabled) + { + cred->securebits = securebits; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS, cred, securebits); +} + +static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, + kernel_cap_t cap_inheritable) +{ + if(!haoc_enabled) + { + cred->cap_inheritable = cap_inheritable; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER, cred, cap_inheritable); +} + +static void __maybe_unused iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted) +{ + if(!haoc_enabled) + { + cred->cap_permitted = cap_permitted; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM, cred, cap_permitted); +} + +static void __maybe_unused iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective) +{ + if(!haoc_enabled) + { + cred->cap_effective = cap_effective; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT, cred, cap_effective); +} + +static void __maybe_unused iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset) +{ + if(!haoc_enabled) + { + cred->cap_bset = cap_bset; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET, cred, cap_bset); +} + +static void __maybe_unused iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient) +{ + if(!haoc_enabled) + { + cred->cap_ambient = cap_ambient; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT, cred, cap_ambient); +} + +static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i) +{ + if(!haoc_enabled) + { + atomic_long_set(&cred->usage, i); + return; + } + iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE, cred, i); +} + +static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) +{ + iee_rw_gate(IEE_OP_SET_CRED_RCU, cred, rcu); +} + +#ifdef CONFIG_KEYS +static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) +{ + if(!haoc_enabled) + { + cred->jit_keyring = jit_keyring; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING, cred, jit_keyring); +} + +static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, + struct key *session_keyring) +{ + if(!haoc_enabled) + { + cred->session_keyring = session_keyring; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING, cred, session_keyring); +} + +static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, + struct key *process_keyring) +{ + if(!haoc_enabled) + { + cred->process_keyring = process_keyring; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING, cred, process_keyring); +} + +static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, + struct key *thread_keyring) +{ + if(!haoc_enabled) + { + cred->thread_keyring = thread_keyring; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING, cred, thread_keyring); +} + +static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, + struct key *request_key_auth) +{ + if(!haoc_enabled) + { + cred->request_key_auth = request_key_auth; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING, cred, request_key_auth); +} +#endif + +#ifdef CONFIG_SECURITY +static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security) +{ + if(!haoc_enabled) + { + cred->security = security; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_SECURITY, cred, security); +} +#endif + +#endif \ No newline at end of file diff --git a/arch/x86/include/asm/haoc/iee-func.h b/arch/x86/include/asm/haoc/iee-func.h index 523ebecf7f38..ffad94ecb8b6 100644 --- a/arch/x86/include/asm/haoc/iee-func.h +++ b/arch/x86/include/asm/haoc/iee-func.h @@ -10,9 +10,15 @@ #ifndef _LINUX_IEE_FUNC_H #define _LINUX_IEE_FUNC_H +#define HUGE_PMD_ORDER 9 + #include extern void set_iee_page(unsigned long addr, unsigned int order); extern void unset_iee_page(unsigned long addr, unsigned int order); extern bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int order); extern unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order); + +extern void iee_free_slab(struct kmem_cache *s, struct slab *slab, + void (*do_free_slab)(struct work_struct *work)); +extern void iee_free_cred_slab(struct work_struct *work); #endif /* _LINUX_IEE_FUNC_H */ diff --git a/arch/x86/kernel/haoc/Makefile b/arch/x86/kernel/haoc/Makefile index eada289a033f..367af97ba37e 100644 --- a/arch/x86/kernel/haoc/Makefile +++ b/arch/x86/kernel/haoc/Makefile @@ -1,2 +1,3 @@ obj-y += haoc.o obj-y += iee/ +obj-$(CONFIG_CREDP) += credp/ \ No newline at end of file diff --git a/arch/x86/kernel/haoc/credp/Makefile b/arch/x86/kernel/haoc/credp/Makefile new file mode 100644 index 000000000000..cc494acaa7a1 --- /dev/null +++ b/arch/x86/kernel/haoc/credp/Makefile @@ -0,0 +1,3 @@ +obj-y += credp.o + +ccflags-y += -I$(srctree)/mm \ No newline at end of file diff --git a/arch/x86/kernel/haoc/credp/credp.c b/arch/x86/kernel/haoc/credp/credp.c new file mode 100644 index 000000000000..7ad23a6973db --- /dev/null +++ b/arch/x86/kernel/haoc/credp/credp.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include +#include +#include "slab.h" + +extern struct cred init_cred; + +void _iee_set_cred_rcu(unsigned long __unused, struct cred *cred, struct rcu_head *rcu) +{ + *((struct rcu_head **)(&(cred->rcu.func))) = rcu; +} + +void _iee_set_cred_security(unsigned long __unused, struct cred *cred, void *security) +{ + cred->security = security; +} + +unsigned long _iee_set_cred_atomic_op_usage(unsigned long __unused, + struct cred *cred, int flag, int nr) +{ + switch (flag) { + case AT_ADD: { + atomic_long_add(nr, &cred->usage); + return 0; + } + case AT_INC_NOT_ZERO: { + return atomic_long_inc_not_zero(&cred->usage); + } + case AT_SUB_AND_TEST: { + return atomic_long_sub_and_test(nr, &cred->usage); + } + } + return 0; +} + +void _iee_set_cred_atomic_set_usage(unsigned long __unused, struct cred *cred, int i) +{ + atomic_long_set(&cred->usage, i); +} + +void _iee_set_cred_non_rcu(unsigned long __unused, struct cred *cred, int non_rcu) +{ + cred->non_rcu = non_rcu; +} + +void _iee_set_cred_session_keyring(unsigned long __unused, struct cred *cred, + struct key *session_keyring) +{ + cred->session_keyring = session_keyring; +} + +void _iee_set_cred_process_keyring(unsigned long __unused, struct cred *cred, + struct key *process_keyring) +{ + cred->process_keyring = process_keyring; +} + +void _iee_set_cred_thread_keyring(unsigned long __unused, struct cred *cred, + struct key *thread_keyring) +{ + cred->thread_keyring = thread_keyring; +} + +void _iee_set_cred_request_key_auth(unsigned long __unused, struct cred *cred, + struct key *request_key_auth) +{ + cred->request_key_auth = request_key_auth; +} + +void _iee_set_cred_jit_keyring(unsigned long __unused, struct cred *cred, unsigned char jit_keyring) +{ + cred->jit_keyring = jit_keyring; +} + +void _iee_set_cred_cap_inheritable(unsigned long __unused, struct cred *cred, + kernel_cap_t cap_inheritable) +{ + cred->cap_inheritable = cap_inheritable; +} + +void _iee_set_cred_cap_permitted(unsigned long __unused, struct cred *cred, + kernel_cap_t cap_permitted) +{ + cred->cap_permitted = cap_permitted; +} + +void _iee_set_cred_cap_effective(unsigned long __unused, struct cred *cred, + kernel_cap_t cap_effective) +{ + cred->cap_effective = cap_effective; +} + +void _iee_set_cred_cap_bset(unsigned long __unused, struct cred *cred, kernel_cap_t cap_bset) +{ + cred->cap_bset = cap_bset; +} + +void _iee_set_cred_cap_ambient(unsigned long __unused, struct cred *cred, kernel_cap_t cap_ambient) +{ + cred->cap_ambient = cap_ambient; +} + +void _iee_set_cred_securebits(unsigned long __unused, struct cred *cred, + unsigned int securebits) +{ + cred->securebits = securebits; +} + +void _iee_set_cred_group_info(unsigned long __unused, struct cred *cred, + struct group_info *group_info) +{ + cred->group_info = group_info; +} + +void _iee_set_cred_ucounts(unsigned long __unused, struct cred *cred, struct ucounts *ucounts) +{ + cred->ucounts = ucounts; +} + +void _iee_set_cred_user_ns(unsigned long __unused, struct cred *cred, + struct user_namespace *user_ns) +{ + cred->user_ns = user_ns; +} + +void _iee_set_cred_user(unsigned long __unused, struct cred *cred, struct user_struct *user) +{ + cred->user = user; +} + +void _iee_set_cred_fsgid(unsigned long __unused, struct cred *cred, kgid_t fsgid) +{ + cred->fsgid = fsgid; +} + +void _iee_set_cred_fsuid(unsigned long __unused, struct cred *cred, kuid_t fsuid) +{ + cred->fsuid = fsuid; +} + +void _iee_set_cred_egid(unsigned long __unused, struct cred *cred, kgid_t egid) +{ + cred->egid = egid; +} + +void _iee_set_cred_euid(unsigned long __unused, struct cred *cred, kuid_t euid) +{ + cred->euid = euid; +} + +void _iee_set_cred_sgid(unsigned long __unused, struct cred *cred, kgid_t sgid) +{ + cred->sgid = sgid; +} + +void _iee_set_cred_suid(unsigned long __unused, struct cred *cred, kuid_t suid) +{ + cred->suid = suid; +} + +void _iee_copy_cred(unsigned long __unused, struct cred *old, struct cred *new) +{ + if (new == &init_cred) + panic("copy_cred for init_cred: %lx\n", (unsigned long)new); + struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); + + memcpy(new, old, sizeof(struct cred)); + *(struct rcu_head **)(&(new->rcu.func)) = rcu; + *(struct rcu_head *)(new->rcu.func) = *(struct rcu_head *)(old->rcu.func); +} + +void _iee_set_cred_gid(unsigned long __unused, struct cred *cred, kgid_t gid) +{ + cred->gid = gid; +} + +void _iee_set_cred_uid(unsigned long __unused, struct cred *cred, kuid_t uid) +{ + cred->uid = uid; +} + +struct iee_free_slab_work { + struct work_struct work; + struct kmem_cache *s; + struct slab *slab; +}; + +void iee_free_cred_slab(struct work_struct *work) +{ + struct iee_free_slab_work *iee_free_slab_work = + container_of(work, struct iee_free_slab_work, work); + struct slab *slab = iee_free_slab_work->slab; + struct folio *folio = slab_folio(slab); + int order = folio_order(folio); + + unset_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), order); + __free_pages(&folio->page, order); + kfree(iee_free_slab_work); +} diff --git a/arch/x86/kernel/haoc/haoc.c b/arch/x86/kernel/haoc/haoc.c index 79ebc303ae99..28d20186b010 100644 --- a/arch/x86/kernel/haoc/haoc.c +++ b/arch/x86/kernel/haoc/haoc.c @@ -19,6 +19,37 @@ iee_func iee_funcs[] = { (iee_func)_iee_set_token_pgd, (iee_func)_iee_invalidate_token, (iee_func)_iee_validate_token, +#endif +#ifdef CONFIG_CREDP + (iee_func)_iee_copy_cred, + (iee_func)_iee_set_cred_uid, + (iee_func)_iee_set_cred_gid, + (iee_func)_iee_set_cred_suid, + (iee_func)_iee_set_cred_sgid, + (iee_func)_iee_set_cred_euid, + (iee_func)_iee_set_cred_egid, + (iee_func)_iee_set_cred_fsuid, + (iee_func)_iee_set_cred_fsgid, + (iee_func)_iee_set_cred_user, + (iee_func)_iee_set_cred_user_ns, + (iee_func)_iee_set_cred_group_info, + (iee_func)_iee_set_cred_securebits, + (iee_func)_iee_set_cred_cap_inheritable, + (iee_func)_iee_set_cred_cap_permitted, + (iee_func)_iee_set_cred_cap_effective, + (iee_func)_iee_set_cred_cap_bset, + (iee_func)_iee_set_cred_cap_ambient, + (iee_func)_iee_set_cred_jit_keyring, + (iee_func)_iee_set_cred_session_keyring, + (iee_func)_iee_set_cred_process_keyring, + (iee_func)_iee_set_cred_thread_keyring, + (iee_func)_iee_set_cred_request_key_auth, + (iee_func)_iee_set_cred_non_rcu, + (iee_func)_iee_set_cred_atomic_set_usage, + (iee_func)_iee_set_cred_atomic_op_usage, + (iee_func)_iee_set_cred_security, + (iee_func)_iee_set_cred_rcu, + (iee_func)_iee_set_cred_ucounts, #endif NULL }; diff --git a/arch/x86/kernel/haoc/iee/iee-func.c b/arch/x86/kernel/haoc/iee/iee-func.c index c18dbecb696d..0054c1b3d616 100644 --- a/arch/x86/kernel/haoc/iee/iee-func.c +++ b/arch/x86/kernel/haoc/iee/iee-func.c @@ -13,6 +13,7 @@ #ifdef CONFIG_IEE_PTRP #include #endif +extern bool haoc_enabled; void set_iee_page(unsigned long addr, unsigned int order) { @@ -33,6 +34,8 @@ struct iee_free_slab_work { void iee_free_slab(struct kmem_cache *s, struct slab *slab, void (*do_free_slab)(struct work_struct *work)) { + if(haoc_enabled) + return; struct iee_free_slab_work *iee_free_slab_work = kmalloc(sizeof(struct iee_free_slab_work), GFP_ATOMIC); @@ -72,9 +75,17 @@ bool iee_free_slab_data(struct kmem_cache *s, struct slab *slab, unsigned int iee_calculate_order(struct kmem_cache *s, unsigned int order) { + if(!haoc_enabled) + { + return order; + } #ifdef CONFIG_IEE_PTRP if (strcmp(s->name, "task_struct") == 0) return IEE_DATA_ORDER; +#endif +#ifdef CONFIG_CREDP + if (strcmp(s->name, "cred_jar") == 0) + order = IEE_DATA_ORDER; #endif return order; } \ No newline at end of file diff --git a/arch/x86/kernel/haoc/iee/iee-gate.S b/arch/x86/kernel/haoc/iee/iee-gate.S index 974cdd707b25..5497c3788b02 100644 --- a/arch/x86/kernel/haoc/iee/iee-gate.S +++ b/arch/x86/kernel/haoc/iee/iee-gate.S @@ -71,6 +71,7 @@ SYM_FUNC_START(iee_rw_gate) jmp __x86_return_thunk /* ret */ SYM_FUNC_END(iee_rw_gate) +EXPORT_SYMBOL(iee_rw_gate) #ifdef CONFIG_IEE_SIP SYM_FUNC_START(iee_rwx_gate) diff --git a/arch/x86/kernel/haoc/iee/iee-init.c b/arch/x86/kernel/haoc/iee/iee-init.c index 9d9366609390..7ac4704769f4 100644 --- a/arch/x86/kernel/haoc/iee/iee-init.c +++ b/arch/x86/kernel/haoc/iee/iee-init.c @@ -165,6 +165,7 @@ void __init iee_init(void) } bool __ro_after_init haoc_enabled; +EXPORT_SYMBOL(haoc_enabled); #ifdef CONFIG_IEE_SIP extern unsigned long cr4_pinned_mask; #endif diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index f58362f004e5..cec6da2e4712 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -114,6 +114,12 @@ __iee_si_text_end = .; . = ALIGN(PAGE_SIZE); \ __iee_si_data_end = .; #endif +#ifdef CONFIG_CREDP +#define CRED_DATA \ + . = ALIGN(PAGE_SIZE); \ + *(.iee.cred) \ + . = ALIGN(PAGE_SIZE); +#endif PHDRS { text PT_LOAD FLAGS(5); /* R_E */ @@ -202,6 +208,9 @@ SECTIONS DATA_DATA #ifdef CONFIG_IEE_SIP IEE_SI_DATA +#endif +#ifdef CONFIG_CREDP + CRED_DATA #endif CONSTRUCTORS diff --git a/fs/coredump.c b/fs/coredump.c index d3a4f5dc2e36..8d9d9ede316a 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -54,6 +54,10 @@ #include +#ifdef CONFIG_CREDP +#include +#endif + static bool dump_vma_snapshot(struct coredump_params *cprm); static void free_vma_snapshot(struct coredump_params *cprm); @@ -633,7 +637,11 @@ void do_coredump(const kernel_siginfo_t *siginfo) */ if (__get_dumpable(cprm.mm_flags) == SUID_DUMP_ROOT) { /* Setuid core dump mode */ + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(cred, GLOBAL_ROOT_UID); + #else cred->fsuid = GLOBAL_ROOT_UID; /* Dump root private */ + #endif need_suid_safe = true; } diff --git a/fs/exec.c b/fs/exec.c index 006c43f03523..45ae26671641 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -80,6 +80,9 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#ifdef CONFIG_CREDP +#include +#endif static int bprm_creds_from_file(struct linux_binprm *bprm); @@ -1680,12 +1683,20 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) if (mode & S_ISUID) { bprm->per_clear |= PER_CLEAR_ON_SETID; + #ifdef CONFIG_CREDP + iee_set_cred_euid(bprm->cred, vfsuid_into_kuid(vfsuid)); + #else bprm->cred->euid = vfsuid_into_kuid(vfsuid); + #endif } if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { bprm->per_clear |= PER_CLEAR_ON_SETID; + #ifdef CONFIG_CREDP + iee_set_cred_egid(bprm->cred, vfsgid_into_kgid(vfsgid)); + #else bprm->cred->egid = vfsgid_into_kgid(vfsgid); + #endif } } diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 0a26444fe202..eeef79c593ee 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -15,6 +15,10 @@ #include +#ifdef CONFIG_CREDP +#include +#endif + #include "flexfilelayout.h" #include "../nfs4session.h" #include "../nfs4idmap.h" @@ -502,8 +506,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, rc = -ENOMEM; if (!kcred) goto out_err_free; + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(kcred, uid); + iee_set_cred_fsgid(kcred, gid); + #else kcred->fsuid = uid; kcred->fsgid = gid; + #endif cred = RCU_INITIALIZER(kcred); if (lgr->range.iomode == IOMODE_READ) diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c index 25a7c771cfd8..820a7c45d633 100644 --- a/fs/nfs/nfs4idmap.c +++ b/fs/nfs/nfs4idmap.c @@ -48,6 +48,10 @@ #include #include +#ifdef CONFIG_CREDP +#include +#endif + #include "internal.h" #include "netns.h" #include "nfs4idmap.h" @@ -226,8 +230,13 @@ int nfs_idmap_init(void) goto failed_reg_legacy; set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(cred, keyring); + iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); + #else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif id_resolver_cache = cred; return 0; diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c index e6beaaf4f170..79250fcc1288 100644 --- a/fs/nfsd/auth.c +++ b/fs/nfsd/auth.c @@ -2,6 +2,9 @@ /* Copyright (C) 1995, 1996 Olaf Kirch */ #include +#ifdef CONFIG_CREDP +#include +#endif #include "nfsd.h" #include "auth.h" @@ -32,22 +35,40 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) if (!new) return -ENOMEM; + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, rqstp->rq_cred.cr_uid); + iee_set_cred_fsgid(new, rqstp->rq_cred.cr_gid); + #else new->fsuid = rqstp->rq_cred.cr_uid; new->fsgid = rqstp->rq_cred.cr_gid; + #endif rqgi = rqstp->rq_cred.cr_group_info; if (flags & NFSEXP_ALLSQUASH) { + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, exp->ex_anon_uid); + iee_set_cred_fsgid(new, exp->ex_anon_gid); + #else new->fsuid = exp->ex_anon_uid; new->fsgid = exp->ex_anon_gid; + #endif gi = groups_alloc(0); if (!gi) goto oom; } else if (flags & NFSEXP_ROOTSQUASH) { if (uid_eq(new->fsuid, GLOBAL_ROOT_UID)) + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, exp->ex_anon_uid); + #else new->fsuid = exp->ex_anon_uid; + #endif if (gid_eq(new->fsgid, GLOBAL_ROOT_GID)) + #ifdef CONFIG_CREDP + iee_set_cred_fsgid(new, exp->ex_anon_gid); + #else new->fsgid = exp->ex_anon_gid; + #endif gi = groups_alloc(rqgi->ngroups); if (!gi) @@ -67,18 +88,35 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) } if (uid_eq(new->fsuid, INVALID_UID)) + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, exp->ex_anon_uid); + #else new->fsuid = exp->ex_anon_uid; + #endif if (gid_eq(new->fsgid, INVALID_GID)) + #ifdef CONFIG_CREDP + iee_set_cred_fsgid(new, exp->ex_anon_gid); + #else new->fsgid = exp->ex_anon_gid; + #endif set_groups(new, gi); put_group_info(gi); if (!uid_eq(new->fsuid, GLOBAL_ROOT_UID)) + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, cap_drop_nfsd_set(new->cap_effective)); + #else new->cap_effective = cap_drop_nfsd_set(new->cap_effective); + #endif else + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted)); + #else new->cap_effective = cap_raise_nfsd_set(new->cap_effective, new->cap_permitted); + #endif put_cred(override_creds(new)); put_cred(new); return 0; diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index d47173d98eef..eb977c4d55cc 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -35,6 +35,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "nfsd.h" #include "state.h" #include "netns.h" @@ -946,8 +949,13 @@ static const struct cred *get_backchannel_cred(struct nfs4_client *clp, struct r if (!kcred) return NULL; + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(kcred, ses->se_cb_sec.uid); + iee_set_cred_fsgid(kcred, ses->se_cb_sec.gid); + #else kcred->fsuid = ses->se_cb_sec.uid; kcred->fsgid = ses->se_cb_sec.gid; + #endif return kcred; } } diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index 892fecce18b8..65d446239ed5 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -44,6 +44,10 @@ #include #include +#ifdef CONFIG_CREDP +#include +#endif + #include "nfsd.h" #include "state.h" #include "vfs.h" @@ -78,8 +82,13 @@ nfs4_save_creds(const struct cred **original_creds) if (!new) return -ENOMEM; + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, GLOBAL_ROOT_UID); + iee_set_cred_fsgid(new, GLOBAL_ROOT_GID); + #else new->fsuid = GLOBAL_ROOT_UID; new->fsgid = GLOBAL_ROOT_GID; + #endif *original_creds = override_creds(new); put_cred(new); return 0; diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index c2495d98c189..8503eab9e0ee 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -11,6 +11,9 @@ #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "nfsd.h" #include "vfs.h" #include "auth.h" @@ -223,9 +226,14 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) error = nfserrno(-ENOMEM); goto out; } + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted)); + #else new->cap_effective = cap_raise_nfsd_set(new->cap_effective, new->cap_permitted); + #endif put_cred(override_creds(new)); put_cred(new); } else { diff --git a/fs/open.c b/fs/open.c index 4679db501d43..f96e3be84a39 100644 --- a/fs/open.c +++ b/fs/open.c @@ -34,6 +34,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "internal.h" @@ -414,17 +417,35 @@ static const struct cred *access_override_creds(void) * routine. */ + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(override_cred, override_cred->uid); + iee_set_cred_fsgid(override_cred, override_cred->gid); + #else override_cred->fsuid = override_cred->uid; override_cred->fsgid = override_cred->gid; + #endif if (!issecure(SECURE_NO_SETUID_FIXUP)) { /* Clear the capabilities if we switch to a non-root user */ kuid_t root_uid = make_kuid(override_cred->user_ns, 0); if (!uid_eq(override_cred->uid, root_uid)) + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = override_cred->cap_effective; + + tmp_cap.val = 0; + iee_set_cred_cap_effective(override_cred, tmp_cap); + } while (0); + #else cap_clear(override_cred->cap_effective); + #endif else + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(override_cred, override_cred->cap_permitted); + #else override_cred->cap_effective = override_cred->cap_permitted; + #endif } /* @@ -444,7 +465,11 @@ static const struct cred *access_override_creds(void) * expecting RCU freeing. But normal thread-synchronous * cred accesses will keep things non-RCY. */ + #ifdef CONFIG_CREDP + iee_set_cred_non_rcu(override_cred, 1); + #else override_cred->non_rcu = 1; + #endif old_cred = override_creds(override_cred); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index d1e0488ad0d8..90e07084a3df 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -14,6 +14,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "overlayfs.h" static unsigned short ovl_redirect_max = 256; @@ -586,8 +589,13 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode, * create a new inode, so just use the ovl mounter's * fs{u,g}id. */ + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(override_cred, inode->i_uid); + iee_set_cred_fsgid(override_cred, inode->i_gid); + #else override_cred->fsuid = inode->i_uid; override_cred->fsgid = inode->i_gid; + #endif err = security_dentry_create_files_as(dentry, attr->mode, &dentry->d_name, old_cred, override_cred); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 416b1a61105c..d8bd6438cd42 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -18,6 +18,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "overlayfs.h" #include "params.h" @@ -1459,7 +1462,16 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_export_op = &ovl_export_fid_operations; /* Never override disk quota limits or use reserved space */ + #ifdef CONFIG_CREDP + { + kernel_cap_t tmp = cred->cap_effective; + + cap_lower(tmp, CAP_SYS_RESOURCE); + iee_set_cred_cap_effective(cred, tmp); + } + #else cap_lower(cred->cap_effective, CAP_SYS_RESOURCE); + #endif sb->s_magic = OVERLAYFS_SUPER_MAGIC; sb->s_xattr = ovl_xattr_handlers(ofs); diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index bc1c1e9b288a..2c5ce8b8c31b 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -14,6 +14,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "cifsglob.h" #include "cifs_spnego.h" #include "cifs_debug.h" @@ -230,8 +233,13 @@ init_cifs_spnego(void) * the results it looks up */ set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(cred, keyring); + iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); + #else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif spnego_cred = cred; cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index bf32bc22ebd6..0a6042d01a34 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -17,6 +17,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "cifspdu.h" #include "cifsglob.h" #include "cifsacl.h" @@ -491,8 +494,13 @@ init_cifs_idmap(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(cred, keyring); + iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); + #else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif root_cred = cred; cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index af4489f76a71..c667a36c0423 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -1100,6 +1100,14 @@ * They will fit only a subset of the architectures */ +#ifdef CONFIG_CREDP +#define CRED_DATA \ +. = ALIGN(PAGE_SIZE); \ +*(.iee.cred) \ +. = ALIGN(PAGE_SIZE); +#else +#define CRED_DATA +#endif /* * Writeable data. @@ -1117,6 +1125,7 @@ . = ALIGN(PAGE_SIZE); \ .data : AT(ADDR(.data) - LOAD_OFFSET) { \ INIT_TASK_DATA(inittask) \ + CRED_DATA \ NOSAVE_DATA \ PAGE_ALIGNED_DATA(pagealigned) \ CACHELINE_ALIGNED_DATA(cacheline) \ diff --git a/include/linux/cred.h b/include/linux/cred.h index de8d8cb71ad1..8e7617cf6287 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h @@ -16,6 +16,10 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +extern unsigned long long iee_rw_gate(int flag, ...); +#endif struct cred; struct inode; @@ -182,6 +186,27 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred) cred->cap_inheritable)); } +#ifdef CONFIG_CREDP +static void __maybe_unused iee_set_cred_non_rcu(struct cred *cred, int non_rcu) +{ + if(!haoc_enabled) + { + cred->non_rcu = 0; + return; + } + iee_rw_gate(IEE_OP_SET_CRED_NON_RCU, cred, non_rcu); + *(int *)(&(((struct rcu_head *)(cred->rcu.func))->next)) = non_rcu; +} + +static bool __maybe_unused iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr) +{ + bool ret; + + ret = iee_rw_gate(IEE_OP_SET_CRED_ATOP_USAGE, cred, flag, nr); + return ret; +} +#endif + /** * get_new_cred - Get a reference on a new set of credentials * @cred: The new credentials to reference @@ -191,7 +216,14 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred) */ static inline struct cred *get_new_cred(struct cred *cred) { + #ifdef CONFIG_CREDP + if(haoc_enabled) + iee_set_cred_atomic_op_usage(cred, AT_ADD, 1); + else + atomic_long_inc(&cred->usage); + #else atomic_long_inc(&cred->usage); + #endif return cred; } @@ -213,7 +245,11 @@ static inline const struct cred *get_cred(const struct cred *cred) struct cred *nonconst_cred = (struct cred *) cred; if (!cred) return cred; + #ifdef CONFIG_CREDP + iee_set_cred_non_rcu(nonconst_cred, 0); + #else nonconst_cred->non_rcu = 0; + #endif return get_new_cred(nonconst_cred); } @@ -222,9 +258,25 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred) struct cred *nonconst_cred = (struct cred *) cred; if (!cred) return NULL; + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + if (!iee_set_cred_atomic_op_usage(nonconst_cred, AT_INC_NOT_ZERO, 0)) + return NULL; + } + else{ + if (!atomic_long_inc_not_zero(&nonconst_cred->usage)) + return NULL; + } + #else if (!atomic_long_inc_not_zero(&nonconst_cred->usage)) return NULL; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_non_rcu(nonconst_cred, 0); + #else nonconst_cred->non_rcu = 0; + #endif return cred; } @@ -244,8 +296,20 @@ static inline void put_cred(const struct cred *_cred) struct cred *cred = (struct cred *) _cred; if (cred) { + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + if (iee_set_cred_atomic_op_usage(cred, AT_SUB_AND_TEST, 1)) + __put_cred(cred); + } + else{ + if (atomic_long_dec_and_test(&(cred)->usage)) + __put_cred(cred); + } + #else if (atomic_long_dec_and_test(&(cred)->usage)) __put_cred(cred); + #endif } } diff --git a/kernel/cred.c b/kernel/cred.c index 64404d51c052..78c1e9606b0b 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -19,6 +19,10 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#include +#endif #if 0 #define kdebug(FMT, ...) \ @@ -33,7 +37,12 @@ do { \ } while (0) #endif +#ifdef CONFIG_CREDP +struct kmem_cache *cred_jar; +static struct kmem_cache *rcu_jar; +#else static struct kmem_cache *cred_jar; +#endif /* init to 2 - one for init_task, one to ensure it is never freed */ static struct group_info init_groups = { .usage = ATOMIC_INIT(2) }; @@ -41,6 +50,32 @@ static struct group_info init_groups = { .usage = ATOMIC_INIT(2) }; /* * The initial credentials for the initial task */ +#ifdef CONFIG_CREDP +struct cred init_cred __section(".iee.cred") = { + .usage = ATOMIC_INIT(4), +#ifdef CONFIG_DEBUG_CREDENTIALS + .subscribers = ATOMIC_INIT(2), + .magic = CRED_MAGIC, +#endif + .uid = GLOBAL_ROOT_UID, + .gid = GLOBAL_ROOT_GID, + .suid = GLOBAL_ROOT_UID, + .sgid = GLOBAL_ROOT_GID, + .euid = GLOBAL_ROOT_UID, + .egid = GLOBAL_ROOT_GID, + .fsuid = GLOBAL_ROOT_UID, + .fsgid = GLOBAL_ROOT_GID, + .securebits = SECUREBITS_DEFAULT, + .cap_inheritable = CAP_EMPTY_SET, + .cap_permitted = CAP_FULL_SET, + .cap_effective = CAP_FULL_SET, + .cap_bset = CAP_FULL_SET, + .user = INIT_USER, + .user_ns = &init_user_ns, + .group_info = &init_groups, + .ucounts = &init_ucounts, +}; +#else struct cred init_cred = { .usage = ATOMIC_INIT(4), .uid = GLOBAL_ROOT_UID, @@ -61,13 +96,22 @@ struct cred init_cred = { .group_info = &init_groups, .ucounts = &init_ucounts, }; +#endif /* * The RCU callback to actually dispose of a set of credentials */ static void put_cred_rcu(struct rcu_head *rcu) { + #ifdef CONFIG_CREDP + struct cred *cred = NULL; + if(haoc_enabled) + cred = *(struct cred **)(rcu + 1); + else + cred = container_of(rcu, struct cred, rcu); + #else struct cred *cred = container_of(rcu, struct cred, rcu); + #endif kdebug("put_cred_rcu(%p)", cred); @@ -86,6 +130,12 @@ static void put_cred_rcu(struct rcu_head *rcu) if (cred->ucounts) put_ucounts(cred->ucounts); put_user_ns(cred->user_ns); + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + kmem_cache_free(rcu_jar, (struct rcu_head *)(cred->rcu.func)); + } + #endif kmem_cache_free(cred_jar, cred); } @@ -104,10 +154,26 @@ void __put_cred(struct cred *cred) BUG_ON(cred == current->cred); BUG_ON(cred == current->real_cred); + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + if (*(int *)(&(((struct rcu_head *)(cred->rcu.func))->next))) + put_cred_rcu((struct rcu_head *)(cred->rcu.func)); + else + call_rcu((struct rcu_head *)(cred->rcu.func), put_cred_rcu); + } + else{ + if (cred->non_rcu) + put_cred_rcu(&cred->rcu); + else + call_rcu(&cred->rcu, put_cred_rcu); + } + #else if (cred->non_rcu) put_cred_rcu(&cred->rcu); else call_rcu(&cred->rcu, put_cred_rcu); + #endif } EXPORT_SYMBOL(__put_cred); @@ -173,7 +239,21 @@ struct cred *cred_alloc_blank(void) if (!new) return NULL; + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + iee_set_cred_rcu(new, kmem_cache_zalloc(rcu_jar, GFP_KERNEL)); + *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; + } + iee_set_cred_atomic_set_usage(new, 1); + #else atomic_long_set(&new->usage, 1); + + #ifdef CONFIG_DEBUG_CREDENTIALS + new->magic = CRED_MAGIC; + #endif + #endif + if (security_cred_alloc_blank(new, GFP_KERNEL_ACCOUNT) < 0) goto error; @@ -208,13 +288,27 @@ struct cred *prepare_creds(void) if (!new) return NULL; + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + iee_set_cred_rcu(new, kmem_cache_alloc(rcu_jar, GFP_KERNEL)); + *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; + } + #endif + kdebug("prepare_creds() alloc %p", new); old = task->cred; + #ifdef CONFIG_CREDP + iee_copy_cred(old, new); + iee_set_cred_non_rcu(new, 0); + iee_set_cred_atomic_set_usage(new, 1); + #else memcpy(new, old, sizeof(struct cred)); new->non_rcu = 0; atomic_long_set(&new->usage, 1); + #endif get_group_info(new->group_info); get_uid(new->user); get_user_ns(new->user_ns); @@ -227,10 +321,18 @@ struct cred *prepare_creds(void) #endif #ifdef CONFIG_SECURITY + #ifdef CONFIG_CREDP + iee_set_cred_security(new, NULL); + #else new->security = NULL; + #endif #endif + #ifdef CONFIG_CREDP + iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); + #else new->ucounts = get_ucounts(new->ucounts); + #endif if (!new->ucounts) goto error; @@ -260,15 +362,30 @@ struct cred *prepare_exec_creds(void) #ifdef CONFIG_KEYS /* newly exec'd tasks don't get a thread keyring */ key_put(new->thread_keyring); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(new, NULL); + #else new->thread_keyring = NULL; + #endif /* inherit the session keyring; new process keyring */ key_put(new->process_keyring); + #ifdef CONFIG_CREDP + iee_set_cred_process_keyring(new, NULL); + #else new->process_keyring = NULL; + #endif #endif + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, new->euid); + iee_set_cred_suid(new, new->euid); + iee_set_cred_fsgid(new, new->egid); + iee_set_cred_sgid(new, new->egid); + #else new->suid = new->fsuid = new->euid; new->sgid = new->fsgid = new->egid; + #endif return new; } @@ -323,7 +440,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) * had one */ if (new->thread_keyring) { key_put(new->thread_keyring); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(new, NULL); + #else new->thread_keyring = NULL; + #endif if (clone_flags & CLONE_THREAD) install_thread_keyring_to_cred(new); } @@ -333,7 +454,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) */ if (!(clone_flags & CLONE_THREAD)) { key_put(new->process_keyring); + #ifdef CONFIG_CREDP + iee_set_cred_process_keyring(new, NULL); + #else new->process_keyring = NULL; + #endif } #endif @@ -591,7 +716,11 @@ int set_cred_ucounts(struct cred *new) if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid))) return -EAGAIN; + #ifdef CONFIG_CREDP + iee_set_cred_ucounts(new, new_ucounts); + #else new->ucounts = new_ucounts; + #endif put_ucounts(old_ucounts); return 0; @@ -603,8 +732,23 @@ int set_cred_ucounts(struct cred *new) void __init cred_init(void) { /* allocate a slab in which we can store credentials */ + #ifdef CONFIG_CREDP + cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + + rcu_jar = kmem_cache_create("rcu_jar", sizeof(struct rcu_head) + sizeof(struct cred *), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + if(haoc_enabled) + { + // Map init_cred + *((struct rcu_head **)(&(init_cred.rcu.func))) = + (struct rcu_head *)kmem_cache_zalloc(rcu_jar, GFP_KERNEL); + *(struct cred **)(((struct rcu_head *)(init_cred.rcu.func)) + 1) = &init_cred; + } + #else cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + #endif } /** @@ -635,29 +779,59 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon) if (!new) return NULL; + #ifdef CONFIG_CREDP + if(haoc_enabled) + { + iee_set_cred_rcu(new, kmem_cache_alloc(rcu_jar, GFP_KERNEL)); + *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; + } + #endif + kdebug("prepare_kernel_cred() alloc %p", new); old = get_task_cred(daemon); + #ifdef CONFIG_CREDP + iee_copy_cred(old, new); + iee_set_cred_non_rcu(new, 0); + iee_set_cred_atomic_set_usage(new, 1); + #else *new = *old; new->non_rcu = 0; atomic_long_set(&new->usage, 1); + #endif get_uid(new->user); get_user_ns(new->user_ns); get_group_info(new->group_info); #ifdef CONFIG_KEYS + #ifdef CONFIG_CREDP + iee_set_cred_session_keyring(new, NULL); + iee_set_cred_process_keyring(new, NULL); + iee_set_cred_thread_keyring(new, NULL); + iee_set_cred_request_key_auth(new, NULL); + iee_set_cred_jit_keyring(new, KEY_REQKEY_DEFL_THREAD_KEYRING); + #else new->session_keyring = NULL; new->process_keyring = NULL; new->thread_keyring = NULL; new->request_key_auth = NULL; new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif #endif #ifdef CONFIG_SECURITY + #ifdef CONFIG_CREDP + iee_set_cred_security(new, NULL); + #else new->security = NULL; + #endif #endif + #ifdef CONFIG_CREDP + iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); + #else new->ucounts = get_ucounts(new->ucounts); + #endif if (!new->ucounts) goto error; @@ -724,8 +898,13 @@ int set_create_files_as(struct cred *new, struct inode *inode) { if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) return -EINVAL; + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, inode->i_uid); + iee_set_cred_fsgid(new, inode->i_gid); + #else new->fsuid = inode->i_uid; new->fsgid = inode->i_gid; + #endif return security_kernel_create_files_as(new, inode); } EXPORT_SYMBOL(set_create_files_as); diff --git a/kernel/groups.c b/kernel/groups.c index 9aaed2a31073..86715829d5a0 100644 --- a/kernel/groups.c +++ b/kernel/groups.c @@ -11,6 +11,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif struct group_info *groups_alloc(int gidsetsize) { @@ -119,7 +122,11 @@ void set_groups(struct cred *new, struct group_info *group_info) { put_group_info(new->group_info); get_group_info(group_info); + #ifdef CONFIG_CREDP + iee_set_cred_group_info(new, group_info); + #else new->group_info = group_info; + #endif } EXPORT_SYMBOL(set_groups); diff --git a/kernel/sys.c b/kernel/sys.c index 355de0b65c23..35d8c4e46570 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -74,6 +74,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "uid16.h" @@ -395,7 +398,11 @@ long __sys_setregid(gid_t rgid, gid_t egid) if (gid_eq(old->gid, krgid) || gid_eq(old->egid, krgid) || ns_capable_setid(old->user_ns, CAP_SETGID)) + #ifdef CONFIG_CREDP + iee_set_cred_gid(new, krgid); + #else new->gid = krgid; + #endif else goto error; } @@ -404,15 +411,27 @@ long __sys_setregid(gid_t rgid, gid_t egid) gid_eq(old->egid, kegid) || gid_eq(old->sgid, kegid) || ns_capable_setid(old->user_ns, CAP_SETGID)) + #ifdef CONFIG_CREDP + iee_set_cred_egid(new, kegid); + #else new->egid = kegid; + #endif else goto error; } if (rgid != (gid_t) -1 || (egid != (gid_t) -1 && !gid_eq(kegid, old->gid))) + #ifdef CONFIG_CREDP + iee_set_cred_sgid(new, new->egid); + #else new->sgid = new->egid; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_fsgid(new, new->egid); + #else new->fsgid = new->egid; + #endif retval = security_task_fix_setgid(new, old, LSM_SETID_RE); if (retval < 0) @@ -454,9 +473,25 @@ long __sys_setgid(gid_t gid) retval = -EPERM; if (ns_capable_setid(old->user_ns, CAP_SETGID)) + #ifdef CONFIG_CREDP + { + iee_set_cred_fsgid(new, kgid); + iee_set_cred_sgid(new, kgid); + iee_set_cred_egid(new, kgid); + iee_set_cred_gid(new, kgid); + } + #else new->gid = new->egid = new->sgid = new->fsgid = kgid; + #endif else if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->sgid)) + #ifdef CONFIG_CREDP + { + iee_set_cred_fsgid(new, kgid); + iee_set_cred_egid(new, kgid); + } + #else new->egid = new->fsgid = kgid; + #endif else goto error; @@ -488,7 +523,11 @@ static int set_user(struct cred *new) return -EAGAIN; free_uid(new->user); + #ifdef CONFIG_CREDP + iee_set_cred_user(new, new_user); + #else new->user = new_user; + #endif return 0; } @@ -549,7 +588,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) retval = -EPERM; if (ruid != (uid_t) -1) { + #ifdef CONFIG_CREDP + iee_set_cred_uid(new, kruid); + #else new->uid = kruid; + #endif if (!uid_eq(old->uid, kruid) && !uid_eq(old->euid, kruid) && !ns_capable_setid(old->user_ns, CAP_SETUID)) @@ -557,7 +600,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) } if (euid != (uid_t) -1) { + #ifdef CONFIG_CREDP + iee_set_cred_euid(new, keuid); + #else new->euid = keuid; + #endif if (!uid_eq(old->uid, keuid) && !uid_eq(old->euid, keuid) && !uid_eq(old->suid, keuid) && @@ -572,8 +619,16 @@ long __sys_setreuid(uid_t ruid, uid_t euid) } if (ruid != (uid_t) -1 || (euid != (uid_t) -1 && !uid_eq(keuid, old->uid))) + #ifdef CONFIG_CREDP + iee_set_cred_suid(new, new->euid); + #else new->suid = new->euid; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, new->euid); + #else new->fsuid = new->euid; + #endif retval = security_task_fix_setuid(new, old, LSM_SETID_RE); if (retval < 0) @@ -626,7 +681,12 @@ long __sys_setuid(uid_t uid) retval = -EPERM; if (ns_capable_setid(old->user_ns, CAP_SETUID)) { + #ifdef CONFIG_CREDP + iee_set_cred_uid(new, kuid); + iee_set_cred_suid(new, kuid); + #else new->suid = new->uid = kuid; + #endif if (!uid_eq(kuid, old->uid)) { retval = set_user(new); if (retval < 0) @@ -636,7 +696,12 @@ long __sys_setuid(uid_t uid) goto error; } + #ifdef CONFIG_CREDP + iee_set_cred_euid(new, kuid); + iee_set_cred_fsuid(new, kuid); + #else new->fsuid = new->euid = kuid; + #endif retval = security_task_fix_setuid(new, old, LSM_SETID_ID); if (retval < 0) @@ -710,7 +775,11 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) return -ENOMEM; if (ruid != (uid_t) -1) { + #ifdef CONFIG_CREDP + iee_set_cred_uid(new, kruid); + #else new->uid = kruid; + #endif if (!uid_eq(kruid, old->uid)) { retval = set_user(new); if (retval < 0) @@ -718,10 +787,22 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) } } if (euid != (uid_t) -1) + #ifdef CONFIG_CREDP + iee_set_cred_euid(new, keuid); + #else new->euid = keuid; + #endif if (suid != (uid_t) -1) + #ifdef CONFIG_CREDP + iee_set_cred_suid(new, ksuid); + #else new->suid = ksuid; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, new->euid); + #else new->fsuid = new->euid; + #endif retval = security_task_fix_setuid(new, old, LSM_SETID_RES); if (retval < 0) @@ -810,12 +891,28 @@ long __sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid) return -ENOMEM; if (rgid != (gid_t) -1) + #ifdef CONFIG_CREDP + iee_set_cred_gid(new, krgid); + #else new->gid = krgid; + #endif if (egid != (gid_t) -1) + #ifdef CONFIG_CREDP + iee_set_cred_egid(new, kegid); + #else new->egid = kegid; + #endif if (sgid != (gid_t) -1) + #ifdef CONFIG_CREDP + iee_set_cred_sgid(new, ksgid); + #else new->sgid = ksgid; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_fsgid(new, new->egid); + #else new->fsgid = new->egid; + #endif retval = security_task_fix_setgid(new, old, LSM_SETID_RES); if (retval < 0) @@ -882,7 +979,11 @@ long __sys_setfsuid(uid_t uid) uid_eq(kuid, old->suid) || uid_eq(kuid, old->fsuid) || ns_capable_setid(old->user_ns, CAP_SETUID)) { if (!uid_eq(kuid, old->fsuid)) { + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, kuid); + #else new->fsuid = kuid; + #endif if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0) goto change_okay; } @@ -926,7 +1027,11 @@ long __sys_setfsgid(gid_t gid) gid_eq(kgid, old->sgid) || gid_eq(kgid, old->fsgid) || ns_capable_setid(old->user_ns, CAP_SETGID)) { if (!gid_eq(kgid, old->fsgid)) { + #ifdef CONFIG_CREDP + iee_set_cred_fsgid(new, kgid); + #else new->fsgid = kgid; + #endif if (security_task_fix_setgid(new,old,LSM_SETID_FS) == 0) goto change_okay; } diff --git a/kernel/umh.c b/kernel/umh.c index 1b13c5d34624..3202d889635e 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -31,6 +31,9 @@ #include #include +#ifdef CONFIG_CREDP +#include +#endif static kernel_cap_t usermodehelper_bset = CAP_FULL_SET; static kernel_cap_t usermodehelper_inheritable = CAP_FULL_SET; @@ -91,9 +94,15 @@ static int call_usermodehelper_exec_async(void *data) goto out; spin_lock(&umh_sysctl_lock); + #ifdef CONFIG_CREDP + iee_set_cred_cap_bset(new, cap_intersect(usermodehelper_bset, new->cap_bset)); + iee_set_cred_cap_inheritable(new, cap_intersect(usermodehelper_inheritable, + new->cap_inheritable)); + #else new->cap_bset = cap_intersect(usermodehelper_bset, new->cap_bset); new->cap_inheritable = cap_intersect(usermodehelper_inheritable, new->cap_inheritable); + #endif spin_unlock(&umh_sysctl_lock); if (sub_info->init) { diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 1d8e47bed3f1..fdbc4d1d1b19 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -21,6 +21,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif static struct kmem_cache *user_ns_cachep __read_mostly; static DEFINE_MUTEX(userns_state_mutex); @@ -45,6 +48,19 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) /* Start with the same capabilities as init but useless for doing * anything as the capabilities are bound to the new user namespace. */ + #ifdef CONFIG_CREDP + iee_set_cred_securebits(cred, SECUREBITS_DEFAULT); + iee_set_cred_cap_inheritable(cred, CAP_EMPTY_SET); + iee_set_cred_cap_permitted(cred, CAP_FULL_SET); + iee_set_cred_cap_effective(cred, CAP_FULL_SET); + iee_set_cred_cap_ambient(cred, CAP_EMPTY_SET); + iee_set_cred_cap_bset(cred, CAP_FULL_SET); +#ifdef CONFIG_KEYS + key_put(cred->request_key_auth); + iee_set_cred_request_key_auth(cred, NULL); +#endif + iee_set_cred_user_ns(cred, user_ns); + #else cred->securebits = SECUREBITS_DEFAULT; cred->cap_inheritable = CAP_EMPTY_SET; cred->cap_permitted = CAP_FULL_SET; @@ -57,6 +73,7 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) #endif /* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */ cred->user_ns = user_ns; + #endif } static unsigned long enforced_nproc_rlimit(void) diff --git a/mm/slab.h b/mm/slab.h index 62df6eeeb5ea..49e803d16b5d 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -25,6 +25,9 @@ typedef u64 freelist_full_t; #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) #undef system_has_freelist_aba #endif +#ifdef CONFIG_CREDP +extern struct kmem_cache *cred_jar; +#endif /* * Freelist pointer and counter to cmpxchg together, avoids the typical ABA @@ -721,7 +724,9 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, return s; } - +#ifdef CONFIG_CREDP +#include +#endif static inline void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, gfp_t flags, size_t size, void **p, bool init, @@ -766,7 +771,16 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, for (i = 0; i < size; i++) { p[i] = kasan_slab_alloc(s, p[i], flags, kasan_init); if (p[i] && init && (!kasan_init || !kasan_has_integrated_init())) + { + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) + iee_memset(p[i], 0, zero_size); + else + memset(p[i], 0, zero_size); + #else memset(p[i], 0, zero_size); + #endif + } kmemleak_alloc_recursive(p[i], s->object_size, 1, s->flags, flags); kmsan_slab_alloc(s, p[i], flags); diff --git a/mm/slub.c b/mm/slub.c index 105e86b5a3d4..32e6cd1f2bca 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -50,6 +50,9 @@ #ifdef CONFIG_IEE_PTRP #include #endif +#if defined(CONFIG_CREDP) +#include +#endif #include "internal.h" @@ -466,6 +469,13 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) #endif freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) { + iee_set_freeptr((void **)freeptr_addr, + (void *)(freelist_ptr_encode(s, fp, freeptr_addr).v)); + return; + } + #endif WRITE_ONCE(*(freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); } @@ -854,6 +864,29 @@ static void set_track_update(struct kmem_cache *s, void *object, { struct track *p = get_track(s, object, alloc); +#ifdef CONFIG_CREDP + struct track tmp; + + if (haoc_enabled && s == cred_jar) { + tmp = *p; + #ifdef CONFIG_STACKDEPOT + tmp.handle = handle; + #endif + tmp.addr = addr; + tmp.cpu = smp_processor_id(); + tmp.pid = current->pid; + tmp.when = jiffies; + iee_memcpy(p, &tmp, sizeof(struct track)); + } else { + #ifdef CONFIG_STACKDEPOT + p->handle = handle; + #endif + p->addr = addr; + p->cpu = smp_processor_id(); + p->pid = current->pid; + p->when = jiffies; + } + #else #ifdef CONFIG_STACKDEPOT p->handle = handle; #endif @@ -861,6 +894,7 @@ static void set_track_update(struct kmem_cache *s, void *object, p->cpu = smp_processor_id(); p->pid = current->pid; p->when = jiffies; + #endif } static __always_inline void set_track(struct kmem_cache *s, void *object, @@ -879,7 +913,14 @@ static void init_tracking(struct kmem_cache *s, void *object) return; p = get_track(s, object, TRACK_ALLOC); + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) + iee_memset(p, 0, 2*sizeof(struct track)); + else + memset(p, 0, 2*sizeof(struct track)); + #else memset(p, 0, 2*sizeof(struct track)); + #endif } static void print_track(const char *s, struct track *t, unsigned long pr_time) @@ -1045,7 +1086,14 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) unsigned int poison_size = s->object_size; if (s->flags & SLAB_RED_ZONE) { + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) + iee_memset(p - s->red_left_pad, val, s->red_left_pad); + else + memset(p - s->red_left_pad, val, s->red_left_pad); + #else memset(p - s->red_left_pad, val, s->red_left_pad); + #endif if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { /* @@ -1058,12 +1106,31 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) } if (s->flags & __OBJECT_POISON) { + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) { + iee_memset(p, POISON_FREE, poison_size - 1); + iee_memset(&p[poison_size - 1], POISON_END, 1); + } else { + memset(p, POISON_FREE, poison_size - 1); + p[poison_size - 1] = POISON_END; + } + #else memset(p, POISON_FREE, poison_size - 1); + #endif p[poison_size - 1] = POISON_END; } if (s->flags & SLAB_RED_ZONE) + #ifdef CONFIG_CREDP + { + if (haoc_enabled && s == cred_jar) + iee_memset(p + poison_size, val, s->inuse - poison_size); + else + memset(p + poison_size, val, s->inuse - poison_size); + } + #else memset(p + poison_size, val, s->inuse - poison_size); + #endif } static void restore_bytes(struct kmem_cache *s, char *message, u8 data, @@ -1438,7 +1505,14 @@ void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) return; metadata_access_enable(); + #ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) + iee_memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); + else + memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); + #else memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); + #endif metadata_access_disable(); } @@ -2076,6 +2150,11 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) #ifdef CONFIG_IEE if(haoc_enabled) iee_allocate_slab_data(s, slab, oo_order(oo)); +#endif +#ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) + set_iee_page((unsigned long)page_address(folio_page(slab_folio(slab), 0)), + oo_order(oo)); #endif account_slab(slab, oo_order(oo), s, flags); @@ -2135,6 +2214,16 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) if (iee_free_slab_data(s, slab, order)) return; } +#endif +#ifdef CONFIG_CREDP + if (haoc_enabled && s == cred_jar) { + #ifdef CONFIG_X86_64 + iee_free_slab(s, slab, iee_free_cred_slab); + return; + #else + unset_iee_page((unsigned long)page_address(folio_page(folio, 0)), order); + #endif + } #endif __free_pages(&folio->page, order); } @@ -3514,10 +3603,17 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list if (!s) return NULL; + #ifdef CONFIG_CREDP + if(haoc_enabled) + goto slab_alloc; + #endif object = kfence_alloc(s, orig_size, gfpflags); if (unlikely(object)) goto out; +#ifdef CONFIG_CREDP +slab_alloc: +#endif object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); maybe_wipe_obj_freeptr(s, object); @@ -4000,6 +4096,11 @@ static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, local_lock_irqsave(&s->cpu_slab->lock, irqflags); for (i = 0; i < size; i++) { + #ifdef CONFIG_CREDP + /* Skip kfence_alloc for iee kmem caches. */ + if(haoc_enabled) + goto slab_alloc; + #endif void *object = kfence_alloc(s, s->object_size, flags); if (unlikely(object)) { @@ -4007,6 +4108,9 @@ static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, continue; } +#ifdef CONFIG_CREDP +slab_alloc: +#endif object = c->freelist; if (unlikely(!object)) { /* @@ -4527,8 +4631,7 @@ static int calculate_sizes(struct kmem_cache *s) s->reciprocal_size = reciprocal_value(size); order = calculate_order(size); #ifdef CONFIG_IEE - if(haoc_enabled) - order = iee_calculate_order(s, order); + order = iee_calculate_order(s, order); #endif if ((int)order < 0) diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c index c42ddd85ff1f..a22c036b6050 100644 --- a/net/dns_resolver/dns_key.c +++ b/net/dns_resolver/dns_key.c @@ -32,6 +32,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "internal.h" MODULE_DESCRIPTION("DNS Resolver"); @@ -365,8 +368,13 @@ static int __init init_dns_resolver(void) /* instruct request_key() to use this special keyring as a cache for * the results it looks up */ set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(cred, keyring); + iee_set_cred_jit_keyring(cred, KEY_REQKEY_DEFL_THREAD_KEYRING); + #else cred->thread_keyring = keyring; cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif dns_resolver_cache = cred; kdebug("DNS resolver keyring: %d\n", key_serial(keyring)); diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index ec41b26af76e..7204bdb018f7 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -38,9 +38,13 @@ static const struct rpc_authops __rcu *auth_flavors[RPC_AUTH_MAXFLAVOR] = { static LIST_HEAD(cred_unused); static unsigned long number_cred_unused; +#ifdef CONFIG_CREDP +static struct cred *machine_cred; +#else static struct cred machine_cred = { .usage = ATOMIC_INIT(1), }; +#endif /* * Return the machine_cred pointer to be used whenever @@ -48,7 +52,11 @@ static struct cred machine_cred = { */ const struct cred *rpc_machine_cred(void) { + #ifdef CONFIG_CREDP + return machine_cred; + #else return &machine_cred; + #endif } EXPORT_SYMBOL_GPL(rpc_machine_cred); @@ -659,15 +667,27 @@ rpcauth_bindcred(struct rpc_task *task, const struct cred *cred, int flags) if (task->tk_op_cred) /* Task must use exactly this rpc_cred */ new = get_rpccred(task->tk_op_cred); + #ifdef CONFIG_CREDP + else if (cred && cred != machine_cred) + #else else if (cred != NULL && cred != &machine_cred) + #endif new = auth->au_ops->lookup_cred(auth, &acred, lookupflags); + #ifdef CONFIG_CREDP + else if (cred == machine_cred) + #else else if (cred == &machine_cred) + #endif new = rpcauth_bind_machine_cred(task, lookupflags); /* If machine cred couldn't be bound, try a root cred */ if (new) ; + #ifdef CONFIG_CREDP + else if (cred == machine_cred) + #else else if (cred == &machine_cred) + #endif new = rpcauth_bind_root_cred(task, lookupflags); else if (flags & RPC_TASK_NULLCREDS) new = authnull_ops.lookup_cred(NULL, NULL, 0); @@ -871,6 +891,11 @@ static struct shrinker rpc_cred_shrinker = { int __init rpcauth_init_module(void) { + #ifdef CONFIG_CREDP + machine_cred = prepare_creds(); + if (!machine_cred) + panic("RPCAUTH: fail to allocate machine_cred"); + #endif int err; err = rpc_init_authunix(); @@ -888,6 +913,9 @@ int __init rpcauth_init_module(void) void rpcauth_remove_module(void) { + #ifdef CONFIG_CREDP + abort_creds(machine_cred); + #endif rpc_destroy_authunix(); unregister_shrinker(&rpc_cred_shrinker); } diff --git a/security/commoncap.c b/security/commoncap.c index bc0521104197..200748841e3f 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -25,6 +25,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif /* * If a non-root user executes a setuid-root binary in @@ -265,7 +268,14 @@ int cap_capset(struct cred *new, /* verify the _new_Effective_ is a subset of the _new_Permitted_ */ if (!cap_issubset(*effective, *permitted)) return -EPERM; - + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, *effective); + iee_set_cred_cap_inheritable(new, *inheritable); + iee_set_cred_cap_permitted(new, *permitted); + + iee_set_cred_cap_ambient(new, cap_intersect(new->cap_ambient, + cap_intersect(*permitted, *inheritable))); + #else new->cap_effective = *effective; new->cap_inheritable = *inheritable; new->cap_permitted = *permitted; @@ -277,6 +287,7 @@ int cap_capset(struct cred *new, new->cap_ambient = cap_intersect(new->cap_ambient, cap_intersect(*permitted, *inheritable)); + #endif if (WARN_ON(!cap_ambient_invariant_ok(new))) return -EINVAL; return 0; @@ -601,9 +612,17 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps, * pP' = (X & fP) | (pI & fI) * The addition of pA' is handled later. */ + #ifdef CONFIG_CREDP + kernel_cap_t temp = new->cap_permitted; + + temp.val = (new->cap_bset.val & caps->permitted.val) | + (new->cap_inheritable.val & caps->inheritable.val); + iee_set_cred_cap_permitted(new, temp); + #else new->cap_permitted.val = (new->cap_bset.val & caps->permitted.val) | (new->cap_inheritable.val & caps->inheritable.val); + #endif if (caps->permitted.val & ~new->cap_permitted.val) /* insufficient to execute correctly */ @@ -726,7 +745,16 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, int rc = 0; struct cpu_vfs_cap_data vcaps; + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = bprm->cred->cap_permitted; + + tmp_cap.val = 0; + iee_set_cred_cap_permitted(bprm->cred, tmp_cap); + } while (0); + #else cap_clear(bprm->cred->cap_permitted); + #endif if (!file_caps_enabled) return 0; @@ -757,7 +785,16 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, out: if (rc) + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = bprm->cred->cap_permitted; + + tmp_cap.val = 0; + iee_set_cred_cap_permitted(bprm->cred, tmp_cap); + } while (0); + #else cap_clear(bprm->cred->cap_permitted); + #endif return rc; } @@ -809,8 +846,13 @@ static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap, */ if (__is_eff(root_uid, new) || __is_real(root_uid, new)) { /* pP' = (cap_bset & ~0) | (pI & ~0) */ + #ifdef CONFIG_CREDP + iee_set_cred_cap_permitted(new, cap_combine(old->cap_bset, + old->cap_inheritable)); + #else new->cap_permitted = cap_combine(old->cap_bset, old->cap_inheritable); + #endif } /* * If only the real uid is 0, we do not set the effective bit. @@ -919,34 +961,73 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) /* downgrade; they get no more than they had, and maybe less */ if (!ns_capable(new->user_ns, CAP_SETUID) || (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) { + #ifdef CONFIG_CREDP + iee_set_cred_euid(new, new->uid); + iee_set_cred_egid(new, new->gid); + #else new->euid = new->uid; new->egid = new->gid; + #endif } + #ifdef CONFIG_CREDP + iee_set_cred_cap_permitted(new, cap_intersect(new->cap_permitted, + old->cap_permitted)); + #else new->cap_permitted = cap_intersect(new->cap_permitted, old->cap_permitted); + #endif } + #ifdef CONFIG_CREDP + iee_set_cred_fsuid(new, new->euid); + iee_set_cred_suid(new, new->euid); + iee_set_cred_fsgid(new, new->egid); + iee_set_cred_sgid(new, new->egid); + #else new->suid = new->fsuid = new->euid; new->sgid = new->fsgid = new->egid; + #endif /* File caps or setid cancels ambient. */ if (has_fcap || is_setid) + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = new->cap_ambient; + + tmp_cap.val = 0; + iee_set_cred_cap_ambient(new, tmp_cap); + } while (0); + #else cap_clear(new->cap_ambient); + #endif /* * Now that we've computed pA', update pP' to give: * pP' = (X & fP) | (pI & fI) | pA' */ + #ifdef CONFIG_CREDP + iee_set_cred_cap_permitted(new, + cap_combine(new->cap_permitted, new->cap_ambient)); + #else new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient); + #endif /* * Set pE' = (fE ? pP' : pA'). Because pA' is zero if fE is set, * this is the same as pE' = (fE ? pP' : 0) | pA'. */ if (effective) + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, new->cap_permitted); + #else new->cap_effective = new->cap_permitted; + #endif else + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, new->cap_ambient); + #else new->cap_effective = new->cap_ambient; + #endif if (WARN_ON(!cap_ambient_invariant_ok(new))) return -EPERM; @@ -957,7 +1038,12 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) return ret; } + #ifdef CONFIG_CREDP + iee_set_cred_securebits(new, + new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); + #else new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); + #endif if (WARN_ON(!cap_ambient_invariant_ok(new))) return -EPERM; @@ -1092,8 +1178,17 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) !uid_eq(new->euid, root_uid) && !uid_eq(new->suid, root_uid))) { if (!issecure(SECURE_KEEP_CAPS)) { + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = new->cap_permitted; + + tmp_cap.val = 0; + iee_set_cred_cap_permitted(new, tmp_cap); + } while (0); + #else cap_clear(new->cap_permitted); cap_clear(new->cap_effective); + #endif } /* @@ -1101,12 +1196,34 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) * by exec to drop capabilities. We should make sure that * this remains the case. */ + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = new->cap_ambient; + + tmp_cap.val = 0; + iee_set_cred_cap_ambient(new, tmp_cap); + } while (0); + #else cap_clear(new->cap_ambient); + #endif } if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid)) + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = new->cap_effective; + + tmp_cap.val = 0; + iee_set_cred_cap_effective(new, tmp_cap); + } while (0); + #else cap_clear(new->cap_effective); + #endif if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid)) + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, new->cap_permitted); + #else new->cap_effective = new->cap_permitted; + #endif } /** @@ -1142,13 +1259,23 @@ int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags) if (!issecure(SECURE_NO_SETUID_FIXUP)) { kuid_t root_uid = make_kuid(old->user_ns, 0); if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid)) + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, + cap_drop_fs_set(new->cap_effective)); + #else new->cap_effective = cap_drop_fs_set(new->cap_effective); + #endif if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid)) + #ifdef CONFIG_CREDP + iee_set_cred_cap_effective(new, cap_raise_fs_set(new->cap_effective, + new->cap_permitted)); + #else new->cap_effective = cap_raise_fs_set(new->cap_effective, new->cap_permitted); + #endif } break; @@ -1243,7 +1370,16 @@ static int cap_prctl_drop(unsigned long cap) new = prepare_creds(); if (!new) return -ENOMEM; + #ifdef CONFIG_CREDP + { + kernel_cap_t tmp = new->cap_bset; + + cap_lower(tmp, cap); + iee_set_cred_cap_bset(new, tmp); + } + #else cap_lower(new->cap_bset, cap); + #endif return commit_creds(new); } @@ -1319,7 +1455,11 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, new = prepare_creds(); if (!new) return -ENOMEM; + #ifdef CONFIG_CREDP + iee_set_cred_securebits(new, arg2); + #else new->securebits = arg2; + #endif return commit_creds(new); case PR_GET_SECUREBITS: @@ -1338,9 +1478,19 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, if (!new) return -ENOMEM; if (arg2) + #ifdef CONFIG_CREDP + iee_set_cred_securebits(new, + new->securebits | issecure_mask(SECURE_KEEP_CAPS)); + #else new->securebits |= issecure_mask(SECURE_KEEP_CAPS); + #endif else + #ifdef CONFIG_CREDP + iee_set_cred_securebits(new, + new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); + #else new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); + #endif return commit_creds(new); case PR_CAP_AMBIENT: @@ -1351,7 +1501,16 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, new = prepare_creds(); if (!new) return -ENOMEM; + #ifdef CONFIG_CREDP + do { + kernel_cap_t tmp_cap = new->cap_ambient; + + tmp_cap.val = 0; + iee_set_cred_cap_ambient(new, tmp_cap); + } while (0); + #else cap_clear(new->cap_ambient); + #endif return commit_creds(new); } @@ -1375,9 +1534,27 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, if (!new) return -ENOMEM; if (arg2 == PR_CAP_AMBIENT_RAISE) + #ifdef CONFIG_CREDP + { + kernel_cap_t tmp = new->cap_ambient; + + cap_raise(tmp, arg3); + iee_set_cred_cap_ambient(new, tmp); + } + #else cap_raise(new->cap_ambient, arg3); + #endif else + #ifdef CONFIG_CREDP + { + kernel_cap_t tmp = new->cap_ambient; + + cap_lower(tmp, arg3); + iee_set_cred_cap_ambient(new, tmp); + } + #else cap_lower(new->cap_ambient, arg3); + #endif return commit_creds(new); } diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index aa1dc43b16dd..60856f2f4220 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -22,6 +22,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "internal.h" #define KEY_MAX_DESC_SIZE 4096 @@ -1155,7 +1158,11 @@ static int keyctl_change_reqkey_auth(struct key *key) return -ENOMEM; key_put(new->request_key_auth); + #ifdef CONFIG_CREDP + iee_set_cred_request_key_auth(new, key_get(key)); + #else new->request_key_auth = key_get(key); + #endif return commit_creds(new); } @@ -1432,7 +1439,11 @@ long keyctl_set_reqkey_keyring(int reqkey_defl) } set: + #ifdef CONFIG_CREDP + iee_set_cred_jit_keyring(new, reqkey_defl); + #else new->jit_keyring = reqkey_defl; + #endif commit_creds(new); return old_setting; error: @@ -1644,9 +1655,20 @@ long keyctl_session_to_parent(void) cred = cred_alloc_blank(); if (!cred) goto error_keyring; + #ifdef CONFIG_CREDP + if(haoc_enabled) + newwork = (struct rcu_head *)(cred->rcu.func); + else + newwork = &cred->rcu; + #else newwork = &cred->rcu; + #endif + #ifdef CONFIG_CREDP + iee_set_cred_session_keyring(cred, key_ref_to_ptr(keyring_r)); + #else cred->session_keyring = key_ref_to_ptr(keyring_r); + #endif keyring_r = NULL; init_task_work(newwork, key_change_session_keyring); @@ -1704,8 +1726,16 @@ long keyctl_session_to_parent(void) unlock: write_unlock_irq(&tasklist_lock); rcu_read_unlock(); - if (oldwork) + if (oldwork){ + #ifdef CONFIG_CREDP + if(haoc_enabled) + put_cred(*(struct cred **)(oldwork + 1)); + else + put_cred(container_of(oldwork, struct cred, rcu)); + #else put_cred(container_of(oldwork, struct cred, rcu)); + #endif + } if (newwork) put_cred(cred); return ret; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index b5d5333ab330..aa56394380fd 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -17,6 +17,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif #include "internal.h" /* Session keyring create vs join semaphore */ @@ -232,7 +235,11 @@ int install_thread_keyring_to_cred(struct cred *new) if (IS_ERR(keyring)) return PTR_ERR(keyring); + #ifdef CONFIG_CREDP + iee_set_cred_thread_keyring(new, keyring); + #else new->thread_keyring = keyring; + #endif return 0; } @@ -279,7 +286,11 @@ int install_process_keyring_to_cred(struct cred *new) if (IS_ERR(keyring)) return PTR_ERR(keyring); + #ifdef CONFIG_CREDP + iee_set_cred_process_keyring(new, keyring); + #else new->process_keyring = keyring; + #endif return 0; } @@ -338,7 +349,11 @@ int install_session_keyring_to_cred(struct cred *cred, struct key *keyring) /* install the keyring */ old = cred->session_keyring; + #ifdef CONFIG_CREDP + iee_set_cred_session_keyring(cred, keyring); + #else cred->session_keyring = keyring; + #endif if (old) key_put(old); @@ -911,7 +926,15 @@ long join_session_keyring(const char *name) void key_change_session_keyring(struct callback_head *twork) { const struct cred *old = current_cred(); + #ifdef CONFIG_CREDP + struct cred *new =NULL; + if(haoc_enabled) + new = *(struct cred **)(twork + 1); + else + new = container_of(twork, struct cred, rcu); + #else struct cred *new = container_of(twork, struct cred, rcu); + #endif if (unlikely(current->flags & PF_EXITING)) { put_cred(new); @@ -925,6 +948,31 @@ void key_change_session_keyring(struct callback_head *twork) return; } + #ifdef CONFIG_CREDP + iee_set_cred_uid(new, old->uid); + iee_set_cred_euid(new, old->euid); + iee_set_cred_suid(new, old->suid); + iee_set_cred_fsuid(new, old->fsuid); + iee_set_cred_gid(new, old->gid); + iee_set_cred_egid(new, old->egid); + iee_set_cred_sgid(new, old->sgid); + iee_set_cred_fsgid(new, old->fsgid); + iee_set_cred_user(new, get_uid(old->user)); + iee_set_cred_ucounts(new, old->ucounts); + iee_set_cred_user_ns(new, get_user_ns(old->user_ns)); + iee_set_cred_group_info(new, get_group_info(old->group_info)); + + iee_set_cred_securebits(new, old->securebits); + iee_set_cred_cap_inheritable(new, old->cap_inheritable); + iee_set_cred_cap_permitted(new, old->cap_permitted); + iee_set_cred_cap_effective(new, old->cap_effective); + iee_set_cred_cap_ambient(new, old->cap_ambient); + iee_set_cred_cap_bset(new, old->cap_bset); + + iee_set_cred_jit_keyring(new, old->jit_keyring); + iee_set_cred_thread_keyring(new, key_get(old->thread_keyring)); + iee_set_cred_process_keyring(new, key_get(old->process_keyring)); + #else new-> uid = old-> uid; new-> euid = old-> euid; new-> suid = old-> suid; @@ -948,6 +996,7 @@ void key_change_session_keyring(struct callback_head *twork) new->jit_keyring = old->jit_keyring; new->thread_keyring = key_get(old->thread_keyring); new->process_keyring = key_get(old->process_keyring); + #endif security_transfer_creds(new, old); diff --git a/security/security.c b/security/security.c index b6144833c7a8..23206976e2f5 100644 --- a/security/security.c +++ b/security/security.c @@ -30,6 +30,9 @@ #include #include #include +#ifdef CONFIG_CREDP +#include +#endif /* How many LSMs were built into the kernel? */ #define LSM_COUNT (__end_lsm_info - __start_lsm_info) @@ -570,11 +573,19 @@ EXPORT_SYMBOL(unregister_blocking_lsm_notifier); static int lsm_cred_alloc(struct cred *cred, gfp_t gfp) { if (blob_sizes.lbs_cred == 0) { + #ifdef CONFIG_CREDP + iee_set_cred_security(cred, NULL); + #else cred->security = NULL; + #endif return 0; } + #ifdef CONFIG_CREDP + iee_set_cred_security(cred, kzalloc(blob_sizes.lbs_cred, gfp)); + #else cred->security = kzalloc(blob_sizes.lbs_cred, gfp); + #endif if (cred->security == NULL) return -ENOMEM; return 0; @@ -2950,7 +2961,11 @@ void security_cred_free(struct cred *cred) call_void_hook(cred_free, cred); kfree(cred->security); + #ifdef CONFIG_CREDP + iee_set_cred_security(cred, NULL); + #else cred->security = NULL; + #endif } /** -- Gitee From 8cbff83a624d1997570cec87b400dd54bbd6dd60 Mon Sep 17 00:00:00 2001 From: dorso <2434317248@qq.com> Date: Wed, 15 Oct 2025 19:29:10 +0800 Subject: [PATCH 8/8] Update KConfig and cred.c --- arch/x86/Kconfig | 30 +++++++++++++++++++++++------- kernel/cred.c | 11 ++++++----- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 989af2993ab5..4152d8496d1a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1554,22 +1554,38 @@ config AMD_MEM_ENCRYPT config IEE bool "Isolated Execution Environment Framework(IEE)" + depends on X86_64 help Support for Isolated Execution Environment Framework. Foundation of HAOC. Could isolate kernel critical data and enforce all write access made and verified in IEE APIs. - depends on X86_64 -config IEE_SIP - depends on IEE - def_bool y config IEE_PTRP + bool "Pointer Protection for IEE(IEE_PTRP)" depends on IEE - def_bool y - + help + Provide IEE matadata for each process called task_token to allow + verifing pointers inside task_struct, like struct cred* that determines + the capabilities of a process. + Could be a enhancement of other sub-module of HAOC. + +config IEE_SIP + bool "Sensitive Instruction Protection for IEE(IEE_SIP)" + depends on IEE + help + Protects critical instructions that may break the isolation of IEE, + such as writing system control registers. These instructions would + be executated inside IEE. + config CREDP + bool "Struct cred protection(CREDP)" depends on IEE - def_bool y + help + Protects kernel struct cred. All modifications of cred must be made and + verified by IEE APIs, and critical dereferences of cred would be monitored + by IEE either. + If unsure, say N. + # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" diff --git a/kernel/cred.c b/kernel/cred.c index 78c1e9606b0b..14bfbbfbd807 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -733,18 +733,19 @@ void __init cred_init(void) { /* allocate a slab in which we can store credentials */ #ifdef CONFIG_CREDP + if (haoc_enabled){ cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); rcu_jar = kmem_cache_create("rcu_jar", sizeof(struct rcu_head) + sizeof(struct cred *), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); - if(haoc_enabled) - { // Map init_cred - *((struct rcu_head **)(&(init_cred.rcu.func))) = - (struct rcu_head *)kmem_cache_zalloc(rcu_jar, GFP_KERNEL); + iee_set_cred_rcu(&init_cred, kmem_cache_zalloc(rcu_jar, GFP_KERNEL)); *(struct cred **)(((struct rcu_head *)(init_cred.rcu.func)) + 1) = &init_cred; - } + pr_info("HAOC: CONFIG_CREDP enabled."); + } else + cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); #else cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -- Gitee