Many of the Morello helper functions are implemented in assembly due to historical reasons. This patch rewrites all of the helpers defined in morello.S into C to improve their readability. This patch also removes the original morello.S assembly file and edits the Makefile accordingly.

Signed-off-by: Akram Ahmad Akram.Ahmad@arm.com --- arch/arm64/include/asm/assembler.h | 10 -- arch/arm64/include/asm/morello.h | 7 +- arch/arm64/kernel/asm-offsets.c | 8 -- arch/arm64/kernel/morello.c | 150 ++++++++++++++++++--- arch/arm64/lib/Makefile | 3 +- arch/arm64/lib/morello.S | 205 ----------------------------- 6 files changed, 138 insertions(+), 245 deletions(-) delete mode 100644 arch/arm64/lib/morello.S

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 7d4046ded4df..11596bee98fe 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -597,16 +597,6 @@ alternative_endif mrs \rd, sp_el0 .endm

-/* - * Return a pointer to tsk's pt_regs. - */ - .macro get_task_pt_regs, rd:req, tsk:req - /* See task_pt_regs() in processor.h */ - ldr \rd, [\tsk, #TSK_STACK] - add \rd, \rd, #THREAD_SIZE - sub \rd, \rd, #PT_REGS_SIZE - .endm - /* * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. * orr is used as it can cover the immediate value (and is idempotent). diff --git a/arch/arm64/include/asm/morello.h b/arch/arm64/include/asm/morello.h index 357f403d073d..da198622b35e 100644 --- a/arch/arm64/include/asm/morello.h +++ b/arch/arm64/include/asm/morello.h @@ -5,10 +5,6 @@ #ifndef __ASM_MORELLO_H #define __ASM_MORELLO_H

-/* Architectural definitions */ -#define MORELLO_CAP_PERM_EXECUTIVE_BIT 1 -#define MORELLO_CAP_PERM_EXECUTIVE_MASK (1 << MORELLO_CAP_PERM_EXECUTIVE_BIT) - #ifndef __ASSEMBLY__

struct pt_regs; @@ -83,6 +79,9 @@ void morello_flush_cap_regs_to_64_regs(struct task_struct *tsk);

#else /* __ASSEMBLY__ */

+#define MORELLO_CAP_PERM_EXECUTIVE_BIT 1 +#define MORELLO_CAP_PERM_EXECUTIVE_MASK (1 << MORELLO_CAP_PERM_EXECUTIVE_BIT) + /* * Merge an X register into a C register if C's lower 64 bits are not equal to * X. This check is required to avoid untagging sealed capabilities. diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 90ca25039583..defc1f6f5d11 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -55,14 +55,6 @@ int main(void) #endif #ifdef CONFIG_ARM64_MTE DEFINE(THREAD_MTE_CTRL, offsetof(struct task_struct, thread.mte_ctrl)); -#endif -#ifdef CONFIG_ARM64_MORELLO - DEFINE(THREAD_MORELLO_USER_STATE, offsetof(struct task_struct, thread.morello_user_state)); - BLANK(); - DEFINE(MORELLO_STATE_CTPIDR, offsetof(struct morello_state, ctpidr)); - DEFINE(MORELLO_STATE_DDC, offsetof(struct morello_state, ddc)); - DEFINE(MORELLO_STATE_CID, offsetof(struct morello_state, cid)); - DEFINE(MORELLO_STATE_CCTLR, offsetof(struct morello_state, cctlr)); #endif BLANK(); DEFINE(S_X0, offsetof(struct pt_regs, regs[0])); diff --git a/arch/arm64/kernel/morello.c b/arch/arm64/kernel/morello.c index 7dfbca70e11d..fc3efdf5be56 100644 --- a/arch/arm64/kernel/morello.c +++ b/arch/arm64/kernel/morello.c @@ -20,13 +20,6 @@ #include <asm/morello.h> #include <asm/ptrace.h>

-/* Private functions implemented in morello.S */ -void __morello_cap_lo_hi_tag(uintcap_t cap, u64 *lo_val, u64 *hi_val, - u8 *tag); -void __morello_merge_c_x(uintcap_t *creg, u64 xreg); -bool __morello_cap_has_executive(uintcap_t cap); -void __morello_thread_init_user(struct task_struct *tsk, uintcap_t ddc); - static uintcap_t morello_sentry_unsealcap __ro_after_init;

/* DDC_ELx reset value (low/high 64 bits), as defined in the Morello spec */ @@ -35,6 +28,25 @@ static uintcap_t morello_sentry_unsealcap __ro_after_init;

#define CAP_OTYPE_FIELD_BITS 15

+static void cap_lo_hi_tag(uintcap_t cap, u64 *lo_val, u64 *hi_val, + u8 *tag) +{ + *lo_val = (u64)cap; + *hi_val = __builtin_cheri_copy_from_high((void *__capability)cap); + *tag = cheri_tag_get(cap); +} + +static void merge_c_x(uintcap_t *creg, u64 xreg) +{ + if (cheri_address_get(*creg) != xreg) + *creg = cheri_address_set(*creg, xreg); +} + +static bool cap_has_executive(uintcap_t cap) +{ + return cheri_perms_get(cap) & ARM_CAP_PERMISSION_EXECUTIVE; +} + static bool is_pure_task(void) { return IS_ENABLED(CONFIG_CHERI_PURECAP_UABI) && !is_compat_task(); @@ -53,6 +65,26 @@ static void update_regs_c64(struct pt_regs *regs, unsigned long pc) } }

+void morello_cap_get_val_tag(uintcap_t cap, __uint128_t *val, u8 *tag) +{ + *((uintcap_t *)val) = cheri_tag_clear(cap); + *tag = cheri_tag_get(cap); +} + +uintcap_t morello_build_any_user_cap(const __uint128_t *val, u8 tag) +{ + uintcap_t cap = *((uintcap_t *)val); + + if (!tag) + return cheri_tag_clear(cap); + + uintcap_t sealing_cap = cheri_type_copy(cheri_user_root_allperms_cap, cap); + + cap = (user_uintptr_t)cheri_cap_build(cheri_user_root_allperms_cap, cap); + cap = cheri_seal_conditionally(cap, sealing_cap); + return cap; +} + void morello_thread_start(struct pt_regs *regs, unsigned long pc) { update_regs_c64(regs, pc); @@ -78,14 +110,100 @@ void morello_thread_init_user(void) /* TODO [PCuABI] - Set DDC to the null capability */ uintcap_t ddc = is_pure_task() ? cheri_user_root_cap : cheri_user_root_allperms_cap; + struct morello_state *morello_state = &current->thread.morello_user_state; + + /* + * CTPIDR doesn't need to be initialised explicitly: + * - tls_thread_flush() already zeroes tpidr_el0, zeroing ctpidr_el0 as + * well + * - The value stored in thread.morello_user_state will be set the next + * time task_save_user_tls() is called, like thread_struct.uw.tp_value. + * + * tls_thread_flush() does not touch rctpidr_el0 so this must be zeroed + * here. We do not need to initialise its value in morello_user_state. + * Only the ddc_el0 register must be initialised to the specific value; + * RDDC is set to a null capability as processes are always started in + * Executive. + */ + write_cap_sysreg(0, rctpidr_el0); + + write_cap_sysreg(ddc, ddc_el0); + write_cap_sysreg(0, rddc_el0); + morello_state->ddc = ddc; + morello_state->rddc = (uintcap_t)0; + + write_cap_sysreg(0, cid_el0); + morello_state->cid = (uintcap_t)0; + + write_sysreg(0, cctlr_el0); + morello_state->cctlr = 0; +} + +void morello_thread_save_user_state(struct task_struct *tsk) +{ + struct morello_state *morello_state = &tsk->thread.morello_user_state; + + /* (R)CTPIDR is handled by task_save_user_tls */ + morello_state->ddc = read_cap_sysreg(ddc_el0); + morello_state->rddc = read_cap_sysreg(rddc_el0); + morello_state->cid = read_cap_sysreg(cid_el0); + morello_state->cctlr = read_sysreg(cctlr_el0); +} + +void morello_thread_restore_user_state(struct task_struct *tsk) +{ + struct morello_state *morello_state = &tsk->thread.morello_user_state; + + /* (R)CTPIDR is handled by task_restore_user_tls */ + write_cap_sysreg(morello_state->ddc, ddc_el0); + write_cap_sysreg(morello_state->rddc, rddc_el0); + write_cap_sysreg(morello_state->cid, cid_el0); + write_sysreg(morello_state->cctlr, cctlr_el0); +} + +void morello_task_save_user_tls(struct task_struct *tsk, user_uintptr_t *tp_ptr) +{ + struct morello_state *morello_state = &tsk->thread.morello_user_state; + struct pt_regs *regs = task_pt_regs(tsk); + uintcap_t active_ctpidr; + + morello_state->ctpidr = read_cap_sysreg(ctpidr_el0); + morello_state->rctpidr = read_cap_sysreg(rctpidr_el0); + + if (cap_has_executive(regs->pcc)) + active_ctpidr = morello_state->ctpidr; + else + active_ctpidr = morello_state->rctpidr; + + *tp_ptr = (user_uintptr_t)active_ctpidr; +} + +void morello_task_restore_user_tls(struct task_struct *tsk, + const user_uintptr_t *tp_ptr) +{ + struct morello_state *morello_state = &tsk->thread.morello_user_state; + struct pt_regs *regs = task_pt_regs(tsk); + uintcap_t *active_ctpidr; + + if (cap_has_executive(regs->pcc)) + active_ctpidr = &morello_state->ctpidr; + else + active_ctpidr = &morello_state->rctpidr; + +#ifdef CONFIG_CHERI_PURECAP_UABI + *active_ctpidr = *tp_ptr; +#else + merge_c_x(active_ctpidr, *tp_ptr); +#endif

- __morello_thread_init_user(current, ddc); + write_cap_sysreg(morello_state->ctpidr, ctpidr_el0); + write_cap_sysreg(morello_state->rctpidr, rctpidr_el0); }

#ifdef CONFIG_CHERI_PURECAP_UABI void morello_thread_set_csp(struct pt_regs *regs, user_uintptr_t sp) { - uintcap_t *thread_sp = __morello_cap_has_executive(regs->pcc) ? + uintcap_t *thread_sp = cap_has_executive(regs->pcc) ? &regs->csp : &regs->rcsp; *thread_sp = sp; } @@ -124,7 +242,7 @@ static char *format_cap(char *buf, size_t size, uintcap_t cap) u64 lo_val, hi_val; u8 tag;

- __morello_cap_lo_hi_tag(cap, &lo_val, &hi_val, &tag); + cap_lo_hi_tag(cap, &lo_val, &hi_val, &tag);

if (snprintf(buf, size, "%u:%016llx:%016llx", tag, hi_val, lo_val) <= 0) buf[0] = '\0'; @@ -245,16 +363,16 @@ void morello_merge_cap_regs(struct pt_regs *regs) int i; uintcap_t *active_csp;

- if (__morello_cap_has_executive(regs->pcc)) + if (cap_has_executive(regs->pcc)) active_csp = &regs->csp; else active_csp = &regs->rcsp;

for (i = 0; i < ARRAY_SIZE(regs->cregs); i++) - __morello_merge_c_x(&regs->cregs[i], regs->regs[i]); + merge_c_x(&regs->cregs[i], regs->regs[i]);

- __morello_merge_c_x(active_csp, regs->sp); - __morello_merge_c_x(&regs->pcc, regs->pc); + merge_c_x(active_csp, regs->sp); + merge_c_x(&regs->pcc, regs->pc); }

void morello_flush_cap_regs_to_64_regs(struct task_struct *tsk) @@ -266,7 +384,7 @@ void morello_flush_cap_regs_to_64_regs(struct task_struct *tsk) uintcap_t active_ctpidr; int i;

- if (__morello_cap_has_executive(regs->pcc)) { + if (cap_has_executive(regs->pcc)) { active_csp = regs->csp; active_ctpidr = morello_state->ctpidr; } else { @@ -303,7 +421,7 @@ static void __init check_root_cap(uintcap_t cap) u64 lo_val, hi_val; u8 tag;

- __morello_cap_lo_hi_tag(cap, &lo_val, &hi_val, &tag); + cap_lo_hi_tag(cap, &lo_val, &hi_val, &tag);

/* * Check that DDC has the reset value, otherwise root capabilities and diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 5fa6f0645edc..2e46ed437b69 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -27,8 +27,7 @@ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o

obj-$(CONFIG_ARM64_MTE) += mte.o

-obj-$(CONFIG_ARM64_MORELLO) += morello.o \ - copy_from_user_with_captags.o \ +obj-$(CONFIG_ARM64_MORELLO) += copy_from_user_with_captags.o \ copy_to_user_with_captags.o

obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o diff --git a/arch/arm64/lib/morello.S b/arch/arm64/lib/morello.S deleted file mode 100644 index 0800375ef33d..000000000000 --- a/arch/arm64/lib/morello.S +++ /dev/null @@ -1,205 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2020 Arm Ltd. - * - * TODO: most of the functions in this file should be reimplemented in C to - * improve readability. - */ - -#include <linux/linkage.h> - -#include <asm/asm-uaccess.h> -#include <asm/assembler.h> -#include <asm/morello.h> -#include <asm/sysreg.h> - -SYM_FUNC_START(morello_cap_get_val_tag) - gctag x3, c0 - clrtag c0, c0 // Store the 128-bit value without the tag - str c0, [x1] - strb w3, [x2] - - ret -SYM_FUNC_END(morello_cap_get_val_tag) - -SYM_FUNC_START(morello_build_any_user_cap) - ldr c0, [x0] - cbz w1, 1f - /* - * The tag should be set, build a valid capability from the root - * capability. - * In case c0 is sealed (non-zero object type), we need to extract the - * object type first to be able to reseal it after BUILD. The CSEAL - * instruction is used to cover the case where c0 was not sealed. - */ - adr_l x3, cheri_user_root_allperms_cap - ldr c3, [x3] - - cpytype c4, c3, c0 - build c0, c0, c3 - cseal c0, c0, c4 - b 2f -1: - /* The tag should be cleared, make sure it is. */ - clrtag c0, c0 -2: - ret -SYM_FUNC_END(morello_build_any_user_cap) - -SYM_FUNC_START(__morello_thread_init_user) - mov x9, #THREAD_MORELLO_USER_STATE - add x0, x0, x9 // x0 = tsk->thread.morello_user_state - - /* - * CTPIDR doesn't need to be initialised explicitly: - * - tls_thread_flush() already zeroes tpidr_el0, zeroing ctpidr_el0 as - * well - * - The value stored in thread.morello_user_state will be set the next - * time task_save_user_tls() is called, like thread_struct.uw.tp_value. - * - * tls_thread_flush() does not touch rcsp_el0, so we need to zero it - * here, but its value in morello_user_state does not need to be - * initialised here either. - */ - msr rctpidr_el0, czr - - /* DDC: initialised to the specified value */ - msr ddc_el0, c1 - /* RDDC: null capability (processes are always started in Executive) */ - msr rddc_el0, czr - stp c1, czr, [x0, #MORELLO_STATE_DDC] - /* CID: null capability */ - msr cid_el0, czr - str czr, [x0, #MORELLO_STATE_CID] - /* CCTLR: all bits cleared */ - msr cctlr_el0, xzr - str xzr, [x0, #MORELLO_STATE_CCTLR] - - ret -SYM_FUNC_END(__morello_thread_init_user) - -SYM_FUNC_START(morello_thread_save_user_state) - mov x9, #THREAD_MORELLO_USER_STATE - add x0, x0, x9 // x0 = tsk->thread.morello_user_state - - /* (R)CTPIDR is handled by task_save_user_tls */ - mrs c1, ddc_el0 - mrs c2, rddc_el0 - stp c1, c2, [x0, #MORELLO_STATE_DDC] - mrs c1, cid_el0 - str c1, [x0, #MORELLO_STATE_CID] - mrs x1, cctlr_el0 - str x1, [x0, #MORELLO_STATE_CCTLR] - - ret -SYM_FUNC_END(morello_thread_save_user_state) - -SYM_FUNC_START(morello_thread_restore_user_state) - mov x9, #THREAD_MORELLO_USER_STATE - add x0, x0, x9 // x0 = tsk->thread.morello_user_state - - /* (R)CTPIDR is handled by task_restore_user_tls */ - ldp c1, c2, [x0, #MORELLO_STATE_DDC] - msr ddc_el0, c1 - msr rddc_el0, c2 - ldr c1, [x0, #MORELLO_STATE_CID] - msr cid_el0, c1 - ldr x1, [x0, #MORELLO_STATE_CCTLR] - msr cctlr_el0, x1 - - ret -SYM_FUNC_END(morello_thread_restore_user_state) - -SYM_FUNC_START(morello_task_save_user_tls) - get_task_pt_regs x8, x0 - mov x9, #THREAD_MORELLO_USER_STATE - add x0, x0, x9 // x0 = tsk->thread.morello_user_state - - mrs c2, ctpidr_el0 - mrs c3, rctpidr_el0 - /* Save CTPIDR and RCTPIDR */ - stp c2, c3, [x0, #MORELLO_STATE_CTPIDR] - - ldr c5, [x8, #S_PCC] - /* - * If the task is currently running in Restricted, save the lower 64 bits - * of RCTPIDR (RTPIDR) in tsk->thread instead of TPIDR. - */ - morello_tst_cap_has_executive c5, x6 // Task running in Executive? -#ifdef CONFIG_CHERI_PURECAP_UABI - csel c2, c2, c3, ne // If not, save RTPIDR - str c2, [x1] -#else - csel x2, x2, x3, ne // If not, save RTPIDR - str x2, [x1] -#endif - ret -SYM_FUNC_END(morello_task_save_user_tls) - -SYM_FUNC_START(morello_task_restore_user_tls) - get_task_pt_regs x8, x0 - mov x9, #THREAD_MORELLO_USER_STATE - add x0, x0, x9 // x0 = tsk->thread.morello_user_state - - /* Load CTPIDR, RCTPIDR and the 64-bit TLS pointer */ - ldp c2, c3, [x0, #MORELLO_STATE_CTPIDR] -#ifdef CONFIG_CHERI_PURECAP_UABI - ldr c4, [x1] -#else - ldr x4, [x1] -#endif - - /* - * The 64-bit TLS pointer may have been modified from within the kernel - * (e.g. through ptrace) since morello_task_save_user_tls was called. - * Merge it back into the active capability TLS pointer, that is RCTPIDR - * if the task is running in Restricted and CTPIDR otherwise. - */ - ldr c5, [x8, #S_PCC] - morello_tst_cap_has_executive c5, x6 // Task running in Executive? - b.eq 1f // If not, merge into RCTPIDR -#ifdef CONFIG_CHERI_PURECAP_UABI - mov c2, c4 // Save the TLS pointer into CTPIDR -#else - morello_merge_c_x 2, x4 // Merge the TLS pointer into CTPIDR -#endif - b 2f -1: -#ifdef CONFIG_CHERI_PURECAP_UABI - mov c3, c4 // Save the TLS pointer into RCTPIDR -#else - morello_merge_c_x 3, x4 // Merge the TLS pointer into RCTPIDR -#endif -2: - msr ctpidr_el0, c2 - msr rctpidr_el0, c3 - - ret -SYM_FUNC_END(morello_task_restore_user_tls) - - -SYM_FUNC_START(__morello_cap_lo_hi_tag) - str x0, [x1] - cfhi x4, c0 // Extract upper 64 bits - str x4, [x2] - gctag x4, c0 - strb w4, [x3] - - ret -SYM_FUNC_END(__morello_cap_lo_hi_tag) - -SYM_FUNC_START(__morello_merge_c_x) - ldr c2, [x0] - cmp x2, x1 - b.eq 1f - scvalue c2, c2, x1 - str c2, [x0] -1: - ret -SYM_FUNC_END(__morello_merge_c_x) - -SYM_FUNC_START(__morello_cap_has_executive) - gcperm x0, c0 - ubfx x0, x0, #MORELLO_CAP_PERM_EXECUTIVE_BIT, #1 - ret -SYM_FUNC_END(__morello_cap_has_executive)