Hello,
This patch series enables purecap applications to utilise tcp_zerocopy_receive to map packets directly from a network interface card into a shared space with the process and kernel.
I do not think I shall have time to continue revising this patch series and debugging the bus error generated by the latest patch. Hopefully this provides a start to tcp_zerocopy_receive for capability architecture.
v2: - There appears to be a new error generated against musl resulting in a BUS error when using memcpy to copy between allocated shared space and other regions of memory. - Rebase patch order to ensure aarch64 support throughout commits. - Fix naming convention format issues for compat structs and functions. - Remove uaddr_to_user_ptr usage to enforce capability model.
v1: I have tested these changes against musl and there still exists an issue in this implementation with copybuf and potentially msg_control generating an EFAULT error.
Gitlab Issue: - https://git.morello-project.org/morello/kernel/linux/-/issues/46
Review branch: - https://git.morello-project.org/harryramsey/linux/-/commits/tcp_zerocopy
Thanks, Harry
Harry Ramsey (2): tcp: Implement compat version of tcp_zerocopy_receive tcp: Support userspace capabilities for tcp_zerocopy_receive
include/linux/sockptr.h | 28 ++++++++ include/uapi/linux/tcp.h | 6 +- net/ipv4/tcp.c | 135 ++++++++++++++++++++++++++++++++++----- 3 files changed, 149 insertions(+), 20 deletions(-)
This patch introduces a compat version of the tcp_zerocopy_receive structure. Subsequently additional helper functions have been added to convert between compat and purecap strucutres.
v2: - Rebase patch order to ensure aarch64 support throughout commits. - Fix naming convention format issues for compat structs and functions.
Signed-off-by: Harry Ramsey harry.ramsey@arm.com --- include/linux/sockptr.h | 28 +++++++++++ net/ipv4/tcp.c | 106 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 7 deletions(-)
diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index bae5e2369b4f..fd270378e8c8 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -41,6 +41,20 @@ static inline bool sockptr_is_null(sockptr_t sockptr) return !sockptr.user; }
+static inline int copy_from_sockptr_offset_with_ptr(void *dst, sockptr_t src, + size_t offset, size_t size) +{ + if (!sockptr_is_kernel(src)) + return copy_from_user_with_ptr(dst, src.user + offset, size); + memcpy(dst, src.kernel + offset, size); + return 0; +} + +static inline int copy_from_sockptr_with_ptr(void *dst, sockptr_t src, size_t size) +{ + return copy_from_sockptr_offset_with_ptr(dst, src, 0, size); +} + static inline int copy_from_sockptr_offset(void *dst, sockptr_t src, size_t offset, size_t size) { @@ -55,6 +69,20 @@ static inline int copy_from_sockptr(void *dst, sockptr_t src, size_t size) return copy_from_sockptr_offset(dst, src, 0, size); }
+static inline int copy_to_sockptr_offset_with_ptr(sockptr_t dst, size_t offset, + const void *src, size_t size) +{ + if (!sockptr_is_kernel(dst)) + return copy_to_user_with_ptr(dst.user + offset, src, size); + memcpy(dst.kernel + offset, src, size); + return 0; +} + +static inline int copy_to_sockptr_with_ptr(sockptr_t dst, const void *src, size_t size) +{ + return copy_to_sockptr_offset_with_ptr(dst, 0, src, size); +} + static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset, const void *src, size_t size) { diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 90526625671b..96b3dbd7bb4e 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1940,6 +1940,98 @@ static int find_next_mappable_frag(const skb_frag_t *frag, return offset; }
+struct compat_tcp_zerocopy_receive { + __u64 address; + __u32 length; + __u32 recv_skip_hint; + __u32 inq; + __s32 err; + __u64 copybuf_address; + __s32 copybuf_len; + __u32 flags; + __u64 msg_control; + __u64 msg_controllen; + __u32 msg_flags; + __u32 reserved; +}; + +static inline bool in_compat64(void) +{ + return IS_ENABLED(CONFIG_COMPAT64) && in_compat_syscall(); +} + +static inline size_t tcp_zerocopy_receive_size(void) +{ + if (in_compat64()) + return sizeof(struct compat_tcp_zerocopy_receive); + + return sizeof(struct tcp_zerocopy_receive); +} + +static int get_compat64_tcp_zerocopy_receive(struct tcp_zerocopy_receive *zc, + sockptr_t src, size_t size) +{ + struct compat_tcp_zerocopy_receive compat_zc = {}; + + if (copy_from_sockptr(&compat_zc, src, size)) + return -EFAULT; + + zc->address = (__kernel_uintptr_t)compat_ptr(compat_zc.address); + zc->length = compat_zc.length; + zc->recv_skip_hint = compat_zc.recv_skip_hint; + zc->inq = compat_zc.inq; + zc->err = compat_zc.err; + zc->copybuf_address = (__kernel_uintptr_t)compat_ptr(compat_zc.copybuf_address); + zc->copybuf_len = compat_zc.copybuf_len; + zc->flags = compat_zc.flags; + zc->msg_control = (__kernel_uintptr_t)compat_ptr(compat_zc.msg_control); + zc->msg_controllen = compat_zc.msg_controllen; + zc->msg_flags = compat_zc.msg_flags; + zc->reserved = compat_zc.reserved; + return 0; +} + +static int copy_tcp_zerocopy_receive_from_sockptr(struct tcp_zerocopy_receive *zc, + sockptr_t src, size_t size) +{ + if (in_compat64()) + return get_compat64_tcp_zerocopy_receive(zc, src, size); + if (copy_from_sockptr_with_ptr(zc, src, size)) + return -EFAULT; + return 0; +} + +static int set_compat64_tcp_zerocopy_receive(struct tcp_zerocopy_receive *zc, + sockptr_t dst, size_t size) +{ + struct compat_tcp_zerocopy_receive compat_zc = {}; + + compat_zc.address = zc->address; + compat_zc.length = zc->length; + compat_zc.recv_skip_hint = zc->recv_skip_hint; + compat_zc.inq = zc->inq; + compat_zc.err = zc->err; + compat_zc.copybuf_address = zc->copybuf_address; + compat_zc.copybuf_len = zc->copybuf_len; + compat_zc.flags = zc->flags; + compat_zc.msg_control = zc->msg_control; + compat_zc.msg_controllen = zc->msg_controllen; + compat_zc.msg_flags = zc->msg_flags; + compat_zc.reserved = zc->reserved; + + return copy_to_sockptr(dst, &compat_zc, size); +} + +static int copy_tcp_zerocopy_receive_to_sockptr(struct tcp_zerocopy_receive *zc, + sockptr_t dst, size_t size) +{ + if (in_compat64()) + return set_compat64_tcp_zerocopy_receive(zc, dst, size); + if (copy_from_sockptr_with_ptr(zc, dst, size)) + return -EFAULT; + return 0; +} + static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, struct tcp_zerocopy_receive *zc, struct sk_buff *skb, u32 offset) @@ -4321,18 +4413,18 @@ int do_tcp_getsockopt(struct sock *sk, int level, if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len < 0 || - len < offsetofend(struct tcp_zerocopy_receive, length)) + len < offsetofend(struct compat_tcp_zerocopy_receive, length)) return -EINVAL; - if (unlikely(len > sizeof(zc))) { - err = check_zeroed_sockptr(optval, sizeof(zc), - len - sizeof(zc)); + if (unlikely(len > tcp_zerocopy_receive_size())) { + err = check_zeroed_sockptr(optval, tcp_zerocopy_receive_size(), + len - tcp_zerocopy_receive_size()); if (err < 1) return err == 0 ? -EINVAL : err; - len = sizeof(zc); + len = tcp_zerocopy_receive_size(); if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; } - if (copy_from_sockptr(&zc, optval, len)) + if (copy_tcp_zerocopy_receive_from_sockptr(&zc, optval, len)) return -EFAULT; if (zc.reserved) return -EINVAL; @@ -4372,7 +4464,7 @@ int do_tcp_getsockopt(struct sock *sk, int level, zerocopy_rcv_inq: zc.inq = tcp_inq_hint(sk); zerocopy_rcv_out: - if (!err && copy_to_sockptr(optval, &zc, len)) + if (!err && copy_tcp_zerocopy_receive_to_sockptr(&zc, optval, len)) err = -EFAULT; return err; }
The tcp_zerocopy_receive struct is used to create a shared memory region between userspace and the kernel in which received network packets are directly copied from the network interface card.
In PCuABI, a user pointer is a 129 bit capability, the __u64 type is not sufficient enough to store it. Instead __kernel_uinptr_t is used instead which is sufficient enough to store a capability whilst remaining 64-bits on other architectures.
Additional checks have been added to ensure that the owning capability has read access in order to read the received network packets from the shared memory region.
v2: - Remove uaddr_to_user_ptr usage to enforce capability model.
Signed-off-by: Harry Ramsey harry.ramsey@arm.com --- include/uapi/linux/tcp.h | 6 +++--- net/ipv4/tcp.c | 31 ++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-)
diff --git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h index 879eeb0a084b..972db73fd3cf 100644 --- a/include/uapi/linux/tcp.h +++ b/include/uapi/linux/tcp.h @@ -352,15 +352,15 @@ struct tcp_diag_md5sig {
#define TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT 0x1 struct tcp_zerocopy_receive { - __u64 address; /* in: address of mapping */ + __kernel_uintptr_t address; /* in: address of mapping */ __u32 length; /* in/out: number of bytes to map/mapped */ __u32 recv_skip_hint; /* out: amount of bytes to skip */ __u32 inq; /* out: amount of bytes in read queue */ __s32 err; /* out: socket error */ - __u64 copybuf_address; /* in: copybuf address (small reads) */ + __kernel_uintptr_t copybuf_address; /* in: copybuf address (small reads) */ __s32 copybuf_len; /* in/out: copybuf bytes avail/used or error */ __u32 flags; /* in: flags */ - __u64 msg_control; /* ancillary data */ + __kernel_uintptr_t msg_control; /* ancillary data */ __u64 msg_controllen; __u32 msg_flags; __u32 reserved; /* set to 0 for now */ diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 96b3dbd7bb4e..383e74eacf4f 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2076,7 +2076,7 @@ static int receive_fallback_to_copy(struct sock *sk, struct tcp_zerocopy_receive *zc, int inq, struct scm_timestamping_internal *tss) { - unsigned long copy_address = (unsigned long)zc->copybuf_address; + __kernel_uintptr_t copy_address = zc->copybuf_address; struct msghdr msg = {}; struct iovec iov; int err; @@ -2087,7 +2087,10 @@ static int receive_fallback_to_copy(struct sock *sk, if (copy_address != zc->copybuf_address) return -EINVAL;
- err = import_single_range(ITER_DEST, uaddr_to_user_ptr(copy_address), + if (check_user_ptr_read((void __user *)zc->copybuf_address, zc->copybuf_len)) + return -EFAULT; + + err = import_single_range(ITER_DEST, copy_address, inq, &iov, &msg.msg_iter); if (err) return err; @@ -2113,7 +2116,7 @@ static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, struct sk_buff *skb, u32 copylen, u32 *offset, u32 *seq) { - unsigned long copy_address = (unsigned long)zc->copybuf_address; + __kernel_uintptr_t copy_address = zc->copybuf_address; struct msghdr msg = {}; struct iovec iov; int err; @@ -2121,7 +2124,10 @@ static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, if (copy_address != zc->copybuf_address) return -EINVAL;
- err = import_single_range(ITER_DEST, uaddr_to_user_ptr(copy_address), + if (check_user_ptr_read((void __user *)zc->copybuf_address, zc->copybuf_len)) + return -EFAULT; + + err = import_single_range(ITER_DEST, copy_address, copylen, &iov, &msg.msg_iter); if (err) return err; @@ -2164,7 +2170,7 @@ static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, struct page **pending_pages, unsigned long pages_remaining, - unsigned long *address, + __kernel_uintptr_t *address, u32 *length, u32 *seq, struct tcp_zerocopy_receive *zc, @@ -2212,7 +2218,7 @@ static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma, static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma, struct page **pages, unsigned int pages_to_map, - unsigned long *address, + __kernel_uintptr_t *address, u32 *length, u32 *seq, struct tcp_zerocopy_receive *zc, @@ -2246,11 +2252,11 @@ static void tcp_zc_finalize_rx_tstamp(struct sock *sk, struct tcp_zerocopy_receive *zc, struct scm_timestamping_internal *tss) { - unsigned long msg_control_addr; + __kernel_uintptr_t msg_control_addr; struct msghdr cmsg_dummy;
- msg_control_addr = (unsigned long)zc->msg_control; - cmsg_dummy.msg_control_user = uaddr_to_user_ptr(msg_control_addr); + msg_control_addr = zc->msg_control; + cmsg_dummy.msg_control_user = msg_control_addr; cmsg_dummy.msg_controllen = (__kernel_size_t)zc->msg_controllen; cmsg_dummy.msg_flags = in_compat_syscall() @@ -2274,7 +2280,7 @@ static int tcp_zerocopy_receive(struct sock *sk, struct scm_timestamping_internal *tss) { u32 length = 0, offset, vma_len, avail_len, copylen = 0; - unsigned long address = (unsigned long)zc->address; + __kernel_uintptr_t address = zc->address; struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE]; s32 copybuf_len = zc->copybuf_len; struct tcp_sock *tp = tcp_sk(sk); @@ -2290,6 +2296,9 @@ static int tcp_zerocopy_receive(struct sock *sk, zc->copybuf_len = 0; zc->msg_flags = 0;
+ if (!check_user_ptr_read((void __user *)zc->address, zc->length)) + return -EFAULT; + if (address & (PAGE_SIZE - 1) || address != zc->address) return -EINVAL;
@@ -4413,7 +4422,7 @@ int do_tcp_getsockopt(struct sock *sk, int level, if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len < 0 || - len < offsetofend(struct compat_tcp_zerocopy_receive, length)) + (len < offsetofend(struct compat_tcp_zerocopy_receive, length))) return -EINVAL; if (unlikely(len > tcp_zerocopy_receive_size())) { err = check_zeroed_sockptr(optval, tcp_zerocopy_receive_size(),
On 20-09-2023 23:11, Harry Ramsey wrote:
Hello,
This patch series enables purecap applications to utilise tcp_zerocopy_receive to map packets directly from a network interface card into a shared space with the process and kernel.
I do not think I shall have time to continue revising this patch series and debugging the bus error generated by the latest patch. Hopefully this provides a start to tcp_zerocopy_receive for capability architecture.
Many thanks for the series, Harry! Your efforts are greatly appreciated! I'll pick up this series and credit you accordingly, of course.
Thanks, Tudor
v2:
- There appears to be a new error generated against musl resulting in a BUS error when using memcpy to copy between allocated shared space and other regions of memory.
- Rebase patch order to ensure aarch64 support throughout commits.
- Fix naming convention format issues for compat structs and functions.
- Remove uaddr_to_user_ptr usage to enforce capability model.
v1: I have tested these changes against musl and there still exists an issue in this implementation with copybuf and potentially msg_control generating an EFAULT error.
Gitlab Issue:
Review branch:
Thanks, Harry
Harry Ramsey (2): tcp: Implement compat version of tcp_zerocopy_receive tcp: Support userspace capabilities for tcp_zerocopy_receive
include/linux/sockptr.h | 28 ++++++++ include/uapi/linux/tcp.h | 6 +- net/ipv4/tcp.c | 135 ++++++++++++++++++++++++++++++++++----- 3 files changed, 149 insertions(+), 20 deletions(-)
linux-morello@op-lists.linaro.org