Hello everybody,
I'm having a hard time trying to figure out what is going on with my socket filter, for testing I wrote a simple echo client/server in python, client sends a random blob to server, server then send blob back to the client, which then verifies it.
Problem happens when the amount of data I'm trying to send exceeds 500KB, following is the error (stacktrace) I'm getting:
(lldb) bt
* thread #5, name = '0xffffff8021440d48', queue = '0x0', stop reason = EXC_BREAKPOINT (code=3, subcode=0x0)
* frame #0: 0xffffff8014f79a7a kernel.development`panic_trap_to_debugger [inlined] current_cpu_datap at cpu_data.h:401 [opt]
frame #1: 0xffffff8014f79a7a kernel.development`panic_trap_to_debugger [inlined] current_processor at cpu.c:220 [opt]
frame #2: 0xffffff8014f79a7a kernel.development`panic_trap_to_debugger [inlined] DebuggerTrapWithState(db_op=DBOP_PANIC, db_message=<unavailable>, db_panic_str="\"m_free: freeing an already freed mbuf\"@/BuildRoot/Library/Caches/com.apple.xbs/Sources/xnu/xnu-4570.71.13/bsd/kern/uipc_mbuf.c:3689", db_panic_args=0xffffff801c0ebc10, db_panic_options=0, db_proceed_on_sync_failure=1, db_panic_caller=18446743524311097385) at debug.c:463 [opt]
frame #3: 0xffffff8014f79a4a kernel.development`panic_trap_to_debugger(panic_format_str="\"m_free: freeing an already freed mbuf\"@/BuildRoot/Library/Caches/com.apple.xbs/Sources/xnu/xnu-4570.71.13/bsd/kern/uipc_mbuf.c:3689", panic_args=0xffffff801c0ebc10, reason=0, ctx=0x0000000000000000, panic_options_mask=0, panic_caller=18446743524311097385)
at debug.c:724 [opt]
frame #4: 0xffffff8014f7984c kernel.development`panic(str=<unavailable>) at debug.c:611 [opt]
frame #5: 0xffffff80154ce029 kernel.development`m_free(m=0xffffff80a970a600) at uipc_mbuf.c:3689 [opt]
frame #6: 0xffffff80154dafd8 kernel.development`sosend [inlined] m_freem(m=<unavailable>) at uipc_mbuf.c:4821 [opt]
frame #7: 0xffffff80154dafc1 kernel.development`sosend(so=<unavailable>, addr=0x0000000000000000, uio=0x0000000000000000, top=<unavailable>, control=0x0000000000000000, flags=0) at uipc_socket.c:2480 [opt]
frame #8: 0xffffff8015500ad7 kernel.development`sock_send_internal(sock=<unavailable>, msg=0x0000000300000008, data=0xffffff80a970a600, flags=<unavailable>, sentlen=0x000000031c0ebe80) at kpi_socket.c:867 [opt]
frame #9: 0xffffff7f97455995 netfilter`send_notification(agent_sock=0xffffff80240ee580, notification=0xffffff8028528330) at netfilter.c:412
frame #10: 0xffffff7f97456499 netfilter`send_filter_packets_to_userspace(unused=0x0000000000000001) at netfilter.c:650
frame #11: 0xffffff8014fb9f80 kernel.development`thread_call_invoke(func=(netfilter`send_filter_packets_to_userspace at netfilter.c:623), param0=<unavailable>, param1=<unavailable>, call=0xffffff8015a931e0) at thread_call.c:1387 [opt]
frame #12: 0xffffff8014fb9b06 kernel.development`thread_call_thread(group=<unavailable>, wres=<unavailable>) at thread_call.c:1463 [opt]
frame #13: 0xffffff8014f1d5c7 kernel.development`call_continuation + 23
It seems kernel is releasing mbuf packets before I send them to a userspace process.
I'm using TCP sockets between driver/userspace process, I tought that by duplicating mbuf with mbuf_dup I'd be safe, but that didn't happen.
I also saw there are calls to increment reference count of mbufs in the kernel, which aren't available to use in NKE API.
Here is the code registered in the socket filter structure as a callback to data_in and data_out:
errno_t tl_data_fn(void *cookie, socket_t so, const struct sockaddr *addr, mbuf_t *data, mbuf_t *control, sflt_data_flag_t flags, NetFilterSocketDataDirection direction)
{
errno_t error = 0;
struct NetFilterNotification *notification = NULL;
if (!cookie || !data || !*data) {
LOG_DEBUG("[tl_data_fn] null_input: cookie=%s, data=%s, *data=%s", (cookie ? "OK" : "NULL"), (data ? "OK" : "NULL"), (data && *data ? "OK" : "NULL"));
return 0;
}
// avoid ciclical processing if this packet is from/to agents.
if (is_socket_in_cache(g_control_sockets, so)) {
return 0;
}
// check if we already processed this packet (reinjected in the stack).
if (check_tag(data, gidtag, NETFILTER_TAG_TYPE, direction == NetFilterSocketDataDirectionIn ? IN_DONE : OUT_DONE))
{
return 0;
}
netfilter_cookie *nf_cookie = (netfilter_cookie *)cookie;
// check if we need to filter this socket (if it was created by any UID being monitored).
lck_mtx_lock(gmutex);
struct tl_cb *tl_cb = find_ctl_by_ref(nf_cookie->ctl_ref);
lck_mtx_unlock(gmutex);
if (!tl_cb || tl_cb->t_comm_sock == so) {
return 0;
}
notification = OSMalloc((uint32_t)sizeof(struct NetFilterNotification), gOSMallocTag);
if (!notification) {
LOG_ERROR("OSMalloc failed: no memory for notification");
error = ENOMEM;
goto failed;
}
error = mbuf_dup(*data, MBUF_WAITOK, &(notification->data));
if (error) {
LOG_ERROR("mbuf_dup error=%d", error);
goto failed;
}
// Prepend our header into the mbuf
error = mbuf_prepend(&(notification->data), sizeof(struct NetFilterNotificationHeader), MBUF_WAITOK);
if (error) {
LOG_ERROR("mbuf_prepend error=%d", error);
goto failed;
}
// Check if space for header is contiguous
if (mbuf_len(notification->data) < sizeof(struct NetFilterNotificationHeader)) {
error = mbuf_pullup(&(notification->data), sizeof(struct NetFilterNotificationHeader));
if (error) { // mbuf_pullup free the mbuf chain in case of an error
LOG_ERROR("mbuf_pullup error=%d", error);
goto failed;
}
}
struct NetFilterNotificationHeader *header = mbuf_data(notification->data);
LOG_DEBUG("[tl_data_fn] packet_size=%lu", mbuf_pkthdr_len(notification->data));
notification->header = header;
notification->must_free_header = FALSE;
notification->cookie = nf_cookie;
notification->is_busy = FALSE;
if (direction == NetFilterSocketDataDirectionIn) {
header->event = NetFilterEventDataIn;
} else {
header->event = NetFilterEventDataOut;
}
header->socketId = (uint64_t)so;
header->proc_pid = nf_cookie->proc_pid;
header->localAddress = nf_cookie->nf_local;
header->remoteAddress = nf_cookie->nf_remote;
header->payloadSize = mbuf_pkthdr_len(notification->data) - sizeof(struct NetFilterNotificationHeader);
add_to_user_queue(notification);
schedule_userspace_notification();
if (*data) {
mbuf_freem(*data);
}
if (control != NULL && *control != NULL) {
mbuf_freem(*control);
}
LOG_DEBUG("[tl_data_fn] Packet sucessfully swallowed.");
// EJUSTRETURN means we swallowed packet, since we gonna process it in userspace.
// we also tagged packet, so next time we won't process it again after its reinjected.
return EJUSTRETURN;
failed:
if (notification) {
if (notification->data) {
mbuf_freem(notification->data);
}
OSFree(notification, sizeof(struct NetFilterNotification), gOSMallocTag);
}
if (data && *data)
mbuf_freem(*data);
if (control != NULL && *control != NULL)
mbuf_freem(*control);
return error;
}
Please help me, I don't know what else to do to workaround this issue, thanks in advance!
Diego Fronza