Add plumbing to notify RX-bound sockets of any NDIS offload updates

published/external/afxdp_experimental.h

@@ -145,6 +145,16 @@ typedef enum _XSK_POLL_MODE {
 //
 #define XSK_SOCKOPT_RX_OFFLOAD_CHECKSUM 1007
 
+
+//
+// XSK_SOCKOPT_RX_OFFLOAD_CURRENT_CONFIG_CHECKSUM
+//
+// Supports: get
+// Optval type: XDP_CHECKSUM_CONFIGURATION
+// Description: Returns the RX queue's current checksum offload configuration.
+//
+#define XSK_SOCKOPT_RX_OFFLOAD_CURRENT_CONFIG_CHECKSUM 1008
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

published/private/xdpif.h

@@ -85,7 +85,8 @@ XDP_REMOVE_INTERFACE_COMPLETE(
 typedef enum {
     XdpOffloadRss,
     XdpOffloadQeo,
-    XdpOffloadChecksum,
+    XdpRxOffloadChecksum,
+    XdpTxOffloadChecksum,
 } XDP_INTERFACE_OFFLOAD_TYPE;
 
 typedef enum {

published/private/xdprxqueue_internal.h

@@ -13,9 +13,17 @@ XDP_RX_QUEUE_CREATE_GET_HOOK_ID(
     _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig
     );
 
+typedef struct _XDP_RX_QUEUE_NOTIFY_HANDLE *XDP_RX_QUEUE_NOTIFY_HANDLE;
+typedef
+XDP_RX_QUEUE_NOTIFY_HANDLE
+XDP_RX_QUEUE_CREATE_GET_NOTIFY_HANDLE(
+    _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig
+    );
+
 typedef struct _XDP_RX_QUEUE_CONFIG_RESERVED {
     XDP_OBJECT_HEADER               Header;
     XDP_RX_QUEUE_CREATE_GET_HOOK_ID *GetHookId;
+    XDP_RX_QUEUE_CREATE_GET_NOTIFY_HANDLE *GetNotifyHandle;
 } XDP_RX_QUEUE_CONFIG_RESERVED;
 
 #define XDP_RX_QUEUE_CONFIG_RESERVED_REVISION_1 1
@@ -41,3 +49,75 @@ XdpRxQueueGetHookId(
 
     return Reserved->GetHookId(RxQueueConfig);
 }
+
+inline
+XDP_RX_QUEUE_NOTIFY_HANDLE
+XdpRxQueueGetNotifyHandle(
+    _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig
+    )
+{
+    XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *Details = (XDP_RX_QUEUE_CONFIG_CREATE_DETAILS *)RxQueueConfig;
+    const XDP_RX_QUEUE_CONFIG_RESERVED *Reserved = Details->Dispatch->Reserved;
+
+    if (Reserved == NULL ||
+        Reserved->Header.Revision < XDP_RX_QUEUE_CONFIG_RESERVED_REVISION_1 ||
+        Reserved->Header.Size < XDP_SIZEOF_RX_QUEUE_CONFIG_RESERVED_REVISION_1 ||
+        Reserved->GetNotifyHandle == NULL) {
+        return NULL;
+    }
+
+    return Reserved->GetNotifyHandle(RxQueueConfig);
+}
+
+typedef enum _XDP_RX_QUEUE_NOTIFY_CODE {
+    //
+    // The RX queue's current offload configuration has changed.
+    //
+    XDP_RX_QUEUE_NOTIFY_OFFLOAD_CURRENT_CONFIG,
+} XDP_RX_QUEUE_NOTIFY_CODE;
+
+typedef
+_IRQL_requires_max_(DISPATCH_LEVEL)
+VOID
+XDP_RX_QUEUE_NOTIFY(
+    _In_ XDP_RX_QUEUE_NOTIFY_HANDLE RxQueueNotifyHandle,
+    _In_ XDP_RX_QUEUE_NOTIFY_CODE NotifyCode,
+    _In_opt_ const VOID *NotifyBuffer,
+    _In_ SIZE_T NotifyBufferSize
+    );
+
+XDP_RX_QUEUE_NOTIFY XdpRxQueueNotify;
+
+typedef struct _XDP_RX_QUEUE_NOTIFY_DISPATCH {
+    XDP_OBJECT_HEADER   Header;
+    XDP_RX_QUEUE_NOTIFY *Notify;
+} XDP_RX_QUEUE_NOTIFY_DISPATCH;
+
+#define XDP_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1 1
+
+#define XDP_SIZEOF_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1 \
+    RTL_SIZEOF_THROUGH_FIELD(XDP_RX_QUEUE_NOTIFY_DISPATCH, Notify)
+
+typedef struct _XDP_RX_QUEUE_NOTIFY_DETAILS {
+    const XDP_RX_QUEUE_NOTIFY_DISPATCH *Dispatch;
+} XDP_RX_QUEUE_NOTIFY_DETAILS;
+
+inline
+_IRQL_requires_max_(DISPATCH_LEVEL)
+VOID
+XDPEXPORT(XdpRxQueueNotify)(
+    _In_ XDP_RX_QUEUE_NOTIFY_HANDLE RxQueueNotifyHandle,
+    _In_ XDP_RX_QUEUE_NOTIFY_CODE NotifyCode,
+    _In_opt_ const VOID *NotifyBuffer,
+    _In_ SIZE_T NotifyBufferSize
+    )
+{
+    const XDP_RX_QUEUE_NOTIFY_DETAILS *Details = (CONST XDP_RX_QUEUE_NOTIFY_DETAILS *)RxQueueNotifyHandle;
+    const XDP_RX_QUEUE_NOTIFY_DISPATCH *Dispatch = Details->Dispatch;
+
+    ASSERT(Dispatch != NULL);
+    ASSERT(Dispatch->Header.Revision >= XDP_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1);
+    ASSERT(Dispatch->Header.Size >= XDP_SIZEOF_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1);
+
+    Dispatch->Notify(RxQueueNotifyHandle, NotifyCode, NotifyBuffer, NotifyBufferSize);
+}

src/xdp/rx.c

@@ -69,6 +69,7 @@ typedef struct _XDP_RX_QUEUE {
     // Control path fields. TODO: Move to a separate, paged structure.
     //
 
+    RTL_REFERENCE_COUNT InterlockedReferenceCount;
     RTL_REFERENCE_COUNT ReferenceCount;
     XDP_BINDING_HANDLE Binding;
     XDP_RX_QUEUE_KEY Key;
@@ -91,7 +92,14 @@ typedef struct _XDP_RX_QUEUE {
     XDP_IF_OFFLOAD_HANDLE InterfaceOffloadHandle;
     PCW_INSTANCE *PcwInstance;
 
-    LIST_ENTRY NotifyClients;
+    struct {
+        KSPIN_LOCK Lock;
+        BOOLEAN WorkerQueued : 1;
+        BOOLEAN OffloadNeeded : 1;
+        XDP_BINDING_WORKITEM WorkItem;
+        XDP_RX_QUEUE_NOTIFY_DETAILS Details;
+        LIST_ENTRY Clients;
+    } Notify;
 } XDP_RX_QUEUE;
 
 typedef struct _XDP_RX_QUEUE_SWAP_PROGRAM_PARAMS {
@@ -670,12 +678,165 @@ XdppRxQueueGetHookId(
     return &RxQueue->Key.HookId;
 }
 
+static
+XDP_RX_QUEUE_NOTIFY_HANDLE
+XdppRxQueueGetNotifyHandle(
+    _In_ XDP_RX_QUEUE_CONFIG_CREATE RxQueueConfig
+    )
+{
+    XDP_RX_QUEUE *RxQueue = XdpRxQueueFromConfigCreate(RxQueueConfig);
+
+    return (XDP_RX_QUEUE_NOTIFY_HANDLE)&RxQueue->Notify.Details;
+}
+
+static
+XDP_RX_QUEUE *
+XdpRxQueueFromNotify(
+    _In_ XDP_RX_QUEUE_NOTIFY_HANDLE RxQueueNotifyHandle
+    )
+{
+    return CONTAINING_RECORD(RxQueueNotifyHandle, XDP_RX_QUEUE, Notify.Details);
+}
+
+static
+VOID
+XdpRxQueueInterlockedReference(
+    _Inout_ XDP_RX_QUEUE *RxQueue
+    )
+{
+    XdpIncrementReferenceCount(&RxQueue->InterlockedReferenceCount);
+}
+
+static
+VOID
+XdpRxQueueInterlockedDereference(
+    _Inout_ XDP_RX_QUEUE *RxQueue
+    )
+{
+    if (XdpDecrementReferenceCount(&RxQueue->InterlockedReferenceCount)) {
+        ExFreePoolWithTag(RxQueue, XDP_POOLTAG_RXQUEUE);
+    }
+}
+
+static
+VOID
+XdpRxQueueNotifyClients(
+    _In_ XDP_RX_QUEUE *RxQueue,
+    _In_ XDP_RX_QUEUE_NOTIFICATION_TYPE NotificationType
+    )
+{
+    LIST_ENTRY *Entry = RxQueue->Notify.Clients.Flink;
+
+    TraceInfo(
+        TRACE_CORE, "RxQueue=%p NotificationType=%!RX_QUEUE_NOTIFICATION_TYPE!",
+        RxQueue, NotificationType);
+
+    while (Entry != &RxQueue->Notify.Clients) {
+        XDP_RX_QUEUE_NOTIFICATION_ENTRY *NotifyEntry;
+
+        NotifyEntry = CONTAINING_RECORD(Entry, XDP_RX_QUEUE_NOTIFICATION_ENTRY, Link);
+        Entry = Entry->Flink;
+
+        NotifyEntry->NotifyRoutine(NotifyEntry, NotificationType);
+    }
+}
+
+static
+_Requires_lock_held_(RxQueue->Notify.Lock)
+VOID
+XdpRxQueueNotifyClientsUnderNotifyLock(
+    _In_ XDP_RX_QUEUE *RxQueue,
+    _In_ XDP_RX_QUEUE_NOTIFICATION_TYPE NotificationType,
+    _In_ _IRQL_saves_ _IRQL_restores_ KIRQL *OldIrql
+    )
+{
+    //
+    // N.B. This routine releases and re-acquires the notify lock.
+    //
+    ASSERT(*OldIrql == PASSIVE_LEVEL);
+    KeReleaseSpinLock(&RxQueue->Notify.Lock, *OldIrql);
+    XdpRxQueueNotifyClients(RxQueue, NotificationType);
+    KeAcquireSpinLock(&RxQueue->Notify.Lock, OldIrql);
+}
+
+static
+VOID
+XdpRxQueueNotifyWorker(
+    _In_ XDP_BINDING_WORKITEM *Item
+    )
+{
+    // !!!TODO
+    XDP_RX_QUEUE *RxQueue = CONTAINING_RECORD(Item, XDP_RX_QUEUE, Notify.WorkItem);
+    KIRQL OldIrql;
+
+    KeAcquireSpinLock(&RxQueue->Notify.Lock, &OldIrql);
+
+    ASSERT(RxQueue->Notify.WorkerQueued);
+
+    while (TRUE) {
+        if (RxQueue->Notify.OffloadNeeded) {
+            RxQueue->Notify.OffloadNeeded = FALSE;
+            XdpRxQueueNotifyClientsUnderNotifyLock(
+                RxQueue, XDP_RX_QUEUE_NOTIFICATION_OFFLOAD_CURRENT_CONFIG, &OldIrql);
+        } else {
+            break;
+        }
+    }
+
+    KeReleaseSpinLock(&RxQueue->Notify.Lock, OldIrql);
+
+    XdpRxQueueInterlockedDereference(RxQueue);
+}
+
+_IRQL_requires_max_(DISPATCH_LEVEL)
+VOID
+XdpRxQueueNotify(
+    _In_ XDP_RX_QUEUE_NOTIFY_HANDLE RxQueueNotifyHandle,
+    _In_ XDP_RX_QUEUE_NOTIFY_CODE NotifyCode,
+    _In_opt_ const VOID *NotifyBuffer,
+    _In_ SIZE_T NotifyBufferSize
+    )
+{
+    UNREFERENCED_PARAMETER(NotifyBuffer);
+    UNREFERENCED_PARAMETER(NotifyBufferSize);
+
+    XDP_RX_QUEUE *RxQueue = XdpRxQueueFromNotify(RxQueueNotifyHandle);
+    KIRQL OldIrql;
+    BOOLEAN NeedNotification = FALSE;
+
+    // !!!TODO
+    // This routine can be invoked from arbitrary IRQLs, from PASSIVE_LEVEL to DISPATCH_LEVEL.
+    //
+
+    KeAcquireSpinLock(&RxQueue->Notify.Lock, &OldIrql);
+
+    switch (NotifyCode) {
+        case XDP_RX_QUEUE_NOTIFY_OFFLOAD_CURRENT_CONFIG:
+            RxQueue->Notify.OffloadNeeded = TRUE;
+            NeedNotification = TRUE;
+            break;
+    }
+
+    if (NeedNotification) {
+        if (!RxQueue->Notify.WorkerQueued) {
+            RxQueue->Notify.WorkerQueued = TRUE;
+            RxQueue->Notify.WorkItem.BindingHandle = RxQueue->Binding;
+            RxQueue->Notify.WorkItem.WorkRoutine = XdpRxQueueNotifyWorker;
+            XdpRxQueueInterlockedReference(RxQueue);
+            XdpIfQueueWorkItem(&RxQueue->Notify.WorkItem);
+        }
+    }
+
+    KeReleaseSpinLock(&RxQueue->Notify.Lock, OldIrql);
+}
+
 static const XDP_RX_QUEUE_CONFIG_RESERVED XdpRxConfigReservedDispatch = {
     .Header                         = {
         .Revision                   = XDP_RX_QUEUE_CONFIG_RESERVED_REVISION_1,
         .Size                       = XDP_SIZEOF_RX_QUEUE_CONFIG_RESERVED_REVISION_1,
     },
     .GetHookId                      = XdppRxQueueGetHookId,
+    .GetNotifyHandle                = XdppRxQueueGetNotifyHandle,
 };
 
 static const XDP_RX_QUEUE_CONFIG_CREATE_DISPATCH XdpRxConfigCreateDispatch = {
@@ -703,28 +864,13 @@ static const XDP_RX_QUEUE_CONFIG_ACTIVATE_DISPATCH XdpRxConfigActivateDispatch =
     .IsChecksumOffloadEnabled   = XdpRxQueueIsChecksumOffloadEnabled,
 };
 
-static
-VOID
-XdpRxQueueNotifyClients(
-    _In_ XDP_RX_QUEUE *RxQueue,
-    _In_ XDP_RX_QUEUE_NOTIFICATION_TYPE NotificationType
-    )
-{
-    LIST_ENTRY *Entry = RxQueue->NotifyClients.Flink;
-
-    TraceInfo(
-        TRACE_CORE, "RxQueue=%p NotificationType=%!RX_QUEUE_NOTIFICATION_TYPE!",
-        RxQueue, NotificationType);
-
-    while (Entry != &RxQueue->NotifyClients) {
-        XDP_RX_QUEUE_NOTIFICATION_ENTRY *NotifyEntry;
-
-        NotifyEntry = CONTAINING_RECORD(Entry, XDP_RX_QUEUE_NOTIFICATION_ENTRY, Link);
-        Entry = Entry->Flink;
-
-        NotifyEntry->NotifyRoutine(NotifyEntry, NotificationType);
-    }
-}
+static const XDP_RX_QUEUE_NOTIFY_DISPATCH XdpRxNotifyDispatch = {
+    .Header                         = {
+        .Revision                   = XDP_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1,
+        .Size                       = XDP_SIZEOF_RX_QUEUE_NOTIFY_DISPATCH_REVISION_1
+    },
+    .Notify                         = XdpRxQueueNotify,
+};
 
 static
 VOID
@@ -1034,11 +1180,12 @@ XdpRxQueueCreate(
         goto Exit;
     }
 
+    XdpInitializeReferenceCount(&RxQueue->InterlockedReferenceCount);
     XdpInitializeReferenceCount(&RxQueue->ReferenceCount);
     RxQueue->State = XdpRxQueueStateUnbound;
     XdpIfInitializeClientEntry(&RxQueue->BindingClientEntry);
     InitializeListHead(&RxQueue->ProgramBindings);
-    InitializeListHead(&RxQueue->NotifyClients);
+    InitializeListHead(&RxQueue->Notify.Clients);
     XdpQueueSyncInitialize(&RxQueue->Sync);
     RxQueue->Binding = Binding;
     RxQueue->Key = Key;
@@ -1151,11 +1298,11 @@ VOID
 XdpRxQueueRegisterNotifications(
     _In_ XDP_RX_QUEUE *RxQueue,
     _Inout_ XDP_RX_QUEUE_NOTIFICATION_ENTRY *NotifyEntry,
-    _In_ XDP_RX_QUEUE_NOTIFY *NotifyRoutine
+    _In_ XDP_RX_QUEUE_NOTIFICATION_ROUTINE *NotifyRoutine
     )
 {
     NotifyEntry->NotifyRoutine = NotifyRoutine;
-    InsertTailList(&RxQueue->NotifyClients, &NotifyEntry->Link);
+    InsertTailList(&RxQueue->Notify.Clients, &NotifyEntry->Link);
 
     if (RxQueue->InterfaceRxQueue != NULL) {
         NotifyRoutine(NotifyEntry, XDP_RX_QUEUE_NOTIFICATION_ATTACH);
@@ -1345,6 +1492,14 @@ XdpRxQueueGetInterfacePollHandle(
     return RxQueue->InterfaceRxPollHandle;
 }
 
+XDP_IF_OFFLOAD_HANDLE
+XdpRxQueueGetInterfaceOffloadHandle(
+    _In_ XDP_RX_QUEUE *RxQueue
+    )
+{
+    return RxQueue->InterfaceOffloadHandle;
+}
+
 XDP_RX_QUEUE_CONFIG_ACTIVATE
 XdpRxQueueGetConfig(
     _In_ XDP_RX_QUEUE *RxQueue
@@ -1392,7 +1547,7 @@ XdpRxQueueDereference(
             XdpExtensionSetCleanup(RxQueue->FrameExtensionSet);
             RxQueue->FrameExtensionSet = NULL;
         }
-        ExFreePoolWithTag(RxQueue, XDP_POOLTAG_RXQUEUE);
+        XdpRxQueueInterlockedDereference(RxQueue);
     }
 }