Repository navigation
Expand file tree
/
Copy pathtxqueue.cpp
More file actions
904 lines (752 loc) · 32.3 KB
/
Copy pathtxqueue.cpp
File metadata and controls
904 lines (752 loc) · 32.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
/*
* ovpn-dco-win OpenVPN protocol accelerator for Windows
*
* Copyright (C) 2020-2021 OpenVPN Inc <sales@openvpn.net>
* Copyright (C) 2023 Rubicon Communications LLC (Netgate)
*
* Author: Lev Stipakov <lev@openvpn.net>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <ntddk.h>
#include <wdf.h>
#include <netadaptercx.h>
#include <net/virtualaddress.h>
#include "crypto.h"
#include "driver.h"
#include "mss.h"
#include "trace.h"
#include "netringiterator.h"
#include "timer.h"
#include "txqueue.h"
#include "socket.h"
#include "peer.h"
static
BOOLEAN
OvpnTxAreSockaddrEqual(const SOCKADDR* addr1, const SOCKADDR* addr2) {
// First, check if the address families are the same
if (addr1->sa_family != addr2->sa_family) {
return 0; // Not equal if the families are different
}
if (addr1->sa_family == AF_INET) {
// Compare IPv4 addresses
SOCKADDR_IN* ipv4_1 = (SOCKADDR_IN*)addr1;
SOCKADDR_IN* ipv4_2 = (SOCKADDR_IN*)addr2;
return (ipv4_1->sin_addr.s_addr == ipv4_2->sin_addr.s_addr &&
ipv4_1->sin_port == ipv4_2->sin_port);
}
else if (addr1->sa_family == AF_INET6) {
// Compare IPv6 addresses
SOCKADDR_IN6* ipv6_1 = (SOCKADDR_IN6*)addr1;
SOCKADDR_IN6* ipv6_2 = (SOCKADDR_IN6*)addr2;
SIZE_T result = RtlCompareMemory(&ipv6_1->sin6_addr, &ipv6_2->sin6_addr, sizeof(ipv6_1->sin6_addr));
return (result == sizeof(ipv6_1->sin6_addr) &&
ipv6_1->sin6_port == ipv6_2->sin6_port);
}
// If the address family is neither AF_INET nor AF_INET6, return not equal
return 0;
}
BOOLEAN
OvpnCheckRecursiveRoutingIPv4(SOCKADDR_IN* transportAdds, UCHAR* buffer, SIZE_T bufferLength, BOOLEAN tcp)
{
if (transportAdds->sin_family != AF_INET || bufferLength < sizeof(IPV4_HEADER))
return FALSE; // Not an IPv4 peer or packet too short
// Extract pointers to avoid repeated FIELD_OFFSET calculations
IN_ADDR* srcAddr = (IN_ADDR*)(buffer + FIELD_OFFSET(IPV4_HEADER, SourceAddress));
IN_ADDR* dstAddr = (IN_ADDR*)(buffer + FIELD_OFFSET(IPV4_HEADER, DestinationAddress));
UINT8 packetProtocol = buffer[FIELD_OFFSET(IPV4_HEADER, Protocol)];
// Validate the transport protocol
if ((tcp && packetProtocol != IPPROTO_TCP) || (!tcp && packetProtocol != IPPROTO_UDP))
return FALSE;
// Extract IP header length
UINT8 ipHeaderLength = (buffer[0] & 0x0F) << 2;
// Ensure transport header is accessible
if (bufferLength < ipHeaderLength + sizeof(UINT16) * 2)
return FALSE;
// Read transport header efficiently
UCHAR* transportHeader = buffer + ipHeaderLength;
UINT16 packetSrcPort = RtlUshortByteSwap(*(UINT16*)(transportHeader));
UINT16 packetDstPort = RtlUshortByteSwap(*(UINT16*)(transportHeader + 2));
UINT16 peerPort = RtlUshortByteSwap(transportAdds->sin_port);
// Check for recursive routing
if (packetDstPort == peerPort &&
RtlCompareMemory(dstAddr, &transportAdds->sin_addr, sizeof(IN_ADDR)) == sizeof(IN_ADDR))
{
LOG_WARN("Recursive routing detected (IPv4), packet dropped",
TraceLoggingIPv4Address(srcAddr->S_un.S_addr, "saddr"),
TraceLoggingIPv4Address(dstAddr->S_un.S_addr, "daddr"),
TraceLoggingUInt16(packetSrcPort, "sport"),
TraceLoggingUInt16(packetDstPort, "dport"),
TraceLoggingUInt8(packetProtocol, "protocol"));
return TRUE;
}
return FALSE;
}
BOOLEAN
OvpnCheckRecursiveRoutingIPv6(SOCKADDR_IN6* transportAddr, UCHAR* buffer, SIZE_T bufferLength, BOOLEAN tcp)
{
if (transportAddr->sin6_family != AF_INET6 || bufferLength < sizeof(IPV6_HEADER))
return FALSE; // Not an IPv6 peer or packet too short
// Extract pointers to avoid repeated FIELD_OFFSET calculations
IN6_ADDR* srcAddr = (IN6_ADDR*)(buffer + FIELD_OFFSET(IPV6_HEADER, SourceAddress));
IN6_ADDR* dstAddr = (IN6_ADDR*)(buffer + FIELD_OFFSET(IPV6_HEADER, DestinationAddress));
UINT8 packetProtocol = buffer[FIELD_OFFSET(IPV6_HEADER, NextHeader)];
// Validate the transport protocol
if ((tcp && packetProtocol != IPPROTO_TCP) || (!tcp && packetProtocol != IPPROTO_UDP))
return FALSE;
// Ensure transport header is accessible
if (bufferLength < sizeof(IPV6_HEADER) + sizeof(UINT16) * 2)
return FALSE;
// Read transport header efficiently
UCHAR* transportHeader = buffer + sizeof(IPV6_HEADER);
UINT16 packetSrcPort = RtlUshortByteSwap(*(UINT16*)(transportHeader));
UINT16 packetDstPort = RtlUshortByteSwap(*(UINT16*)(transportHeader + 2));
UINT16 peerPort = RtlUshortByteSwap(transportAddr->sin6_port);
// Check for recursive routing
if (packetDstPort == peerPort &&
RtlCompareMemory(dstAddr, &transportAddr->sin6_addr, sizeof(IN6_ADDR)) == sizeof(IN6_ADDR))
{
LOG_WARN("Recursive routing detected (IPv6), packet dropped",
TraceLoggingIPv6Address(srcAddr->u.Byte, "saddr"),
TraceLoggingIPv6Address(dstAddr->u.Byte, "daddr"),
TraceLoggingUInt16(packetSrcPort, "sport"),
TraceLoggingUInt16(packetDstPort, "dport"),
TraceLoggingUInt8(packetProtocol, "protocol"));
return TRUE;
}
return FALSE;
}
// The peer a plaintext packet belongs to, with its MSS clamp applied. NULL means there is
// nowhere to send it.
static
OvpnPeerContext*
OvpnTxFindPeer(_In_ POVPN_DEVICE device, _In_reads_bytes_(len) UCHAR* data, SIZE_T len)
{
OvpnPeerContext* peer = NULL;
if (OvpnMssIsIPv4(data, len)) {
auto addr = ((IPV4_HEADER*)data)->DestinationAddress;
peer = OvpnFindPeerVPN4(device, addr, FALSE);
if (peer == nullptr) {
peer = device->IRoutesIPV4.Find(reinterpret_cast<UCHAR*>(&addr));
}
if ((device->Mode == OVPN_MODE_P2P) && (peer != nullptr) && (OvpnCheckRecursiveRoutingIPv4(&peer->TransportAddrs.Remote.IPv4, data, len, peer->Socket->Tcp))) {
OvpnPeerCtxRelease(peer);
peer = nullptr;
}
if (peer != nullptr) {
OvpnMssDoIPv4(data, len, peer->MSS);
}
} else if (OvpnMssIsIPv6(data, len)) {
auto addr = ((IPV6_HEADER*)data)->DestinationAddress;
peer = OvpnFindPeerVPN6(device, addr, FALSE);
if (peer == nullptr) {
peer = device->IRoutesIPV6.Find(reinterpret_cast<UCHAR*>(&addr));
}
if ((device->Mode == OVPN_MODE_P2P) && (peer != nullptr) && (OvpnCheckRecursiveRoutingIPv6(&peer->TransportAddrs.Remote.IPv6, data, len, peer->Socket->Tcp))) {
OvpnPeerCtxRelease(peer);
peer = nullptr;
}
if (peer != nullptr) {
OvpnMssDoIPv6(data, len, peer->MSS);
}
}
return peer;
}
// Encrypt in place under the peer's transmit lock, held by the caller. On failure the
// buffer is left for the caller to dispose of.
_Must_inspect_result_
static
NTSTATUS
OvpnTxEncryptLocked(_In_ OvpnPeerContext* peer, _In_ OVPN_TX_BUFFER* buffer)
{
OvpnCryptoTxContext* tx = &peer->CryptoContext.Tx;
if (!tx->Encrypt) {
// LOG_WARN("CryptoContext not initialized");
return STATUS_INVALID_DEVICE_STATE;
}
const OvpnCryptoPacketLayout layout = tx->Layout;
if (layout.FrontLen > OVPN_BUFFER_HEADROOM) {
LOG_ERROR("Packet header exceeds tx headroom",
TraceLoggingValue(layout.FrontLen, "front"),
TraceLoggingValue(OVPN_BUFFER_HEADROOM, "headroom"));
return STATUS_INVALID_BUFFER_SIZE;
}
if ((buffer->Len + layout.FrontLen + layout.TailLen) > (OVPN_DCO_MTU_MAX + OVPN_BUFFER_HEADROOM + OVPN_BUFFER_TAILROOM)) {
LOG_ERROR("Packet exceeds tx buffer capacity",
TraceLoggingValue(buffer->Len, "len"),
TraceLoggingValue(layout.FrontLen, "front"),
TraceLoggingValue(layout.TailLen, "tail"));
return STATUS_INVALID_BUFFER_SIZE;
}
OvpnTxBufferPush(buffer, layout.FrontLen);
OvpnBufferPut(buffer, layout.TailLen);
return OvpnCryptoEncrypt(tx, buffer->Data, buffer->Len);
}
// What one pass has chained but not yet sent: the buffers, where they are going and the
// socket they leave by. A different socket or destination flushes what is already there.
struct OVPN_TX_BATCH
{
// the socket the chain goes out on, held by the batch until the pass ends: the
// chain outlives the peer reference it came from
OvpnSocket* Sock;
OVPN_TX_BUFFER* Head;
OVPN_TX_BUFFER* Tail;
SOCKADDR_STORAGE Sockaddr;
};
// Send what is chained, if anything.
static
VOID
OvpnTxBatchFlush(_Inout_ OVPN_TX_BATCH* batch)
{
if (batch->Head == NULL) {
return;
}
if (batch->Sock->Tcp) {
OvpnSocketSendTcpBatch(batch->Sock, batch->Head);
}
else {
// this will use WskSendMessages to send the buffer list we constructed before
LOG_IF_NOT_NT_SUCCESS(OvpnSocketSend(batch->Sock, batch->Head, (SOCKADDR*)&batch->Sockaddr));
}
batch->Head = NULL;
batch->Tail = NULL;
}
// Point the batch at this socket, sending what the previous one still holds. The batch
// takes its own reference: the caller's peer reference is dropped before the chain is
// sent, and the peer may be the last thing keeping the socket open.
static
VOID
OvpnTxBatchUse(_Inout_ OVPN_TX_BATCH* batch, _In_ OvpnSocket* sock)
{
if (batch->Sock == sock) {
return;
}
OvpnTxBatchFlush(batch);
if (batch->Sock != NULL) {
OvpnSocketReleaseNoBlock(batch->Sock);
}
OvpnSocketReference(sock);
batch->Sock = sock;
}
// End of a pass: send whatever is still chained.
static
VOID
OvpnTxBatchEnd(_Inout_ OVPN_TX_BATCH* batch)
{
OvpnTxBatchFlush(batch);
if (batch->Sock != NULL) {
// never inline: this is the transmit pass, and a close waits for WSK
OvpnSocketReleaseNoBlock(batch->Sock);
batch->Sock = NULL;
}
}
// Hand one encrypted buffer to the socket. Datagrams for one destination are chained and
// sent together by the caller; the buffer is consumed either way.
_Must_inspect_result_
static
NTSTATUS
OvpnTxSubmit(_Inout_ OVPN_TX_BATCH* batch, _In_ OVPN_TX_BUFFER* buffer, _In_ const OVPN_REMOTE_ADDR& remoteAddr)
{
// TCP: chain the buffers too, and send up to OVPN_TX_TCP_BATCH_MAX as one stream write. The
// head keeps the chain's length in WskBufList.Buffer.Length: only the UDP path reads that field,
// and the pool clears it for the buffer's next use.
if (batch->Sock->Tcp) {
SIZE_T const len = 2 + buffer->Len;
buffer->WskBufList.Next = NULL;
if ((batch->Head != NULL) && (batch->Head->WskBufList.Buffer.Length + len > OVPN_TX_TCP_BATCH_MAX)) {
OvpnTxBatchFlush(batch);
}
if (batch->Head == NULL) {
batch->Head = buffer;
buffer->WskBufList.Buffer.Length = len;
}
else {
batch->Tail->WskBufList.Next = &buffer->WskBufList;
batch->Head->WskBufList.Buffer.Length += len;
}
batch->Tail = buffer;
return STATUS_SUCCESS;
}
// for UDP we use SendMessages to send multiple datagrams at once
// here we only append WSK_BUF to the list
buffer->WskBufList.Buffer.Length = buffer->Len;
buffer->WskBufList.Buffer.Mdl = buffer->Mdl;
buffer->WskBufList.Buffer.Offset = FIELD_OFFSET(OVPN_TX_BUFFER, Head) + (ULONG)(buffer->Data - buffer->Head);
// If this peer is different (head sockaddr != peer sockaddr) to the previous buffer chain peers,
// then flush those and restart with a new buffer list.
if ((batch->Head != NULL) && !(OvpnTxAreSockaddrEqual((const SOCKADDR*)&batch->Sockaddr, (const SOCKADDR*)&remoteAddr)))
{
OvpnTxBatchFlush(batch);
}
if (batch->Head == NULL) {
batch->Head = buffer;
OvpnSocketCopyRemoteToSockaddr(remoteAddr, &batch->Sockaddr);
}
else {
batch->Tail->WskBufList.Next = &buffer->WskBufList;
}
batch->Tail = buffer;
return STATUS_SUCCESS;
}
// Encrypt one buffer, hand it to the socket and account for it. Consumed either way.
_Must_inspect_result_
static
NTSTATUS
OvpnTxEncryptAndSend(_In_ POVPN_DEVICE device, _Inout_ OVPN_TX_BATCH* batch, _In_ OvpnPeerContext* peer,
_In_ OVPN_TX_BUFFER* buffer)
{
// the chain belongs to one socket: a peer sending by another one flushes it first
OvpnTxBatchUse(batch, peer->Socket);
InterlockedExchangeAddNoFence64(&device->Stats.TunBytesSent, buffer->Len);
InterlockedExchangeAddNoFence64(&peer->VpnTxBytes, buffer->Len);
KIRQL irql = ExAcquireSpinLockShared(&peer->TxLock);
auto remoteAddr = peer->TransportAddrs.Remote;
NTSTATUS status = OvpnTxEncryptLocked(peer, buffer);
ExReleaseSpinLockShared(&peer->TxLock, irql);
// Moving the epoch on is the only thing here that writes the key, so it is the only
// thing that needs the lock to itself. The buffer is already prepared, so only the
// encryption repeats.
if (status == STATUS_RETRY) {
irql = ExAcquireSpinLockExclusive(&peer->TxLock);
status = OvpnCryptoAdvanceSendKey(&peer->CryptoContext.Tx);
ExReleaseSpinLockExclusive(&peer->TxLock, irql);
if (NT_SUCCESS(status)) {
irql = ExAcquireSpinLockShared(&peer->TxLock);
status = OvpnCryptoEncrypt(&peer->CryptoContext.Tx, buffer->Data, buffer->Len);
ExReleaseSpinLockShared(&peer->TxLock, irql);
}
}
if (!NT_SUCCESS(status)) {
OvpnTxBufferPoolPut(buffer);
return status;
}
InterlockedExchangeAddNoFence64(&peer->LinkTxBytes, buffer->Len);
status = OvpnTxSubmit(batch, buffer, remoteAddr);
OvpnTimerResetXmit(peer->Timer);
return status;
}
// The worker a flow belongs to, so its packets keep their order. The ports are in the
// hash because addresses alone would put every flow between two machines together.
static
ULONG
OvpnTxFlowHash(_In_reads_bytes_(len) UCHAR const* data, SIZE_T len)
{
ULONG h = 0;
if ((len >= sizeof(IPV4_HEADER)) && ((data[0] >> 4) == 4)) {
IPV4_HEADER const* ip = (IPV4_HEADER const*)data;
SIZE_T const hdrLen = (SIZE_T)ip->HeaderLength << 2;
h = ip->SourceAddress.S_un.S_addr ^ ip->DestinationAddress.S_un.S_addr;
if (((ip->Protocol == IPPROTO_TCP) || (ip->Protocol == IPPROTO_UDP)) &&
(len >= (hdrLen + sizeof(UINT32)))) {
h ^= *(UINT32 UNALIGNED const*)(data + hdrLen);
}
}
else if ((len >= sizeof(IPV6_HEADER)) && ((data[0] >> 4) == 6)) {
IPV6_HEADER const* ip = (IPV6_HEADER const*)data;
UINT32 UNALIGNED const* addrs = (UINT32 UNALIGNED const*)&ip->SourceAddress;
for (int i = 0; i < 8; ++i) {
h ^= addrs[i];
}
if (((ip->NextHeader == IPPROTO_TCP) || (ip->NextHeader == IPPROTO_UDP)) &&
(len >= (sizeof(IPV6_HEADER) + sizeof(UINT32)))) {
h ^= *(UINT32 UNALIGNED const*)(data + sizeof(IPV6_HEADER));
}
}
h ^= h >> 16;
h ^= h >> 8;
return h;
}
// Encrypt and send everything handed to this worker since it last ran. It outlives the
// ring pass that queued the work, so it holds the socket itself.
// Notification armed means the framework has stopped calling Advance, so once there is
// room again it must be told to resume, or what is left on the ring stays there.
static
VOID
OvpnTxQueueResume(_In_ POVPN_TXQUEUE queue)
{
if (InterlockedExchange(&queue->NotificationEnabled, FALSE) == TRUE) {
NetTxQueueNotifyMoreCompletedPacketsAvailable((NETPACKETQUEUE)WdfObjectContextGetObject(queue));
}
}
// The transmit queue thread copies every packet, and on a worker's core it slows the flow that
// worker encrypts; workers keep off the receive queue thread's home core, so this thread is held
// there for one Advance and reverted after, as that thread is. Only while there is a home: see
// OvpnRxQueueHoldHome.
static
BOOLEAN
OvpnTxQueueHoldHome(_In_ POVPN_DEVICE device, _Out_ PGROUP_AFFINITY previous)
{
RtlZeroMemory(previous, sizeof(*previous));
// not in a DPC: see OvpnRxQueueHoldHome
if (KeGetCurrentIrql() != PASSIVE_LEVEL) {
return FALSE;
}
// group and mask are read apart, so a home that moves between them gives a stale pairing;
// keeping to the group's active processors makes that harmless
USHORT const group = (USHORT)ReadNoFence(&device->RxHomeGroup);
KAFFINITY const home = (KAFFINITY)ReadULong64NoFence((volatile DWORD64*)&device->RxHomeAffinity) &
KeQueryGroupAffinity(group);
if (home == 0) {
return FALSE;
}
PROCESSOR_NUMBER here;
KeGetCurrentProcessorNumberEx(&here);
if ((here.Group == group) && (home & ((KAFFINITY)1 << here.Number))) {
return FALSE;
}
GROUP_AFFINITY homeAffinity = {};
homeAffinity.Group = group;
homeAffinity.Mask = home;
KeSetSystemGroupAffinityThread(&homeAffinity, previous);
return TRUE;
}
KDEFERRED_ROUTINE OvpnTxWorkerDpc;
_Use_decl_annotations_
VOID
OvpnTxWorkerDpc(KDPC* dpc, PVOID context, PVOID arg1, PVOID arg2)
{
UNREFERENCED_PARAMETER(dpc);
UNREFERENCED_PARAMETER(arg1);
UNREFERENCED_PARAMETER(arg2);
POVPN_TX_WORKER worker = (POVPN_TX_WORKER)context;
POVPN_DEVICE device = worker->Device;
// take the whole queue in one go, so the work runs without the lock
LIST_ENTRY work;
KeAcquireSpinLockAtDpcLevel(&worker->Lock);
OvpnListMoveAll(&worker->Queue, &work);
KeReleaseSpinLockFromDpcLevel(&worker->Lock);
if (IsListEmpty(&work)) {
return;
}
OVPN_TX_BATCH batch = {};
ULONG chained = 0;
while (!IsListEmpty(&work)) {
OVPN_TX_BUFFER* buffer = CONTAINING_RECORD(RemoveHeadList(&work), OVPN_TX_BUFFER, PoolListEntry);
OvpnPeerContext* peer = buffer->Peer;
buffer->Peer = NULL;
InterlockedDecrement(&worker->Owner->Queued);
NTSTATUS status = OvpnTxEncryptAndSend(device, &batch, peer, buffer);
if (!NT_SUCCESS(status)) {
InterlockedIncrementNoFence(&device->Stats.LostOutDataPackets);
}
OvpnPeerCtxRelease(peer);
// Send as we go: packet IDs are assigned at encryption, so a batch held back while
// other workers send can fall behind the receiver's replay window.
if ((batch.Head != NULL) && (++chained >= OVPN_TX_SEND_BATCH)) {
OvpnTxBatchFlush(&batch);
chained = 0;
}
}
OvpnTxBatchEnd(&batch);
OvpnTxQueueResume(worker->Owner);
}
// Hand a buffer to the worker that owns its flow, with the caller's reference on the peer.
static
VOID
OvpnTxToWorker(_In_ POVPN_TXQUEUE queue, _In_ OvpnPeerContext* peer, _In_ OVPN_TX_BUFFER* buffer, ULONG hash,
ULONG workerCount)
{
// Not on the core that receives the tunnel, the busiest one: a flow whose ACKs
// are encrypted there too saturates it, and the whole tunnel slows down. Nor on the
// receive queue thread's home core: see OvpnRxQueueChooseHome.
ULONG const rxCpu = ReadULongNoFence(&peer->RxProcessor);
ULONG64 const home = ReadULong64NoFence((volatile DWORD64*)&queue->Workers[0].Device->RxHomeIndexMask);
ULONG index = hash % workerCount;
for (ULONG tries = 0; tries < workerCount; ++tries) {
ULONG const p = queue->Workers[index].Processor;
if ((p != rxCpu) && !((p < 64) && (home & (1ULL << p)))) {
break;
}
index = (index + 1) % workerCount;
}
POVPN_TX_WORKER worker = &queue->Workers[index];
buffer->Peer = peer;
KIRQL irql;
KeAcquireSpinLock(&worker->Lock, &irql);
InsertTailList(&worker->Queue, &buffer->PoolListEntry);
KeReleaseSpinLock(&worker->Lock, irql);
InterlockedIncrement(&queue->Queued);
}
// Wake the workers once per ring pass. Waking per packet costs more than it saves: a
// pass is what makes a batch worth sending.
static
VOID
OvpnTxWakeWorkers(_In_ POVPN_TXQUEUE queue)
{
for (ULONG i = 0; i < queue->WorkerCount; ++i) {
POVPN_TX_WORKER worker = &queue->Workers[i];
// a stale answer here only costs an empty run, which the worker handles
if (!IsListEmpty(&worker->Queue)) {
KeInsertQueueDpc(&worker->Dpc, NULL, NULL);
}
}
}
static
VOID
OvpnTxWorkersInitialize(_In_ POVPN_TXQUEUE queue, _In_ POVPN_DEVICE device)
{
queue->WorkerCount = 0;
ULONG const processors = KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
ULONG const count = min(processors, OVPN_TX_WORKERS_MAX);
if (count < 2) {
return;
}
for (ULONG i = 0; i < count; ++i) {
POVPN_TX_WORKER worker = &queue->Workers[i];
worker->Device = device;
worker->Owner = queue;
worker->Index = i;
KeInitializeSpinLock(&worker->Lock);
InitializeListHead(&worker->Queue);
KeInitializeDpc(&worker->Dpc, OvpnTxWorkerDpc, worker);
// At medium importance a busy target runs it at its next clock tick, so the
// flow's ACKs leave in clumps and the sender bursts past the receiver's queue.
KeSetImportanceDpc(&worker->Dpc, MediumHighImportance);
// Spread out, so hyperthread siblings do not get two workers. The index counts
// across every processor group, so it needs the Ex form, which carries the group.
PROCESSOR_NUMBER target;
worker->Processor = (i * processors) / count;
if (NT_SUCCESS(KeGetProcessorNumberFromIndex(worker->Processor, &target))) {
LOG_IF_NOT_NT_SUCCESS(KeSetTargetProcessorDpcEx(&worker->Dpc, &target));
}
}
queue->WorkerCount = count;
LOG_INFO("Tx workers started", TraceLoggingValue(count, "workers"),
TraceLoggingValue(processors, "processors"));
}
_IRQL_requires_(PASSIVE_LEVEL)
static
VOID
OvpnTxWorkersStop(_In_ POVPN_TXQUEUE queue)
{
ULONG const count = InterlockedExchange((LONG*)&queue->WorkerCount, 0);
if (count == 0) {
return;
}
for (ULONG i = 0; i < count; ++i) {
KeRemoveQueueDpc(&queue->Workers[i].Dpc);
}
// KeRemoveQueueDpc says nothing about a dpc the dispatcher has already taken off its
// list and is about to call, which would then run on the freed queue context. This
// waits for those, and is why the drain belongs at passive level.
KeFlushQueuedDpcs();
for (ULONG i = 0; i < count; ++i) {
POVPN_TX_WORKER worker = &queue->Workers[i];
KIRQL irql;
KeAcquireSpinLock(&worker->Lock, &irql);
while (!IsListEmpty(&worker->Queue)) {
OVPN_TX_BUFFER* buffer = CONTAINING_RECORD(RemoveHeadList(&worker->Queue), OVPN_TX_BUFFER, PoolListEntry);
OvpnPeerContext* peer = buffer->Peer;
buffer->Peer = NULL;
OvpnTxBufferPoolPut(buffer);
InterlockedDecrement(&queue->Queued);
InterlockedIncrementNoFence(&queue->Workers[i].Device->Stats.LostOutDataPackets);
if (peer != NULL) {
OvpnPeerCtxRelease(peer);
}
}
KeReleaseSpinLock(&worker->Lock, irql);
}
}
_Must_inspect_result_
static
NTSTATUS
OvpnTxProcessPacket(_In_ POVPN_DEVICE device, _In_ POVPN_TXQUEUE queue,
_In_ NET_RING_PACKET_ITERATOR *pi, _Inout_ OVPN_TX_BATCH* batch)
{
NET_RING_FRAGMENT_ITERATOR fi = NetPacketIteratorGetFragments(pi);
OvpnPeerContext* peer = NULL;
// Fixed for the queue's life, and read here so the gotos below do not jump over it.
ULONG const workerCount = queue->WorkerCount;
// get buffer into which we gather plaintext fragments and do in-place encryption
OVPN_TX_BUFFER* buffer;
NTSTATUS status;
LOG_IF_NOT_NT_SUCCESS(status = OvpnTxBufferPoolGet(device->TxBufferPool, &buffer));
if (!NT_SUCCESS(status)) {
// Through out: so the fragments go back with the packet. Returning here kept them,
// and enough of those fill the fragment ring and stop the framework posting at all.
status = STATUS_INSUFFICIENT_RESOURCES;
goto out;
}
// gather fragments into single buffer
while (NetFragmentIteratorHasAny(&fi)) {
// get fragment payload
NET_FRAGMENT* fragment = NetFragmentIteratorGetFragment(&fi);
if ((buffer->Len + fragment->ValidLength) > OVPN_DCO_MTU_MAX) {
LOG_WARN("Packet max length exceeded, dropping",
TraceLoggingValue(buffer->Len, "currentLen"),
TraceLoggingValue(fragment->ValidLength, "lenToAdd"),
TraceLoggingValue(OVPN_DCO_MTU_MAX - buffer->Len, "spaceLeft"));
OvpnTxBufferPoolPut(buffer);
status = STATUS_INVALID_BUFFER_SIZE;
goto out;
}
NET_FRAGMENT_VIRTUAL_ADDRESS* virtualAddr = NetExtensionGetFragmentVirtualAddress(
&queue->VirtualAddressExtension, NetFragmentIteratorGetIndex(&fi));
RtlCopyMemory(OvpnBufferPut(buffer, fragment->ValidLength),
(UCHAR const*)virtualAddr->VirtualAddress + fragment->Offset, fragment->ValidLength);
NetFragmentIteratorAdvance(&fi);
}
peer = OvpnTxFindPeer(device, buffer->Data, buffer->Len);
if (peer == nullptr) {
status = STATUS_ADDRESS_NOT_ASSOCIATED;
OvpnTxBufferPoolPut(buffer);
goto out;
}
// a stream has to leave in the order it was encrypted, so TCP stays on this thread
if ((workerCount > 0) && !peer->Socket->Tcp) {
// A worker buffer is not returned to the pool until its send completes, so the
// count in flight - and the pool behind it - grows without bound when sends lag.
// Over the cap, drop this packet rather than grow.
if (InterlockedCompareExchange(&device->TxDataInFlight, 0, 0) >= OVPN_TX_DATA_INFLIGHT_MAX) {
OvpnPeerCtxRelease(peer);
OvpnTxBufferPoolPut(buffer);
status = STATUS_INSUFFICIENT_RESOURCES;
goto out;
}
InterlockedIncrement(&device->TxDataInFlight);
buffer->CountedInFlight = TRUE;
// the reference goes with the buffer
OvpnTxToWorker(queue, peer, buffer, OvpnTxFlowHash(buffer->Data, buffer->Len), workerCount);
goto out;
}
status = OvpnTxEncryptAndSend(device, batch, peer, buffer);
OvpnPeerCtxRelease(peer);
out:
// update fragment ring's BeginIndex to indicate that we've processes all fragments
NET_PACKET* packet = NetPacketIteratorGetPacket(pi);
NET_RING* const fragmentRing = NetRingCollectionGetFragmentRing(fi.Iterator.Rings);
UINT32 const lastFragmentIndex = NetRingAdvanceIndex(fragmentRing, packet->FragmentIndex, packet->FragmentCount);
fragmentRing->BeginIndex = lastFragmentIndex;
return status;
}
_Use_decl_annotations_
VOID
OvpnEvtTxQueueAdvance(NETPACKETQUEUE netPacketQueue)
{
POVPN_TXQUEUE queue = OvpnGetTxQueueContext(netPacketQueue);
NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(queue->Rings);
POVPN_DEVICE device = OvpnGetDeviceContext(queue->Adapter->WdfDevice);
// A relaxed read is enough, it only decides whether to bother; the packets go out on
// whichever socket their peer holds.
if (ReadPointerNoFence((PVOID volatile*)&device->Socket) == NULL) {
// Nowhere to send them, so drop them and hand the ring back. Leaving them on it
// waits for EvtTxQueueCancel, which runs when a queue is deleted and not when a
// session ends, so the framework would poll a ring that never drains.
while (NetPacketIteratorHasAny(&pi)) {
NET_PACKET* packet = NetPacketIteratorGetPacket(&pi);
if (!packet->Ignore && !packet->Scratch) {
InterlockedIncrementNoFence(&device->Stats.LostOutDataPackets);
}
NetPacketIteratorAdvance(&pi);
}
NetPacketIteratorSet(&pi);
// The fragments go back too. Without this they stay ours, the fragment ring
// fills, and the framework stops posting packets for good - the datapath is
// dead from then on, even once a transport is back.
NET_RING* fragmentRing = NetRingCollectionGetFragmentRing(queue->Rings);
fragmentRing->BeginIndex = fragmentRing->EndIndex;
return;
}
GROUP_AFFINITY previous;
BOOLEAN const moved = OvpnTxQueueHoldHome(device, &previous);
OVPN_TX_BATCH batch = {};
// Stop once the workers are a queue-depth behind and let the framework poll again.
// The packets stay on the ring, which is the whole point: a buffer taken off it is
// held until a worker sends it, so without this the pool grows to hold the backlog.
LONG const queuedMax = (LONG)(queue->WorkerCount * OVPN_TX_QUEUED_MAX_PER_WORKER);
while (NetPacketIteratorHasAny(&pi)) {
if ((queue->WorkerCount > 0) &&
(InterlockedCompareExchange(&queue->Queued, 0, 0) >= queuedMax)) {
break;
}
NET_PACKET* packet = NetPacketIteratorGetPacket(&pi);
NTSTATUS status = STATUS_SUCCESS;
if (!packet->Ignore && !packet->Scratch) {
status = OvpnTxProcessPacket(device, queue, &pi, &batch);
if (!NT_SUCCESS(status)) {
InterlockedIncrementNoFence(&device->Stats.LostOutDataPackets);
}
}
NetPacketIteratorAdvance(&pi);
if (!NT_SUCCESS(status)) {
break;
}
}
NetPacketIteratorSet(&pi);
OvpnTxBatchEnd(&batch);
if (queue->WorkerCount > 0) {
OvpnTxWakeWorkers(queue);
}
if (moved) {
KeRevertToUserGroupAffinityThread(&previous);
}
}
_Use_decl_annotations_
VOID
OvpnTxQueueInitialize(NETPACKETQUEUE netPacketQueue, POVPN_ADAPTER adapter)
{
POVPN_TXQUEUE queue = OvpnGetTxQueueContext(netPacketQueue);
queue->Adapter = adapter;
queue->Rings = NetTxQueueGetRingCollection(netPacketQueue);
NET_EXTENSION_QUERY extension;
NET_EXTENSION_QUERY_INIT(&extension, NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_NAME, NET_FRAGMENT_EXTENSION_VIRTUAL_ADDRESS_VERSION_1, NetExtensionTypeFragment);
NetTxQueueGetExtension(netPacketQueue, &extension, &queue->VirtualAddressExtension);
OvpnTxWorkersInitialize(queue, OvpnGetDeviceContext(adapter->WdfDevice));
}
// Cleanup rather than destroy: the drain waits on queued dpcs, which needs passive level,
// and runs while the buffer pool it returns buffers to is still alive. Both come from
// NetAdapterCx, so assert the level - asking WDF for it with ExecutionLevel stops the
// device starting.
_Use_decl_annotations_
VOID
OvpnEvtTxQueueCleanup(WDFOBJECT txQueue)
{
NT_ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
OvpnTxWorkersStop(OvpnGetTxQueueContext((NETPACKETQUEUE)txQueue));
}
_Use_decl_annotations_
VOID
OvpnEvtTxQueueSetNotificationEnabled(NETPACKETQUEUE queue, BOOLEAN notificationEnabled)
{
POVPN_TXQUEUE txQueue = OvpnGetTxQueueContext(queue);
InterlockedExchange(&txQueue->NotificationEnabled, notificationEnabled);
// The workers may have made room before this was armed, and then nothing would wake
// the queue. Only with packets waiting, or an idle queue would poll forever.
if (notificationEnabled && (txQueue->WorkerCount > 0)) {
NET_RING const* packets = NetRingCollectionGetPacketRing(txQueue->Rings);
LONG const queuedMax = (LONG)(txQueue->WorkerCount * OVPN_TX_QUEUED_MAX_PER_WORKER);
if ((packets->BeginIndex != packets->EndIndex) &&
(InterlockedCompareExchange(&txQueue->Queued, 0, 0) < queuedMax)) {
OvpnTxQueueResume(txQueue);
}
}
}
_Use_decl_annotations_
VOID
OvpnEvtTxQueueCancel(NETPACKETQUEUE netPacketQueue)
{
// mark all packets as "ignore"
POVPN_TXQUEUE queue = OvpnGetTxQueueContext(netPacketQueue);
NET_RING_PACKET_ITERATOR pi = NetRingGetAllPackets(queue->Rings);
while (NetPacketIteratorHasAny(&pi)) {
// we cannot modify Ignore here, otherwise Verifier will bark on us
NetPacketIteratorGetPacket(&pi)->Scratch = 1;
NetPacketIteratorAdvance(&pi);
}
NetPacketIteratorSet(&pi);
// return all fragments' ownership back to netadapter
NET_RING* fragmentRing = NetRingCollectionGetFragmentRing(queue->Rings);
fragmentRing->BeginIndex = fragmentRing->EndIndex;
}