Skip to content

Commit 5edd704

Browse files
wencaiwuluewencaiwulue
authoredMar 13, 2025··
feat: batch write route msg to unix socket: 'route ip+net: sysctl: no buffer space available' (#466)
·
v2.4.2v2.4.0
1 parent 24d16b2 commit 5edd704

File tree

5 files changed

+107
-61
lines changed

5 files changed

+107
-61
lines changed
 

‎pkg/handler/connect.go

+30-23
Original file line numberDiff line numberDiff line change
@@ -508,35 +508,42 @@ func (c *ConnectOptions) addRouteDynamic(ctx context.Context) error {
508508
return nil
509509
}
510510

511-
func (c *ConnectOptions) addRoute(ipStr string) error {
511+
func (c *ConnectOptions) addRoute(ipStrList ...string) error {
512512
if c.tunName == "" {
513513
return nil
514514
}
515-
516-
ip := net.ParseIP(ipStr)
517-
if ip == nil {
518-
return nil
519-
}
520-
for _, p := range c.apiServerIPs {
521-
// if pod ip or service ip is equal to apiServer ip, can not add it to route table
522-
if p.Equal(ip) {
523-
return nil
515+
var routes []types.Route
516+
for _, ipStr := range ipStrList {
517+
ip := net.ParseIP(ipStr)
518+
if ip == nil {
519+
continue
524520
}
525-
}
526-
527-
var mask net.IPMask
528-
if ip.To4() != nil {
529-
mask = net.CIDRMask(32, 32)
530-
} else {
531-
mask = net.CIDRMask(128, 128)
532-
}
533-
if r, err := netroute.New(); err == nil {
534-
ifi, _, _, err := r.Route(ip)
535-
if err == nil && ifi.Name == c.tunName {
536-
return nil
521+
var match bool
522+
for _, p := range c.apiServerIPs {
523+
// if pod ip or service ip is equal to apiServer ip, can not add it to route table
524+
if p.Equal(ip) {
525+
match = true
526+
break
527+
}
528+
}
529+
if match {
530+
continue
531+
}
532+
var mask net.IPMask
533+
if ip.To4() != nil {
534+
mask = net.CIDRMask(32, 32)
535+
} else {
536+
mask = net.CIDRMask(128, 128)
537+
}
538+
if r, err := netroute.New(); err == nil {
539+
ifi, _, _, err := r.Route(ip)
540+
if err == nil && ifi.Name == c.tunName {
541+
continue
542+
}
537543
}
544+
routes = append(routes, types.Route{Dst: net.IPNet{IP: ip, Mask: mask}})
538545
}
539-
err := tun.AddRoutes(c.tunName, types.Route{Dst: net.IPNet{IP: ip, Mask: mask}})
546+
err := tun.AddRoutes(c.tunName, routes...)
540547
return err
541548
}
542549

‎pkg/handler/function_test.go

+15
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,14 @@ var (
4040
func TestFunctions(t *testing.T) {
4141
Init()
4242
kubevpnConnect(t)
43+
kubevpnStatus(t)
4344
t.Run(runtime.FuncForPC(reflect.ValueOf(pingPodIP).Pointer()).Name(), pingPodIP)
4445
t.Run(runtime.FuncForPC(reflect.ValueOf(dialUDP).Pointer()).Name(), dialUDP)
4546
t.Run(runtime.FuncForPC(reflect.ValueOf(healthCheckPod).Pointer()).Name(), healthCheckPod)
4647
t.Run(runtime.FuncForPC(reflect.ValueOf(healthCheckService).Pointer()).Name(), healthCheckService)
4748
t.Run(runtime.FuncForPC(reflect.ValueOf(shortDomain).Pointer()).Name(), shortDomain)
4849
t.Run(runtime.FuncForPC(reflect.ValueOf(fullDomain).Pointer()).Name(), fullDomain)
50+
kubevpnStatus(t)
4951
}
5052

5153
func pingPodIP(t *testing.T) {
@@ -338,6 +340,19 @@ func kubevpnConnect(t *testing.T) {
338340
}
339341
}
340342

343+
func kubevpnStatus(t *testing.T) {
344+
cmd := exec.Command("kubevpn", "status")
345+
stdout, stderr, err := util.RunWithRollingOutWithChecker(cmd, nil)
346+
if err != nil {
347+
t.Log(stdout, stderr)
348+
t.Error(err)
349+
t.Fail()
350+
return
351+
}
352+
t.Log(stdout)
353+
t.Log(stderr)
354+
}
355+
341356
func Init() {
342357
var err error
343358

‎pkg/tun/route_darwin.go

+37-22
Original file line numberDiff line numberDiff line change
@@ -10,33 +10,49 @@ import (
1010
"golang.org/x/sys/unix"
1111
)
1212

13-
func addRoute(seq int, r netip.Prefix, gw route.Addr) error {
13+
func addRoute(gw route.Addr, r ...netip.Prefix) error {
14+
if len(r) == 0 {
15+
return nil
16+
}
1417
return withRouteSocket(func(routeSocket int) error {
15-
m := newRouteMessage(unix.RTM_ADD, seq, r, gw)
16-
rb, err := m.Marshal()
17-
if err != nil {
18-
return err
19-
}
20-
_, err = unix.Write(routeSocket, rb)
21-
if errors.Is(err, unix.EEXIST) {
22-
err = nil
18+
for i, prefix := range r {
19+
m := newRouteMessage(unix.RTM_ADD, i+1, prefix, gw)
20+
rb, err := m.Marshal()
21+
if err != nil {
22+
return err
23+
}
24+
_, err = unix.Write(routeSocket, rb)
25+
if errors.Is(err, unix.EEXIST) {
26+
err = nil
27+
}
28+
if err != nil {
29+
return err
30+
}
2331
}
24-
return err
32+
return nil
2533
})
2634
}
2735

28-
func deleteRoute(seq int, r netip.Prefix, gw route.Addr) error {
36+
func deleteRoute(gw route.Addr, r ...netip.Prefix) error {
37+
if len(r) == 0 {
38+
return nil
39+
}
2940
return withRouteSocket(func(routeSocket int) error {
30-
m := newRouteMessage(unix.RTM_DELETE, seq, r, gw)
31-
rb, err := m.Marshal()
32-
if err != nil {
33-
return err
34-
}
35-
_, err = unix.Write(routeSocket, rb)
36-
if errors.Is(err, unix.ESRCH) {
37-
err = nil
41+
for i, prefix := range r {
42+
m := newRouteMessage(unix.RTM_DELETE, i+1, prefix, gw)
43+
rb, err := m.Marshal()
44+
if err != nil {
45+
return err
46+
}
47+
_, err = unix.Write(routeSocket, rb)
48+
if errors.Is(err, unix.ESRCH) {
49+
err = nil
50+
}
51+
if err != nil {
52+
return err
53+
}
3854
}
39-
return err
55+
return nil
4056
})
4157
}
4258

@@ -45,12 +61,11 @@ func withRouteSocket(f func(routeSocket int) error) error {
4561
if err != nil {
4662
return err
4763
}
48-
64+
defer unix.Close(routeSocket)
4965
// Avoid the overhead of echoing messages back to sender
5066
if err = unix.SetsockoptInt(routeSocket, unix.SOL_SOCKET, unix.SO_USELOOPBACK, 0); err != nil {
5167
return err
5268
}
53-
defer unix.Close(routeSocket)
5469
return f(routeSocket)
5570
}
5671

‎pkg/tun/tun_darwin.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ func addTunRoutes(ifName string, routes ...types.Route) error {
9999
}
100100
gw := &route.LinkAddr{Index: tunIfi.Index}
101101

102+
var prefixList []netip.Prefix
102103
for _, r := range routes {
103104
if r.Dst.String() == "" {
104105
continue
@@ -108,10 +109,14 @@ func addTunRoutes(ifName string, routes ...types.Route) error {
108109
if err != nil {
109110
return err
110111
}
111-
err = addRoute(1, prefix, gw)
112-
if err != nil {
113-
return fmt.Errorf("failed to add route: %v", err)
114-
}
112+
prefixList = append(prefixList, prefix)
113+
}
114+
if len(prefixList) == 0 {
115+
return nil
116+
}
117+
err = addRoute(gw, prefixList...)
118+
if err != nil {
119+
return fmt.Errorf("failed to add route: %v", err)
115120
}
116121
return nil
117122
}

‎pkg/util/route.go

+16-12
Original file line numberDiff line numberDiff line change
@@ -50,18 +50,20 @@ func GetNsForListPodAndSvc(ctx context.Context, clientset *kubernetes.Clientset,
5050
return
5151
}
5252

53-
func ListService(ctx context.Context, lister v12.ServiceInterface, addRouteFunc func(ipStr string) error) error {
53+
func ListService(ctx context.Context, lister v12.ServiceInterface, addRouteFunc func(ipStr ...string) error) error {
5454
opts := metav1.ListOptions{Limit: 100, Continue: ""}
5555
for {
5656
serviceList, err := lister.List(ctx, opts)
5757
if err != nil {
5858
return err
5959
}
60+
var ips []string
6061
for _, service := range serviceList.Items {
61-
err = addRouteFunc(service.Spec.ClusterIP)
62-
if err != nil {
63-
log.Errorf("Failed to add service: %s IP: %s to route table: %v", service.Name, service.Spec.ClusterIP, err)
64-
}
62+
ips = append(ips, service.Spec.ClusterIP)
63+
}
64+
err = addRouteFunc(ips...)
65+
if err != nil {
66+
log.Errorf("Failed to add service IP: %s to route table: %v", ips, err)
6567
}
6668
if serviceList.Continue == "" {
6769
return nil
@@ -70,7 +72,7 @@ func ListService(ctx context.Context, lister v12.ServiceInterface, addRouteFunc
7072
}
7173
}
7274

73-
func WatchServiceToAddRoute(ctx context.Context, watcher v12.ServiceInterface, routeFunc func(ipStr string) error) error {
75+
func WatchServiceToAddRoute(ctx context.Context, watcher v12.ServiceInterface, routeFunc func(ipStr ...string) error) error {
7476
defer func() {
7577
if er := recover(); er != nil {
7678
log.Error(er)
@@ -99,21 +101,23 @@ func WatchServiceToAddRoute(ctx context.Context, watcher v12.ServiceInterface, r
99101
}
100102
}
101103

102-
func ListPod(ctx context.Context, lister v12.PodInterface, addRouteFunc func(ipStr string) error) error {
104+
func ListPod(ctx context.Context, lister v12.PodInterface, addRouteFunc func(ipStr ...string) error) error {
103105
opts := metav1.ListOptions{Limit: 100, Continue: ""}
104106
for {
105107
podList, err := lister.List(ctx, opts)
106108
if err != nil {
107109
return err
108110
}
111+
var ips []string
109112
for _, pod := range podList.Items {
110113
if pod.Spec.HostNetwork {
111114
continue
112115
}
113-
err = addRouteFunc(pod.Status.PodIP)
114-
if err != nil {
115-
log.Errorf("Failed to add pod: %s IP: %s to route table: %v", pod.Name, pod.Status.PodIP, err)
116-
}
116+
ips = append(ips, pod.Status.PodIP)
117+
}
118+
err = addRouteFunc(ips...)
119+
if err != nil {
120+
log.Errorf("Failed to add Pod IP: %v route table: %v", ips, err)
117121
}
118122
if podList.Continue == "" {
119123
return nil
@@ -122,7 +126,7 @@ func ListPod(ctx context.Context, lister v12.PodInterface, addRouteFunc func(ipS
122126
}
123127
}
124128

125-
func WatchPodToAddRoute(ctx context.Context, watcher v12.PodInterface, addRouteFunc func(ipStr string) error) error {
129+
func WatchPodToAddRoute(ctx context.Context, watcher v12.PodInterface, addRouteFunc func(ipStrList ...string) error) error {
126130
defer func() {
127131
if er := recover(); er != nil {
128132
log.Errorln(er)

0 commit comments

Comments
 (0)
Please sign in to comment.