Skip to content

Commit 7b97ce2

Browse files
lukasmetznerlukasmetzner
authored
feat(robot): forward InternalIPs by default on Robot nodes (#865)
Whenever a user configures the `--node-ip` flag, we forward the configured IPs. We validate if the IP already exists as an ExternalIP and does not collide with the configured address family. This improves the Robot support for HCCM, as users can configure private IPs for Robot nodes. Forwarding only happens during the initialization phase of HCCM and should therefore not break existing clusters. This feature can be disabled by setting the environment variable `ROBOT_FORWARD_INTERNAL_IPS` to `false`. --------- Co-authored-by: Alexander Block <ablock84@gmail.com> Co-authored-by: Julian Tölle <julian.toelle@hetzner-cloud.de>
1 parent 427ea35 commit 7b97ce2

File tree

6 files changed

+206
-41
lines changed

6 files changed

+206
-41
lines changed

‎docs/robot.md

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ The Node controller adds information about the server to the Node object. The va
5656
- Addresses
5757
- We add the Hostname and (depending on the configuration and availability) the IPv4 and IPv6 addresses of the server in `Node.status.addresses`.
5858
- For the IPv6 address we use the first address in the Network -> For the network `2a01:f48:111:4221::` we add the address `2a01:f48:111:4221::1`.
59+
- By default, we pass along InternalIPs configured via the kubelet flag `--node-ip`. This can be disabled by setting the environment variable `ROBOT_FORWARD_INTERNAL_IPS` to `false`. It is not allowed to configure the same IP for InternalIP and ExternalIP.
5960
- Private IPs in a vSwitch are not supported.
6061

6162
### Node Lifecycle Controller

‎hcloud/instances.go

+62-20
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"net"
2324

2425
hrobotmodels "github.com/syself/hrobot-go/models"
2526
corev1 "k8s.io/api/core/v1"
2627
"k8s.io/client-go/tools/record"
2728
cloudprovider "k8s.io/cloud-provider"
29+
"k8s.io/klog/v2"
2830

2931
"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/config"
3032
"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/metrics"
@@ -34,7 +36,8 @@ import (
3436
)
3537

3638
const (
37-
ProvidedBy = "instance.hetzner.cloud/provided-by"
39+
ProvidedBy = "instance.hetzner.cloud/provided-by"
40+
MisconfiguredInternalIP = "MisconfiguredInternalIP"
3841
)
3942

4043
type instances struct {
@@ -106,7 +109,7 @@ func (i *instances) lookupServer(
106109
return nil, nil
107110
}
108111

109-
return robotServer{server, i.robotClient}, nil
112+
return robotServer{server, i.robotClient, i.recorder}, nil
110113
}
111114

112115
// If the node has no provider ID we try to find the server by name from
@@ -139,7 +142,7 @@ func (i *instances) lookupServer(
139142
case cloudServer != nil:
140143
return hcloudServer{cloudServer}, nil
141144
case hrobotServer != nil:
142-
return robotServer{hrobotServer, i.robotClient}, nil
145+
return robotServer{hrobotServer, i.robotClient, i.recorder}, nil
143146
default:
144147
// Both nil
145148
return nil, nil
@@ -196,7 +199,7 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *corev1.Node) (*c
196199
op, node.Name, errServerNotFound)
197200
}
198201

199-
metadata, err := server.Metadata(i.networkID, i.cfg)
202+
metadata, err := server.Metadata(i.networkID, node, i.cfg)
200203
if err != nil {
201204
return nil, fmt.Errorf("%s: %w", op, err)
202205
}
@@ -254,13 +257,12 @@ func hcloudNodeAddresses(
254257

255258
func robotNodeAddresses(
256259
server *hrobotmodels.Server,
260+
node *corev1.Node,
257261
cfg config.HCCMConfiguration,
262+
recorder record.EventRecorder,
258263
) []corev1.NodeAddress {
259264
var addresses []corev1.NodeAddress
260-
addresses = append(
261-
addresses,
262-
corev1.NodeAddress{Type: corev1.NodeHostName, Address: server.Name},
263-
)
265+
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: server.Name})
264266

265267
dualStack := cfg.Instance.AddressFamily == config.AddressFamilyDualStack
266268
ipv4 := cfg.Instance.AddressFamily == config.AddressFamilyIPv4 || dualStack
@@ -269,18 +271,56 @@ func robotNodeAddresses(
269271
if ipv6 {
270272
// For a given IPv6 network of 2a01:f48:111:4221::, the instance address is 2a01:f48:111:4221::1
271273
hostAddress := server.ServerIPv6Net + "1"
272-
273-
addresses = append(
274-
addresses,
275-
corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: hostAddress},
276-
)
274+
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: hostAddress})
277275
}
278276

279277
if ipv4 {
280-
addresses = append(
281-
addresses,
282-
corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: server.ServerIP},
283-
)
278+
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: server.ServerIP})
279+
}
280+
281+
if cfg.Robot.ForwardInternalIPs {
282+
OUTER:
283+
for _, currentAddress := range node.Status.Addresses {
284+
if currentAddress.Type != corev1.NodeInternalIP {
285+
continue
286+
}
287+
288+
ip := net.ParseIP(currentAddress.Address)
289+
isIPv4 := ip.To4() != nil
290+
291+
var warnMsg string
292+
if isIPv4 && ipv6 && !dualStack {
293+
warnMsg = fmt.Sprintf(
294+
"Configured InternalIP is IPv4 even though IPv6 only is configured. As a result, %s is not added as an InternalIP",
295+
currentAddress.Address,
296+
)
297+
} else if !isIPv4 && ipv4 && !dualStack {
298+
warnMsg = fmt.Sprintf(
299+
"Configured InternalIP is IPv6 even though IPv4 only is configured. As a result, %s is not added as an InternalIP",
300+
currentAddress.Address,
301+
)
302+
}
303+
304+
if warnMsg != "" {
305+
recorder.Event(node, corev1.EventTypeWarning, MisconfiguredInternalIP, warnMsg)
306+
klog.Warning(warnMsg)
307+
continue
308+
}
309+
310+
for _, address := range addresses {
311+
if currentAddress.Address == address.Address {
312+
warnMsg := fmt.Sprintf(
313+
"Configured InternalIP already exists as an ExternalIP. As a result, %s is not added as an InternalIP",
314+
currentAddress.Address,
315+
)
316+
recorder.Event(node, corev1.EventTypeWarning, MisconfiguredInternalIP, warnMsg)
317+
klog.Warning(warnMsg)
318+
continue OUTER
319+
}
320+
}
321+
322+
addresses = append(addresses, currentAddress)
323+
}
284324
}
285325

286326
return addresses
@@ -290,6 +330,7 @@ type genericServer interface {
290330
IsShutdown() (bool, error)
291331
Metadata(
292332
networkID int64,
333+
node *corev1.Node,
293334
cfg config.HCCMConfiguration,
294335
) (*cloudprovider.InstanceMetadata, error)
295336
}
@@ -302,7 +343,7 @@ func (s hcloudServer) IsShutdown() (bool, error) {
302343
return s.Status == hcloud.ServerStatusOff, nil
303344
}
304345

305-
func (s hcloudServer) Metadata(networkID int64, cfg config.HCCMConfiguration) (*cloudprovider.InstanceMetadata, error) {
346+
func (s hcloudServer) Metadata(networkID int64, _ *corev1.Node, cfg config.HCCMConfiguration) (*cloudprovider.InstanceMetadata, error) {
306347
return &cloudprovider.InstanceMetadata{
307348
ProviderID: providerid.FromCloudServerID(s.ID),
308349
InstanceType: s.ServerType.Name,
@@ -318,6 +359,7 @@ func (s hcloudServer) Metadata(networkID int64, cfg config.HCCMConfiguration) (*
318359
type robotServer struct {
319360
*hrobotmodels.Server
320361
robotClient robot.Client
362+
recorder record.EventRecorder
321363
}
322364

323365
func (s robotServer) IsShutdown() (bool, error) {
@@ -331,11 +373,11 @@ func (s robotServer) IsShutdown() (bool, error) {
331373
return resetStatus.OperatingStatus == "shut off", nil
332374
}
333375

334-
func (s robotServer) Metadata(_ int64, cfg config.HCCMConfiguration) (*cloudprovider.InstanceMetadata, error) {
376+
func (s robotServer) Metadata(_ int64, node *corev1.Node, cfg config.HCCMConfiguration) (*cloudprovider.InstanceMetadata, error) {
335377
return &cloudprovider.InstanceMetadata{
336378
ProviderID: providerid.FromRobotServerNumber(s.ServerNumber),
337379
InstanceType: getInstanceTypeOfRobotServer(s.Server),
338-
NodeAddresses: robotNodeAddresses(s.Server, cfg),
380+
NodeAddresses: robotNodeAddresses(s.Server, node, cfg, s.recorder),
339381
Zone: getZoneOfRobotServer(s.Server),
340382
Region: getRegionOfRobotServer(s.Server),
341383
AdditionalLabels: map[string]string{

‎hcloud/instances_test.go

+71-6
Original file line numberDiff line numberDiff line change
@@ -598,11 +598,12 @@ func TestNodeAddresses(t *testing.T) {
598598

599599
func TestNodeAddressesRobotServer(t *testing.T) {
600600
tests := []struct {
601-
name string
602-
addressFamily config.AddressFamily
603-
server *hrobotmodels.Server
604-
privateNetwork int
605-
expected []corev1.NodeAddress
601+
name string
602+
addressFamily config.AddressFamily
603+
server *hrobotmodels.Server
604+
nodeStatusNodeAddresses []corev1.NodeAddress
605+
privateNetwork int
606+
expected []corev1.NodeAddress
606607
}{
607608
{
608609
name: "public ipv4",
@@ -644,6 +645,61 @@ func TestNodeAddressesRobotServer(t *testing.T) {
644645
{Type: corev1.NodeExternalIP, Address: "203.0.113.7"},
645646
},
646647
},
648+
{
649+
name: "public ipv4 and internal ip",
650+
addressFamily: config.AddressFamilyIPv4,
651+
nodeStatusNodeAddresses: []corev1.NodeAddress{
652+
{
653+
Type: corev1.NodeInternalIP,
654+
Address: "10.0.1.2",
655+
},
656+
},
657+
server: &hrobotmodels.Server{
658+
Name: "foobar",
659+
ServerIP: "203.0.113.7",
660+
},
661+
expected: []corev1.NodeAddress{
662+
{Type: corev1.NodeHostName, Address: "foobar"},
663+
{Type: corev1.NodeExternalIP, Address: "203.0.113.7"},
664+
{Type: corev1.NodeInternalIP, Address: "10.0.1.2"},
665+
},
666+
},
667+
{
668+
name: "configured InternalIP is also ExternalIP",
669+
addressFamily: config.AddressFamilyIPv4,
670+
nodeStatusNodeAddresses: []corev1.NodeAddress{
671+
{
672+
Type: corev1.NodeInternalIP,
673+
Address: "203.0.113.7",
674+
},
675+
},
676+
server: &hrobotmodels.Server{
677+
Name: "foobar",
678+
ServerIP: "203.0.113.7",
679+
},
680+
expected: []corev1.NodeAddress{
681+
{Type: corev1.NodeHostName, Address: "foobar"},
682+
{Type: corev1.NodeExternalIP, Address: "203.0.113.7"},
683+
},
684+
},
685+
{
686+
name: "configured InternalIP does not fit configured AddressFamily",
687+
addressFamily: config.AddressFamilyIPv4,
688+
nodeStatusNodeAddresses: []corev1.NodeAddress{
689+
{
690+
Type: corev1.NodeInternalIP,
691+
Address: "2001:db8:1234::",
692+
},
693+
},
694+
server: &hrobotmodels.Server{
695+
Name: "foobar",
696+
ServerIP: "203.0.113.7",
697+
},
698+
expected: []corev1.NodeAddress{
699+
{Type: corev1.NodeHostName, Address: "foobar"},
700+
{Type: corev1.NodeExternalIP, Address: "203.0.113.7"},
701+
},
702+
},
647703
}
648704

649705
for _, test := range tests {
@@ -653,7 +709,16 @@ func TestNodeAddressesRobotServer(t *testing.T) {
653709
cfg, err := config.Read()
654710
assert.NoError(t, err)
655711

656-
addresses := robotNodeAddresses(test.server, cfg)
712+
node := &corev1.Node{
713+
Status: corev1.NodeStatus{
714+
Addresses: []corev1.NodeAddress{},
715+
},
716+
}
717+
if test.nodeStatusNodeAddresses != nil {
718+
node.Status.Addresses = test.nodeStatusNodeAddresses
719+
}
720+
721+
addresses := robotNodeAddresses(test.server, node, cfg, &MockEventRecorder{})
657722

658723
if !reflect.DeepEqual(addresses, test.expected) {
659724
t.Fatalf("%s: expected addresses %+v but got %+v", test.name, test.expected, addresses)

‎hcloud/instances_util.go

+21
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,33 @@ import (
2424

2525
hrobotmodels "github.com/syself/hrobot-go/models"
2626
corev1 "k8s.io/api/core/v1"
27+
"k8s.io/apimachinery/pkg/runtime"
2728

2829
"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/metrics"
2930
"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/robot"
3031
"github.com/hetznercloud/hcloud-go/v2/hcloud"
3132
)
3233

34+
type MockEventRecorder struct{}
35+
36+
func (er *MockEventRecorder) Event(_ runtime.Object, _, _, _ string) {
37+
}
38+
39+
func (er *MockEventRecorder) Eventf(
40+
_ runtime.Object,
41+
_, _, _ string,
42+
_ ...interface{},
43+
) {
44+
}
45+
46+
func (er *MockEventRecorder) AnnotatedEventf(
47+
_ runtime.Object,
48+
_ map[string]string,
49+
_, _, _ string,
50+
_ ...interface{},
51+
) {
52+
}
53+
3354
func getCloudServerByName(ctx context.Context, c *hcloud.Client, name string) (*hcloud.Server, error) {
3455
const op = "hcloud/getCloudServerByName"
3556
metrics.OperationCalled.WithLabelValues(op).Inc()

‎internal/config/config.go

+14-5
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ const (
1818
hcloudNetwork = "HCLOUD_NETWORK"
1919
hcloudDebug = "HCLOUD_DEBUG"
2020

21-
robotEnabled = "ROBOT_ENABLED"
22-
robotUser = "ROBOT_USER"
23-
robotPassword = "ROBOT_PASSWORD"
24-
robotCacheTimeout = "ROBOT_CACHE_TIMEOUT"
25-
robotRateLimitWaitTime = "ROBOT_RATE_LIMIT_WAIT_TIME"
21+
robotEnabled = "ROBOT_ENABLED"
22+
robotUser = "ROBOT_USER"
23+
robotPassword = "ROBOT_PASSWORD"
24+
robotCacheTimeout = "ROBOT_CACHE_TIMEOUT"
25+
robotRateLimitWaitTime = "ROBOT_RATE_LIMIT_WAIT_TIME"
26+
robotForwardInternalIPs = "ROBOT_FORWARD_INTERNAL_IPS"
2627

2728
hcloudInstancesAddressFamily = "HCLOUD_INSTANCES_ADDRESS_FAMILY"
2829

@@ -53,6 +54,8 @@ type RobotConfiguration struct {
5354
Password string
5455
CacheTimeout time.Duration
5556
RateLimitWaitTime time.Duration
57+
// ForwardInternalIPs is enabled by default.
58+
ForwardInternalIPs bool
5659
}
5760

5861
type MetricsConfiguration struct {
@@ -143,6 +146,12 @@ func Read() (HCCMConfiguration, error) {
143146
if err != nil {
144147
errs = append(errs, err)
145148
}
149+
cfg.Robot.ForwardInternalIPs, err = getEnvBool(robotForwardInternalIPs, true)
150+
if err != nil {
151+
errs = append(errs, err)
152+
}
153+
// Robot needs to be enabled
154+
cfg.Robot.ForwardInternalIPs = cfg.Robot.ForwardInternalIPs && cfg.Robot.Enabled
146155

147156
cfg.Metrics.Enabled, err = getEnvBool(hcloudMetricsEnabled, true)
148157
if err != nil {

0 commit comments

Comments
 (0)