From baf2f951e2e0eb9a8f14ed35b1762e64eebb0e0e Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 4 Jun 2019 14:57:09 +0200 Subject: [PATCH 01/13] feat: run etcd-proxy as apiserver sidecar Signed-off-by: Martin Linkhorst --- .../master-default/userdata.clc.yaml | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index de8b7f98dc..3e5a66da3f 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -59,25 +59,6 @@ systemd: [Install] WantedBy=multi-user.target - - name: etcd-member.service - enable: true - contents: | - [Unit] - Wants=network.target - - [Service] - Type=simple - Restart=on-failure - RestartSec=5s - StartLimitIntervalSec=0 - ExecStartPre=/usr/bin/mkdir --parents /var/lib/coreos - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/lib/coreos/etcd-member-wrapper.uuid - ExecStart=/usr/bin/rkt run --uuid-file-save=/var/lib/coreos/etcd-member-wrapper.uuid --port=2379-tcp:2379 --mount volume=dns,target=/etc/resolv.conf --volume dns,kind=host,source=/run/systemd/resolve/resolv.conf,readOnly=true --insecure-options=image docker://registry.opensource.zalan.do/teapot/etcd-proxy:master-2 -- {{ .Cluster.ConfigItems.etcd_endpoints }} - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/lib/coreos/etcd-member-wrapper.uuid - - [Install] - WantedBy=multi-user.target - - name: docker.service dropins: - name: 40-flannel.conf @@ -221,7 +202,7 @@ systemd: contents: | [Unit] Description=drain this k8s node to make running pods time to gracefully shut down before stopping kubelet - After=docker.service kubelet.service etcd-member.service + After=docker.service kubelet.service [Service] Type=oneshot @@ -558,6 +539,18 @@ storage: volumeMounts: - name: config-volume mountPath: /etc/nginx + - name: etcd-proxy + image: registry.opensource.zalan.do/teapot/etcd-proxy:master-2 + - {{ .Cluster.ConfigItems.etcd_endpoints }} + ports: + - containerPort: 2379 + resources: + requests: + cpu: 25m + memory: 25Mi + limits: + cpu: 25m + memory: 25Mi volumes: - hostPath: path: /etc/kubernetes/ssl From 46ca3c707013adee537e63772823b5e5c1d9cace Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 4 Jun 2019 16:15:02 +0200 Subject: [PATCH 02/13] fix: add missing args yaml key Signed-off-by: Martin Linkhorst --- cluster/node-pools/master-default/userdata.clc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index 3e5a66da3f..ad802bc75e 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -541,6 +541,7 @@ storage: mountPath: /etc/nginx - name: etcd-proxy image: registry.opensource.zalan.do/teapot/etcd-proxy:master-2 + args: - {{ .Cluster.ConfigItems.etcd_endpoints }} ports: - containerPort: 2379 From 07198415de06c57613bf7b221950507d1bfb106c Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 4 Jun 2019 16:15:29 +0200 Subject: [PATCH 03/13] chore: drop resource limits on master pods Signed-off-by: Martin Linkhorst --- cluster/node-pools/master-default/userdata.clc.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index ad802bc75e..1a3528755e 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -549,9 +549,6 @@ storage: requests: cpu: 25m memory: 25Mi - limits: - cpu: 25m - memory: 25Mi volumes: - hostPath: path: /etc/kubernetes/ssl From 9050699c6eafc5634ad349497559d7740415233e Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 4 Jun 2019 17:06:35 +0200 Subject: [PATCH 04/13] chore: update etcd-proxy to the latest version Signed-off-by: Martin Linkhorst --- cluster/node-pools/master-default/userdata.clc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index 1a3528755e..ae0bfbf0cd 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -540,7 +540,7 @@ storage: - name: config-volume mountPath: /etc/nginx - name: etcd-proxy - image: registry.opensource.zalan.do/teapot/etcd-proxy:master-2 + image: registry.opensource.zalan.do/teapot/etcd-proxy:master-3 args: - {{ .Cluster.ConfigItems.etcd_endpoints }} ports: From c53652eba5aeb12cf95582382cdbf7474f9a64e6 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Fri, 7 Jun 2019 11:21:29 +0200 Subject: [PATCH 05/13] feat: make etcd-proxy sidecar feature configurable Signed-off-by: Martin Linkhorst --- cluster/config-defaults.yaml | 3 +++ .../master-default/userdata.clc.yaml | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 047cfad1d3..ae7b5b7ae6 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -194,6 +194,9 @@ teapot_admission_controller_ignore_namespaces: "^kube-system$" etcd_instance_count: "3" {{end}} +# toggle host vs. sidecar etcd-proxy +etcd_proxy_as_sidecar: "false" + dynamodb_service_link_enabled: "false" cluster_dns: "coredns" diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index ae0bfbf0cd..209e75f40a 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -59,6 +59,27 @@ systemd: [Install] WantedBy=multi-user.target +{{ if ne .Cluster.ConfigItems.etcd_proxy_as_sidecar "true" }} + - name: etcd-member.service + enable: true + contents: | + [Unit] + Wants=network.target + + [Service] + Type=simple + Restart=on-failure + RestartSec=5s + StartLimitIntervalSec=0 + ExecStartPre=/usr/bin/mkdir --parents /var/lib/coreos + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/lib/coreos/etcd-member-wrapper.uuid + ExecStart=/usr/bin/rkt run --uuid-file-save=/var/lib/coreos/etcd-member-wrapper.uuid --port=2379-tcp:2379 --mount volume=dns,target=/etc/resolv.conf --volume dns,kind=host,source=/run/systemd/resolve/resolv.conf,readOnly=true --insecure-options=image docker://registry.opensource.zalan.do/teapot/etcd-proxy:master-2 -- {{ .Cluster.ConfigItems.etcd_endpoints }} + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/lib/coreos/etcd-member-wrapper.uuid + + [Install] + WantedBy=multi-user.target +{{ end }} + - name: docker.service dropins: - name: 40-flannel.conf @@ -539,6 +560,7 @@ storage: volumeMounts: - name: config-volume mountPath: /etc/nginx +{{ if eq .Cluster.ConfigItems.etcd_proxy_as_sidecar "true"}} - name: etcd-proxy image: registry.opensource.zalan.do/teapot/etcd-proxy:master-3 args: @@ -549,6 +571,7 @@ storage: requests: cpu: 25m memory: 25Mi +{{ end }} volumes: - hostPath: path: /etc/kubernetes/ssl From 788425e364afa8a938f912c4179e7ed7483e3af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Mon, 24 Jun 2019 11:47:23 +0200 Subject: [PATCH 06/13] roll out restrictive TLS policy to only support >= TLS1.2 to test clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/config-defaults.yaml | 5 +++++ cluster/manifests/ingress-controller/deployment.yaml | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index ba9581f55b..87530109a2 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -11,6 +11,11 @@ autoscaling_buffer_pods: "1" autoscaling_buffer_pods: "0" {{end}} +# ALB config created by kube-aws-ingress-controller +{{if eq .Environment "test"}} +kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS-1-2-2017-01" +{{end}} + # skipper resource settings skipper_limits_mem: "250Mi" skipper_requests_cpu: "150m" diff --git a/cluster/manifests/ingress-controller/deployment.yaml b/cluster/manifests/ingress-controller/deployment.yaml index 069034f6e6..7d031b8ee6 100644 --- a/cluster/manifests/ingress-controller/deployment.yaml +++ b/cluster/manifests/ingress-controller/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-ingress-aws-controller - version: v0.8.6 + version: v0.8.7 spec: replicas: 1 selector: @@ -15,7 +15,7 @@ spec: metadata: labels: application: kube-ingress-aws-controller - version: v0.8.6 + version: v0.8.7 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "false"}} annotations: iam.amazonaws.com/role: "{{ .LocalID }}-app-ingr-ctrl" @@ -29,9 +29,12 @@ spec: serviceAccountName: kube-ingress-aws-controller containers: - name: controller - image: registry.opensource.zalan.do/teapot/kube-ingress-aws-controller:v0.8.6 + image: registry.opensource.zalan.do/teapot/kube-ingress-aws-controller:v0.8.7 args: - -stack-termination-protection +{{ if index .ConfigItems "kube_aws_ingress_controller_ssl_policy" }} + - -ssl-policy={{ .ConfigItems.kube_aws_ingress_controller_ssl_policy }} +{{ end }} env: - name: AWS_REGION value: {{ .Region }} From 63fe8e35591ed08f24487e1359fe9683e3d9325a Mon Sep 17 00:00:00 2001 From: Arjun Naik Date: Wed, 26 Jun 2019 12:05:44 +0200 Subject: [PATCH 07/13] Updated the Ubuntu images Signed-off-by: Arjun Naik --- cluster/config-defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 497145d4ba..29670333d2 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -207,7 +207,7 @@ cluster_dns: "coredns" coredns_log_svc_names: "true" coreos_image: "ami-0d1579b60bb706fb7" # Container Linux 2079.6.0 (HVM, eu-central-1) -kuberuntu_image: "ami-0d856c4c2daf9b569" # Kuberuntu (dev) (HVM, eu-central-1) +kuberuntu_image: "ami-0d0d49d08e198103c" # Kuberuntu (dev) (HVM, eu-central-1) # Feature toggle to allow gradual decommissioning of ingress-template-controller enable_ingress_template_controller: "false" From 2f2b8ca4b82e874e869be2681dab13821a6afa66 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Wed, 26 Jun 2019 13:42:40 +0200 Subject: [PATCH 08/13] make external service account token usage configurable Signed-off-by: Martin Linkhorst --- cluster/config-defaults.yaml | 4 ++++ cluster/node-pools/master-default/userdata.clc.yaml | 2 +- cluster/node-pools/master-ubuntu-default/userdata.yaml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 497145d4ba..5630864886 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -242,5 +242,9 @@ node_cidr_mask_size: "24" # Default: 24 # when set to true, routes external traffic to the apiserver through a skipper sidecar apiserver_proxy: "true" +# when set to true, service account tokens can be used from outside the cluster +# requires apiserver_proxy to be set to "true" +allow_external_service_accounts: "true" + # use kube-aws-iam-controller for kube-system components kube_aws_iam_controller_kube_system_enable: "false" diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index 81bd5886cd..c29be8ad80 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -575,7 +575,7 @@ storage: - -enable-prometheus-metrics - -write-timeout-server=60m - -inline-routes - - 'z: JWTPayloadAnyKV("iss", "kubernetes/serviceaccount") -> enableAccessLog() -> "https://127.0.0.1:443"; h: Path("/kube-system/healthz") -> setPath("/healthz") -> disableAccessLog() -> "http://127.0.0.1:8080"; all: * -> disableAccessLog() -> "https://127.0.0.1:443";' + - 's: JWTPayloadAllKV("iss", "kubernetes/serviceaccount") -> enableAccessLog() -> {{ if eq .ConfigItems.allow_external_service_accounts "true" }}"https://127.0.0.1:443"{{ else }}status(401) -> {{ end }}; h: Path("/kube-system/healthz") -> setPath("/healthz") -> disableAccessLog() -> "http://127.0.0.1:8080"; all: * -> disableAccessLog() -> "https://127.0.0.1:443";' ports: - containerPort: 8443 readinessProbe: diff --git a/cluster/node-pools/master-ubuntu-default/userdata.yaml b/cluster/node-pools/master-ubuntu-default/userdata.yaml index 31c93b089c..a9ba1d68f5 100644 --- a/cluster/node-pools/master-ubuntu-default/userdata.yaml +++ b/cluster/node-pools/master-ubuntu-default/userdata.yaml @@ -311,7 +311,7 @@ write_files: - -enable-prometheus-metrics - -write-timeout-server=60m - -inline-routes - - 'z: JWTPayloadAnyKV("iss", "kubernetes/serviceaccount") -> enableAccessLog() -> "https://127.0.0.1:443"; h: Path("/kube-system/healthz") -> setPath("/healthz") -> disableAccessLog() -> "http://127.0.0.1:8080"; all: * -> disableAccessLog() -> "https://127.0.0.1:443";' + - 's: JWTPayloadAllKV("iss", "kubernetes/serviceaccount") -> enableAccessLog() -> {{ if eq .ConfigItems.allow_external_service_accounts "true" }}"https://127.0.0.1:443"{{ else }}status(401) -> {{ end }}; h: Path("/kube-system/healthz") -> setPath("/healthz") -> disableAccessLog() -> "http://127.0.0.1:8080"; all: * -> disableAccessLog() -> "https://127.0.0.1:443";' ports: - containerPort: 8443 readinessProbe: From d8a4acd17cde2a2d2113c0aabaf86a0bf64d8d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Wed, 26 Jun 2019 15:51:38 +0200 Subject: [PATCH 09/13] via ingress overwritable ssl policy, set defaults to test (target policy) and prod (current poilcy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/config-defaults.yaml | 2 ++ cluster/manifests/ingress-controller/deployment.yaml | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 87530109a2..9502ef3385 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -14,6 +14,8 @@ autoscaling_buffer_pods: "0" # ALB config created by kube-aws-ingress-controller {{if eq .Environment "test"}} kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-TLS-1-2-2017-01" +{{else}} +kube_aws_ingress_controller_ssl_policy: "ELBSecurityPolicy-2016-08" {{end}} # skipper resource settings diff --git a/cluster/manifests/ingress-controller/deployment.yaml b/cluster/manifests/ingress-controller/deployment.yaml index 7d031b8ee6..eb7cb16fd8 100644 --- a/cluster/manifests/ingress-controller/deployment.yaml +++ b/cluster/manifests/ingress-controller/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-ingress-aws-controller - version: v0.8.7 + version: v0.8.8 spec: replicas: 1 selector: @@ -15,7 +15,7 @@ spec: metadata: labels: application: kube-ingress-aws-controller - version: v0.8.7 + version: v0.8.8 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "false"}} annotations: iam.amazonaws.com/role: "{{ .LocalID }}-app-ingr-ctrl" @@ -29,12 +29,10 @@ spec: serviceAccountName: kube-ingress-aws-controller containers: - name: controller - image: registry.opensource.zalan.do/teapot/kube-ingress-aws-controller:v0.8.7 + image: registry.opensource.zalan.do/teapot/kube-ingress-aws-controller:v0.8.8 args: - -stack-termination-protection -{{ if index .ConfigItems "kube_aws_ingress_controller_ssl_policy" }} - -ssl-policy={{ .ConfigItems.kube_aws_ingress_controller_ssl_policy }} -{{ end }} env: - name: AWS_REGION value: {{ .Region }} From 1d3b1633bf2f8f5d601f081061d67a4712fa2167 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Tue, 25 Jun 2019 13:43:37 +0200 Subject: [PATCH 10/13] Give CDP controller access to CRDs that it needs to manage Signed-off-by: Mikkel Oscar Lyderik Larsen --- .../manifests/roles/cdp-controller-rbac.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cluster/manifests/roles/cdp-controller-rbac.yaml b/cluster/manifests/roles/cdp-controller-rbac.yaml index 4a8e934cae..983831ed45 100644 --- a/cluster/manifests/roles/cdp-controller-rbac.yaml +++ b/cluster/manifests/roles/cdp-controller-rbac.yaml @@ -34,6 +34,27 @@ rules: - list - watch - patch +- apiGroups: + - "zalando.org" + resources: + - awsiamroles + verbs: + - get + - list + - watch + - create +- apiGroups: + - "zalando.org" + resources: + - gradualdeployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding From 70674b05e0bcb595a22f8cd9ffc3c81be41cfdfb Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Thu, 27 Jun 2019 11:07:16 +0200 Subject: [PATCH 11/13] external-dns: reduce route53 batch size to 100 Signed-off-by: Martin Linkhorst --- cluster/manifests/external-dns/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/external-dns/deployment.yaml b/cluster/manifests/external-dns/deployment.yaml index 6fdd369c3f..4c28bbeb8a 100644 --- a/cluster/manifests/external-dns/deployment.yaml +++ b/cluster/manifests/external-dns/deployment.yaml @@ -35,7 +35,7 @@ spec: - --provider=aws - --registry=txt - --txt-owner-id={{ .Region }}:{{ .LocalID }} - - --aws-batch-change-size=350 + - --aws-batch-change-size=100 resources: limits: cpu: 50m From 97adece3ad82d56b7001bb3dd1c4621d8878721a Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Thu, 27 Jun 2019 11:39:33 +0200 Subject: [PATCH 12/13] feat: enable etcd-proxy sidecar for ubuntu-based node pools Signed-off-by: Martin Linkhorst --- .../master-ubuntu-default/userdata.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cluster/node-pools/master-ubuntu-default/userdata.yaml b/cluster/node-pools/master-ubuntu-default/userdata.yaml index a9ba1d68f5..0d2b95d807 100644 --- a/cluster/node-pools/master-ubuntu-default/userdata.yaml +++ b/cluster/node-pools/master-ubuntu-default/userdata.yaml @@ -1,13 +1,17 @@ #cloud-config runcmd: - [ systemctl, start, gen-controller-manager-config.service ] +{{ if ne .Cluster.ConfigItems.etcd_proxy_as_sidecar "true"}} - [ systemctl, start, etcd-member.service ] +{{ end }} write_files: +{{ if ne .Cluster.ConfigItems.etcd_proxy_as_sidecar "true"}} - owner: root:root path: /etc/etcd-member/environment content: | ETCD_ENDPOINT={{ .Cluster.ConfigItems.etcd_endpoints }} +{{ end }} - owner: root:root path: /etc/kubernetes/secrets.env @@ -332,6 +336,18 @@ write_files: - mountPath: /etc/kubernetes/ssl name: ssl-certs-kubernetes readOnly: true +{{ if eq .Cluster.ConfigItems.etcd_proxy_as_sidecar "true"}} + - name: etcd-proxy + image: registry.opensource.zalan.do/teapot/etcd-proxy:master-3 + args: + - {{ .Cluster.ConfigItems.etcd_endpoints }} + ports: + - containerPort: 2379 + resources: + requests: + cpu: 25m + memory: 25Mi +{{ end }} volumes: - hostPath: path: /etc/kubernetes/ssl From 6bdd5381dc0fbfcc20f46e4b0ccd62528adc51fe Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Thu, 27 Jun 2019 11:41:00 +0200 Subject: [PATCH 13/13] chore: enable etcd-proxy sidecar by default Signed-off-by: Martin Linkhorst --- cluster/config-defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 6e097e2458..1244c0fe41 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -209,7 +209,7 @@ etcd_instance_count: "3" {{end}} # toggle host vs. sidecar etcd-proxy -etcd_proxy_as_sidecar: "false" +etcd_proxy_as_sidecar: "true" dynamodb_service_link_enabled: "false"